From 37d4213732affc53f4bdd0b43e87254e325e45ea Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 24 Jun 2025 09:52:52 +0800 Subject: [PATCH 1/2] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20=E9=9B=86=E6=88=90?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=AE=B0=E5=BD=95=E6=94=AF=E6=8C=81=E5=85=A5?= =?UTF-8?q?=E5=8F=82=E3=80=81=E7=BB=93=E6=9E=9C=E3=80=81=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E7=9A=84=E5=86=85=E5=AE=B9=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1436241948540928]集成调用记录支持入参、结果、错误信息的内容进行过滤 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1436241948540928 --- .../core/FullTextIndexHandlerBase.java | 22 ++- .../core/FullTextIndexInitializer.java | 1 + .../core/IFullTextIndexHandler.java | 3 + .../dto/fulltextindex/FullTextIndexVo.java | 16 ++ .../enums/FrameworkFullTextIndexType.java | 59 ++++++ .../core/IntegrationHandlerBase.java | 30 ++- .../IntegrationCrossoverService.java | 7 + .../dao/mapper/IntegrationMapper.java | 14 ++ .../dao/mapper/IntegrationMapper.xml | 134 ++++++++++--- .../integration/dto/IntegrationAuditVo.java | 18 ++ .../IntegrationAuditFullTextIndexHandler.java | 177 ++++++++++++++++++ .../service/IntegrationService.java | 7 + .../service/IntegrationServiceImpl.java | 22 +++ 13 files changed, 483 insertions(+), 27 deletions(-) create mode 100644 src/main/java/neatlogic/framework/fulltextindex/enums/FrameworkFullTextIndexType.java create mode 100644 src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java diff --git a/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexHandlerBase.java b/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexHandlerBase.java index da5d656e2..c30b2f0c2 100644 --- a/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexHandlerBase.java +++ b/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexHandlerBase.java @@ -15,6 +15,7 @@ along with this program. If not, see .*/ package neatlogic.framework.fulltextindex.core; +import com.alibaba.fastjson.JSONObject; import neatlogic.framework.asynchronization.thread.NeatLogicThread; import neatlogic.framework.asynchronization.threadlocal.TenantContext; import neatlogic.framework.asynchronization.threadlocal.UserContext; @@ -91,6 +92,10 @@ public abstract class FullTextIndexHandlerBase implements IFullTextIndexHandler createIndex(targetId, isSync, null); } + protected final void createIndex(Long targetId, JSONObject dataObj, boolean isSync) { + createIndex(targetId, isSync, null, dataObj); + } + /** * 给重建索引使用的方法，以同步方式执行索引创建 * @@ -98,9 +103,19 @@ public abstract class FullTextIndexHandlerBase implements IFullTextIndexHandler * @param isSync 是否同步 */ protected final void createIndex(Long targetId, boolean isSync, Semaphore lock) { + this.createIndex(targetId, isSync, lock, null); + } + + /** + * 给重建索引使用的方法，以同步方式执行索引创建 + * + * @param targetId 目标id + * @param isSync 是否同步 + */ + protected final void createIndex(Long targetId, boolean isSync, Semaphore lock, JSONObject dataObj) { AfterTransactionJob job = new AfterTransactionJob<>("FULLTEXTINDEX-CREATE-" + this.getType().getType().toUpperCase(Locale.ROOT) + "-" + targetId); String moduleId = this.getModuleId(); - job.execute(new FullTextIndexVo(targetId, this.getType().getType()), fullTextIndexVo -> { + job.execute(new FullTextIndexVo(targetId, this.getType().getType(), dataObj), fullTextIndexVo -> { //System.out.println("创建索引"); //删除索引 fullTextIndexMapper.deleteFullTextIndexByTargetIdAndType(fullTextIndexVo, moduleId); @@ -194,6 +209,11 @@ public abstract class FullTextIndexHandlerBase implements IFullTextIndexHandler createIndex(targetId, false, null); } + @Override + public final void createIndex(Long targetId, JSONObject dataObj) { + createIndex(targetId, false, null, dataObj); + } + @Override public final void createIndex(Long targetId, Semaphore lock) { //System.out.println("重建索引开始"); diff --git a/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexInitializer.java b/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexInitializer.java index cfa5ae835..17829790f 100644 --- a/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexInitializer.java +++ b/src/main/java/neatlogic/framework/fulltextindex/core/FullTextIndexInitializer.java @@ -50,6 +50,7 @@ public class FullTextIndexInitializer extends ModuleInitializedListenerBase { FULLTEXT_INDEX_MODULE_MAP.add("cmdb"); FULLTEXT_INDEX_MODULE_MAP.add("autoexec"); FULLTEXT_INDEX_MODULE_MAP.add("rdm"); + FULLTEXT_INDEX_MODULE_MAP.add("framework"); } @Override diff --git a/src/main/java/neatlogic/framework/fulltextindex/core/IFullTextIndexHandler.java b/src/main/java/neatlogic/framework/fulltextindex/core/IFullTextIndexHandler.java index 5c0f1afc5..25bf516f0 100644 --- a/src/main/java/neatlogic/framework/fulltextindex/core/IFullTextIndexHandler.java +++ b/src/main/java/neatlogic/framework/fulltextindex/core/IFullTextIndexHandler.java @@ -16,6 +16,7 @@ along with this program. If not, see .*/ package neatlogic.framework.fulltextindex.core; +import com.alibaba.fastjson.JSONObject; import neatlogic.framework.fulltextindex.dto.globalsearch.DocumentVo; import java.util.concurrent.Semaphore; @@ -49,6 +50,8 @@ public interface IFullTextIndexHandler { **/ void createIndex(Long targetId); + void createIndex(Long targetId, JSONObject dataObj); + void createIndex(Long targetId, Semaphore lock); /* diff --git a/src/main/java/neatlogic/framework/fulltextindex/dto/fulltextindex/FullTextIndexVo.java b/src/main/java/neatlogic/framework/fulltextindex/dto/fulltextindex/FullTextIndexVo.java index 1f8d6f672..7393cf9b8 100644 --- a/src/main/java/neatlogic/framework/fulltextindex/dto/fulltextindex/FullTextIndexVo.java +++ b/src/main/java/neatlogic/framework/fulltextindex/dto/fulltextindex/FullTextIndexVo.java @@ -15,6 +15,7 @@ along with this program. If not, see .*/ package neatlogic.framework.fulltextindex.dto.fulltextindex; +import com.alibaba.fastjson.JSONObject; import neatlogic.framework.common.constvalue.ApiParamType; import neatlogic.framework.fulltextindex.utils.FullTextIndexUtil; import neatlogic.framework.restful.annotation.EntityField; @@ -40,6 +41,7 @@ public class FullTextIndexVo { private List wordOffsetVoList; private final Map fieldContentMap = new HashMap<>(); private Map> wordOffsetMap = new HashMap<>(); + private JSONObject dataObj; public FullTextIndexVo() { } @@ -49,6 +51,12 @@ public class FullTextIndexVo { targetType = _targetType; } + public FullTextIndexVo(Long _targetId, String _targetType, JSONObject _dataObj) { + targetId = _targetId; + targetType = _targetType; + dataObj = _dataObj; + } + public Long getTargetId() { return targetId; } @@ -198,6 +206,14 @@ public class FullTextIndexVo { this.targetField = targetField; } + public JSONObject getDataObj() { + return dataObj; + } + + public void setDataObj(JSONObject dataObj) { + this.dataObj = dataObj; + } + public Map getFieldContentMap() { return fieldContentMap; } diff --git a/src/main/java/neatlogic/framework/fulltextindex/enums/FrameworkFullTextIndexType.java b/src/main/java/neatlogic/framework/fulltextindex/enums/FrameworkFullTextIndexType.java new file mode 100644 index 000000000..2d3506d10 --- /dev/null +++ b/src/main/java/neatlogic/framework/fulltextindex/enums/FrameworkFullTextIndexType.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.fulltextindex.enums; + +import neatlogic.framework.fulltextindex.core.IFullTextIndexType; +import neatlogic.framework.util.$; + +public enum FrameworkFullTextIndexType implements IFullTextIndexType { + INTEGRATION_AUDIT("integration_audit", "集成审计"), + ; + + private final String type; + private final String typeName; + + FrameworkFullTextIndexType(String type, String typeName) { + this.type = type; + this.typeName = typeName; + } + + @Override + public String getType() { + return this.type; + } + + @Override + public String getTypeName() { + return $.t(typeName); + } + + @Override + public String getTypeName(String type) { + for (FrameworkFullTextIndexType t : values()) { + if (t.getType().equals(type)) { + return t.getTypeName(); + } + } + return ""; + } + + @Override + public boolean isActiveGlobalSearch() { + return false; + } +} diff --git a/src/main/java/neatlogic/framework/integration/core/IntegrationHandlerBase.java b/src/main/java/neatlogic/framework/integration/core/IntegrationHandlerBase.java index b451669c0..a9ad979eb 100644 --- a/src/main/java/neatlogic/framework/integration/core/IntegrationHandlerBase.java +++ b/src/main/java/neatlogic/framework/integration/core/IntegrationHandlerBase.java @@ -30,8 +30,12 @@ import neatlogic.framework.exception.type.ParamTypeNotFoundException; import neatlogic.framework.file.core.AuditType; import neatlogic.framework.file.core.Event; import neatlogic.framework.file.core.appender.AppenderManager; +import neatlogic.framework.fulltextindex.core.FullTextIndexHandlerFactory; +import neatlogic.framework.fulltextindex.core.IFullTextIndexHandler; +import neatlogic.framework.fulltextindex.enums.FrameworkFullTextIndexType; import neatlogic.framework.integration.authentication.core.AuthenticateHandlerFactory; import neatlogic.framework.integration.authentication.core.IAuthenticateHandler; +import neatlogic.framework.integration.crossover.IntegrationCrossoverService; import neatlogic.framework.integration.dto.IntegrationAuditVo; import neatlogic.framework.integration.dto.IntegrationResultVo; import neatlogic.framework.integration.dto.IntegrationVo; @@ -199,11 +203,22 @@ public abstract class IntegrationHandlerBase implements IIntegrationHandler { } + List fullIndexParamValueList = new ArrayList<>(); integrationAuditVo.setRequestFrom(iRequestFrom.toString()); integrationAuditVo.setUserUuid(UserContext.get().getUserUuid());// 用户非必填，因作业不存在登录用户 integrationAuditVo.setIntegrationUuid(integrationVo.getUuid()); integrationAuditVo.setStartTime(new Date()); if (MapUtils.isNotEmpty(requestParamObj)) { + IntegrationCrossoverService integrationCrossoverService = CrossoverServiceFactory.getApi(IntegrationCrossoverService.class); + List searchAbleInputParamNameList = integrationCrossoverService.getSearchAbleInputParamNameList(integrationVo); + if (CollectionUtils.isNotEmpty(searchAbleInputParamNameList)) { + for (String paramName : searchAbleInputParamNameList) { + Object paramValue = requestParamObj.get(paramName); + if (paramValue != null) { + fullIndexParamValueList.add(paramValue.toString()); + } + } + } integrationAuditVo.setParam(requestParamObj.toJSONString()); } @@ -364,7 +379,7 @@ public abstract class IntegrationHandlerBase implements IIntegrationHandler { // NeatLogicThread thread = new IntegrationAuditSaveThread(integrationAuditVo); // thread.setThreadName("INTEGRATION-AUDIT-SAVER-" + integrationVo.getUuid()); // CachedThreadPool.execute(thread); - + Long auditId = integrationAuditVo.getId(); JSONObject data = new JSONObject(); data.put("integrationAudit", integrationAuditVo); String param = integrationAuditVo.getParam(); @@ -389,10 +404,15 @@ public abstract class IntegrationHandlerBase implements IIntegrationHandler { IntegrationAuditAppendPreProcessor appendPreProcessor = CrossoverServiceFactory.getApi(IntegrationAuditAppendPreProcessor.class); AppenderManager.execute(new Event(integrationVo.getName(), integrationAuditVo.getStartTime().getTime(), data, appendPreProcessor, appendPostProcessor, AuditType.INTEGRATION_AUDIT)); - // connection.disconnect(); //Indicates that other requests to the - // server are unlikely in the near future. Calling disconnect() should - // not imply that this HttpURLConnection - // instance can be reused for other requests. + //创建全文检索索引 + if (CollectionUtils.isNotEmpty(fullIndexParamValueList)) { + IFullTextIndexHandler indexHandler = FullTextIndexHandlerFactory.getHandler(FrameworkFullTextIndexType.INTEGRATION_AUDIT); + if (indexHandler != null) { + JSONObject dataObj = new JSONObject(); + dataObj.put("param", String.join(",", fullIndexParamValueList)); + indexHandler.createIndex(auditId, dataObj); + } + } return resultVo; } } diff --git a/src/main/java/neatlogic/framework/integration/crossover/IntegrationCrossoverService.java b/src/main/java/neatlogic/framework/integration/crossover/IntegrationCrossoverService.java index 51639ffcb..da7193f08 100644 --- a/src/main/java/neatlogic/framework/integration/crossover/IntegrationCrossoverService.java +++ b/src/main/java/neatlogic/framework/integration/crossover/IntegrationCrossoverService.java @@ -50,4 +50,11 @@ public interface IntegrationCrossoverService extends ICrossoverService { * @param sourceColumnList */ boolean mergeFilterListAndSourceColumnList(JSONArray filterList, List sourceColumnList); + + /** + * 找出允许搜索的输入参数名称列表 + * @param integrationVo + * @return + */ + List getSearchAbleInputParamNameList(IntegrationVo integrationVo); } diff --git a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.java b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.java index 2e7a1aaed..3c3918bc3 100644 --- a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.java +++ b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.java @@ -18,15 +18,22 @@ package neatlogic.framework.integration.dao.mapper; import neatlogic.framework.common.dto.ValueTextVo; import neatlogic.framework.integration.dto.IntegrationAuditVo; import neatlogic.framework.integration.dto.IntegrationVo; +import org.apache.ibatis.annotations.Param; import java.util.List; public interface IntegrationMapper { + IntegrationAuditVo getIntegrationAuditById(Long id); + + List getIntegrationAuditListByIdList(List idList); + List searchIntegrationAudit(IntegrationAuditVo integrationAuditVo); int getIntegrationAuditCount(IntegrationAuditVo integrationAuditVo); + List getIntegrationAuditIdList(IntegrationAuditVo integrationAuditVo); + IntegrationVo getIntegrationByUuid(String uuid); IntegrationVo getIntegrationByName(String name); @@ -45,6 +52,13 @@ public interface IntegrationMapper { List checkUuidListExists(List uuidList); + List getNotIndexIntegrationAuditIdList( + @Param("integrationUuid") String integrationUuid, + @Param("id") Long startId, + @Param("type") String type, + @Param("pageSize") Integer pageSize + ); + int insertIntegration(IntegrationVo integrationVo); int updateIntegration(IntegrationVo integrationVo); diff --git a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml index 0b584d2fe..91ae694f9 100644 --- a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml +++ b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml @@ -16,33 +16,77 @@ along with this program. If not, see .--> + + + + + `start_time` >= #{startTime} AND `end_time` <= #{endTime} - AND id in + AND a.id in #{item} - AND integration_uuid = #{integrationUuid} + AND a.integration_uuid = #{integrationUuid} - AND integration_uuid in + AND a.integration_uuid in #{item} - AND `user_uuid` IN + AND a.`user_uuid` IN #{uuid} - AND `status` IN + AND a.`status` IN #{status} + + and fw.word in + + #{item} + + @@ -50,23 +94,23 @@ along with this program. If not, see .--> + + + + + UPDATE `integration` diff --git a/src/main/java/neatlogic/framework/integration/dto/IntegrationAuditVo.java b/src/main/java/neatlogic/framework/integration/dto/IntegrationAuditVo.java index b3b5c2520..ea6ec38ca 100644 --- a/src/main/java/neatlogic/framework/integration/dto/IntegrationAuditVo.java +++ b/src/main/java/neatlogic/framework/integration/dto/IntegrationAuditVo.java @@ -7,6 +7,7 @@ import neatlogic.framework.common.config.Config; import neatlogic.framework.common.constvalue.ApiParamType; import neatlogic.framework.common.constvalue.GroupSearch; import neatlogic.framework.common.dto.BasePageVo; +import neatlogic.framework.fulltextindex.utils.FullTextIndexUtil; import neatlogic.framework.restful.annotation.EntityField; import neatlogic.framework.util.SnowflakeUtil; import org.apache.commons.collections4.CollectionUtils; @@ -62,6 +63,8 @@ public class IntegrationAuditVo extends BasePageVo implements AuditVoHandler { private JSONObject headers; @EntityField(name = "请求头字符串", type = ApiParamType.STRING) private String headersStr; + @EntityField(name = "入参关键字", type = ApiParamType.STRING) + private String paramKeyword; public Long getId() { if (id == null) { @@ -266,4 +269,19 @@ public class IntegrationAuditVo extends BasePageVo implements AuditVoHandler { } return null; } + + public String getParamKeyword() { + return paramKeyword; + } + + public void setParamKeyword(String paramKeyword) { + this.paramKeyword = paramKeyword; + } + + public final List getParamWordList() { + if (StringUtils.isNotBlank(this.paramKeyword)) { + return FullTextIndexUtil.sliceKeyword(this.paramKeyword); + } + return null; + } } diff --git a/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java b/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java new file mode 100644 index 000000000..402bd19bc --- /dev/null +++ b/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.module.framework.fulltextindex; + +import com.alibaba.fastjson.JSONObject; +import neatlogic.framework.common.config.Config; +import neatlogic.framework.crossover.CrossoverServiceFactory; +import neatlogic.framework.crossover.IFileCrossoverService; +import neatlogic.framework.file.dto.AuditFilePathVo; +import neatlogic.framework.fulltextindex.core.FullTextIndexHandlerBase; +import neatlogic.framework.fulltextindex.core.IFullTextIndexType; +import neatlogic.framework.fulltextindex.dto.fulltextindex.FullTextIndexTypeVo; +import neatlogic.framework.fulltextindex.dto.fulltextindex.FullTextIndexVo; +import neatlogic.framework.fulltextindex.dto.globalsearch.DocumentVo; +import neatlogic.framework.fulltextindex.enums.FrameworkFullTextIndexType; +import neatlogic.framework.integration.crossover.IntegrationCrossoverService; +import neatlogic.framework.integration.dao.mapper.IntegrationMapper; +import neatlogic.framework.integration.dto.IntegrationAuditVo; +import neatlogic.framework.integration.dto.IntegrationVo; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +@Component +public class IntegrationAuditFullTextIndexHandler extends FullTextIndexHandlerBase { + + @Resource + private IntegrationMapper integrationMapper; + + @Override + protected String getModuleId() { + return "framework"; + } + + @Override + protected void myCreateIndex(FullTextIndexVo fullTextIndexVo) { + JSONObject dataObj = fullTextIndexVo.getDataObj(); + if (MapUtils.isNotEmpty(dataObj)) { + String value = dataObj.getString("param"); + if (StringUtils.isNotBlank(value)) { + fullTextIndexVo.addFieldContent("param", new FullTextIndexVo.WordVo(value)); + } + } +// IntegrationAuditVo integrationAuditVo = integrationMapper.getIntegrationAuditById(fullTextIndexVo.getTargetId()); +// if (integrationAuditVo != null && StringUtils.isNotBlank(integrationAuditVo.getParamFilePath())) { +// IntegrationVo integrationVo = integrationMapper.getIntegrationByUuid(integrationAuditVo.getIntegrationUuid()); +// if (integrationVo != null) { +// JSONObject config = integrationVo.getConfig(); +// JSONObject param = config.getJSONObject("param"); +// if (MapUtils.isNotEmpty(param)) { +// JSONArray paramList = param.getJSONArray("paramList"); +// if (CollectionUtils.isNotEmpty(paramList)) { +// List searchAbleParamNameList = new ArrayList<>(); +// for (int i = 0; i < paramList.size(); i++) { +// JSONObject paramObj = paramList.getJSONObject(i); +// String name = paramObj.getString("name"); +// String mode = paramObj.getString("mode"); +// Integer isSearchAble = paramObj.getInteger("isSearchAble"); +// if (StringUtils.isNotBlank(name) && Objects.equals(mode, "input") && Objects.equals(isSearchAble, 1)) { +// searchAbleParamNameList.add(name); +// } +// } +// if (CollectionUtils.isNotEmpty(searchAbleParamNameList)) { +// List paramValueList = new ArrayList<>(); +// String content = getContent(integrationAuditVo.getParamFilePath()); +// if (StringUtils.isNotBlank(content) && content.startsWith("{") && content.endsWith("}")) { +// JSONObject paramObj = JSONObject.parseObject(content); +// for (String paramName : searchAbleParamNameList) { +// Object paramValue = paramObj.get(paramName); +// if (paramValue != null) { +// paramValueList.add(paramValue.toString()); +// } +// } +// } +// if (CollectionUtils.isNotEmpty(paramValueList)) { +// fullTextIndexVo.addFieldContent("param", new FullTextIndexVo.WordVo(String.join(",", paramValueList))); +// } +// } +// } +// } +// } +// } + } + + @Override + protected void myMakeupDocument(DocumentVo documentVo) { + + } + + @Override + protected void myRebuildIndex(FullTextIndexTypeVo fullTextIndexTypeVo) { + IntegrationVo searchVo = new IntegrationVo(); + int rowNum = integrationMapper.searchIntegrationCount(searchVo); + if (rowNum > 0) { + searchVo.setRowNum(rowNum); + searchVo.setPageSize(100); + Integer pageCount = searchVo.getPageCount(); + for (int currentPage = 1; currentPage < pageCount; currentPage++) { + searchVo.setCurrentPage(currentPage); + List integrationList = integrationMapper.searchIntegration(searchVo); + for (IntegrationVo integrationVo : integrationList) { + IntegrationCrossoverService integrationCrossoverService = CrossoverServiceFactory.getApi(IntegrationCrossoverService.class); + List searchAbleInputParamNameList = integrationCrossoverService.getSearchAbleInputParamNameList(integrationVo); + if (CollectionUtils.isNotEmpty(searchAbleInputParamNameList)) { + Long startId = 0L; + List notIndexApiAuditIdList = integrationMapper.getNotIndexIntegrationAuditIdList(integrationVo.getUuid(), startId, fullTextIndexTypeVo.getType(), 100); + while (CollectionUtils.isNotEmpty(notIndexApiAuditIdList)) { + List integrationAuditList = integrationMapper.getIntegrationAuditListByIdList(notIndexApiAuditIdList); + for (IntegrationAuditVo integrationAuditVo : integrationAuditList) { + if (StringUtils.isNotBlank(integrationAuditVo.getParamFilePath())) { + String content = getContent(integrationAuditVo.getParamFilePath()); + if (StringUtils.isNotBlank(content) && content.startsWith("{") && content.endsWith("}")) { + List fullIndexParamValueList = new ArrayList<>(); + JSONObject requestParamObj = JSONObject.parseObject(content); + for (String paramName : searchAbleInputParamNameList) { + Object paramValue = requestParamObj.get(paramName); + if (paramValue != null) { + fullIndexParamValueList.add(paramValue.toString()); + } + } + if (CollectionUtils.isNotEmpty(fullIndexParamValueList)) { + JSONObject dataObj = new JSONObject(); + dataObj.put("param", String.join(",", fullIndexParamValueList)); + this.createIndex(integrationAuditVo.getId(), dataObj, true); + } + } + } + } + startId = notIndexApiAuditIdList.get(notIndexApiAuditIdList.size() - 1); + notIndexApiAuditIdList = integrationMapper.getNotIndexIntegrationAuditIdList(integrationVo.getUuid(), startId, fullTextIndexTypeVo.getType(), 100); + } + } + } + } + } + } + + @Override + public IFullTextIndexType getType() { + return FrameworkFullTextIndexType.INTEGRATION_AUDIT; + } + + private String getContent(String filePath) { + AuditFilePathVo auditFilePathVo = new AuditFilePathVo(filePath); + IFileCrossoverService fileCrossoverService = CrossoverServiceFactory.getApi(IFileCrossoverService.class); + if (Objects.equals(auditFilePathVo.getServerId(), Config.SCHEDULE_SERVER_ID)) { + JSONObject jsonObj = fileCrossoverService.readLocalFile(auditFilePathVo.getPath(), auditFilePathVo.getStartIndex(), auditFilePathVo.getOffset()); + return jsonObj.getString("content"); + } else { + JSONObject paramObj = new JSONObject(); + paramObj.put("filePath", filePath); + JSONObject jsonObj = fileCrossoverService.readRemoteFile(paramObj, auditFilePathVo.getServerId()); + return jsonObj.getString("content"); + } + } +} diff --git a/src/main/java/neatlogic/module/framework/integration/service/IntegrationService.java b/src/main/java/neatlogic/module/framework/integration/service/IntegrationService.java index f3d95445d..8d4bbf27b 100644 --- a/src/main/java/neatlogic/module/framework/integration/service/IntegrationService.java +++ b/src/main/java/neatlogic/module/framework/integration/service/IntegrationService.java @@ -51,4 +51,11 @@ public interface IntegrationService { * @param sourceColumnList */ boolean mergeFilterListAndSourceColumnList(JSONArray filterList, List sourceColumnList); + + /** + * 找出允许搜索的输入参数名称列表 + * @param integrationVo + * @return + */ + List getSearchAbleInputParamNameList(IntegrationVo integrationVo); } diff --git a/src/main/java/neatlogic/module/framework/integration/service/IntegrationServiceImpl.java b/src/main/java/neatlogic/module/framework/integration/service/IntegrationServiceImpl.java index c343719d3..576eece89 100644 --- a/src/main/java/neatlogic/module/framework/integration/service/IntegrationServiceImpl.java +++ b/src/main/java/neatlogic/module/framework/integration/service/IntegrationServiceImpl.java @@ -339,6 +339,28 @@ public class IntegrationServiceImpl implements IntegrationService, IntegrationCr return true; } + @Override + public List getSearchAbleInputParamNameList(IntegrationVo integrationVo) { + List searchAbleInputParamNameList = new ArrayList<>(); + JSONObject config = integrationVo.getConfig(); + JSONObject param = config.getJSONObject("param"); + if (MapUtils.isNotEmpty(param)) { + JSONArray paramList = param.getJSONArray("paramList"); + if (CollectionUtils.isNotEmpty(paramList)) { + for (int i = 0; i < paramList.size(); i++) { + JSONObject paramObj = paramList.getJSONObject(i); + String name = paramObj.getString("name"); + String mode = paramObj.getString("mode"); + Integer isSearchAble = paramObj.getInteger("isSearchAble"); + if (StringUtils.isNotBlank(name) && Objects.equals(mode, "input") && Objects.equals(isSearchAble, 1)) { + searchAbleInputParamNameList.add(name); + } + } + } + } + return searchAbleInputParamNameList; + } + private List getSearchColumnDetailList(String integrationUuid, List columnVoList, JSONArray searchColumnArray) { Map columnVoMap = new HashMap<>(); for (ColumnVo columnVo : columnVoList) { -- Gitee From 627dbb67ff1f562d9ee739a4e4fa488d3fc871a6 Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 24 Jun 2025 18:18:37 +0800 Subject: [PATCH 2/2] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20=E9=9B=86=E6=88=90?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=AE=B0=E5=BD=95=E6=94=AF=E6=8C=81=E5=85=A5?= =?UTF-8?q?=E5=8F=82=E3=80=81=E7=BB=93=E6=9E=9C=E3=80=81=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E7=9A=84=E5=86=85=E5=AE=B9=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1436241948540928]集成调用记录支持入参、结果、错误信息的内容进行过滤 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1436241948540928 --- .../dao/mapper/IntegrationMapper.xml | 102 ++++++++++-------- .../IntegrationAuditFullTextIndexHandler.java | 70 ++++-------- 2 files changed, 76 insertions(+), 96 deletions(-) diff --git a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml index 91ae694f9..91601c457 100644 --- a/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml +++ b/src/main/java/neatlogic/framework/integration/dao/mapper/IntegrationMapper.xml @@ -55,39 +55,31 @@ along with this program. If not, see .--> - - `start_time` >= #{startTime} - AND `end_time` <= #{endTime} - - AND a.id in - #{item} - - - AND a.integration_uuid = #{integrationUuid} - - - AND a.integration_uuid in - #{item} - - - AND a.`user_uuid` IN - - #{uuid} - - - - AND a.`status` IN - - #{status} - - - - and fw.word in - - #{item} - - - + `start_time` >= #{startTime} + AND `end_time` <= #{endTime} + + AND a.id in + #{item} + + + AND a.integration_uuid = #{integrationUuid} + + + AND a.integration_uuid in + #{item} + + + AND a.`user_uuid` IN + + #{uuid} + + + + AND a.`status` IN + + #{status} + + @@ -109,7 +101,9 @@ along with this program. If not, see .--> (UNIX_TIMESTAMP(IFNULL(a.end_time, NOW()))*1000 - UNIX_TIMESTAMP(a.start_time)*1000) AS timeCost FROM `integration_audit` a - + + + ORDER BY a.id DESC LIMIT #{startNum}, #{pageSize} @@ -123,11 +117,19 @@ along with this program. If not, see .--> FROM `integration_audit` a - join fulltextindex_target_framework ftf on a.id = ftf.target_id and ftf.target_type = 'integration_audit' - join fulltextindex_field_framework fff on ftf.target_id = fff.target_id - join fulltextindex_word fw on fff.word_id = fw.id + join `fulltextindex_target_framework` ftf on a.id = ftf.target_id and ftf.target_type = 'integration_audit' + join `fulltextindex_field_framework` fff on ftf.target_id = fff.target_id AND fff.target_field = 'param' + join `fulltextindex_word` fw on fff.word_id = fw.id - + + + + and fw.word in + + #{item} + + + @@ -136,22 +138,30 @@ along with this program. If not, see .--> select b.id from ( SELECT a.id - + ,COUNT(distinct fw.word) AS match_count FROM `integration_audit` a - join fulltextindex_target_framework ftf on a.id = ftf.target_id and ftf.target_type = 'integration_audit' - join fulltextindex_field_framework fff on ftf.target_id = fff.target_id - join fulltextindex_word fw on fff.word_id = fw.id + join `fulltextindex_target_framework` ftf on a.id = ftf.target_id and ftf.target_type = 'integration_audit' + join `fulltextindex_field_framework` fff on ftf.target_id = fff.target_id AND fff.target_field = 'param' + join `fulltextindex_word` fw on fff.word_id = fw.id - - + + + + and fw.word in + + #{item} + + + + GROUP BY a.`id` order by - + match_count DESC, a.`id` DESC @@ -303,7 +313,7 @@ along with this program. If not, see .--> - SELECT a.`id` FROM `integration_audit` a WHERE a.`integration_uuid` = #{integrationUuid} diff --git a/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java b/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java index 402bd19bc..6f60d14de 100644 --- a/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java +++ b/src/main/java/neatlogic/module/framework/fulltextindex/IntegrationAuditFullTextIndexHandler.java @@ -35,6 +35,8 @@ import neatlogic.framework.integration.dto.IntegrationVo; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; import javax.annotation.Resource; @@ -45,6 +47,8 @@ import java.util.Objects; @Component public class IntegrationAuditFullTextIndexHandler extends FullTextIndexHandlerBase { + private final Logger logger = LoggerFactory.getLogger(IntegrationAuditFullTextIndexHandler.class); + @Resource private IntegrationMapper integrationMapper; @@ -62,45 +66,6 @@ public class IntegrationAuditFullTextIndexHandler extends FullTextIndexHandlerBa fullTextIndexVo.addFieldContent("param", new FullTextIndexVo.WordVo(value)); } } -// IntegrationAuditVo integrationAuditVo = integrationMapper.getIntegrationAuditById(fullTextIndexVo.getTargetId()); -// if (integrationAuditVo != null && StringUtils.isNotBlank(integrationAuditVo.getParamFilePath())) { -// IntegrationVo integrationVo = integrationMapper.getIntegrationByUuid(integrationAuditVo.getIntegrationUuid()); -// if (integrationVo != null) { -// JSONObject config = integrationVo.getConfig(); -// JSONObject param = config.getJSONObject("param"); -// if (MapUtils.isNotEmpty(param)) { -// JSONArray paramList = param.getJSONArray("paramList"); -// if (CollectionUtils.isNotEmpty(paramList)) { -// List searchAbleParamNameList = new ArrayList<>(); -// for (int i = 0; i < paramList.size(); i++) { -// JSONObject paramObj = paramList.getJSONObject(i); -// String name = paramObj.getString("name"); -// String mode = paramObj.getString("mode"); -// Integer isSearchAble = paramObj.getInteger("isSearchAble"); -// if (StringUtils.isNotBlank(name) && Objects.equals(mode, "input") && Objects.equals(isSearchAble, 1)) { -// searchAbleParamNameList.add(name); -// } -// } -// if (CollectionUtils.isNotEmpty(searchAbleParamNameList)) { -// List paramValueList = new ArrayList<>(); -// String content = getContent(integrationAuditVo.getParamFilePath()); -// if (StringUtils.isNotBlank(content) && content.startsWith("{") && content.endsWith("}")) { -// JSONObject paramObj = JSONObject.parseObject(content); -// for (String paramName : searchAbleParamNameList) { -// Object paramValue = paramObj.get(paramName); -// if (paramValue != null) { -// paramValueList.add(paramValue.toString()); -// } -// } -// } -// if (CollectionUtils.isNotEmpty(paramValueList)) { -// fullTextIndexVo.addFieldContent("param", new FullTextIndexVo.WordVo(String.join(",", paramValueList))); -// } -// } -// } -// } -// } -// } } @Override @@ -116,7 +81,7 @@ public class IntegrationAuditFullTextIndexHandler extends FullTextIndexHandlerBa searchVo.setRowNum(rowNum); searchVo.setPageSize(100); Integer pageCount = searchVo.getPageCount(); - for (int currentPage = 1; currentPage < pageCount; currentPage++) { + for (int currentPage = 1; currentPage <= pageCount; currentPage++) { searchVo.setCurrentPage(currentPage); List integrationList = integrationMapper.searchIntegration(searchVo); for (IntegrationVo integrationVo : integrationList) { @@ -162,16 +127,21 @@ public class IntegrationAuditFullTextIndexHandler extends FullTextIndexHandlerBa } private String getContent(String filePath) { - AuditFilePathVo auditFilePathVo = new AuditFilePathVo(filePath); - IFileCrossoverService fileCrossoverService = CrossoverServiceFactory.getApi(IFileCrossoverService.class); - if (Objects.equals(auditFilePathVo.getServerId(), Config.SCHEDULE_SERVER_ID)) { - JSONObject jsonObj = fileCrossoverService.readLocalFile(auditFilePathVo.getPath(), auditFilePathVo.getStartIndex(), auditFilePathVo.getOffset()); - return jsonObj.getString("content"); - } else { - JSONObject paramObj = new JSONObject(); - paramObj.put("filePath", filePath); - JSONObject jsonObj = fileCrossoverService.readRemoteFile(paramObj, auditFilePathVo.getServerId()); - return jsonObj.getString("content"); + try { + AuditFilePathVo auditFilePathVo = new AuditFilePathVo(filePath); + IFileCrossoverService fileCrossoverService = CrossoverServiceFactory.getApi(IFileCrossoverService.class); + if (Objects.equals(auditFilePathVo.getServerId(), Config.SCHEDULE_SERVER_ID)) { + JSONObject jsonObj = fileCrossoverService.readLocalFile(auditFilePathVo.getPath(), auditFilePathVo.getStartIndex(), auditFilePathVo.getOffset()); + return jsonObj.getString("content"); + } else { + JSONObject paramObj = new JSONObject(); + paramObj.put("filePath", filePath); + JSONObject jsonObj = fileCrossoverService.readRemoteFile(paramObj, auditFilePathVo.getServerId()); + return jsonObj.getString("content"); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); } + return null; } } -- Gitee