From 296f6b2aad86032514f98b01a4db63b31696b215 Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 3 Jun 2025 14:19:03 +0800 Subject: [PATCH 1/6] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20neatlogic=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BB=93=E5=BA=93=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE=E5=BA=93=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1432467032998096]neatlogic数据仓库支持配置第三方数据库获取数据 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1432467032998096 --- .../dao/mapper/DataBaseMapper.java | 35 +++ .../dao/mapper/DataBaseMapper.xml | 75 ++++++ .../mapper/DataWarehouseDataSourceMapper.xml | 5 + .../datawarehouse/dto/DataSourceVo.java | 10 + .../datawarehouse/dto/DatabaseVo.java | 81 ++++++ .../DatabaseConnectionFailedException.java | 39 +++ .../exceptions/DatabaseNotFoundException.java | 27 ++ .../scheduler/core/SchedulerManager.java | 2 +- .../handler/JDBCDataSourceHandler.java | 239 ++++++++++++++++++ .../changelog/2025-06-03/neatlogic_tenant.sql | 10 + .../changelog/2025-06-03/version.json | 12 + .../resources/framework/sqlscript/ddl.sql | 12 + 12 files changed, 546 insertions(+), 1 deletion(-) create mode 100644 src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java create mode 100644 src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml create mode 100644 src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java create mode 100644 src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java create mode 100644 src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java create mode 100644 src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java create mode 100644 src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql create mode 100644 src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java new file mode 100644 index 000000000..3024448b6 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.dao.mapper; + +import neatlogic.framework.datawarehouse.dto.DatabaseVo; + +import java.util.List; + +public interface DataBaseMapper { + + DatabaseVo getDataBaseById(Long id); + + int getDataBaseCount(DatabaseVo DataBaseVo); + + List getDataBaseList(DatabaseVo DataBaseVo); + + int insertDataBase(DatabaseVo DataBaseVo); + + int deleteDataBaseById(Long id); +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml new file mode 100644 index 000000000..9cd5425b7 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + INSERT INTO `database` (`id`, `name`, `type`, `config`) + VALUES (#{id}, #{name}, #{type}, #{configStr, typeHandler=CompressHandler}) + ON DUPLICATE KEY UPDATE + `name` = #{name}, + `type` = #{type}, + `config` = #{configStr, typeHandler=CompressHandler} + + + + DELETE FROM `database` WHERE `id` = #{value} + + \ No newline at end of file diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml index c2ae1ffca..f6f45dcca 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataWarehouseDataSourceMapper.xml @@ -52,6 +52,7 @@ along with this program. If not, see .--> + @@ -104,6 +105,7 @@ along with this program. If not, see .--> a.module_id as moduleId, a.data_count as dataCount, a.db_type as dbType, + a.database_id as databaseId, b.id as fieldId, b.name as fieldName, b.label as fieldLabel, @@ -203,6 +205,7 @@ along with this program. If not, see .--> expire_unit = #{expireUnit}, is_active = #{isActive}, db_type = #{dbType}, + database_id = #{databaseId}, xml = #{xml,typeHandler=CompressHandler} where id = #{id} @@ -238,6 +241,7 @@ along with this program. If not, see .--> expire_unit, xml, db_type, + database_id, is_active) values (#{id}, #{name}, @@ -250,6 +254,7 @@ along with this program. If not, see .--> #{expireUnit}, #{xml,typeHandler=CompressHandler}, #{dbType}, + #{databaseId}, #{isActive}) diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java index 085b83b26..59367fb06 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java @@ -76,6 +76,8 @@ public class DataSourceVo extends BasePageVo { private List fieldList = new ArrayList<>();//需要默认值为空数组，避免空指针异常 @EntityField(name = "nfdd.datasourcevo.entityfield.name.connectionid", type = ApiParamType.LONG) private Long connectionId; + @EntityField(name = "nfdd.datasourcedatavo.entityfield.name.datasourceid", type = ApiParamType.LONG) + private Long databaseId; @JSONField(serialize = false)//数据列表 private List dataList; @EntityField(name = "nfdd.datasourcevo.entityfield.name.paramlist", type = ApiParamType.JSONARRAY) @@ -249,6 +251,14 @@ public class DataSourceVo extends BasePageVo { this.connectionId = connectionId; } + public Long getDatabaseId() { + return databaseId; + } + + public void setDatabaseId(Long databaseId) { + this.databaseId = databaseId; + } + public void setQueryTimeout(Integer queryTimeout) { this.queryTimeout = queryTimeout; } diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java new file mode 100644 index 000000000..6146e4030 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.dto; + +import com.alibaba.fastjson.JSONObject; +import com.alibaba.fastjson.annotation.JSONField; +import neatlogic.framework.common.dto.BasePageVo; + +public class DatabaseVo extends BasePageVo { + private Long id; + private String name; + private String type; + private JSONObject config; + @JSONField(serialize = false) + private String configStr; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public JSONObject getConfig() { + if (config == null && configStr != null) { + try { + config = JSONObject.parseObject(configStr); + } catch (Exception ignored) { + + } + } + return this.config; + } + + public void setConfig(JSONObject config) { + this.config = config; + } + + public String getConfigStr() { + if (config != null) { + configStr = config.toJSONString(); + } + return configStr; + } + + public void setConfigStr(String configStr) { + this.configStr = configStr; + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java new file mode 100644 index 000000000..eaca60a58 --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseConnectionFailedException.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.exceptions; + +import neatlogic.framework.exception.core.ApiRuntimeException; +import neatlogic.framework.util.$; + +public class DatabaseConnectionFailedException extends ApiRuntimeException { + + public enum Type { + CONFIG_IS_EMPTY, FILE_ID_LIST_IS_EMPTY + } + public DatabaseConnectionFailedException(Type type, String name) { + super(getMessage(type, name)); + } + + private static String getMessage(Type type, String name) { + if (type == Type.FILE_ID_LIST_IS_EMPTY) { + return $.t("nfde.databaseconnectionfailedexception.fileidlistisempty", name); + } else { + return $.t("{0}nfde.databaseconnectionfailedexception.configisempty", name); + } + } +} diff --git a/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java new file mode 100644 index 000000000..d5c412c4c --- /dev/null +++ b/src/main/java/neatlogic/framework/datawarehouse/exceptions/DatabaseNotFoundException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.framework.datawarehouse.exceptions; + +import neatlogic.framework.exception.core.ApiRuntimeException; + +public class DatabaseNotFoundException extends ApiRuntimeException { + + public DatabaseNotFoundException(Long id) { + super("nfde.databasenotfoundexception.databasenotfoundexception", id); + } +} diff --git a/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java b/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java index 3c3f3e26c..47baf3fbf 100644 --- a/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java +++ b/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java @@ -262,7 +262,7 @@ public class SchedulerManager extends ModuleInitializedListenerBase { schedulerMapper.deleteJobLockByServerId(Config.SCHEDULE_SERVER_ID); UserContext.init(SystemUser.SYSTEM); for (IJob jobHandler : jobHandlerList) { - jobHandler.initJob(tenantUuid); +// jobHandler.initJob(tenantUuid); } schedulerMapper.deleteUnusedJobStatus(); } catch (Exception e) { diff --git a/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java new file mode 100644 index 000000000..496fbc510 --- /dev/null +++ b/src/main/java/neatlogic/module/framework/datawarehouse/handler/JDBCDataSourceHandler.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2025 深圳极向量科技有限公司 All Rights Reserved. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package neatlogic.module.framework.datawarehouse.handler; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import neatlogic.framework.common.util.FileUtil; +import neatlogic.framework.datawarehouse.core.DataSourceServiceHandlerBase; +import neatlogic.framework.datawarehouse.dao.mapper.DataBaseMapper; +import neatlogic.framework.datawarehouse.dao.mapper.DataWarehouseDataSourceMapper; +import neatlogic.framework.datawarehouse.dto.*; +import neatlogic.framework.datawarehouse.exceptions.DatabaseConnectionFailedException; +import neatlogic.framework.datawarehouse.exceptions.DatabaseNotFoundException; +import neatlogic.framework.datawarehouse.exceptions.ReportDataSourceSyncException; +import neatlogic.framework.exception.file.FileNotFoundException; +import neatlogic.framework.file.dao.mapper.FileMapper; +import neatlogic.framework.file.dto.FileVo; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.dom4j.DocumentException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.File; +import java.io.InputStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.sql.*; +import java.util.*; + +@Component +public class JDBCDataSourceHandler extends DataSourceServiceHandlerBase { + static Logger logger = LoggerFactory.getLogger(JDBCDataSourceHandler.class); + int FETCH_SIZE = 1000; + @Resource + private DataWarehouseDataSourceMapper dataSourceMapper; + @Resource + private DataBaseMapper dataBaseMapper; + + @Resource + private FileMapper fileMapper; + + @Override + public String getHandler() { + return "jdbc"; + } + + @Override + public void mySyncData(DataSourceVo dataSourceVo, DataSourceAuditVo reportDataSourceAuditVo) { + Connection conn = null; + PreparedStatement queryStatement = null; + ResultSet resultSet = null; + + try { + List selectList = getSqlFromDataSource(dataSourceVo); + conn = getConnection(dataSourceVo); + for (SelectVo select : selectList) { + String sqlText = select.getSql(); + queryStatement = conn.prepareStatement(sqlText, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); + queryStatement.setFetchSize(FETCH_SIZE); + queryStatement.setFetchDirection(ResultSet.FETCH_FORWARD); + if (dataSourceVo.getQueryTimeout() != null && dataSourceVo.getQueryTimeout() > 0) { + queryStatement.setQueryTimeout(dataSourceVo.getQueryTimeout()); + } + + if (CollectionUtils.isNotEmpty(select.getParamList())) { + for (int p = 0; p < select.getParamList().size(); p++) { + if (select.getParamList().get(p) instanceof String || select.getParamList().get(p) instanceof Number) { + queryStatement.setObject(p + 1, select.getParamList().get(p)); + } else if (select.getParamList().get(p) instanceof String[]) { + // 数组参数有待处理 + queryStatement.setObject(p + 1, ((String[]) select.getParamList().get(p))[0]); + } else if (select.getParamList().get(p) instanceof Number[]) { + // 数组参数有待处理 + queryStatement.setObject(p + 1, ((Number[]) select.getParamList().get(p))[0]); + } + } + } + /* + 新增日志记录 + */ + if (logger.isInfoEnabled()) { + logger.info("REPORT RUN SQL::" + sqlText); + } + + resultSet = queryStatement.executeQuery(); + + ResultSetMetaData metaData = resultSet.getMetaData(); + Map fieldMap = new HashMap<>(); + for (int i = 1; i <= metaData.getColumnCount(); i++) { + fieldMap.put(metaData.getColumnLabel(i).toLowerCase(), i); + } + + while (resultSet.next()) { + DataSourceDataVo reportDataSourceDataVo = new DataSourceDataVo(dataSourceVo.getId()); + reportDataSourceDataVo.setExpireMinute(dataSourceVo.getExpireMinute()); + List aggregateFieldList = new ArrayList<>(); + List keyFieldList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(dataSourceVo.getParamList())) { + for (DataSourceParamVo paramVo : dataSourceVo.getParamList()) { + if (fieldMap.containsKey(paramVo.getName().toLowerCase())) { + Object v = resultSet.getObject(fieldMap.get(paramVo.getName().toLowerCase())); + Long lv = null; + try { + lv = (Long) v; + } catch (Exception ex) { + logger.error(ex.getMessage(), ex); + } + if (lv != null) { + if (paramVo.getCurrentValue() == null) { + paramVo.setCurrentValue(lv); + } else if (lv > paramVo.getCurrentValue()) { + paramVo.setCurrentValue(lv); + } + } + } + } + } + for (DataSourceFieldVo fieldVo : dataSourceVo.getFieldList()) { + if (fieldMap.containsKey(fieldVo.getName().toLowerCase())) { + Object v = resultSet.getObject(fieldMap.get(fieldVo.getName().toLowerCase())); + fieldVo.setValue(v != null ? v : "");//把所有的null值都转成空字符串 + } + reportDataSourceDataVo.addField(fieldVo); + if (StringUtils.isNotBlank(fieldVo.getAggregate())) { + aggregateFieldList.add(fieldVo); + } + if (fieldVo.getIsKey().equals(1)) { + keyFieldList.add(fieldVo); + } + } + aggregateAndInsertData(aggregateFieldList, keyFieldList, reportDataSourceDataVo, reportDataSourceAuditVo); + } + if (CollectionUtils.isNotEmpty(dataSourceVo.getParamList())) { + for (DataSourceParamVo param : dataSourceVo.getParamList()) { + dataSourceMapper.updateDataSourceParamCurrentValue(param); + } + } + } + } catch (SQLException | DocumentException | InstantiationException | IllegalAccessException | + ClassNotFoundException e) { + logger.error(e.getMessage(), e); + reportDataSourceAuditVo.setError(e.getMessage()); + throw new ReportDataSourceSyncException(dataSourceVo, e); + } finally { + try { + if (resultSet != null) { + resultSet.close(); + } + if (queryStatement != null) { + queryStatement.close(); + } + if (conn != null) { + ClassLoader classLoader = conn.getClass().getClassLoader(); + conn.close(); + if (classLoader instanceof URLClassLoader) { + ((URLClassLoader) classLoader).close(); + } + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } + } + + private Connection getConnection(DataSourceVo dataSourceVo) throws SQLException, InstantiationException, IllegalAccessException, ClassNotFoundException { + DatabaseVo dataBaseVo = dataBaseMapper.getDataBaseById(dataSourceVo.getDatabaseId()); + if (dataBaseVo == null) { + throw new DatabaseNotFoundException(dataSourceVo.getDatabaseId()); + } + JSONObject config = dataBaseVo.getConfig(); + if (MapUtils.isNotEmpty(config)) { + String user = config.getString("user"); + String password = config.getString("password"); + String url = config.getString("url"); + String driverClassName = config.getString("driverClassName"); + Properties props = new Properties(); + if (StringUtils.isNoneBlank(user)) { + props.put("user", user); + } + if (StringUtils.isNotBlank(password)) { + props.put("password", password); + } + JSONArray fileIdList = config.getJSONArray("fileIdList"); + if (CollectionUtils.isNotEmpty(fileIdList)) { + URL[] urls = new URL[fileIdList.size()]; + for (int i = 0; i < fileIdList.size(); i++) { + Long fileId = fileIdList.getLong(i); + FileVo fileVo = fileMapper.getFileById(fileId); + if (fileVo == null) { + throw new FileNotFoundException(fileId); + } + try (InputStream is = FileUtil.getData(fileVo.getPath())) { + String fileName = fileVo.getName(); + String suffix = ".jar"; + String prefix = fileName.substring(0, fileName.length() - suffix.length()); + File file = new File(prefix + "-" + fileVo.getId() + suffix); + Path path = Paths.get(file.toURI()); + Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING); + urls[i] = file.toURI().toURL(); + }catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + URLClassLoader loader = new URLClassLoader(urls, null); + Class clazz = loader.loadClass(driverClassName); + Driver driver = ((Driver) clazz.newInstance()); + return driver.connect(url, props); + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, dataBaseVo.getName()); + } + } else { + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, dataBaseVo.getName()); + } + } +} diff --git a/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql new file mode 100644 index 000000000..f3424461b --- /dev/null +++ b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/neatlogic_tenant.sql @@ -0,0 +1,10 @@ +CREATE TABLE `database` ( + `id` bigint NOT NULL COMMENT '主键ID', + `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL COMMENT '名称', + `type` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '类型', + `config` text COLLATE utf8mb4_general_ci NOT NULL COMMENT '配置信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + +ALTER TABLE `datawarehouse_datasource` + ADD COLUMN `database_id` BIGINT NULL COMMENT '数据源' AFTER `db_type`; \ No newline at end of file diff --git a/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json new file mode 100644 index 000000000..6ac9cae50 --- /dev/null +++ b/src/main/resources/neatlogic/resources/framework/changelog/2025-06-03/version.json @@ -0,0 +1,12 @@ +{ + "content": [ + { + "type": "功能", + "detail": [ + { + "msg": "1.数据仓库支持配置第三方数据库获取数据" + } + ] + } + ] +} diff --git a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql index 8987dc4f7..4644fe354 100644 --- a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql +++ b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql @@ -112,6 +112,17 @@ CREATE TABLE IF NOT EXISTS `database_view_info` CHARACTER SET = utf8mb4 COLLATE = utf8mb4_general_ci COMMENT = '视图信息表'; +-- ---------------------------- +-- Table structure for database +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `database` ( + `id` bigint NOT NULL COMMENT '主键ID', + `name` varchar(255) COLLATE utf8mb4_general_ci NOT NULL COMMENT '名称', + `type` varchar(100) COLLATE utf8mb4_general_ci NOT NULL COMMENT '类型', + `config` text COLLATE utf8mb4_general_ci NOT NULL COMMENT '配置信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; + -- ---------------------------- -- Table structure for datawarehouse_datasource -- ---------------------------- @@ -131,6 +142,7 @@ CREATE TABLE IF NOT EXISTS `datawarehouse_datasource` `data_count` int NULL DEFAULT NULL COMMENT '数据量', `expire_unit` enum ('minute','hour','day','month','year') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '过期单位', `db_type` enum ('mysql','mongodb') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'mysql' COMMENT '数据库类型', + `database_id` bigint NULL DEFAULT NULL COMMENT '数据源', `last_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次激活时间', `last_finish_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次完成时间', `next_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '下一次激活时间', -- Gitee From 3e49ceb68996e0a4499f6d3e4a1b07fa5a98e60a Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 3 Jun 2025 14:20:42 +0800 Subject: [PATCH 2/6] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20neatlogic=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BB=93=E5=BA=93=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE=E5=BA=93=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1432467032998096]neatlogic数据仓库支持配置第三方数据库获取数据 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1432467032998096 --- .../neatlogic/framework/scheduler/core/SchedulerManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java b/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java index 47baf3fbf..3c3f3e26c 100644 --- a/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java +++ b/src/main/java/neatlogic/framework/scheduler/core/SchedulerManager.java @@ -262,7 +262,7 @@ public class SchedulerManager extends ModuleInitializedListenerBase { schedulerMapper.deleteJobLockByServerId(Config.SCHEDULE_SERVER_ID); UserContext.init(SystemUser.SYSTEM); for (IJob jobHandler : jobHandlerList) { -// jobHandler.initJob(tenantUuid); + jobHandler.initJob(tenantUuid); } schedulerMapper.deleteUnusedJobStatus(); } catch (Exception e) { -- Gitee From e17e47249600c6212eeb59f2d8a8d97e38cb408b Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 3 Jun 2025 16:07:20 +0800 Subject: [PATCH 3/6] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20neatlogic=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BB=93=E5=BA=93=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE=E5=BA=93=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1432467032998096]neatlogic数据仓库支持配置第三方数据库获取数据 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1432467032998096 --- .../neatlogic/framework/datawarehouse/dto/DataSourceVo.java | 2 +- .../resources/neatlogic/resources/framework/sqlscript/ddl.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java index 59367fb06..d7f9c9b31 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DataSourceVo.java @@ -76,7 +76,7 @@ public class DataSourceVo extends BasePageVo { private List fieldList = new ArrayList<>();//需要默认值为空数组，避免空指针异常 @EntityField(name = "nfdd.datasourcevo.entityfield.name.connectionid", type = ApiParamType.LONG) private Long connectionId; - @EntityField(name = "nfdd.datasourcedatavo.entityfield.name.datasourceid", type = ApiParamType.LONG) + @EntityField(name = "nfdd.datasourcevo.entityfield.name.databaseid", type = ApiParamType.LONG) private Long databaseId; @JSONField(serialize = false)//数据列表 private List dataList; diff --git a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql index 4644fe354..2881a7a9a 100644 --- a/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql +++ b/src/main/resources/neatlogic/resources/framework/sqlscript/ddl.sql @@ -142,7 +142,7 @@ CREATE TABLE IF NOT EXISTS `datawarehouse_datasource` `data_count` int NULL DEFAULT NULL COMMENT '数据量', `expire_unit` enum ('minute','hour','day','month','year') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT NULL COMMENT '过期单位', `db_type` enum ('mysql','mongodb') CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NULL DEFAULT 'mysql' COMMENT '数据库类型', - `database_id` bigint NULL DEFAULT NULL COMMENT '数据源', + `database_id` bigint NULL DEFAULT NULL COMMENT '数据库ID', `last_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次激活时间', `last_finish_time` timestamp(3) NULL DEFAULT NULL COMMENT '最后一次完成时间', `next_fire_time` timestamp(3) NULL DEFAULT NULL COMMENT '下一次激活时间', -- Gitee From 1c65ad0b94de63a58f75930f33f5a28f863f6048 Mon Sep 17 00:00:00 2001 From: "1437892690@qq.com" <1437892690@qq.com> Date: Tue, 3 Jun 2025 16:34:38 +0800 Subject: [PATCH 4/6] =?UTF-8?q?[=E5=8A=9F=E8=83=BD]=20neatlogic=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=BB=93=E5=BA=93=E6=94=AF=E6=8C=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E6=95=B0=E6=8D=AE=E5=BA=93=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 关联 #[1432467032998096]neatlogic数据仓库支持配置第三方数据库获取数据 http://192.168.0.96:8090/demo/rdm.html#/story-detail/939050947543040/939050947543042/1432467032998096 --- ...ataBaseMapper.java => DatabaseMapper.java} | 2 +- ...{DataBaseMapper.xml => DatabaseMapper.xml} | 2 +- .../service/DatabaseService.java | 26 ++++ .../service/DatabaseServiceImpl.java | 115 ++++++++++++++++++ .../handler/JDBCDataSourceHandler.java | 78 ++---------- 5 files changed, 150 insertions(+), 73 deletions(-) rename src/main/java/neatlogic/framework/datawarehouse/dao/mapper/{DataBaseMapper.java => DatabaseMapper.java} (97%) rename src/main/java/neatlogic/framework/datawarehouse/dao/mapper/{DataBaseMapper.xml => DatabaseMapper.xml} (99%) create mode 100644 src/main/java/neatlogic/framework/datawarehouse/service/DatabaseService.java create mode 100644 src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java similarity index 97% rename from src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java rename to src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java index 3024448b6..7ff6afe22 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.java +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.java @@ -21,7 +21,7 @@ import neatlogic.framework.datawarehouse.dto.DatabaseVo; import java.util.List; -public interface DataBaseMapper { +public interface DatabaseMapper { DatabaseVo getDataBaseById(Long id); diff --git a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml similarity index 99% rename from src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml rename to src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml index 9cd5425b7..74ccc3736 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DataBaseMapper.xml +++ b/src/main/java/neatlogic/framework/datawarehouse/dao/mapper/DatabaseMapper.xml @@ -17,7 +17,7 @@ --> - + + + @@ -71,12 +72,13 @@ - INSERT INTO `database` (`id`, `name`, `type`, `config`) - VALUES (#{id}, #{name}, #{type}, #{configStr, typeHandler=CompressHandler}) + INSERT INTO `database` (`id`, `name`, `type`, `config`, `file_id_list`) + VALUES (#{id}, #{name}, #{type}, #{configStr, typeHandler=CompressHandler}, #{fileIdListStr}) ON DUPLICATE KEY UPDATE `name` = #{name}, `type` = #{type}, - `config` = #{configStr, typeHandler=CompressHandler} + `config` = #{configStr, typeHandler=CompressHandler}, + `file_id_list` = #{fileIdListStr} diff --git a/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java index 6146e4030..71a736110 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java +++ b/src/main/java/neatlogic/framework/datawarehouse/dto/DatabaseVo.java @@ -17,17 +17,25 @@ package neatlogic.framework.datawarehouse.dto; +import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.annotation.JSONField; import neatlogic.framework.common.dto.BasePageVo; +import neatlogic.framework.file.dto.FileVo; + +import java.util.List; public class DatabaseVo extends BasePageVo { private Long id; private String name; private String type; private JSONObject config; + private List fileIdList; + private List fileList; @JSONField(serialize = false) private String configStr; + @JSONField(serialize = false) + private String fileIdListStr; public Long getId() { return id; @@ -78,4 +86,38 @@ public class DatabaseVo extends BasePageVo { public void setConfigStr(String configStr) { this.configStr = configStr; } + + public List getFileIdList() { + if (fileIdList == null && fileIdListStr != null) { + try { + fileIdList = JSONArray.parseArray(fileIdListStr, Long.class); + } catch (Exception ignored) { + + } + } + return fileIdList; + } + + public void setFileIdList(List fileIdList) { + this.fileIdList = fileIdList; + } + + public List getFileList() { + return fileList; + } + + public void setFileList(List fileList) { + this.fileList = fileList; + } + + public String getFileIdListStr() { + if (fileIdListStr == null && fileIdList != null) { + fileIdListStr = JSONArray.toJSONString(fileIdList); + } + return fileIdListStr; + } + + public void setFileIdListStr(String fileIdListStr) { + this.fileIdListStr = fileIdListStr; + } } diff --git a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java index 93cc6166a..d171a6c6c 100644 --- a/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java +++ b/src/main/java/neatlogic/framework/datawarehouse/service/DatabaseServiceImpl.java @@ -17,7 +17,6 @@ package neatlogic.framework.datawarehouse.service; -import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import neatlogic.framework.common.util.FileUtil; import neatlogic.framework.datawarehouse.dao.mapper.DatabaseMapper; @@ -42,10 +41,10 @@ import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.sql.Connection; import java.sql.Driver; import java.sql.SQLException; +import java.util.List; import java.util.Properties; @Service @@ -66,50 +65,52 @@ public class DatabaseServiceImpl implements DatabaseService { if (dataBaseVo == null) { throw new DatabaseNotFoundException(databaseId); } - JSONObject config = dataBaseVo.getConfig(); - if (MapUtils.isNotEmpty(config)) { - String user = config.getString("user"); - String password = config.getString("password"); - String url = config.getString("url"); - String driverClassName = config.getString("driverClassName"); - Properties props = new Properties(); - if (StringUtils.isNoneBlank(user)) { - props.put("user", user); - } - if (StringUtils.isNotBlank(password)) { - props.put("password", password); - } - JSONArray fileIdList = config.getJSONArray("fileIdList"); - if (CollectionUtils.isNotEmpty(fileIdList)) { - URL[] urls = new URL[fileIdList.size()]; - for (int i = 0; i < fileIdList.size(); i++) { - Long fileId = fileIdList.getLong(i); - FileVo fileVo = fileMapper.getFileById(fileId); - if (fileVo == null) { - throw new FileNotFoundException(fileId); - } - try (InputStream is = FileUtil.getData(fileVo.getPath())) { - String fileName = fileVo.getName(); - String suffix = ".jar"; - String prefix = fileName.substring(0, fileName.length() - suffix.length()); - File file = new File(prefix + "-" + fileVo.getId() + suffix); + List fileIdList = dataBaseVo.getFileIdList(); + if (CollectionUtils.isNotEmpty(fileIdList)) { + URL[] urls = new URL[fileIdList.size()]; + for (int i = 0; i < fileIdList.size(); i++) { + Long fileId = fileIdList.get(i); + FileVo fileVo = fileMapper.getFileById(fileId); + if (fileVo == null) { + throw new FileNotFoundException(fileId); + } + try (InputStream is = FileUtil.getData(fileVo.getPath())) { + String fileName = fileVo.getName(); + String suffix = ".jar"; + String prefix = fileName.substring(0, fileName.length() - suffix.length()); + File file = new File(prefix + "-" + fileVo.getId() + suffix); + if (!file.exists()) { Path path = Paths.get(file.toURI()); - Files.copy(is, path, StandardCopyOption.REPLACE_EXISTING); - urls[i] = file.toURI().toURL(); - }catch (Exception e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(e); + Files.copy(is, path); } + urls[i] = file.toURI().toURL(); + }catch (Exception e) { + logger.error(e.getMessage(), e); + throw new RuntimeException(e); + } + } + JSONObject config = dataBaseVo.getConfig(); + if (MapUtils.isNotEmpty(config)) { + String user = config.getString("user"); + String password = config.getString("password"); + String url = config.getString("url"); + String driverClassName = config.getString("driverClassName"); + Properties props = new Properties(); + if (StringUtils.isNoneBlank(user)) { + props.put("user", user); + } + if (StringUtils.isNotBlank(password)) { + props.put("password", password); } URLClassLoader loader = new URLClassLoader(urls, null); Class clazz = loader.loadClass(driverClassName); Driver driver = ((Driver) clazz.newInstance()); return driver.connect(url, props); } else { - throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, dataBaseVo.getName()); + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, dataBaseVo.getName()); } } else { - throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.CONFIG_IS_EMPTY, dataBaseVo.getName()); + throw new DatabaseConnectionFailedException(DatabaseConnectionFailedException.Type.FILE_ID_LIST_IS_EMPTY, dataBaseVo.getName()); } } } -- Gitee