From 5b83cccf7f9d0f83c612981eb83b0a96faf29b6c Mon Sep 17 00:00:00 2001 From: panbaolin Date: Wed, 1 Jul 2026 15:34:02 +0800 Subject: [PATCH] =?UTF-8?q?fix:1.=E6=B7=BB=E5=8A=A0jeecg-model-consumer?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=BA=90=E9=A1=B9?= =?UTF-8?q?=E9=87=8D=E5=BB=BA=E3=80=81=E8=BE=93=E8=BF=90=E6=A8=A1=E6=8B=9F?= =?UTF-8?q?=E3=80=81=E5=A4=A9=E6=B0=94=E9=A2=84=E6=8A=A5=E8=AE=A1=E7=AE=97?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BD=BF=E4=BB=BB=E5=8A=A1=E4=B8=8E?= =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E9=80=BB=E8=BE=91=E5=88=86=E5=BC=80=EF=BC=8C?= =?UTF-8?q?=E9=80=82=E5=BA=94=E5=A4=9A=E6=9C=8D=E5=8A=A1=E5=99=A8=E9=83=A8?= =?UTF-8?q?=E7=BD=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../enums/SourceRebuildTaskStatusEnum.java | 15 +- .../constant/enums/WeatherTaskStatusEnum.java | 26 +- .../common}/properties/ServerProperties.java | 2 +- .../properties/SourceRebuildProperties.java | 5 - .../properties/SystemStorageProperties.java | 48 +- .../properties/T1hDownloadProperties.java | 5 + .../jeecg/common/util/Grib2TimeReader.java | 81 ++++ .../java/org/jeecg/common/util/NcUtil.java | 1 - .../config/mybatis/MybatisInterceptor.java | 16 +- .../entity/SourceRebuildMonitoringData.java | 5 +- .../base/entity/SourceRebuildTask.java | 34 +- .../base/entity/SourceRebuildTaskLog.java | 3 +- .../modules/base/entity/TransportTask.java | 2 +- .../entity/TransportTaskBackwardChild.java | 2 +- .../entity/TransportTaskForwardChild.java | 4 +- .../entity/TransportTaskForwardRelease.java | 2 +- .../entity/TransportTaskForwardSpecies.java | 4 +- .../modules/base/entity/TransportTaskLog.java | 3 +- .../modules/base/entity/WeatherData.java | 46 +- .../base/entity/WeatherLinkedDataLog.java | 45 ++ .../modules/base/entity/WeatherTask.java | 18 +- .../modules/base/entity/WeatherTaskLog.java | 9 +- .../mapper/WeatherLinkedDataLogMapper.java | 7 + jeecg-model-consumer/pom.xml | 16 + .../java/org/jeecg/jsch/JSchRemoteRunner.java | 298 ++++++++++++ .../consumer/RebuildTaskConsumerHandler.java | 102 ++++ .../rebuild/consumer/china/AbstractChain.java | 27 ++ .../china/AbstractTaskMsgHandler.java | 87 ++++ .../consumer/china/Server11TaskHandler.java | 33 ++ .../consumer/china/Server12TaskHandler.java | 32 ++ .../consumer/china/Server13TaskHandler.java | 32 ++ .../consumer/china/Server14TaskHandler.java | 27 ++ .../SourceRebuildMonitoringDataService.java | 19 + .../service/SourceRebuildTaskLogService.java | 29 ++ .../service/SourceRebuildTaskService.java | 31 ++ ...ourceRebuildMonitoringDataServiceImpl.java | 32 ++ .../impl/SourceRebuildTaskLogServiceImpl.java | 49 ++ .../impl/SourceRebuildTaskServiceImpl.java | 65 +++ .../rebuild}/task/SourceRebuildTaskExec.java | 331 +++++++------ ...dler.java => TranTaskConsumerHandler.java} | 19 +- .../consumer/china/AbstractChain.java | 2 +- .../china/AbstractTaskMsgHandler.java | 7 +- .../consumer/china/Server13TaskHandler.java | 2 +- .../consumer/china/Server14TaskHandler.java | 1 + .../flexparttask/AbstractTaskExec.java | 16 +- .../flexparttask/BackwardTaskExec.java | 7 +- .../flexparttask/ForwardTaskExec.java | 4 +- .../flexparttask/ProgressMonitor.java | 3 +- .../impl/TransportTaskServiceImpl.java | 3 +- .../transport/util/JSchRemoteRunner.java | 125 ----- .../consumer/WeatherTaskConsumerHandler.java | 100 ++++ .../weather/consumer/china/AbstractChain.java | 27 ++ .../china/AbstractTaskMsgHandler.java | 83 ++++ .../consumer/china/Server11TaskHandler.java | 33 ++ .../consumer/china/Server12TaskHandler.java | 32 ++ .../consumer/china/Server13TaskHandler.java | 32 ++ .../consumer/china/Server14TaskHandler.java | 27 ++ .../weather/service/WeatherTaskService.java | 45 ++ .../service/impl/WeatherTaskServiceImpl.java | 93 ++++ .../weather/task/AbstractWeatherTask.java | 108 +++++ .../jeecg/weather}/task/ProgressEvent.java | 7 +- .../jeecg/weather}/task/ProgressMonitor.java | 6 +- .../jeecg/weather}/task/ProgressQueue.java | 2 +- .../weather/task/WeatherForecastTaskExec.java | 402 ++++++++++++++++ .../service/SourceRebuildTaskLogService.java | 19 - .../service/SourceRebuildTaskService.java | 14 - .../impl/SourceRebuildTaskLogServiceImpl.java | 35 -- .../impl/SourceRebuildTaskServiceImpl.java | 49 +- .../impl/TaskResultDataServiceImpl.java | 4 +- .../jeecg/properties/ServerProperties.java | 19 - .../impl/TransportTaskServiceImpl.java | 5 - .../controller/WeatherDataController.java | 47 +- .../controller/WeatherTaskController.java | 9 +- .../java/org/jeecg/job/DownloadT1hJob.java | 76 ++- .../org/jeecg/service/WeatherDataService.java | 32 +- .../org/jeecg/service/WeatherTaskService.java | 36 +- .../service/impl/WeatherDataServiceImpl.java | 412 ++++++---------- .../service/impl/WeatherTaskServiceImpl.java | 139 ++---- .../jeecg/task/GraphcastWeatherTaskExec.java | 410 ---------------- .../org/jeecg/task/PanGuWeatherTaskExec.java | 438 ------------------ .../jeecg/JeecgConsumerCloudApplication.java | 12 +- 81 files changed, 2601 insertions(+), 1934 deletions(-) rename {jeecg-model-consumer/src/main/java/org/jeecg => jeecg-boot-base-core/src/main/java/org/jeecg/common}/properties/ServerProperties.java (95%) create mode 100644 jeecg-boot-base-core/src/main/java/org/jeecg/common/util/Grib2TimeReader.java create mode 100644 jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherLinkedDataLog.java create mode 100644 jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/mapper/WeatherLinkedDataLogMapper.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/jsch/JSchRemoteRunner.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/RebuildTaskConsumerHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractChain.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractTaskMsgHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server11TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server12TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server13TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server14TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildMonitoringDataService.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskLogService.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskService.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildMonitoringDataServiceImpl.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskLogServiceImpl.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskServiceImpl.java rename {jeecg-module-source-rebuild/src/main/java/org/jeecg => jeecg-model-consumer/src/main/java/org/jeecg/rebuild}/task/SourceRebuildTaskExec.java (51%) rename jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/{TranTaskMessageConsumerHandler.java => TranTaskConsumerHandler.java} (88%) delete mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/transport/util/JSchRemoteRunner.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/WeatherTaskConsumerHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractChain.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractTaskMsgHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server11TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server12TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server13TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server14TaskHandler.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/service/WeatherTaskService.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/service/impl/WeatherTaskServiceImpl.java create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/task/AbstractWeatherTask.java rename {jeecg-module-weather/src/main/java/org/jeecg => jeecg-model-consumer/src/main/java/org/jeecg/weather}/task/ProgressEvent.java (75%) rename {jeecg-module-weather/src/main/java/org/jeecg => jeecg-model-consumer/src/main/java/org/jeecg/weather}/task/ProgressMonitor.java (90%) rename {jeecg-module-weather/src/main/java/org/jeecg => jeecg-model-consumer/src/main/java/org/jeecg/weather}/task/ProgressQueue.java (95%) create mode 100644 jeecg-model-consumer/src/main/java/org/jeecg/weather/task/WeatherForecastTaskExec.java delete mode 100644 jeecg-module-transport/src/main/java/org/jeecg/properties/ServerProperties.java delete mode 100644 jeecg-module-weather/src/main/java/org/jeecg/task/GraphcastWeatherTaskExec.java delete mode 100644 jeecg-module-weather/src/main/java/org/jeecg/task/PanGuWeatherTaskExec.java diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/SourceRebuildTaskStatusEnum.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/SourceRebuildTaskStatusEnum.java index cc5f51e..9f1c936 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/SourceRebuildTaskStatusEnum.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/SourceRebuildTaskStatusEnum.java @@ -8,19 +8,28 @@ public enum SourceRebuildTaskStatusEnum { /** * 执行失败 */ - ERROR(-1), + FAILURE(-1), /** * 未开始 */ NOT_STARTED(0), + /** + * 等待中 + */ + WAITING(1), /** * 执行中 */ - IN_OPERATION(1), + IN_OPERATION(2), /** * 已完成 */ - COMPLETED(2); + COMPLETED(3), + + /** + * 检查未通过 + */ + INSPECTION_FAILED(4); private Integer value; diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/WeatherTaskStatusEnum.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/WeatherTaskStatusEnum.java index 5392436..9203f21 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/WeatherTaskStatusEnum.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/enums/WeatherTaskStatusEnum.java @@ -14,21 +14,31 @@ public enum WeatherTaskStatusEnum { */ NOT_STARTED(0), /** - * 进行中 + * 等待中 */ - IN_OPERATION(1), + WAITING(1), + /** + * 执行中 + */ + IN_OPERATION(2), /** * 已完成 */ - COMPLETED(2); + COMPLETED(3), - private Integer key; + /** + * 检查未通过 + */ + INSPECTION_FAILED(4); + ; - WeatherTaskStatusEnum(Integer key) { - this.key = key; + private Integer value; + + WeatherTaskStatusEnum(Integer value) { + this.value = value; } - public Integer getKey(){ - return this.key; + public Integer getValue(){ + return this.value; } } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/properties/ServerProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/ServerProperties.java similarity index 95% rename from jeecg-model-consumer/src/main/java/org/jeecg/properties/ServerProperties.java rename to jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/ServerProperties.java index c7399f3..3ed2a8d 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/properties/ServerProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/ServerProperties.java @@ -1,4 +1,4 @@ -package org.jeecg.properties; +package org.jeecg.common.properties; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SourceRebuildProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SourceRebuildProperties.java index 0e2d8cd..f018a44 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SourceRebuildProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SourceRebuildProperties.java @@ -9,11 +9,6 @@ import org.springframework.stereotype.Component; @ConfigurationProperties(prefix = "source-rebuild") public class SourceRebuildProperties { - /** - * Rserve 服务地址 - */ - private String serverAddr; - /** * Rserve端口 */ diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SystemStorageProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SystemStorageProperties.java index 61d16a9..940b33f 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SystemStorageProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SystemStorageProperties.java @@ -6,43 +6,38 @@ import org.springframework.stereotype.Component; @Data @Component -@ConfigurationProperties(prefix = "filesystem") +@ConfigurationProperties(prefix = "weather") public class SystemStorageProperties { - /** - * 系统存储根路径 - */ - private String rootPath; - /** * 盘古模型预测数据存储路径 */ - private String pangu; + private String panguDataPath; /** * CRA40数据存储路径 */ - private String cra40; + private String cra40DataPath; /** * NCEP CFSv2数据存储路径 */ - private String ncep; + private String ncepDataPath; /** * graphcast模型预测数据存储路径 */ - private String graphcast; + private String graphcastDataPath; /** * t1h数据存储路径 */ - private String t1h; + private String t1hDataPath; /** * fnl数据存储路径 */ - private String fnl; + private String fnlDataPath; /** * WRF数据存储路径 @@ -55,14 +50,9 @@ public class SystemStorageProperties { private String cmaqPath; /** - * grib文件格式化脚本使用的python环境 + * python环境路径 */ - private String formatScriptPythonEnv; - - /** - * pangu的ai-models 安装地址 - */ - private String panguEnvPath; + private String pythonEnvPath; /** * graphcast的ai-models 安装地址 @@ -83,4 +73,24 @@ public class SystemStorageProperties { * 盘古数据格式化脚本路径（格式化成flexpart能识别的） */ private String panguFormatScriptPath; + + /** + * graphcast数据格式化脚本路径（格式化成flexpart能识别的） + */ + private String graphcastFormatScriptPath; + + /** + * 预测模式为本地时，预测文件的临时存储路径 + */ + private String forecastFileTmpPath; + + /** + * 天气预测docker容器http请求 baseurl + */ + private String weatherForecastBaseUrl; + + /** + * 天气预测docker容器http请求 url + */ + private String weatherForecastUri; } diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/T1hDownloadProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/T1hDownloadProperties.java index cc67e7a..f551a6d 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/T1hDownloadProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/T1hDownloadProperties.java @@ -9,6 +9,11 @@ import org.springframework.stereotype.Component; @ConfigurationProperties(prefix = "t1h-download") public class T1hDownloadProperties { + /** + * 是否下载t1h数据开关 + */ + private boolean downloadSwitch; + /** * 定时器cron */ diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/Grib2TimeReader.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/Grib2TimeReader.java new file mode 100644 index 0000000..dbd9bc0 --- /dev/null +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/Grib2TimeReader.java @@ -0,0 +1,81 @@ +package org.jeecg.common.util; + +import ucar.ma2.Array; +import ucar.nc2.Attribute; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; + +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Grib2TimeReader { + + public static LocalDateTime readValidTime(String gribFile) throws Exception { + try (NetcdfFile nc = NetcdfFiles.open(gribFile)) { + Variable timeVar = nc.findVariable("time"); + if (timeVar == null) { + throw new RuntimeException("未找到 time 变量，请先用 ncdump/ToolsUI 查看 GRIB 映射后的变量名"); + } + + Attribute unitsAttr = timeVar.findAttribute("units"); + if (unitsAttr == null) { + throw new RuntimeException("time 变量没有 units 属性"); + } + + String units = unitsAttr.getStringValue(); + Array data = timeVar.read(); + + double offset = data.getDouble(data.getIndex()); + return parseTimeUnits(units, offset); + } + } + + private static LocalDateTime parseTimeUnits(String units, double offset) { + Pattern pattern = Pattern.compile("(?i)(\\w+)\\s+since\\s+(.+)"); + Matcher matcher = pattern.matcher(units.trim()); + + if (!matcher.matches()) { + throw new RuntimeException("无法解析 time units: " + units); + } + + String unit = matcher.group(1).toLowerCase(); + String baseText = matcher.group(2).trim(); + + LocalDateTime baseTime; + if (baseText.endsWith("Z")) { + baseTime = OffsetDateTime.parse(baseText).toLocalDateTime(); + } else { + baseText = baseText.replace(" ", "T"); + baseTime = LocalDateTime.parse(baseText); + } + + long seconds; + if (unit.startsWith("hour")) { + seconds = Math.round(offset * 3600); + } else if (unit.startsWith("minute")) { + seconds = Math.round(offset * 60); + } else if (unit.startsWith("second")) { + seconds = Math.round(offset); + } else if (unit.startsWith("day")) { + seconds = Math.round(offset * 86400); + } else { + throw new RuntimeException("不支持的时间单位: " + unit); + } + + return baseTime.plusSeconds(seconds); + } + + public static void main(String[] args) throws Exception { + LocalDateTime validTime = readValidTime("C:\\Users\\cnndc\\Desktop\\pangu_20260605_12_00.grib2"); + + String result = validTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); + System.out.println(result); + } +} + + + diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/NcUtil.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/NcUtil.java index 240b671..282e242 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/NcUtil.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/NcUtil.java @@ -1,6 +1,5 @@ package org.jeecg.common.util; -import lombok.val; import ucar.ma2.*; import ucar.nc2.Attribute; import ucar.nc2.NetcdfFile; diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/config/mybatis/MybatisInterceptor.java b/jeecg-boot-base-core/src/main/java/org/jeecg/config/mybatis/MybatisInterceptor.java index 4946c8c..afea824 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/config/mybatis/MybatisInterceptor.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/config/mybatis/MybatisInterceptor.java @@ -58,13 +58,17 @@ public class MybatisInterceptor implements Interceptor { } } // 注入创建时间 - if ("createTime".equals(field.getName()) && Date.class.equals(field.getType())) { + if ("createTime".equals(field.getName())) { field.setAccessible(true); Object localCreateDate = field.get(parameter); field.setAccessible(false); if (localCreateDate == null || "".equals(localCreateDate)) { field.setAccessible(true); - field.set(parameter, new Date()); + if (Date.class.equals(field.getType())){ + field.set(parameter, new Date()); + }else if (LocalDateTime.class.equals(field.getType())){ + field.set(parameter, LocalDateTime.now()); + } field.setAccessible(false); } } @@ -109,9 +113,13 @@ public class MybatisInterceptor implements Interceptor { field.setAccessible(false); } } - if ("updateTime".equals(field.getName()) && Date.class.equals(field.getType())) { + if ("updateTime".equals(field.getName())) { field.setAccessible(true); - field.set(parameter, new Date()); + if (Date.class.equals(field.getType())){ + field.set(parameter, new Date()); + }else if (LocalDateTime.class.equals(field.getType())){ + field.set(parameter, LocalDateTime.now()); + } field.setAccessible(false); } } catch (Exception e) { diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildMonitoringData.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildMonitoringData.java index 90a29a0..60e37f4 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildMonitoringData.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildMonitoringData.java @@ -13,6 +13,7 @@ import org.jeecg.common.validgroup.UpdateGroup; import org.jeecgframework.poi.excel.annotation.Excel; import org.springframework.format.annotation.DateTimeFormat; import java.io.Serializable; +import java.time.LocalDateTime; import java.util.Date; /** @@ -57,7 +58,7 @@ public class SourceRebuildMonitoringData implements Serializable { @NotNull(message = "测量停止时间不能为空", groups = {InsertGroup.class, UpdateGroup.class}) @TableField(value = "collect_stop") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") - private Date collectStop; + private LocalDateTime collectStop; /** * 活度浓度 @@ -89,5 +90,5 @@ public class SourceRebuildMonitoringData implements Serializable { @TableField(value = "create_time") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; } \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTask.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTask.java index fa1f7ba..8907430 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTask.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTask.java @@ -13,6 +13,8 @@ import org.jeecg.common.validgroup.InsertGroup; import org.jeecg.common.validgroup.UpdateGroup; import org.springframework.format.annotation.DateTimeFormat; import java.io.Serializable; +import java.time.LocalDate; +import java.time.LocalDateTime; import java.util.Date; /** @@ -38,7 +40,7 @@ public class SourceRebuildTask implements Serializable { private String taskName; /** - * 任务状态-1执行错误，0-未开始，1进行中，2已完成 + * 任务状态-1执行错误，0-未开始，1等待中，2进行中，3已完成，4检查未通过 */ @Null(message = "任务状态必须为空",groups = {InsertGroup.class, UpdateGroup.class}) @TableField(value = "task_status") @@ -85,7 +87,7 @@ public class SourceRebuildTask implements Serializable { @NotNull(message = "srs时间周期-开始日期不能为空",groups = {InsertGroup.class, UpdateGroup.class}) @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd") @TableField(value = "srs_start_time") - private Date srsStartTime; + private LocalDate srsStartTime; /** * srs时间周期-结束日期 @@ -93,7 +95,7 @@ public class SourceRebuildTask implements Serializable { @NotNull(message = "srs时间周期-结束日期不能为空",groups = {InsertGroup.class, UpdateGroup.class}) @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd") @TableField(value = "srs_end_time") - private Date srsEndTime; + private LocalDate srsEndTime; /** * 释放类型,1-持续释放，2-均匀释放，3-非均匀释放 @@ -126,14 +128,14 @@ public class SourceRebuildTask implements Serializable { */ @TableField(value = "release_start_time") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd") - private Date releaseStartTime; + private LocalDateTime releaseStartTime; /** * 释放源结束释放时间 */ @TableField(value = "release_end_time") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd") - private Date releaseEndTime; + private LocalDateTime releaseEndTime; /** * 源强 @@ -175,25 +177,13 @@ public class SourceRebuildTask implements Serializable { @TableField(value = "result_address") private String resultAddress; - /** - * 创建人 - */ - @TableField(value = "create_by") - private String createBy; - /** * 创建时间 */ @TableField(value = "create_time") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; - - /** - * 更新人 - */ - @TableField(value = "update_by") - private String updateBy; + private LocalDateTime createTime; /** * 更新时间 @@ -201,5 +191,11 @@ public class SourceRebuildTask implements Serializable { @TableField(value = "update_time") @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date updateTime; + private LocalDateTime updateTime; + + /** + * 任务置顶（0未置顶，1置顶） + */ + @TableField(value = "top_task") + private Integer topTask; } \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTaskLog.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTaskLog.java index a3bdc89..76d0f3f 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTaskLog.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/SourceRebuildTaskLog.java @@ -9,6 +9,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.format.annotation.DateTimeFormat; import java.io.Serializable; +import java.time.LocalDateTime; import java.util.Date; /** @@ -36,7 +37,7 @@ public class SourceRebuildTaskLog implements Serializable { */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; /** * 日志内容 diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTask.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTask.java index 723d8a1..78bbee6 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTask.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTask.java @@ -135,7 +135,7 @@ public class TransportTask{ private Integer releaseDataSource; /** - * 任务置顶 + * 任务置顶（0未置顶，1置顶） */ @TableField(value = "top_task") private Integer topTask; diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskBackwardChild.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskBackwardChild.java index b7abc07..003d267 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskBackwardChild.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskBackwardChild.java @@ -59,7 +59,7 @@ public class TransportTaskBackwardChild { @TableField(value = "create_time") @JsonFormat(timezone = "Asia/Shanghai", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; /** * 样品id diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardChild.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardChild.java index 74602bf..f7e2d51 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardChild.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardChild.java @@ -7,6 +7,8 @@ import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; import lombok.Data; import org.springframework.format.annotation.DateTimeFormat; + +import java.time.LocalDateTime; import java.util.Date; import java.util.List; @@ -53,7 +55,7 @@ public class TransportTaskForwardChild { @TableField(value = "create_time") @JsonFormat(timezone = "Asia/Shanghai", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; /** * 正演子表信息 diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardRelease.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardRelease.java index 8418805..eeb825e 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardRelease.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardRelease.java @@ -63,6 +63,6 @@ public class TransportTaskForwardRelease { @TableField(value = "create_time") @JsonFormat(timezone = "Asia/Shanghai", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; } diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardSpecies.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardSpecies.java index 6db9216..b5a7554 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardSpecies.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskForwardSpecies.java @@ -5,6 +5,8 @@ import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; import lombok.Data; import org.springframework.format.annotation.DateTimeFormat; + +import java.time.LocalDateTime; import java.util.Date; /** @@ -32,6 +34,6 @@ public class TransportTaskForwardSpecies { @TableField(value = "create_time") @JsonFormat(timezone = "Asia/Shanghai", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; } diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskLog.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskLog.java index 6ebd4d4..e1a94dd 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskLog.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/TransportTaskLog.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import java.io.Serializable; +import java.time.LocalDateTime; import java.util.Date; /** @@ -40,5 +41,5 @@ public class TransportTaskLog implements Serializable { * 创建时间 */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; } \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherData.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherData.java index cd2cafb..f167ecb 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherData.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherData.java @@ -5,12 +5,9 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.fasterxml.jackson.annotation.JsonFormat; -import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; - import java.io.Serializable; import java.time.LocalDateTime; -import java.util.Date; /** * 气象数据文件表 @@ -22,9 +19,14 @@ public class WeatherData implements Serializable { /** * ID */ - @TableId(type = IdType.ASSIGN_ID) - @Schema(description = "ID") - private String id; + @TableId(type = IdType.AUTO) + private Integer id; + + /** + * 预测任务产生的数据有taskId + */ + @TableField(value = "task_id") + private Integer taskId; /** * 文件名称 @@ -32,12 +34,6 @@ public class WeatherData implements Serializable { @TableField(value = "file_name") private String fileName; - /** - * 文件大小 - */ - @TableField(value = "file_size") - private Double fileSize; - /** * 文件类型 */ @@ -62,30 +58,18 @@ public class WeatherData implements Serializable { @TableField(value = "file_path") private String filePath; + /** + * 对于盘古模型，graphcast模型预测结果文件，格式化后的数据存储路径 + */ + @TableField(value = "format_file_path") + private String formatFilePath; + /** * 创建时间 */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @TableField(value = "create_time") - private Date createTime; - - /** - * 文件MD5值 - */ - @TableField(value = "md5_value") - private String md5Value; - - /** - * 总分片数 - */ - @TableField(value = "share_total") - private Integer shareTotal; - - /** - * 分片索引 - */ - @TableField(value = "share_index") - private Integer shareIndex; + private LocalDateTime createTime; /** * 日期批次 diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherLinkedDataLog.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherLinkedDataLog.java new file mode 100644 index 0000000..16aa18f --- /dev/null +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherLinkedDataLog.java @@ -0,0 +1,45 @@ +package org.jeecg.modules.base.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.annotation.JsonFormat; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + * 关联气象数据文件日志表 + */ +@Data +@TableName("stas_weather_linked_data_log") +public class WeatherLinkedDataLog implements Serializable { + + /** + * ID + */ + @TableId(type = IdType.AUTO) + @Schema(description = "ID") + private Integer id; + + /** + * 数据类型（3-cra40，4-ncep,5-fnl） + */ + @TableField(value = "data_type") + private Integer dataType; + + /** + * 任务运行过程日志 + */ + @TableField(value = "log_content") + private String logContent; + + /** + * 创建时间 + */ + @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") + private LocalDateTime createTime; + +} \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTask.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTask.java index 93fb638..4fd8cae 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTask.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTask.java @@ -15,7 +15,7 @@ import org.jeecg.common.validgroup.UpdateGroup; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.web.multipart.MultipartFile; import java.time.LocalDate; -import java.util.Date; +import java.time.LocalDateTime; /** * 天气预测任务表 @@ -29,9 +29,9 @@ public class WeatherTask{ */ @Null(message = "ID必须为空",groups = { InsertGroup.class}) @NotNull(message = "ID不能为空",groups = { UpdateGroup.class}) - @TableId(type = IdType.INPUT) + @TableId(type = IdType.AUTO) @Schema(description = "ID") - private java.lang.String id; + private Integer id; /** * 任务名称 @@ -41,7 +41,7 @@ public class WeatherTask{ private String taskName; /** - * 任务状态（-1失败，0-未开始，1-运行中，2-已完成） + * 任务状态（-1失败，0-未开始，1-等待中，2-执行中，3-已完成） */ @Null(message = "任务状态必须为空",groups = {InsertGroup.class, UpdateGroup.class}) @TableField(value = "task_status") @@ -113,7 +113,7 @@ public class WeatherTask{ */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; /** * 更新人 @@ -125,6 +125,12 @@ public class WeatherTask{ */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") - private Date updateTime; + private LocalDateTime updateTime; + + /** + * 任务置顶（0未置顶，1置顶） + */ + @TableField(value = "top_task") + private Integer topTask; } \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTaskLog.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTaskLog.java index e438cf6..b9c029e 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTaskLog.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTaskLog.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonFormat; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import java.io.Serializable; +import java.time.LocalDateTime; import java.util.Date; /** @@ -20,15 +21,15 @@ public class WeatherTaskLog implements Serializable { /** * ID */ - @TableId(type = IdType.ASSIGN_ID) + @TableId(type = IdType.AUTO) @Schema(description = "ID") - private String id; + private Integer id; /** * 任务表主键 */ @TableField(value = "task_id") - private String taskId; + private Integer taskId; /** * 任务运行过程日志 @@ -40,6 +41,6 @@ public class WeatherTaskLog implements Serializable { * 创建时间 */ @JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm:ss") - private Date createTime; + private LocalDateTime createTime; } \ No newline at end of file diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/mapper/WeatherLinkedDataLogMapper.java b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/mapper/WeatherLinkedDataLogMapper.java new file mode 100644 index 0000000..8f4ba03 --- /dev/null +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/mapper/WeatherLinkedDataLogMapper.java @@ -0,0 +1,7 @@ +package org.jeecg.modules.base.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.jeecg.modules.base.entity.WeatherLinkedDataLog; + +public interface WeatherLinkedDataLogMapper extends BaseMapper { +} diff --git a/jeecg-model-consumer/pom.xml b/jeecg-model-consumer/pom.xml index 226dca5..817ff47 100644 --- a/jeecg-model-consumer/pom.xml +++ b/jeecg-model-consumer/pom.xml @@ -15,6 +15,8 @@ 17 17 UTF-8 + 2.1.0 + 1.8.1 @@ -23,5 +25,19 @@ jeecg-boot-base-core ${jeecgboot.version} + + org.rosuda.REngine + REngine + ${rEngine.version} + + + org.rosuda.REngine + Rserve + ${rserve.version} + + + org.springframework.boot + spring-boot-starter-webflux + \ No newline at end of file diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/jsch/JSchRemoteRunner.java b/jeecg-model-consumer/src/main/java/org/jeecg/jsch/JSchRemoteRunner.java new file mode 100644 index 0000000..ddaed43 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/jsch/JSchRemoteRunner.java @@ -0,0 +1,298 @@ +package org.jeecg.jsch; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.util.StrUtil; +import com.jcraft.jsch.*; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * ssh远程操作工具 + */ +@Slf4j +public class JSchRemoteRunner { + + private final JSch jsch = new JSch(); + + @Getter + private Session session = null; + + /** + * 登录 + * + * @param host + * @param port + * @param username + * @param password + * @throws JSchException + */ + public void login(String host, int port, String username, String password) throws JSchException { + Properties config = new Properties(); + config.put("StrictHostKeyChecking","no"); + config.put("PreferredAuthentications", "password");//只用秘密登录 + session = jsch.getSession(username, host, port); + session.setUserInfo(new UserInfo() { + @Override + public String getPassphrase() { + return null; + } + @Override + public String getPassword() { + return password; + } + @Override + public boolean promptPassword(String s) { + return true; + } + @Override + public boolean promptPassphrase(String s) { + return false; + } + @Override + public boolean promptYesNo(String s) { + return true; + } + @Override + public void showMessage(String message) { + log.info("SSH Message: {}", message); + } + }); + session.setConfig(config); + session.setServerAliveInterval(30); + session.setServerAliveCountMax(3); + session.setTimeout(0); + session.connect(); + } + + /** + * 写入文件内容，如果文件不存在则创建 + * @param path + * @param content + */ + public void writeFile(String path, String content) { + ChannelSftp sftp = null; + try { + sftp = (ChannelSftp)session.openChannel("sftp"); + sftp.connect(); + sftp.put(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)),path,ChannelSftp.OVERWRITE); + } catch (JSchException | SftpException e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(sftp)) { + sftp.disconnect(); + } + } + } + + /** + * 清空目录下所有文件 + * @param path + */ + public void clearDir(String path){ + if (StrUtil.isBlank(path) || StrUtil.equals("/", path)){ + throw new RuntimeException("需要清空的目录路径不能为空"); + } + ChannelExec channel = null; + try{ + //打开一个执行通道 + channel = (ChannelExec) session.openChannel("exec"); + String fullCommand = "rm -f "+path+"/*"; + channel.setCommand(fullCommand); + // 连接通道 + channel.connect(); + }catch(JSchException e){ + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(channel)) { + channel.disconnect(); + } + } + } + + /** + * 判断目录是否存在 + * @param dirPath + * @return + */ + public boolean dirExist(String dirPath){ + if (StrUtil.isBlank(dirPath)){ + return false; + } + ChannelSftp sftp = null; + try { + sftp = (ChannelSftp)session.openChannel("sftp"); + sftp.connect(); + SftpATTRS stat = sftp.stat(dirPath); + return stat != null && stat.isDir(); + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + return false; + } + throw new RuntimeException("SFTP检查目录失败:",e); + } catch (JSchException e) { + throw new RuntimeException("ssh连接异常:",e); + } finally { + if (Objects.nonNull(sftp)) { + sftp.disconnect(); + } + } + } + + /** + * 判断目录是否存在 + * @param dirPaths + * @return + */ + public List getNotExistDirs(List dirPaths){ + if (CollUtil.isEmpty(dirPaths)) { + return new ArrayList<>(); + } + ChannelSftp sftp = null; + try { + List notExistDirs = new ArrayList<>(); + sftp = (ChannelSftp)session.openChannel("sftp"); + sftp.connect(); + for (String dirPath : dirPaths) { + try { + SftpATTRS stat = sftp.stat(dirPath); + if (stat == null || !stat.isDir()){ + notExistDirs.add(dirPath); + } + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + log.warn("{}文件不存在",dirPath); + } + } + } + return notExistDirs; + } catch (JSchException e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(sftp)) { + sftp.disconnect(); + } + } + } + + /** + * 获取远程主机指定目录下的所有匹配规则的文件 + * @param path + * @param matchingSuffix + * @return + */ + public List listFiles(String path,String matchingSuffix){ + ChannelSftp sftp = null; + try { + sftp = (ChannelSftp)session.openChannel("sftp"); + sftp.connect(); + Vector fileList = sftp.ls(path); + List matchedFiles = new ArrayList<>(); + for (ChannelSftp.LsEntry entry : fileList) { + String fileName = entry.getFilename(); + if (fileName.endsWith(matchingSuffix)) { + matchedFiles.add(new File(path+File.separator+fileName)); + } + } + return CollUtil.emptyIfNull(matchedFiles); + } catch (JSchException | SftpException e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(sftp)) { + sftp.disconnect(); + } + } + } + + /** + * 获取远程主机指定目录下的所有匹配规则的文件 + * @param paths + * @param matchingSuffix + * @return + */ + public List listFiles(List paths,String matchingSuffix){ + ChannelSftp sftp = null; + try { + sftp = (ChannelSftp)session.openChannel("sftp"); + sftp.connect(); + List matchedFiles = new ArrayList<>(); + for (String path : paths) { + Vector fileList = sftp.ls(path); + for (ChannelSftp.LsEntry entry : fileList) { + String fileName = entry.getFilename(); + if (fileName.endsWith(matchingSuffix)) { + matchedFiles.add(path+File.separator+fileName); + } + } + } + return CollUtil.emptyIfNull(matchedFiles); + } catch (JSchException | SftpException e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(sftp)) { + sftp.disconnect(); + } + } + } + + /** + * 创建目录，可多层创建 + * @param path + */ + public void mkdir(String path){ + ChannelExec channelExec = null; + try { + channelExec = (ChannelExec) session.openChannel("exec"); + channelExec.setCommand("mkdir -p "+path); + channelExec.connect(); + while (!channelExec.isClosed()) { + Thread.sleep(100); + } + } catch (Exception e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(channelExec)){ + channelExec.disconnect(); + } + } + + } + + /** + * 移动文件 + * @param srcPath + * @param destPath + */ + public void moveFiles(String srcPath,String destPath){ + if (StrUtil.isBlank(srcPath) || StrUtil.equals("/", srcPath) || StrUtil.isBlank(destPath) || StrUtil.equals("/", destPath)){ + throw new RuntimeException("需要移动的目录路径不能为空"); + } + ChannelExec channelExec = null; + try { + channelExec = (ChannelExec) session.openChannel("exec"); + channelExec.setCommand("mv "+srcPath+" "+destPath); + channelExec.connect(); + while (!channelExec.isClosed()) { + Thread.sleep(100); + } + } catch (Exception e) { + throw new RuntimeException(e); + }finally { + if (Objects.nonNull(channelExec)){ + channelExec.disconnect(); + } + } + } + + /** + * 关闭会话 + */ + public void close(){ + if (Objects.nonNull(session)) { + session.disconnect(); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/RebuildTaskConsumerHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/RebuildTaskConsumerHandler.java new file mode 100644 index 0000000..fcb246b --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/RebuildTaskConsumerHandler.java @@ -0,0 +1,102 @@ +package org.jeecg.rebuild.consumer; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jeecg.common.constant.CommonConstant; +import org.jeecg.common.constant.enums.SourceRebuildTaskStatusEnum; +import org.jeecg.common.constant.enums.TransportTaskStatusEnum; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.properties.SourceRebuildProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.SourceRebuildTask; +import org.jeecg.modules.base.mapper.*; +import org.jeecg.rebuild.consumer.china.AbstractTaskMsgHandler; +import org.jeecg.rebuild.consumer.china.Server11TaskHandler; +import org.jeecg.rebuild.service.SourceRebuildMonitoringDataService; +import org.jeecg.rebuild.service.SourceRebuildTaskLogService; +import org.jeecg.rebuild.service.SourceRebuildTaskService; +import org.springframework.stereotype.Component; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * 源项重建任务控制器 + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class RebuildTaskConsumerHandler { + + private final RedisUtil redisUtil; + private final SourceRebuildTaskLogService sourceRebuildTaskLogService; + private final SourceRebuildProperties sourceRebuildProperties; + private final SourceRebuildTaskService sourceRebuildTaskService; + private final ServerProperties serverProperties; + private final SourceRebuildMonitoringDataService monitoringDataService; + + public void startConsumerThread(){ + MessageConsumerThread messageConsumerThread = new MessageConsumerThread(); + messageConsumerThread.setName("rebuild-task-thread"); + messageConsumerThread.start(); + log.info("启动源项重建任务消费线程----------------------"); + } + + /** + * 处理源项重建任务 + */ + private void handlerTask(SourceRebuildTask message){ + AbstractTaskMsgHandler taskMsgHandler = new Server11TaskHandler(); + taskMsgHandler.init( + sourceRebuildTaskLogService, + sourceRebuildProperties, + sourceRebuildTaskService, + monitoringDataService, + serverProperties, + redisUtil); + taskMsgHandler.handler(message); + } + + /** + * 获取等待中的任务 + * @return + */ + private SourceRebuildTask getTopMessage(){ + LambdaQueryWrapper topQueryWrapper = new LambdaQueryWrapper<>(); + topQueryWrapper.eq(SourceRebuildTask::getTaskStatus, SourceRebuildTaskStatusEnum.WAITING.getValue()); + topQueryWrapper.orderByDesc(SourceRebuildTask::getTopTask); + topQueryWrapper.orderByDesc(SourceRebuildTask::getUpdateTime); + topQueryWrapper.last("LIMIT 1"); + SourceRebuildTask rebuildTask = sourceRebuildTaskService.getOne(topQueryWrapper); + //如果没有要置顶执行的任务，则按照普通创建时间排序查询 + if(Objects.isNull(rebuildTask)){ + LambdaQueryWrapper timeQueryWrapper = new LambdaQueryWrapper<>(); + timeQueryWrapper.eq(SourceRebuildTask::getTaskStatus, SourceRebuildTaskStatusEnum.WAITING.getValue()); + timeQueryWrapper.orderByAsc(SourceRebuildTask::getCreateTime); + timeQueryWrapper.last("LIMIT 1"); + rebuildTask = sourceRebuildTaskService.getOne(timeQueryWrapper); + } + return rebuildTask; + } + + private class MessageConsumerThread extends Thread{ + @Override + public void run() { + while (true) { + try { + //获取本机项数据，如果为空，表示本机没有任务在运行 + boolean flag = redisUtil.hHasKey(CommonConstant.HOST_TASK_STATE,CommonConstant.BUILD_TASK_STATE_PRE+serverProperties.getHost()); + if (!flag) { + SourceRebuildTask topMessage = getTopMessage(); + if (Objects.nonNull(topMessage)) { + handlerTask(topMessage); + } + } + TimeUnit.SECONDS.sleep(60); + } catch (InterruptedException e) { + log.error("执行源项重建任务数据异常，原因为：",e); + } + } + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractChain.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractChain.java new file mode 100644 index 0000000..2c3271c --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractChain.java @@ -0,0 +1,27 @@ +package org.jeecg.rebuild.consumer.china; + +import lombok.Setter; + +/** + * 执行链路 + */ +public abstract class AbstractChain { + + /** + * 上一任处理链 + */ + @Setter + protected AbstractTaskMsgHandler previous; + + /** + * 下一任处理链 + */ + @Setter + protected AbstractTaskMsgHandler next; + + /** + * 设置过滤器链 + */ + protected abstract void setChina(); + +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractTaskMsgHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractTaskMsgHandler.java new file mode 100644 index 0000000..8046533 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/AbstractTaskMsgHandler.java @@ -0,0 +1,87 @@ +package org.jeecg.rebuild.consumer.china; + +import lombok.extern.slf4j.Slf4j; +import org.jeecg.common.constant.CommonConstant; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.properties.SourceRebuildProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.SourceRebuildTask; +import org.jeecg.rebuild.service.SourceRebuildMonitoringDataService; +import org.jeecg.rebuild.service.SourceRebuildTaskLogService; +import org.jeecg.rebuild.service.SourceRebuildTaskService; +import org.jeecg.rebuild.task.SourceRebuildTaskExec; + +/** + * 处理任务策略父类 + */ +@Slf4j +public abstract class AbstractTaskMsgHandler extends AbstractChain { + + protected RedisUtil redisUtil; + protected SourceRebuildTaskLogService sourceRebuildTaskLogService; + protected SourceRebuildProperties sourceRebuildProperties; + protected SourceRebuildTaskService sourceRebuildTaskService; + protected SourceRebuildMonitoringDataService monitoringDataService; + protected ServerProperties serverProperties; + + /** + * 初始化bean引用 + */ + public void init(SourceRebuildTaskLogService sourceRebuildTaskLogService, + SourceRebuildProperties sourceRebuildProperties, + SourceRebuildTaskService sourceRebuildTaskService, + SourceRebuildMonitoringDataService monitoringDataService, + ServerProperties serverProperties, + RedisUtil redisUtil) { + this.sourceRebuildTaskLogService = sourceRebuildTaskLogService; + this.sourceRebuildProperties = sourceRebuildProperties; + this.sourceRebuildTaskService = sourceRebuildTaskService; + this.monitoringDataService = monitoringDataService; + this.serverProperties = serverProperties; + this.redisUtil = redisUtil; + this.setChina(); + } + + /** + * 初始化下个节点 + * @param serverProperties + */ + protected void initNext(ServerProperties serverProperties){ + this.serverProperties = serverProperties; + this.setChina(); + } + + /** + * 处理消息 + */ + public abstract void handler(SourceRebuildTask message); + + /** + * 运行任务 + * @param rebuildTask + */ + protected void runTask(SourceRebuildTask rebuildTask,String ip){ + try { + redisUtil.hset(CommonConstant.HOST_TASK_STATE,CommonConstant.BUILD_TASK_STATE_PRE+serverProperties.getHost(),rebuildTask.getId()); + boolean flag = false; + SourceRebuildTaskExec taskExec = new SourceRebuildTaskExec(); + taskExec.init( + rebuildTask, + sourceRebuildTaskLogService, + sourceRebuildProperties, + sourceRebuildTaskService, + serverProperties, + monitoringDataService); + taskExec.setName("source-rebuild-task"); + flag = taskExec.checkTask(); + if (flag){ + log.info("任务：{}，检查通过，当前在主机：{}上执行。", rebuildTask.getTaskName(),ip); + taskExec.start(); + }else { + sourceRebuildTaskService.setInpectionFailed(rebuildTask); + } + }catch (Exception e){ + log.error("检查任务或开启任务执行线程出现错误",e); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server11TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server11TaskHandler.java new file mode 100644 index 0000000..daf201c --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server11TaskHandler.java @@ -0,0 +1,33 @@ +package org.jeecg.rebuild.consumer.china; + +import org.jeecg.modules.base.entity.SourceRebuildTask; + +/** + * 处理重建任务 + */ +public class Server11TaskHandler extends AbstractTaskMsgHandler { + + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server12TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(SourceRebuildTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp11())){ + super.runTask(message,serverProperties.getIp11()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server12TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server12TaskHandler.java new file mode 100644 index 0000000..5cb00fb --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server12TaskHandler.java @@ -0,0 +1,32 @@ +package org.jeecg.rebuild.consumer.china; + +import org.jeecg.modules.base.entity.SourceRebuildTask; + +/** + * 处理重建任务 + */ +public class Server12TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server13TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(SourceRebuildTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp12())){ + super.runTask(message,serverProperties.getIp12()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server13TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server13TaskHandler.java new file mode 100644 index 0000000..c218e2b --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server13TaskHandler.java @@ -0,0 +1,32 @@ +package org.jeecg.rebuild.consumer.china; + +import org.jeecg.modules.base.entity.SourceRebuildTask; + +/** + * 处理重建任务 + */ +public class Server13TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server14TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(SourceRebuildTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp13())){ + super.runTask(message,serverProperties.getIp13()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server14TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server14TaskHandler.java new file mode 100644 index 0000000..a0cf27e --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/consumer/china/Server14TaskHandler.java @@ -0,0 +1,27 @@ +package org.jeecg.rebuild.consumer.china; + +import org.jeecg.modules.base.entity.SourceRebuildTask; + +/** + * 处理重建任务 + */ +public class Server14TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + + } + + /** + * 处理消息 + */ + @Override + public void handler(SourceRebuildTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp14())){ + super.runTask(message,serverProperties.getIp14()); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildMonitoringDataService.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildMonitoringDataService.java new file mode 100644 index 0000000..7f614c4 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildMonitoringDataService.java @@ -0,0 +1,19 @@ +package org.jeecg.rebuild.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import org.jeecg.modules.base.entity.SourceRebuildMonitoringData; +import java.util.List; + +/** + * 源项重建任务监测数据 + */ +public interface SourceRebuildMonitoringDataService extends IService { + + /** + * 获取任务所属监测数据 + * @param taskId + * @return + */ + List getTaskMonitoringData(Integer taskId); + +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskLogService.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskLogService.java new file mode 100644 index 0000000..e7f74c4 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskLogService.java @@ -0,0 +1,29 @@ +package org.jeecg.rebuild.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import org.jeecg.modules.base.entity.SourceRebuildTaskLog; +import java.util.List; + +/** + * 源项重建任务日志数据 + */ +public interface SourceRebuildTaskLogService extends IService { + + /** + * 保存源项重建任务运行过程日志 + * @param sourceRebuildTaskLog + */ + void cteate(SourceRebuildTaskLog sourceRebuildTaskLog); + + /** + * 删除任务运行过程日志 + * @param taskId + */ + void delete(Integer taskId); + + /** + * 批量保存日志 + * @param logs + */ + void batchCteate(List logs); +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskService.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskService.java new file mode 100644 index 0000000..30e79d9 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/SourceRebuildTaskService.java @@ -0,0 +1,31 @@ +package org.jeecg.rebuild.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import org.jeecg.modules.base.entity.SourceRebuildTask; + +/** + * 源项重建任务 + */ +public interface SourceRebuildTaskService extends IService { + + /** + * 修改任务状态 + * @param taskId + * @param taskStatus + */ + void updateTaskStatus(Integer taskId, Integer taskStatus); + + /** + * 任务耗时 + * @param taskId + * @param minute + * @param taskStatus + */ + void updateTaskTimeConsuming(Integer taskId, Double minute,Integer taskStatus); + + /** + * 设置检查失败 + * @param task + */ + void setInpectionFailed(SourceRebuildTask task); +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildMonitoringDataServiceImpl.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildMonitoringDataServiceImpl.java new file mode 100644 index 0000000..3a07a1d --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildMonitoringDataServiceImpl.java @@ -0,0 +1,32 @@ +package org.jeecg.rebuild.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import lombok.RequiredArgsConstructor; +import org.jeecg.modules.base.entity.SourceRebuildMonitoringData; +import org.jeecg.modules.base.mapper.SourceRebuildMonitoringDataMapper; +import org.jeecg.rebuild.service.SourceRebuildMonitoringDataService; +import org.springframework.stereotype.Service; +import java.util.List; + +/** + * 源项重建任务监测数据 + */ +@Service +@RequiredArgsConstructor +public class SourceRebuildMonitoringDataServiceImpl extends ServiceImpl implements SourceRebuildMonitoringDataService { + + /** + * 获取任务所属监测数据 + * + * @param taskId + * @return + */ + @Override + public List getTaskMonitoringData(Integer taskId) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(SourceRebuildMonitoringData::getTaskId, taskId); + queryWrapper.orderByAsc(SourceRebuildMonitoringData::getCollectStop); + return this.baseMapper.selectList(queryWrapper); + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskLogServiceImpl.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskLogServiceImpl.java new file mode 100644 index 0000000..2606c3c --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskLogServiceImpl.java @@ -0,0 +1,49 @@ +package org.jeecg.rebuild.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import org.jeecg.modules.base.entity.SourceRebuildTaskLog; +import org.jeecg.modules.base.mapper.SourceRebuildTaskLogMapper; +import org.jeecg.rebuild.service.SourceRebuildTaskLogService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import java.util.List; + +@Service +public class SourceRebuildTaskLogServiceImpl extends ServiceImpl implements SourceRebuildTaskLogService { + + /** + * 保存源项重建任务运行过程日志 + * + * @param sourceRebuildTaskLog + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void cteate(SourceRebuildTaskLog sourceRebuildTaskLog) { + this.baseMapper.insert(sourceRebuildTaskLog); + } + + /** + * 删除任务运行过程日志 + * + * @param taskId + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void delete(Integer taskId) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(SourceRebuildTaskLog::getTaskId, taskId); + this.baseMapper.delete(queryWrapper); + } + + /** + * 批量保存日志 + * + * @param logs + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void batchCteate(List logs) { + this.saveBatch(logs); + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskServiceImpl.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskServiceImpl.java new file mode 100644 index 0000000..e06f850 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/service/impl/SourceRebuildTaskServiceImpl.java @@ -0,0 +1,65 @@ +package org.jeecg.rebuild.service.impl; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import lombok.RequiredArgsConstructor; +import org.jeecg.common.constant.CommonConstant; +import org.jeecg.common.constant.enums.SourceRebuildTaskStatusEnum; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.SourceRebuildTask; +import org.jeecg.modules.base.mapper.SourceRebuildTaskMapper; +import org.jeecg.rebuild.service.SourceRebuildTaskService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * 源项重建任务 + */ +@Service +@RequiredArgsConstructor +public class SourceRebuildTaskServiceImpl extends ServiceImpl implements SourceRebuildTaskService { + + private final RedisUtil redisUtil; + private final ServerProperties serverProperties; + + /** + * 修改任务状态 + * + * @param taskId + * @param taskStatus + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void updateTaskStatus(Integer taskId, Integer taskStatus) { + SourceRebuildTask sourceRebuildTask = this.baseMapper.selectById(taskId); + sourceRebuildTask.setTaskStatus(taskStatus); + this.updateById(sourceRebuildTask); + } + + /** + * 任务耗时 + * + * @param taskId + * @param minute + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void updateTaskTimeConsuming(Integer taskId, Double minute,Integer taskStatus) { + SourceRebuildTask sourceRebuildTask = this.baseMapper.selectById(taskId); + sourceRebuildTask.setTimeConsuming(minute); + sourceRebuildTask.setTaskStatus(taskStatus); + this.updateById(sourceRebuildTask); + redisUtil.hdel(CommonConstant.HOST_TASK_STATE,CommonConstant.BUILD_TASK_STATE_PRE+serverProperties.getHost()); + } + + /** + * 设置检查失败 + * @param task + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void setInpectionFailed(SourceRebuildTask task) { + task.setTaskStatus(SourceRebuildTaskStatusEnum.INSPECTION_FAILED.getValue()); + this.baseMapper.updateById(task); + } +} diff --git a/jeecg-module-source-rebuild/src/main/java/org/jeecg/task/SourceRebuildTaskExec.java b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/task/SourceRebuildTaskExec.java similarity index 51% rename from jeecg-module-source-rebuild/src/main/java/org/jeecg/task/SourceRebuildTaskExec.java rename to jeecg-model-consumer/src/main/java/org/jeecg/rebuild/task/SourceRebuildTaskExec.java index e26664b..f5d6527 100644 --- a/jeecg-module-source-rebuild/src/main/java/org/jeecg/task/SourceRebuildTaskExec.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/rebuild/task/SourceRebuildTaskExec.java @@ -1,17 +1,26 @@ -package org.jeecg.task; +package org.jeecg.rebuild.task; +import cn.hutool.core.collection.CollUtil; import cn.hutool.core.date.DateUtil; import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.ArrayUtil; +import com.jcraft.jsch.JSchException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.StopWatch; import org.jeecg.common.constant.enums.SourceRebuildReleaseSourceEnum; import org.jeecg.common.constant.enums.SourceRebuildTaskStatusEnum; +import org.jeecg.common.constant.enums.TransportTaskStatusEnum; +import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.properties.SourceRebuildProperties; +import org.jeecg.jsch.JSchRemoteRunner; import org.jeecg.modules.base.entity.SourceRebuildMonitoringData; import org.jeecg.modules.base.entity.SourceRebuildTask; import org.jeecg.modules.base.entity.SourceRebuildTaskLog; -import org.jeecg.service.SourceRebuildTaskLogService; -import org.jeecg.service.SourceRebuildTaskService; +import org.jeecg.rebuild.service.SourceRebuildMonitoringDataService; +import org.jeecg.rebuild.service.SourceRebuildTaskLogService; +import org.jeecg.rebuild.service.SourceRebuildTaskService; +import org.jeecg.transport.flexparttask.ProgressEvent; +import org.jeecg.transport.flexparttask.ProgressQueue; import org.rosuda.REngine.REXP; import org.rosuda.REngine.REXPDouble; import org.rosuda.REngine.REXPInteger; @@ -25,32 +34,45 @@ import java.math.RoundingMode; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.DateFormat; import java.text.DecimalFormat; +import java.time.Duration; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; +@Slf4j public class SourceRebuildTaskExec extends Thread{ - private List taskMonitoringDatas; private SourceRebuildTaskLogService sourceRebuildTaskLogService; private SourceRebuildTask sourceRebuildTask; private SourceRebuildProperties sourceRebuildProperties; private SourceRebuildTaskService sourceRebuildTaskService; + private ServerProperties serverProperties; + private SourceRebuildMonitoringDataService monitoringDataService; + private List taskMonitoringDatas; + private List srsFilesPath; + private JSchRemoteRunner jschRemoteRunner = null; + private boolean taskRunError = false; + private String errorFlag = "__R_ERROR__"; /** * 初始化 */ - public void init(List taskMonitoringDatas, - SourceRebuildTask sourceRebuildTask, + public void init(SourceRebuildTask sourceRebuildTask, SourceRebuildTaskLogService sourceRebuildTaskLogService, SourceRebuildProperties sourceRebuildProperties, - SourceRebuildTaskService sourceRebuildTaskService){ - this.taskMonitoringDatas = taskMonitoringDatas; + SourceRebuildTaskService sourceRebuildTaskService, + ServerProperties serverProperties, + SourceRebuildMonitoringDataService monitoringDataService){ this.sourceRebuildTask = sourceRebuildTask; this.sourceRebuildTaskLogService = sourceRebuildTaskLogService; this.sourceRebuildProperties = sourceRebuildProperties; this.sourceRebuildTaskService = sourceRebuildTaskService; + this.serverProperties = serverProperties; + this.monitoringDataService = monitoringDataService; } @Override @@ -58,6 +80,92 @@ public class SourceRebuildTaskExec extends Thread{ this.execute(); } + /** + * 校验任务 + * @return + */ + public boolean checkTask(){ + boolean flag = true; + try { + List msgList = new ArrayList<>(); + //校验监测数据 + List taskMonitoringDatas = this.monitoringDataService.getTaskMonitoringData(sourceRebuildTask.getId()); + if(CollectionUtils.isEmpty(taskMonitoringDatas)){ + flag = false; + msgList.add("监测数据为空，请补充监测数据"); + }else { + this.taskMonitoringDatas = taskMonitoringDatas; + jschRemoteRunner = new JSchRemoteRunner(); + //登录ssh + jschRemoteRunner.login(this.serverProperties.getHost(),this.serverProperties.getPort(), + this.serverProperties.getUsername(),this.serverProperties.getPassword()); + //校验SRS目录是否存在 + List srsDirsPath = new ArrayList<>(); + for (SourceRebuildMonitoringData monitoringData : taskMonitoringDatas) { + StringBuilder srsPath = new StringBuilder(); + srsPath.append(sourceRebuildProperties.getSrsFilePath()); +// srsPath.append(File.separator); + srsPath.append("/"); + srsPath.append(DateUtil.format(monitoringData.getCollectStop(),"yyyyMMdd")); +// srsPath.append(File.separator); + srsPath.append("/"); + srsPath.append(sourceRebuildProperties.getXeGZFileDirSign()); + srsDirsPath.add(srsPath.toString()); + } + //获取不存在的srs目录 + List notExistDirs = jschRemoteRunner.getNotExistDirs(srsDirsPath); + if (CollUtil.isNotEmpty(notExistDirs)){ + flag = false; + String[] notExistDirArr = new String[notExistDirs.size()]; + for(int i=0;i notExistFiles = new ArrayList<>(); + List srsFilesPath = new ArrayList<>(); + List filesPath = jschRemoteRunner.listFiles(srsDirsPath, ".srm.gz"); + taskMonitoringDatas.forEach(data->{ + Map srsFilePathMap = new HashMap<>(); + for (String filePath : filesPath){ + String fileName = filePath.substring(filePath.lastIndexOf("/")+1); + String[] fileNameSplit = fileName.split("\\."); + LocalDateTime date = LocalDateTime.parse(fileNameSplit[2], DateTimeFormatter.ofPattern("yyyyMMddHH")); + if(data.getStation().equals(fileNameSplit[0])){ + Duration duration = Duration.between(data.getCollectStop(), date).abs(); + long hours = duration.toHours(); + if (hours <= 6){ + String relativePath = filePath.substring(sourceRebuildProperties.getSrsFilePath().length()+1); + srsFilePathMap.put(hours,relativePath); + } + } + } + if (CollUtil.isNotEmpty(srsFilePathMap)){ + Long min = CollUtil.min(srsFilePathMap.keySet()); + srsFilesPath.add(srsFilePathMap.get(min)+"\n"); + }else { + String collectStopStr = DateUtil.format(data.getCollectStop(),"yyyy-MM-dd HH:mm:ss"); + notExistFiles.add(collectStopStr+"样品对应的SRS文件不存在"); + } + }); + //如果notExistFiles为空说明样品对应的srs文件都找到了 + if (CollUtil.isEmpty(notExistFiles)){ + this.srsFilesPath = srsFilesPath; + }else { + flag = false; + this.batchGenerateLog(notExistFiles.toArray(new String[0])); + } + } + } + }catch (Exception e){ + this.sourceRebuildTaskService.updateTaskStatus(this.sourceRebuildTask.getId(), SourceRebuildTaskStatusEnum.INSPECTION_FAILED.getValue()); + log.error("源项重建任务检查出现错误，任务名称：{}",this.sourceRebuildTask.getTaskName(),e); + jschRemoteRunner.close(); + } + return flag; + } + /** * 执行任务 */ @@ -72,30 +180,30 @@ public class SourceRebuildTaskExec extends Thread{ String startRunLog = "----------------------------------------开始执行任务----------------------------------------"; this.generateLog(startRunLog); //生成重建需要的监测数据 - String monitorDataLog = "----------------------------------------生成监测数据输入文件----------------------------------------"; - String titleLog = "Entity,Metric,Date,Value,Uncertainty,MDC Value"; - this.generateLog(monitorDataLog); - this.generateLog(titleLog); this.generateMonitoringDataFile(); //生成SRS关系文件 - String srsDataLog = "----------------------------------------生成SRS数据输入文件----------------------------------------"; - this.generateLog(srsDataLog); this.generateSRSDataFile(); - String sourceProcessDataLog = "----------------------------------------配置源项分析参数并执行源项重建----------------------------------------"; - this.generateLog(sourceProcessDataLog); this.execSourceRebuild(); - String genStopLog = "----------------------------------------任务执行结束----------------------------------------"; - this.generateLog(genStopLog); }catch (Exception e){ - String taskErrorLog = "任务执行失败，原因："+e.getMessage(); - this.generateLog(taskErrorLog); - throw e; + taskRunError = true; + this.generateLog("任务执行失败，原因："+e.getMessage()); + log.error("任务执行失败，原因为：",e); }finally { + jschRemoteRunner.close(); //添加任务耗时 stopWatch.stop(); long seconds = stopWatch.getTime(TimeUnit.SECONDS); Double min = seconds/60D; - this.sourceRebuildTaskService.updateTaskTimeConsuming(this.sourceRebuildTask.getId(),min); + Integer taskState; + if (taskRunError){ + //如果taskRunError等于true，说明运行过程出错，这里把任务状态改为失败 + taskState = SourceRebuildTaskStatusEnum.FAILURE.getValue(); + }else{ + taskState = SourceRebuildTaskStatusEnum.COMPLETED.getValue(); + } + this.sourceRebuildTaskService.updateTaskTimeConsuming(this.sourceRebuildTask.getId(),min,taskState); + String taskCompletedLog = "任务执行结束，耗时："+min+"分钟"; + ProgressQueue.getInstance().offer(new ProgressEvent(this.sourceRebuildTask.getId(),taskCompletedLog)); } } @@ -103,17 +211,16 @@ public class SourceRebuildTaskExec extends Thread{ * 生成重建需要的监测数据 */ private void generateMonitoringDataFile(){ + String monitorDataLog = "----------------------------------------生成监测数据输入文件----------------------------------------"; + String title = "Entity,Metric,Date,Value,Uncertainty,MDC Value\n"; + this.generateLog(monitorDataLog); + this.generateLog(title); //根据监测数据生成文件（input_subexp1.dat） - String inputPath = this.getInputPath(); - File inputSubexp1 = new File(inputPath +File.separator+ "input_subexp1.dat"); - if(!FileUtil.exist(inputSubexp1)){ - FileUtil.touch(inputSubexp1); - } +// String inputPath = sourceRebuildProperties.getRInput()+File.separator+ "input_subexp1.dat"; + String inputPath = sourceRebuildProperties.getRInput()+"/"+ "input_subexp1.dat"; //格式化监测数据 - String title = "Entity,Metric,Date,Value,Uncertainty,MDC Value"; List lines = new ArrayList<>(); lines.add(title); - for(SourceRebuildMonitoringData taskMonitoringData : taskMonitoringDatas){ String station = taskMonitoringData.getStation(); String nuclide = taskMonitoringData.getNuclide(); @@ -122,93 +229,35 @@ public class SourceRebuildTaskExec extends Thread{ String uncertainty = this.formatSCI(taskMonitoringData.getUncertainty()); String mdc = this.formatSCI(taskMonitoringData.getMdc()); - String formattedLine = String.format("%s, %s, %s, %s, %s, %s",station,nuclide,date,activity,uncertainty,mdc); + String formattedLine = String.format("%s, %s, %s, %s, %s, %s\n",station,nuclide,date,activity,uncertainty,mdc); lines.add(formattedLine); //把监测数据详情保存到日志 this.generateLog(formattedLine); } - FileUtil.writeLines(lines,inputSubexp1,"UTF-8"); + jschRemoteRunner.writeFile(inputPath,String.join("",lines)); } /** * 生成SRS关系数据文件 */ private void generateSRSDataFile(){ - //以日期为key，以文件相对路径为value - Map allSRSFileMap = new HashMap<>(); - Map finalSRSFileMap = new TreeMap<>(); - //根据监测数据校验SRS文件，如果缺少SRS文件也报错 - String srsFilePath = this.getSRSFilePath(); - Path dirpath = Paths.get(srsFilePath); - try(Stream walk = Files.walk(dirpath)) { - walk.filter(Files::isRegularFile) - .filter(path -> path.toAbsolutePath().toString().contains(sourceRebuildProperties.getXeGZFileDirSign())) - .forEach(filePath -> { - //USX77.fp.2024112115.f9.srm - String[] nameArr = filePath.getFileName().toString().split("\\."); - Date date = DateUtil.parse(nameArr[2],"yyyyMMddHH"); - String relativePath = filePath.toAbsolutePath().toString().substring(filePath.toAbsolutePath().toString().indexOf("srs")); - allSRSFileMap.put(date,relativePath); - }); - //如果遍历指定目录没有找到SRS数据文件则结束 - if(CollectionUtils.isEmpty(allSRSFileMap)){ - String srsNotExistLog = "SRS 文件存储目录中没有SRS数据文件"; - throw new RuntimeException(srsNotExistLog); - } - //根据监测数据多线程找对应的SRS数据文件，每一个没找到都写入日志 - int srsNotExistCount = 0; - for(SourceRebuildMonitoringData monitoringData : this.taskMonitoringDatas){ - boolean flag = false; - String station = monitoringData.getStation(); - Date collectStop = monitoringData.getCollectStop(); - for (Map.Entry entry : allSRSFileMap.entrySet()) { - Date date = entry.getKey(); - String srsFile = entry.getValue(); - if(srsFile.contains(station)){ - boolean withinSixHours = this.isWithinSixHours(collectStop, date); - if (withinSixHours) { - finalSRSFileMap.put(collectStop, srsFile); - flag = true; - break; - } - } - } - if (!flag){ - srsNotExistCount ++; - String targetSRSNotExist = "监测数据："+DateUtil.format(collectStop,"yyyyMMddHH")+"所对应的SRS文件不存在"; - this.generateLog(targetSRSNotExist); - } - } - //当监测数据都寻找完对应的SRS数据文件后，未寻找到的结果如果大于0，则结束流程，并写入日志 - if(srsNotExistCount > 0){ - String countSRSNotExist = "共有："+srsNotExistCount+"条监测数据，找不到对应的SRS数据文件"; - this.generateLog(countSRSNotExist); - throw new RuntimeException(countSRSNotExist); - } - //每条监测数据都找到SRS数据文件后生成文件（srsfilelist_subexp1.dat） - String inputPath = getInputPath(); - File srsSubexp1 = new File(inputPath +File.separator+ "srsfilelist_subexp1.dat"); - if(!FileUtil.exist(srsSubexp1)){ - FileUtil.touch(srsSubexp1); - } - //去重并格式化监测数据 - List lines = new ArrayList<>(finalSRSFileMap.values()); - //把SRS数据详情保存到日志 - lines.forEach(this::generateLog); - //写入到文件 - FileUtil.writeLines(lines,srsSubexp1,"UTF-8"); - }catch (Exception e){ - throw new RuntimeException(e.getMessage()); - } + String srsDataLog = "----------------------------------------生成SRS数据输入文件----------------------------------------"; + this.generateLog(srsDataLog); + this.srsFilesPath.forEach(this::generateLog); +// String inputPath = sourceRebuildProperties.getRInput()+File.separator+ "srsfilelist_subexp1.dat"; + String inputPath = sourceRebuildProperties.getRInput()+"/"+ "srsfilelist_subexp1.dat"; + jschRemoteRunner.writeFile(inputPath,String.join("",this.srsFilesPath)); } /** * 执行源项重建 */ - private void execSourceRebuild(){ + private void execSourceRebuild() throws Exception { + String sourceProcessDataLog = "----------------------------------------配置源项分析参数并执行源项重建----------------------------------------"; + this.generateLog(sourceProcessDataLog); RConnection conn = null; try { - conn = new RConnection(this.sourceRebuildProperties.getServerAddr(), this.sourceRebuildProperties.getPort()); + conn = new RConnection(this.serverProperties.getHost(), this.sourceRebuildProperties.getPort()); conn.login(this.sourceRebuildProperties.getUsername(), this.sourceRebuildProperties.getPassword()); if(conn.isConnected()){ String log = "Rserve 连接成功，设置运行参数"; @@ -216,8 +265,8 @@ public class SourceRebuildTaskExec extends Thread{ //重建任务名称 conn.assign("experiment", "FREAR_syntheticTestCase"); //重建任务输入数据目录 - conn.assign("expdir", this.getInputPath()); - conn.assign("datadir", this.getSRSFilePath()); + conn.assign("expdir", this.sourceRebuildProperties.getRInput()); + conn.assign("datadir", this.sourceRebuildProperties.getSrsFilePath()); //重建任务输出数据目录 conn.assign("outbasedir", this.getOutputPath()); //数据库相关 @@ -243,8 +292,8 @@ public class SourceRebuildTaskExec extends Thread{ String domain = String.format("domain <- list(lonmin=%s,latmin=%s,lonmax=%s,latmax=%s,dx=%s,dy=%s)",lonmin,latmin,lonmax,latmax,resolution,resolution); conn.eval(domain); //设置时间范围 - String srsStartTime = DateUtil.format(this.sourceRebuildTask.getSrsStartTime(), "yyyy-MM-dd"); - String srsEndTime = DateUtil.format(this.sourceRebuildTask.getSrsEndTime(), "yyyy-MM-dd"); + String srsStartTime = this.sourceRebuildTask.getSrsStartTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + String srsEndTime = this.sourceRebuildTask.getSrsEndTime().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); String times = String.format("times <- expression(seq(as.POSIXct('%s'),as.POSIXct('%s'),by=3*3600))",srsStartTime,srsEndTime); conn.eval(times); //如果已知排放源位置 @@ -269,19 +318,28 @@ public class SourceRebuildTaskExec extends Thread{ conn.eval("setwd(\""+this.sourceRebuildProperties.getWorkSpace()+"\")"); //执行脚本 conn.eval("source(\"R_scripts/set_up.R\")"); - REXP outputREXP = conn.eval("capture.output({source(\"R_scripts/main.R\")})"); + String rCmd = + "local({" + + " tryCatch({" + + " out <- capture.output(source('R_scripts/main.R', echo=TRUE, print.eval=TRUE));" + + " c('__R_OK__', out)" + + " }, error=function(e) {" + + " c('__R_ERROR__'," + + " paste('message:', conditionMessage(e))," + + " paste('call:', paste(deparse(conditionCall(e)), collapse=' '))" + + " )" + + " })" + + "})"; +// REXP outputREXP = conn.eval("capture.output({source(\"R_scripts/main.R\")})"); + REXP outputREXP = conn.eval(rCmd); // 将输出作为字符串数组获取 String[] outputLines = outputREXP.asStrings(); if(ArrayUtil.isNotEmpty(outputLines)){ this.batchGenerateLog(outputLines); } } - } catch (RserveException e) { - String log = "Rserve 执行过程报错，请检查配置参数"; - e.printStackTrace(); - throw new RuntimeException(log); - } catch (REXPMismatchException e) { - throw new RuntimeException(e); + }catch (Exception e) { + throw new Exception(e); }finally { if(Objects.nonNull(conn) && conn.isConnected()){ conn.close(); @@ -289,18 +347,6 @@ public class SourceRebuildTaskExec extends Thread{ } } - /** - * 判断两个时间误差是否在2小时内 - * @param date1 - * @param date2 - * @return - */ - private boolean isWithinSixHours(Date date1, Date date2){ - long diffInMillis = Math.abs(date2.getTime() - date1.getTime());// 毫秒差（绝对值） - long sixHoursInMillis = 6 * 60 * 60 * 1000; // 6小时 - return diffInMillis <= sixHoursInMillis; - } - /** * 格式化为科学计数法 * @param value @@ -317,33 +363,6 @@ public class SourceRebuildTaskExec extends Thread{ return df.format(result); } - /** - * 获取R项目运行输入监测数据及监测数据和srs对应关系文件地址 - * @return - */ - private String getInputPath(){ - StringBuilder inputPath = new StringBuilder(); - inputPath.append(sourceRebuildProperties.getRInput()); - if(!FileUtil.exist(inputPath.toString())){ - FileUtil.mkdir(inputPath.toString()); - } - return inputPath.toString(); - } - - /** - * 获取SRS 文件存储路径 - * @return - */ - private String getSRSFilePath(){ - StringBuilder srsFilePath = new StringBuilder(); - srsFilePath.append(sourceRebuildProperties.getSrsFilePath()); - if(!FileUtil.exist(srsFilePath.toString())){ - String srsNotExistLog = "SRS 文件存储目录不存在"; - throw new RuntimeException(srsNotExistLog); - } - return srsFilePath.toString(); - } - /** * 获取R项目运行结果输出地址 * @return @@ -351,11 +370,11 @@ public class SourceRebuildTaskExec extends Thread{ private String getOutputPath(){ StringBuilder outputPath = new StringBuilder(); outputPath.append(sourceRebuildProperties.getROutput()); - outputPath.append(File.separator); + outputPath.append("/"); +// outputPath.append(File.separator); outputPath.append(sourceRebuildTask.getTaskName()); - if(!FileUtil.exist(outputPath.toString())){ - FileUtil.mkdir(outputPath.toString()); - } + //创建任务输出目录 + this.jschRemoteRunner.mkdir(outputPath.toString()); return outputPath.toString(); } @@ -364,6 +383,9 @@ public class SourceRebuildTaskExec extends Thread{ * @param logContent */ private void generateLog(String logContent){ + if(logContent.contains(errorFlag)){ + taskRunError = true; + } sourceRebuildTaskLogService.cteate(new SourceRebuildTaskLog(sourceRebuildTask.getId(),logContent)); } @@ -374,6 +396,9 @@ public class SourceRebuildTaskExec extends Thread{ private void batchGenerateLog(String[] logContents){ List logs = new ArrayList<>(); for(String logContent : logContents){ + if(logContent.contains(errorFlag)){ + taskRunError = true; + } logs.add(new SourceRebuildTaskLog(sourceRebuildTask.getId(),logContent)); } sourceRebuildTaskLogService.batchCteate(logs); diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskMessageConsumerHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskConsumerHandler.java similarity index 88% rename from jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskMessageConsumerHandler.java rename to jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskConsumerHandler.java index fd82f38..70b0de8 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskMessageConsumerHandler.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/TranTaskConsumerHandler.java @@ -13,13 +13,13 @@ import org.jeecg.common.constant.RocketMQTopConstant; import org.jeecg.common.constant.enums.TransportTaskStatusEnum; import org.jeecg.common.constant.enums.TransportTaskTopEnum; import org.jeecg.common.properties.DataFusionProperties; +import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.properties.SystemStorageProperties; import org.jeecg.common.properties.TransportSimulationProperties; import org.jeecg.common.util.RedisUtil; import org.jeecg.modules.base.dto.TransportTaskDTO; import org.jeecg.modules.base.entity.TransportTask; import org.jeecg.modules.base.mapper.*; -import org.jeecg.properties.ServerProperties; import org.jeecg.transport.consumer.china.AbstractTaskMsgHandler; import org.jeecg.transport.consumer.china.Server11TaskHandler; import org.jeecg.transport.service.StationDataService; @@ -38,7 +38,7 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component @RequiredArgsConstructor -public class TranTaskMessageConsumerHandler{ +public class TranTaskConsumerHandler{ private final RedisUtil redisUtil; private final TransportTaskMapper transportTaskMapper; @@ -106,19 +106,14 @@ public class TranTaskMessageConsumerHandler{ //获取本机项数据，如果为空，表示本机没有任务在运行 boolean flag = redisUtil.hHasKey(CommonConstant.HOST_TASK_STATE,CommonConstant.TRAN_TASK_STATE_PRE+serverProperties.getHost()); if (!flag) { - try { - TransportTask topMessage = getTopMessage(); - if (Objects.nonNull(topMessage)) { - handlerTask(topMessage); - } - }catch (Exception e){ - log.error("消费输运任务数据异常，原因为：{}",e.getMessage(),e); + TransportTask topMessage = getTopMessage(); + if (Objects.nonNull(topMessage)) { + handlerTask(topMessage); } } - log.info("60秒钟处理一次"); - TimeUnit.SECONDS.sleep(30); + TimeUnit.SECONDS.sleep(60); } catch (InterruptedException e) { - throw new RuntimeException(e); + log.error("执行输运任务数据异常，原因为：",e); } } } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractChain.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractChain.java index 08bd7e1..f06b7d8 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractChain.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractChain.java @@ -3,7 +3,7 @@ package org.jeecg.transport.consumer.china; import lombok.Setter; /** - * 能谱执行链路 + * 执行链路 */ public abstract class AbstractChain { diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractTaskMsgHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractTaskMsgHandler.java index 7494153..c525fcf 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractTaskMsgHandler.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/AbstractTaskMsgHandler.java @@ -1,23 +1,20 @@ package org.jeecg.transport.consumer.china; import lombok.extern.slf4j.Slf4j; -import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.enums.TransportTaskModeEnum; import org.jeecg.common.properties.DataFusionProperties; +import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.properties.SystemStorageProperties; import org.jeecg.common.properties.TransportSimulationProperties; import org.jeecg.common.util.RedisUtil; -import org.jeecg.modules.base.dto.TransportTaskDTO; import org.jeecg.modules.base.entity.TransportTask; import org.jeecg.modules.base.mapper.*; -import org.jeecg.properties.ServerProperties; import org.jeecg.transport.flexparttask.AbstractTaskExec; import org.jeecg.transport.flexparttask.BackwardTaskExec; import org.jeecg.transport.flexparttask.ForwardTaskExec; import org.jeecg.transport.service.StationDataService; import org.jeecg.transport.service.StationsModValService; import org.jeecg.transport.service.TransportTaskService; -import java.util.Objects; /** * 处理任务策略父类 @@ -120,7 +117,7 @@ public abstract class AbstractTaskMsgHandler extends AbstractChain{ } } if (flag){ - log.info("任务：{}，匹配成功，当前在主机：{}上执行。", transportTask.getTaskName(),ip); + log.info("任务：{}，检查通过，当前在主机：{}上执行。", transportTask.getTaskName(),ip); }else { transportTaskService.setInpectionFailed(transportTask); } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server13TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server13TaskHandler.java index 53c77dd..22bac0b 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server13TaskHandler.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server13TaskHandler.java @@ -24,7 +24,7 @@ public class Server13TaskHandler extends AbstractTaskMsgHandler { @Override public void handler(TransportTask message) { if(serverProperties.getHost().equals(serverProperties.getIp13())){ - super.runTask(message,serverProperties.getIp12()); + super.runTask(message,serverProperties.getIp13()); }else { super.next.handler(message); } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server14TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server14TaskHandler.java index 6b49a9e..c46c244 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server14TaskHandler.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/consumer/china/Server14TaskHandler.java @@ -21,6 +21,7 @@ public class Server14TaskHandler extends AbstractTaskMsgHandler { @Override public void handler(TransportTask message) { if(serverProperties.getHost().equals(serverProperties.getIp14())){ + super.runTask(message,serverProperties.getIp14()); //这里需要再加个判断，14服务器处理反演和永久任务 // if (TransportTaskModeEnum.BACK_FORWARD.getKey().equals(message.getTaskMode())){ // super.runTask(message,serverProperties.getIp11()); diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/AbstractTaskExec.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/AbstractTaskExec.java index be7aee8..92eff22 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/AbstractTaskExec.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/AbstractTaskExec.java @@ -10,13 +10,13 @@ import org.apache.logging.log4j.util.Strings; import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.enums.WeatherDataSourceEnum; import org.jeecg.common.properties.DataFusionProperties; +import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.properties.SystemStorageProperties; import org.jeecg.common.properties.TransportSimulationProperties; import org.jeecg.common.util.RedisUtil; import org.jeecg.modules.base.entity.TransportTask; import org.jeecg.modules.base.entity.WeatherData; import org.jeecg.modules.base.mapper.*; -import org.jeecg.properties.ServerProperties; import org.jeecg.transport.service.StationDataService; import org.jeecg.transport.service.StationsModValService; import org.jeecg.transport.service.TransportTaskService; @@ -147,20 +147,18 @@ public abstract class AbstractTaskExec extends Thread{ */ protected String getMetDataPath(Integer dataSource){ StringBuilder path = new StringBuilder(); - path.append(systemStorageProperties.getRootPath()); - path.append(File.separator); if(WeatherDataSourceEnum.PANGU.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getPangu()); + path.append(systemStorageProperties.getPanguDataPath()+"/format"); }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getGraphcast()); + path.append(systemStorageProperties.getGraphcastDataPath()+"/format"); }else if(WeatherDataSourceEnum.CRA40.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getCra40()); + path.append(systemStorageProperties.getCra40DataPath()); }else if(WeatherDataSourceEnum.NCEP.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getNcep()); + path.append(systemStorageProperties.getNcepDataPath()); }else if(WeatherDataSourceEnum.FNL.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getFnl()); + path.append(systemStorageProperties.getFnlDataPath()); }else if(WeatherDataSourceEnum.T1H.getKey().equals(dataSource)){ - path.append(systemStorageProperties.getT1h()); + path.append(systemStorageProperties.getT1hDataPath()); } return path.toString(); } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/BackwardTaskExec.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/BackwardTaskExec.java index 4d080ee..84f82db 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/BackwardTaskExec.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/BackwardTaskExec.java @@ -9,7 +9,7 @@ import org.jeecg.common.constant.enums.FlexpartSpeciesType; import org.jeecg.common.constant.enums.TransportTaskStatusEnum; import org.jeecg.common.constant.enums.WeatherDataSourceEnum; import org.jeecg.modules.base.entity.*; -import org.jeecg.transport.util.JSchRemoteRunner; +import org.jeecg.jsch.JSchRemoteRunner; import java.io.File; import java.math.BigDecimal; import java.math.RoundingMode; @@ -119,7 +119,7 @@ public class BackwardTaskExec extends AbstractTaskExec { taskState = TransportTaskStatusEnum.COMPLETED.getValue(); } super.transportTaskService.updateTaskStatusToCompleted(super.transportTask.getId(),result.doubleValue(),taskState); - String taskCompletedLog = "任务执行完成，耗时："+min+"分钟"; + String taskCompletedLog = "任务执行结束，耗时："+min+"分钟"; ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskCompletedLog)); } } @@ -150,7 +150,8 @@ public class BackwardTaskExec extends AbstractTaskExec { paramContent.append(super.transportTask.getZ1()).append("\n"); paramContent.append(super.transportTask.getZ2()).append("\n"); paramContent.append(metDataPath).append("\n"); - paramContent.append(super.simulationProperties.getOutputPath()+ File.separator+super.transportTask.getTaskName()).append("\n"); + paramContent.append(super.simulationProperties.getOutputPath()+ "/" +super.transportTask.getTaskName()).append("\n"); +// paramContent.append(super.simulationProperties.getOutputPath()+ File.separator+super.transportTask.getTaskName()).append("\n"); paramContent.append(FlexpartSpeciesType.NOT_SPECIES.getId()).append("\n");//反演固定61，不显示具体核素，只显示Xe jschRemoteRunner.writeFile(paramConfigPath,paramContent.toString()); //处理台站数据文件 diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ForwardTaskExec.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ForwardTaskExec.java index 2e75041..1134de6 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ForwardTaskExec.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ForwardTaskExec.java @@ -8,7 +8,7 @@ import org.apache.logging.log4j.util.Strings; import org.jeecg.common.constant.enums.TransportTaskStatusEnum; import org.jeecg.common.constant.enums.WeatherDataSourceEnum; import org.jeecg.modules.base.entity.*; -import org.jeecg.transport.util.JSchRemoteRunner; +import org.jeecg.jsch.JSchRemoteRunner; import java.io.File; import java.math.BigDecimal; import java.math.RoundingMode; @@ -136,7 +136,7 @@ public class ForwardTaskExec extends AbstractTaskExec { taskState = TransportTaskStatusEnum.COMPLETED.getValue(); } super.transportTaskService.updateTaskStatusToCompleted(super.transportTask.getId(),result.doubleValue(),taskState); - String taskCompletedLog = "任务执行完成，耗时："+min+"分钟"; + String taskCompletedLog = "任务执行结束，耗时："+min+"分钟"; ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskCompletedLog)); } } diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ProgressMonitor.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ProgressMonitor.java index bf0386f..d725037 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ProgressMonitor.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/flexparttask/ProgressMonitor.java @@ -3,7 +3,6 @@ package org.jeecg.transport.flexparttask; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.jeecg.common.constant.enums.TransportTaskStatusEnum; import org.jeecg.modules.base.entity.TransportTaskLog; import org.jeecg.transport.service.TransportTaskService; import org.springframework.stereotype.Component; @@ -14,7 +13,7 @@ import java.util.Objects; * @author 86187 * */ -@Component +@Component("transportProgressMonitor") @RequiredArgsConstructor @Slf4j public class ProgressMonitor{ diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/transport/service/impl/TransportTaskServiceImpl.java b/jeecg-model-consumer/src/main/java/org/jeecg/transport/service/impl/TransportTaskServiceImpl.java index b15ddee..df637cc 100644 --- a/jeecg-model-consumer/src/main/java/org/jeecg/transport/service/impl/TransportTaskServiceImpl.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/transport/service/impl/TransportTaskServiceImpl.java @@ -6,10 +6,10 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.RequiredArgsConstructor; import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.enums.*; +import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.util.RedisUtil; import org.jeecg.modules.base.entity.*; import org.jeecg.modules.base.mapper.*; -import org.jeecg.properties.ServerProperties; import org.jeecg.transport.service.TransportTaskService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -126,7 +126,6 @@ public class TransportTaskServiceImpl extends ServiceImpl topQueryWrapper = new LambdaQueryWrapper<>(); + topQueryWrapper.eq(WeatherTask::getTaskStatus, WeatherTaskStatusEnum.WAITING.getValue()); + topQueryWrapper.orderByDesc(WeatherTask::getTopTask); + topQueryWrapper.orderByDesc(WeatherTask::getUpdateTime); + topQueryWrapper.last("LIMIT 1"); + WeatherTask weatherTask = this.weatherTaskService.getOne(topQueryWrapper); + //如果没有要置顶执行的任务，则按照普通创建时间排序查询 + if(Objects.isNull(weatherTask)){ + LambdaQueryWrapper timeQueryWrapper = new LambdaQueryWrapper<>(); + timeQueryWrapper.eq(WeatherTask::getTaskStatus, SourceRebuildTaskStatusEnum.WAITING.getValue()); + timeQueryWrapper.orderByAsc(WeatherTask::getCreateTime); + timeQueryWrapper.last("LIMIT 1"); + weatherTask = this.weatherTaskService.getOne(timeQueryWrapper); + } + return weatherTask; + } + + private class MessageConsumerThread extends Thread{ + @Override + public void run() { + while (true) { + try { + //获取本机项数据，如果为空，表示本机没有任务在运行 + boolean flag = redisUtil.hHasKey(CommonConstant.HOST_TASK_STATE,CommonConstant.WEATHER_TASK_STATE_PRE+serverProperties.getHost()); + if (!flag) { + WeatherTask topMessage = getTopMessage(); + if (Objects.nonNull(topMessage)) { + handlerTask(topMessage); + } + } + TimeUnit.SECONDS.sleep(30); + } catch (InterruptedException e) { + log.error("执行源项重建任务数据异常，原因为：",e); + } + } + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractChain.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractChain.java new file mode 100644 index 0000000..97591df --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractChain.java @@ -0,0 +1,27 @@ +package org.jeecg.weather.consumer.china; + +import lombok.Setter; + +/** + * 执行链路 + */ +public abstract class AbstractChain { + + /** + * 上一任处理链 + */ + @Setter + protected AbstractTaskMsgHandler previous; + + /** + * 下一任处理链 + */ + @Setter + protected AbstractTaskMsgHandler next; + + /** + * 设置过滤器链 + */ + protected abstract void setChina(); + +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractTaskMsgHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractTaskMsgHandler.java new file mode 100644 index 0000000..f87e665 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/AbstractTaskMsgHandler.java @@ -0,0 +1,83 @@ +package org.jeecg.weather.consumer.china; + +import lombok.extern.slf4j.Slf4j; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.properties.SystemStorageProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.WeatherTask; +import org.jeecg.modules.base.mapper.WeatherDataMapper; +import org.jeecg.weather.service.WeatherTaskService; +import org.jeecg.weather.task.AbstractWeatherTask; +import org.jeecg.weather.task.WeatherForecastTaskExec; + +/** + * 处理任务策略父类 + */ +@Slf4j +public abstract class AbstractTaskMsgHandler extends AbstractChain { + + protected RedisUtil redisUtil; + protected WeatherTaskService weatherTaskService; + protected SystemStorageProperties systemStorageProperties; + protected WeatherDataMapper weatherDataMapper; + protected ServerProperties serverProperties; + + /** + * 初始化bean引用 + */ + public void init(RedisUtil redisUtil, + WeatherTaskService weatherTaskService, + SystemStorageProperties systemStorageProperties, + WeatherDataMapper weatherDataMapper, + ServerProperties serverProperties) { + this.redisUtil = redisUtil; + this.weatherTaskService = weatherTaskService; + this.systemStorageProperties = systemStorageProperties; + this.weatherDataMapper = weatherDataMapper; + this.serverProperties = serverProperties; + + this.setChina(); + } + + /** + * 初始化下个节点 + * @param serverProperties + */ + protected void initNext(ServerProperties serverProperties){ + this.serverProperties = serverProperties; + this.setChina(); + } + + /** + * 处理消息 + */ + public abstract void handler(WeatherTask message); + + /** + * 运行任务 + * @param weatherTask + */ + protected void runTask(WeatherTask weatherTask,String ip){ + try { + boolean flag; + AbstractWeatherTask taskExec = new WeatherForecastTaskExec(); + taskExec.init( + redisUtil, + weatherTask, + weatherTaskService, + systemStorageProperties, + weatherDataMapper, + serverProperties); + taskExec.setName("weather-forecast-task"); + flag = taskExec.checkTask(weatherTask); + if (flag){ + log.info("任务：{}，检查通过，当前在主机：{}上执行。", weatherTask.getTaskName(),ip); + taskExec.start(); + }else { + weatherTaskService.setInpectionFailed(weatherTask); + } + }catch (Exception e){ + log.error("检查任务或开启任务执行线程出现错误",e); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server11TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server11TaskHandler.java new file mode 100644 index 0000000..bef5c72 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server11TaskHandler.java @@ -0,0 +1,33 @@ +package org.jeecg.weather.consumer.china; + +import org.jeecg.modules.base.entity.WeatherTask; + +/** + * 处理天气预报任务 + */ +public class Server11TaskHandler extends AbstractTaskMsgHandler { + + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server12TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(WeatherTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp11())){ + super.runTask(message,serverProperties.getIp11()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server12TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server12TaskHandler.java new file mode 100644 index 0000000..2835ac2 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server12TaskHandler.java @@ -0,0 +1,32 @@ +package org.jeecg.weather.consumer.china; + +import org.jeecg.modules.base.entity.WeatherTask; + +/** + * 处理天气预报任务 + */ +public class Server12TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server13TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(WeatherTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp12())){ + super.runTask(message,serverProperties.getIp12()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server13TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server13TaskHandler.java new file mode 100644 index 0000000..0af3917 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server13TaskHandler.java @@ -0,0 +1,32 @@ +package org.jeecg.weather.consumer.china; + +import org.jeecg.modules.base.entity.WeatherTask; + +/** + * 处理天气预报任务 + */ +public class Server13TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + AbstractTaskMsgHandler taskHandler = new Server14TaskHandler(); + taskHandler.initNext(super.serverProperties); + taskHandler.setPrevious(this); + super.setNext(taskHandler); + } + + /** + * 处理消息 + */ + @Override + public void handler(WeatherTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp13())){ + super.runTask(message,serverProperties.getIp13()); + }else { + super.next.handler(message); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server14TaskHandler.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server14TaskHandler.java new file mode 100644 index 0000000..07690e8 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/consumer/china/Server14TaskHandler.java @@ -0,0 +1,27 @@ +package org.jeecg.weather.consumer.china; + +import org.jeecg.modules.base.entity.WeatherTask; + +/** + * 处理天气预报任务 + */ +public class Server14TaskHandler extends AbstractTaskMsgHandler { + + /** + * 设置过滤器链 + */ + @Override + protected void setChina() { + + } + + /** + * 处理消息 + */ + @Override + public void handler(WeatherTask message) { + if(serverProperties.getHost().equals(serverProperties.getIp14())){ + super.runTask(message,serverProperties.getIp14()); + } + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/WeatherTaskService.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/WeatherTaskService.java new file mode 100644 index 0000000..3a92fd1 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/WeatherTaskService.java @@ -0,0 +1,45 @@ +package org.jeecg.weather.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import org.jeecg.modules.base.entity.WeatherTask; +import org.jeecg.modules.base.entity.WeatherTaskLog; + +/** + * 天气预报预测任务管理 + */ +public interface WeatherTaskService extends IService { + + /** + * 保存任务过程日志 + * @param log + */ + void saveLog(WeatherTaskLog log); + + /** + * 修改任务运行状态 + * @param taskId + * @param taskStatus + */ + void updateTaskStatus(Integer taskId,Integer taskStatus); + + /** + * 根据任务id删除任务日志 + * @param taskId + */ + void deleteTaskLog(Integer taskId); + + /** + * 修改任务状态为结束(完成或异常停止) + * @param taskId + * @param timeConsuming + * @param taskStatus + */ + void updateTaskStatusToCompleted(Integer taskId, Double timeConsuming,Integer taskStatus); + + /** + * 设置检查失败 + * @param weatherTask + */ + void setInpectionFailed(WeatherTask weatherTask); + +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/impl/WeatherTaskServiceImpl.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/impl/WeatherTaskServiceImpl.java new file mode 100644 index 0000000..fbfae12 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/service/impl/WeatherTaskServiceImpl.java @@ -0,0 +1,93 @@ +package org.jeecg.weather.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import lombok.RequiredArgsConstructor; +import org.jeecg.common.constant.CommonConstant; +import org.jeecg.common.constant.enums.WeatherTaskStatusEnum; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.WeatherTask; +import org.jeecg.modules.base.entity.WeatherTaskLog; +import org.jeecg.modules.base.mapper.WeatherTaskLogMapper; +import org.jeecg.modules.base.mapper.WeatherTaskMapper; +import org.jeecg.weather.service.WeatherTaskService; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * 天气预报预测任务管理 + */ +@Service +@RequiredArgsConstructor +public class WeatherTaskServiceImpl extends ServiceImpl implements WeatherTaskService { + + private final WeatherTaskLogMapper weatherTaskLogMapper; + private final RedisUtil redisUtil; + private final ServerProperties serverProperties; + + /** + * 保存任务过程日志 + * + * @param log + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void saveLog(WeatherTaskLog log) { + this.weatherTaskLogMapper.insert(log); + } + + /** + * 修改任务运行状态 + * + * @param taskId + * @param taskStatus + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void updateTaskStatus(Integer taskId, Integer taskStatus) { + WeatherTask weatherTask = this.baseMapper.selectById(taskId); + weatherTask.setTaskStatus(taskStatus); + this.updateById(weatherTask); + } + + /** + * 根据任务id删除任务日志 + * + * @param taskId + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void deleteTaskLog(Integer taskId) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(WeatherTaskLog::getTaskId,taskId); + this.weatherTaskLogMapper.delete(queryWrapper); + } + + /** + * 修改任务状态为结束(完成或异常停止) + * @param taskId + * @param timeConsuming + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void updateTaskStatusToCompleted(Integer taskId, Double timeConsuming,Integer taskStatus) { + WeatherTask weatherTask = this.baseMapper.selectById(taskId); + weatherTask.setTaskStatus(taskStatus); + weatherTask.setTimeConsuming(timeConsuming); + this.updateById(weatherTask); + redisUtil.hdel(CommonConstant.HOST_TASK_STATE,CommonConstant.WEATHER_TASK_STATE_PRE+serverProperties.getHost(),taskId.toString()); + } + + /** + * 设置检查失败 + * + * @param weatherTask + */ + @Transactional(rollbackFor = RuntimeException.class) + @Override + public void setInpectionFailed(WeatherTask weatherTask) { + weatherTask.setTaskStatus(WeatherTaskStatusEnum.INSPECTION_FAILED.getValue()); + this.updateById(weatherTask); + } +} diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/AbstractWeatherTask.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/AbstractWeatherTask.java new file mode 100644 index 0000000..6f60945 --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/AbstractWeatherTask.java @@ -0,0 +1,108 @@ +package org.jeecg.weather.task; + +import cn.hutool.core.io.FileUtil; +import cn.hutool.core.util.StrUtil; +import com.jcraft.jsch.ChannelExec; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import org.jeecg.common.constant.CommonConstant; +import org.jeecg.common.constant.enums.WeatherForecastDatasourceEnum; +import org.jeecg.common.properties.ServerProperties; +import org.jeecg.common.properties.SystemStorageProperties; +import org.jeecg.common.util.RedisUtil; +import org.jeecg.modules.base.entity.WeatherTask; +import org.jeecg.modules.base.mapper.WeatherDataMapper; +import org.jeecg.weather.service.WeatherTaskService; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Objects; + +/** + * 天气预报模型执行父类 + */ +public abstract class AbstractWeatherTask extends Thread{ + + protected RedisUtil redisUtil; + protected WeatherTask weatherTask; + protected WeatherTaskService weatherTaskService; + protected SystemStorageProperties systemStorageProperties; + protected WeatherDataMapper weatherDataMapper; + protected ServerProperties serverProperties; + + /** + * 初始化 + */ + public void init(RedisUtil redisUtil, + WeatherTask weatherTask, + WeatherTaskService weatherTaskService, + SystemStorageProperties systemStorageProperties, + WeatherDataMapper weatherDataMapper, + ServerProperties serverProperties){ + this.redisUtil = redisUtil; + this.weatherTask = weatherTask; + this.weatherTaskService = weatherTaskService; + this.systemStorageProperties = systemStorageProperties; + this.weatherDataMapper = weatherDataMapper; + this.serverProperties = serverProperties; + } + + /** + * 前置检查任务 + * @param weatherTask + * @return + */ + public boolean checkTask(WeatherTask weatherTask){ + if(Objects.isNull(weatherTask)){ + return false; + } + if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.LOCATION_FILE.getKey())){ + if(StrUtil.isBlank(weatherTask.getInputFile())){ + return false; + } + if(!FileUtil.exist(weatherTask.getInputFile())){ + return false; + } + } + return true; + } + + /** + * 设置redis标记 + */ + protected void setRedisFlag(){ + redisUtil.hset(CommonConstant.HOST_TASK_STATE,CommonConstant.WEATHER_TASK_STATE_PRE+serverProperties.getHost(),weatherTask.getId().toString()); + } + + /** + * 持续读取日志 + */ + protected void execCommand(String command, Session session){ + ChannelExec channel = null; + try{ + //打开一个执行通道 + channel = (ChannelExec) session.openChannel("exec"); + String fullCommand = command + " 2>&1"; + channel.setCommand(fullCommand); + // 获取脚本的标准输出流，包含错误输出流 + InputStream in = channel.getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(in)); + // 连接通道 + channel.connect(); + + String line; + while ((line = reader.readLine()) != null) { + if(StrUtil.isNotBlank(line)){ + ProgressQueue.getInstance().offer(new ProgressEvent(weatherTask.getId(),line)); + } + } + }catch(JSchException | IOException e){ + throw new RuntimeException(e); + }finally { + if (channel != null) { + channel.disconnect(); + } + } + } +} diff --git a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressEvent.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressEvent.java similarity index 75% rename from jeecg-module-weather/src/main/java/org/jeecg/task/ProgressEvent.java rename to jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressEvent.java index 11440f0..d7249a0 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressEvent.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressEvent.java @@ -1,8 +1,7 @@ -package org.jeecg.task; +package org.jeecg.weather.task; import lombok.Data; import lombok.NoArgsConstructor; - import java.io.Serializable; /** @@ -14,14 +13,14 @@ public class ProgressEvent implements Serializable{ private static final long serialVersionUID = 1L; - private String taskId; + private Integer taskId; /** * 过程日志 */ private String content; - public ProgressEvent(String taskId, String content) { + public ProgressEvent(Integer taskId, String content) { this.taskId = taskId; this.content = content; } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressMonitor.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressMonitor.java similarity index 90% rename from jeecg-module-weather/src/main/java/org/jeecg/task/ProgressMonitor.java rename to jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressMonitor.java index 63b3861..7ed65b1 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressMonitor.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressMonitor.java @@ -1,10 +1,10 @@ -package org.jeecg.task; +package org.jeecg.weather.task; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jeecg.modules.base.entity.WeatherTaskLog; -import org.jeecg.service.WeatherTaskService; +import org.jeecg.weather.service.WeatherTaskService; import org.springframework.stereotype.Component; import java.util.Objects; @@ -13,7 +13,7 @@ import java.util.Objects; * @author 86187 * */ -@Component +@Component("weatherProgressMonitor") @RequiredArgsConstructor @Slf4j public class ProgressMonitor{ diff --git a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressQueue.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressQueue.java similarity index 95% rename from jeecg-module-weather/src/main/java/org/jeecg/task/ProgressQueue.java rename to jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressQueue.java index 058f07b..05d73ba 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/task/ProgressQueue.java +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/ProgressQueue.java @@ -1,4 +1,4 @@ -package org.jeecg.task; +package org.jeecg.weather.task; import java.util.LinkedList; diff --git a/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/WeatherForecastTaskExec.java b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/WeatherForecastTaskExec.java new file mode 100644 index 0000000..fce9e0b --- /dev/null +++ b/jeecg-model-consumer/src/main/java/org/jeecg/weather/task/WeatherForecastTaskExec.java @@ -0,0 +1,402 @@ +package org.jeecg.weather.task; + +import cn.hutool.core.collection.CollUtil; +import cn.hutool.core.io.FileUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.time.StopWatch; +import org.jeecg.common.constant.enums.WeatherDataSourceEnum; +import org.jeecg.common.constant.enums.WeatherForecastDatasourceEnum; +import org.jeecg.common.constant.enums.WeatherTaskStatusEnum; +import org.jeecg.common.util.Grib2TimeReader; +import org.jeecg.jsch.JSchRemoteRunner; +import org.jeecg.modules.base.entity.WeatherData; +import org.springframework.http.MediaType; +import org.springframework.web.reactive.function.client.WebClient; +import java.io.*; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.concurrent.TimeUnit; + +/** + * 气象预测任务执行线程 + */ +@Slf4j +public class WeatherForecastTaskExec extends AbstractWeatherTask { + + /** + * 切片目录（临时） + */ + private final static String GRIB_COPY_DIR = "grib_copy_split"; + /** + * 预测单个大文件输出目录（临时） + */ + private final static String OUTPUT_DIR = "output"; + /** + * 适配flexpart格式化存储路径（临时） + */ + private final static String FLEXPART_FORMAT_DIR = "flexpart_format"; + + @Override + public void run() { + this.execute(); + } + + /** + * 执行任务 + */ + public void execute() { + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + try{ + super.setRedisFlag(); + //修改任务状态为执行中 + this.weatherTaskService.updateTaskStatus(this.weatherTask.getId(), WeatherTaskStatusEnum.IN_OPERATION.getValue()); + //如果此任务已存在历史日志，先清除 + this.weatherTaskService.deleteTaskLog(this.weatherTask.getId()); + //执行模拟 + this.execSimulation(); + //执行完成 + this.execComplete(stopWatch,WeatherTaskStatusEnum.COMPLETED.getValue(), ""); + }catch (Exception e){ + String taskErrorLog = "任务执行失败，原因："+e.getMessage(); + //执行失败 + this.execComplete(stopWatch,WeatherTaskStatusEnum.FAILURE.getValue(), taskErrorLog); + throw e; + } + } + + /** + * 任务执行完成 + * @param stopWatch + * @param taskStatus + * @param taskErrorLog + */ + private void execComplete(StopWatch stopWatch,Integer taskStatus,String taskErrorLog){ + //添加任务耗时 + stopWatch.stop(); + long seconds = stopWatch.getTime(TimeUnit.SECONDS); + double min = seconds/60D; + BigDecimal bgMin = new BigDecimal(min); + BigDecimal result = bgMin.setScale(2, RoundingMode.HALF_UP); + this.weatherTaskService.updateTaskStatusToCompleted(this.weatherTask.getId(),result.doubleValue(),taskStatus); + if (WeatherTaskStatusEnum.COMPLETED.getValue().equals(taskStatus)){ + String taskCompletedLog = "任务执行完成，耗时："+result+"分钟"; + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskCompletedLog)); + }else if (WeatherTaskStatusEnum.FAILURE.getValue().equals(taskStatus)){ + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskErrorLog)); + } + } + + /** + * 执行模拟 + */ + private void execSimulation(){ + JSchRemoteRunner jschRemoteRunner = new JSchRemoteRunner(); + try { + //登录ssh + jschRemoteRunner.login(super.serverProperties.getHost(),super.serverProperties.getPort(), + super.serverProperties.getUsername(),super.serverProperties.getPassword()); + //预测气象数据 + this.forecast(); + //适配flexpart + this.convertGrib(jschRemoteRunner); + //检查预测结果数据并存储到数据库 + this.checkDataAndSaveToDB(); + //删除以任务id命名的目录 + String delPath = ""; + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + delPath = this.systemStorageProperties.getPanguModelExecPath()+File.separator+this.weatherTask.getId(); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + delPath = this.systemStorageProperties.getGraphcastModelExecPath()+File.separator+this.weatherTask.getId(); + } + File outputDir = new File(delPath); + if (outputDir.exists() && outputDir.isDirectory()){ + FileUtils.deleteDirectory(new File(delPath)); + } + } catch (Exception e) { + log.error("盘古模型执行预测任务出现异常",e); + throw new RuntimeException(e); + }finally { + jschRemoteRunner.close(); + } + } + + /** + * 天气预测 + */ + private void forecast(){ + try { + //处理运行命令 + Map commandParam = null; + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + commandParam = this.getPanguRequestJson(); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + commandParam = this.getGraphcastRequestJson(); + } + if (Objects.nonNull(commandParam)){ + WebClient webClient = WebClient.builder() + .baseUrl(super.systemStorageProperties.getWeatherForecastBaseUrl()) + .build(); + webClient.post() + .uri(super.systemStorageProperties.getWeatherForecastUri()) + .contentType(MediaType.APPLICATION_JSON) + .accept(MediaType.TEXT_PLAIN) + .bodyValue(commandParam) + .retrieve() + .bodyToFlux(String.class) + .doOnNext(log->{ + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),log)); + }) + .doOnError(e->{ + throw new RuntimeException(e); + }) + .blockLast(); + } + }catch (RuntimeException e){ + throw new RuntimeException(e); + } + } + + /** + * 转换grib + * @param jschRemoteRunner + */ + private void convertGrib(JSchRemoteRunner jschRemoteRunner){ + try { + String panguFormatScriptPath = systemStorageProperties.getPanguFormatScriptPath(); + String graphcastFormatScriptPath = systemStorageProperties.getGraphcastFormatScriptPath(); + String flexpartFormatPath = this.getFlexpartFormatPath(); + String pythonEnvPath = systemStorageProperties.getPythonEnvPath(); + String gribCopyPath = this.getGribCopyPath(); + boolean exist = jschRemoteRunner.dirExist(flexpartFormatPath); + if (!exist){ + jschRemoteRunner.mkdir(flexpartFormatPath); + } + //构建转换命令 + String startDate = this.weatherTask.getStartDate().format(DateTimeFormatter.ofPattern("yyyyMMdd")); + String endDate = this.weatherTask.getStartDate().atTime(this.weatherTask.getStartTime(), 0, 0).plusHours(this.weatherTask.getLeadTime()).format(DateTimeFormatter.ofPattern("yyyyMMdd")); + StringBuilder convertCommand = new StringBuilder(); + convertCommand.append("cd "); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + convertCommand.append(panguFormatScriptPath); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + convertCommand.append(graphcastFormatScriptPath); + } + convertCommand.append(" && "); + convertCommand.append(pythonEnvPath); + convertCommand.append(File.separator); + convertCommand.append("python3 -u "); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + convertCommand.append("pangu_reformat.py "); + convertCommand.append("--ini_dir "); + convertCommand.append(panguFormatScriptPath); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + convertCommand.append(" graphcast_reformat.py "); + convertCommand.append("--ini_dir "); + convertCommand.append(graphcastFormatScriptPath); + } + convertCommand.append(" --output_dir "); + convertCommand.append(flexpartFormatPath); + convertCommand.append(" --input_dir "); + convertCommand.append(gribCopyPath); + convertCommand.append(" --start_date "); + convertCommand.append(startDate); + convertCommand.append(" --end_date "); + convertCommand.append(endDate); + super.execCommand(convertCommand.toString(),jschRemoteRunner.getSession()); + }catch (Exception e){ + throw new RuntimeException("适配flexpart，转换grib异常",e); + } + } + + /** + * 检查预测结果数据并存储到数据库 + */ + private void checkDataAndSaveToDB(){ + String flexpartFormatPath = this.getFlexpartFormatPath(); + String gribCopyPath = this.getGribCopyPath(); + List sourceFiles = FileUtil.loopFiles(gribCopyPath); + List formatFiles = FileUtil.loopFiles(flexpartFormatPath); + if(CollUtil.isEmpty(sourceFiles)){ + String log = "天气预测任务可能出现问题，未找到预测结果切片"; + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),log)); + return; + } + if(CollUtil.isEmpty(formatFiles)){ + String log = "天气预测任务可能出现问题，适配flexpart切片目录为空"; + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),log)); + return; + } + int gribNum = this.weatherTask.getLeadTime()/6; + if(sourceFiles.size() != gribNum || formatFiles.size() != gribNum){ + String log = "天气预测任务可能出现问题，预测结果文件数量和预测时间不一致"; + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),log)); + return; + } + //把预测好的及格式化后的气象文件移动到最终目录 + String formatFilesFinalPath = this.getFormatFilesFinalStoragePath(); + String sourceFilesFinalPath = this.getSourceFilesFinalStoragePath(); + FileUtil.move(new File(gribCopyPath),new File(sourceFilesFinalPath),true); + FileUtil.move(new File(flexpartFormatPath),new File(formatFilesFinalPath),true); + //处理文件入库 + List dataList = new ArrayList<>(); + for(File sourceFile : sourceFiles){ + try{ + //构建文件信息 + WeatherData weatherData = new WeatherData(); + weatherData.setFileName(sourceFile.getName()); + weatherData.setFileExt(sourceFile.getName().substring(sourceFile.getName().lastIndexOf(".")+1)); + weatherData.setDataStartTime(Grib2TimeReader.readValidTime(sourceFilesFinalPath+File.separator+sourceFile.getName())); + weatherData.setDataSource(weatherTask.getPredictionModel()); + weatherData.setFilePath(sourceFilesFinalPath+File.separator+sourceFile.getName()); + weatherData.setFormatFilePath(formatFilesFinalPath+File.separator+sourceFile.getName()); + weatherData.setTaskId(this.weatherTask.getId()); + dataList.add(weatherData); + }catch (Exception e){ + String logContent = "读取"+sourceFilesFinalPath+File.separator+sourceFile.getName()+"文件时间参数出现错误"; + log.error(logContent,e); + ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),logContent)); + } + } + if (CollUtil.isNotEmpty(dataList)){ + weatherDataMapper.insert(dataList); + } + } + + /** + * 获取盘古模型请求命令 + * @return + */ + private Map getPanguRequestJson(){ + Map map = new HashMap<>(); + map.put("model","pangu"); + if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ + map.put("input_model","cds"); + }else if (this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.LOCATION_FILE.getKey())){ + map.put("input_model","file"); + map.put("file",this.weatherTask.getInputFile()); + } + map.put("date",this.weatherTask.getStartDate().format(DateTimeFormatter.ofPattern("yyyyMMdd"))); + map.put("time",this.weatherTask.getStartTime()); + map.put("lead_time",this.weatherTask.getLeadTime()); + map.put("assets","assets-panguweather"); + map.put("workdir",systemStorageProperties.getPanguModelExecPath()); + map.put("split_dir",getGribCopyPath()); + map.put("path",buildOutputFilePath()); + return map; + } + + /** + * 获取Graphcast模型请求命令 + * @return + */ + private Map getGraphcastRequestJson(){ + Map map = new HashMap<>(); + map.put("model","graphcast"); + if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ + map.put("input_model","cds"); + }else if (this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.LOCATION_FILE.getKey())){ + map.put("input_model","file"); + map.put("file",this.weatherTask.getInputFile()); + } + map.put("date",this.weatherTask.getStartDate().format(DateTimeFormatter.ofPattern("yyyyMMdd"))); + map.put("time",this.weatherTask.getStartTime()); + map.put("lead_time",this.weatherTask.getLeadTime()); + map.put("class","od"); + map.put("assets","assets-graphcast"); + map.put("workdir",systemStorageProperties.getPanguModelExecPath()); + map.put("split_dir",getGribCopyPath()); + map.put("path",buildOutputFilePath()); + + return map; + } + + /** + * 构建输出文件地址 + * @return + */ + private String buildOutputFilePath(){ + StringBuilder path = new StringBuilder(); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + path.append(this.systemStorageProperties.getPanguModelExecPath()); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + path.append(this.systemStorageProperties.getGraphcastModelExecPath()); + } + path.append(File.separator); + path.append(this.weatherTask.getId()); + path.append(File.separator); + path.append(OUTPUT_DIR); + path.append(File.separator); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + path.append("panguweather_output.grib"); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + path.append("graphcast_output.grib"); + } + return path.toString(); + } + + /** + * 获取Pangu/Graphcast数据切割存储路径 + * @return + */ + private String getGribCopyPath(){ + StringBuilder path = new StringBuilder(); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + path.append(systemStorageProperties.getPanguModelExecPath()); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + path.append(systemStorageProperties.getPanguModelExecPath()); + } + path.append(File.separator); + path.append(this.weatherTask.getId()); + path.append(File.separator); + path.append(GRIB_COPY_DIR); + return path.toString(); + } + + /** + * 获取盘古数据存储路径 + * @return + */ + private String getFlexpartFormatPath(){ + StringBuilder path = new StringBuilder(); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + path.append(systemStorageProperties.getPanguModelExecPath()); + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + path.append(systemStorageProperties.getGraphcastModelExecPath()); + } + path.append(File.separator); + path.append(this.weatherTask.getId()); + path.append(File.separator); + path.append(FLEXPART_FORMAT_DIR); + return path.toString(); + } + + /** + * 获取预测原文件最终存储路径 + * @return + */ + private String getSourceFilesFinalStoragePath(){ + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + return systemStorageProperties.getPanguModelExecPath()+"/source"; + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + return systemStorageProperties.getGraphcastModelExecPath()+"/source"; + } + return ""; + } + + /** + * 获取预测并格式化后的文件最终存储路径 + * @return + */ + private String getFormatFilesFinalStoragePath(){ + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ + return systemStorageProperties.getPanguModelExecPath()+"/format"; + }else if(WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())){ + return systemStorageProperties.getGraphcastModelExecPath()+"/format"; + } + return ""; + } +} diff --git a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskLogService.java b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskLogService.java index ce10863..64160b8 100644 --- a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskLogService.java +++ b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskLogService.java @@ -2,35 +2,16 @@ package org.jeecg.service; import com.baomidou.mybatisplus.extension.service.IService; import org.jeecg.modules.base.entity.SourceRebuildTaskLog; -import java.util.List; /** * 源项重建任务日志数据 */ public interface SourceRebuildTaskLogService extends IService { - /** - * 保存源项重建任务运行过程日志 - * @param sourceRebuildTaskLog - */ - void cteate(SourceRebuildTaskLog sourceRebuildTaskLog); - /** * 删除任务运行过程日志 * @param taskId */ void delete(Integer taskId); - /** - * 获取任务运行过程日志 - * @param taskId - * @return - */ - List getTaskLogs(Integer taskId); - - /** - * 批量保存日志 - * @param logs - */ - void batchCteate(List logs); } diff --git a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskService.java b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskService.java index 369c542..3a76cbc 100644 --- a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskService.java +++ b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/SourceRebuildTaskService.java @@ -62,18 +62,4 @@ public interface SourceRebuildTaskService extends IService { * @return */ List getTaskLog(Integer taskId); - - /** - * 修改任务状态 - * @param taskId - * @param taskStatus - */ - void updateTaskStatus(Integer taskId, Integer taskStatus); - - /** - * 任务耗时 - * @param taskId - * @param minute - */ - void updateTaskTimeConsuming(Integer taskId, Double minute); } diff --git a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskLogServiceImpl.java b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskLogServiceImpl.java index 61a6e6c..433724e 100644 --- a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskLogServiceImpl.java +++ b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskLogServiceImpl.java @@ -13,17 +13,6 @@ import java.util.List; @Service public class SourceRebuildTaskLogServiceImpl extends ServiceImpl implements SourceRebuildTaskLogService { - /** - * 保存源项重建任务运行过程日志 - * - * @param sourceRebuildTaskLog - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void cteate(SourceRebuildTaskLog sourceRebuildTaskLog) { - this.baseMapper.insert(sourceRebuildTaskLog); - } - /** * 删除任务运行过程日志 * @@ -36,28 +25,4 @@ public class SourceRebuildTaskLogServiceImpl extends ServiceImpl getTaskLogs(Integer taskId) { - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.eq(SourceRebuildTaskLog::getTaskId, taskId); - return this.baseMapper.selectList(queryWrapper); - } - - /** - * 批量保存日志 - * - * @param logs - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void batchCteate(List logs) { - this.saveBatch(logs); - } } diff --git a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskServiceImpl.java b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskServiceImpl.java index cd758c3..775e305 100644 --- a/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskServiceImpl.java +++ b/jeecg-module-source-rebuild/src/main/java/org/jeecg/service/impl/SourceRebuildTaskServiceImpl.java @@ -9,7 +9,6 @@ import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.jeecg.common.constant.enums.SourceRebuildTaskStatusEnum; import org.jeecg.common.properties.SourceRebuildProperties; -import org.jeecg.common.properties.SystemStorageProperties; import org.jeecg.common.system.query.PageRequest; import org.jeecg.modules.base.entity.SourceRebuildMonitoringData; import org.jeecg.modules.base.entity.SourceRebuildTask; @@ -19,11 +18,9 @@ import org.jeecg.modules.base.mapper.SourceRebuildTaskMapper; import org.jeecg.service.SourceRebuildMonitoringDataService; import org.jeecg.service.SourceRebuildTaskLogService; import org.jeecg.service.SourceRebuildTaskService; -import org.jeecg.task.SourceRebuildTaskExec; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; - import java.io.File; import java.time.LocalDate; import java.time.LocalDateTime; @@ -40,7 +37,6 @@ public class SourceRebuildTaskServiceImpl extends ServiceImpl taskMonitoringDatas = this.monitoringDataService.getTaskMonitoringData(id); if(CollectionUtils.isEmpty(taskMonitoringDatas)){ throw new RuntimeException("监测数据为空，请补充监测数据"); } - - SourceRebuildTaskExec sourceRebuildTaskExec = new SourceRebuildTaskExec(); - sourceRebuildTaskExec.init( - taskMonitoringDatas, - sourceRebuildTask, - sourceRebuildTaskLogService, - sourceRebuildProperties, - this); - sourceRebuildTaskExec.setName(sourceRebuildTask.getId()+"_任务执行线程"); - sourceRebuildTaskExec.start(); + sourceRebuildTask.setTaskStatus(SourceRebuildTaskStatusEnum.WAITING.getValue()); + this.updateById(sourceRebuildTask); } /** @@ -202,33 +192,4 @@ public class SourceRebuildTaskServiceImpl extends ServiceImpl verifyFileExist(@NotBlank(message = "文件MD5值不能为空") String fileMD5Value) { - FileExistVo fileExist = weatherDataService.verifyFileExist(fileMD5Value); - return Result.ok(fileExist); - } - - @AutoLog(value = "上传文件") - @Operation(summary = "上传文件") - @PostMapping("uploadFile") - public Result uploadFile(FileVo fileVo){ - if (!fileVo.getFileExt().equals(WeatherFileSuffixEnum.GRIB.getValue()) && !fileVo.getFileExt().equals(WeatherFileSuffixEnum.GRIB2.getValue())){ - throw new RuntimeException("不支持当前上传的文件类型！"); - }else{ - FileUploadResultVo resultVo = weatherDataService.uploadFile(fileVo); - return Result.ok(resultVo); - } - } /** * 查询批次列表 * @return @@ -112,7 +89,7 @@ public class WeatherDataController { @AutoLog(value = "气象预测-气象预览") @Operation(summary = "气象预测-气象预览") @GetMapping(value = "getWeatherDataPreview") - public Result getWeatherDataPreview(String weatherId,Integer weatherType) { + public Result getWeatherDataPreview(Integer weatherId,Integer weatherType) { return Result.OK(weatherDataService.getWeatherDataPreview(weatherId, weatherType)); } @@ -151,7 +128,7 @@ public class WeatherDataController { @AutoLog(value = "删除气象数据") @Operation(summary = "删除气象数据") @DeleteMapping("delete") - public Result delete(@RequestBody List ids){ + public Result delete(@RequestBody List ids){ weatherDataService.delete(ids); return Result.OK(); } @@ -166,4 +143,22 @@ public class WeatherDataController { weatherDataService.handleStaticDataToDB(path,dataSource); return Result.OK(); } + + @AutoLog(value = "关联气象数据") + @Operation(summary = "关联气象数据") + @PutMapping("linkedData") + public Result linkedData(Integer dataType, + @DateTimeFormat(iso=DateTimeFormat.ISO.DATE) LocalDate startDate, + @DateTimeFormat(iso=DateTimeFormat.ISO.DATE)LocalDate endDate){ + weatherDataService.linkedData(dataType,startDate,endDate); + return Result.OK(); + } + + @AutoLog(value = "查询关联气象数据日志") + @Operation(summary = "查询关联气象数据日志") + @GetMapping("getLinkedDataLog") + public Result getLinkedDataLog(){ + List result = weatherDataService.getLinkedDataLog(); + return Result.OK(result); + } } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherTaskController.java b/jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherTaskController.java index cc0c012..2726e12 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherTaskController.java +++ b/jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherTaskController.java @@ -12,7 +12,6 @@ import org.jeecg.common.validgroup.InsertGroup; import org.jeecg.common.validgroup.UpdateGroup; import org.jeecg.modules.base.entity.WeatherTask; import org.jeecg.service.WeatherTaskService; -import org.jeecg.vo.ForecastFileVO; import org.springframework.format.annotation.DateTimeFormat; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; @@ -53,7 +52,7 @@ public class WeatherTaskController { @AutoLog(value = "获取单条天气预测任务记录") @Operation(summary = "获取单条天气预测任务记录") @GetMapping("getById") - public Result getById(@NotNull(message = "ID不能为空") String id){ + public Result getById(@NotNull(message = "ID不能为空") Integer id){ return Result.OK(weatherTaskService.getById(id)); } @@ -68,7 +67,7 @@ public class WeatherTaskController { @AutoLog(value = "删除天气预测任务") @Operation(summary = "删除天气预测任务") @DeleteMapping("delete") - public Result delete(@RequestBody List ids){ + public Result delete(@RequestBody List ids){ weatherTaskService.delete(ids); return Result.OK(); } @@ -76,7 +75,7 @@ public class WeatherTaskController { @AutoLog(value = "启动任务") @Operation(summary = "启动任务") @PutMapping("runTask") - public Result runTask(@NotBlank(message = "任务ID不能为空") String taskId){ + public Result runTask(@NotBlank(message = "任务ID不能为空") Integer taskId){ weatherTaskService.runTask(taskId); return Result.OK(); } @@ -84,7 +83,7 @@ public class WeatherTaskController { @AutoLog(value = "获取天气预测任务过程日志") @Operation(summary = "获取天气预测任务过程日志") @GetMapping("getTaskLog") - public Result getTaskLog(@NotBlank(message = "预测任务ID不能为空") String taskId){ + public Result getTaskLog(@NotBlank(message = "预测任务ID不能为空") Integer taskId){ return Result.OK(weatherTaskService.getTaskLog(taskId)); } } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/job/DownloadT1hJob.java b/jeecg-module-weather/src/main/java/org/jeecg/job/DownloadT1hJob.java index 697e5aa..abdd0f3 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/job/DownloadT1hJob.java +++ b/jeecg-module-weather/src/main/java/org/jeecg/job/DownloadT1hJob.java @@ -42,29 +42,31 @@ public class DownloadT1hJob { @Scheduled(cron = "#{@t1hDownloadProperties.cron}") // @Scheduled(cron = "0 0 1 * * ?") public void downloadT1HFile() { - log.info("开始执行T1H文件下载任务"); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try { - String baseTime = getBaseTime(); - // 第一阶段：下载文件 - downloadAllT1hFiles(baseTime); - // 第二阶段：合并为NC文件 - mergeT1hNCFiles(baseTime); - // 第三阶段：解析为Grib2文件 - reformat(baseTime); - // 合并后删除原始文件 - Arrays.stream(new File(getFullPath(t1hDownloadProperties.getT1hPath(), baseTime)).listFiles()).filter(File::isFile).forEach(File::delete); - Arrays.stream(new File(getFullPath(t1hDownloadProperties.getT1hNcPath(), baseTime)).listFiles()).filter(File::isFile).forEach(File::delete); - // 更新气象文件信息 - saveWeatherData(baseTime); - log.info("T1H文件下载任务执行完成"); - } catch (Exception e) { - log.error("T1H文件下载任务执行失败", e); - throw new RuntimeException("T1H文件下载任务失败", e); - } finally { - stopWatch.stop(); - log.info("T1H文件下载任务执行耗时: {} 毫秒", stopWatch.getTotalTimeMillis()); + if (t1hDownloadProperties.isDownloadSwitch()){ + log.info("开始执行T1H文件下载任务"); + StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + try { + String baseTime = getBaseTime(); + // 第一阶段：下载文件 + downloadAllT1hFiles(baseTime); + // 第二阶段：合并为NC文件 + mergeT1hNCFiles(baseTime); + // 第三阶段：解析为Grib2文件 + reformat(baseTime); + // 合并后删除原始文件 + Arrays.stream(new File(getFullPath(t1hDownloadProperties.getT1hPath(), baseTime)).listFiles()).filter(File::isFile).forEach(File::delete); + Arrays.stream(new File(getFullPath(t1hDownloadProperties.getT1hNcPath(), baseTime)).listFiles()).filter(File::isFile).forEach(File::delete); + // 更新气象文件信息 + saveWeatherData(baseTime); + log.info("T1H文件下载任务执行完成"); + } catch (Exception e) { + log.error("T1H文件下载任务执行失败", e); + throw new RuntimeException("T1H文件下载任务失败", e); + } finally { + stopWatch.stop(); + log.info("T1H文件下载任务执行耗时: {} 毫秒", stopWatch.getTotalTimeMillis()); + } } } @@ -90,7 +92,7 @@ public class DownloadT1hJob { List forecastTimes = generateForecastTimes(); ProcessBuilder processBuilder = new ProcessBuilder( t1hDownloadProperties.getPythonPath(), - getPythonScriptPath(t1hDownloadProperties.getMergeT1hPy()), + t1hDownloadProperties.getMergeT1hPy(), "--indir", getFullPath(t1hDownloadProperties.getT1hPath(), baseTime), "--output_dir", getFullPath(t1hDownloadProperties.getT1hNcPath(), baseTime), "--variables", String.join(",", variables), @@ -108,13 +110,13 @@ public class DownloadT1hJob { private void reformat(String baseTime) { ProcessBuilder processBuilder = new ProcessBuilder( t1hDownloadProperties.getPythonPath(), - getPythonScriptPath(t1hDownloadProperties.getReformatPy()), - "--ini_dir", getPythonScriptPath(t1hDownloadProperties.getScriptPath()), - "--output_dir", getFullPath(systemStorageProperties.getT1h(), baseTime), + t1hDownloadProperties.getReformatPy(), + "--ini_dir", t1hDownloadProperties.getScriptPath(), + "--output_dir", getFullPath(systemStorageProperties.getT1hDataPath(), baseTime), "--cra_dir", getFullPath(t1hDownloadProperties.getT1hNcPath(), baseTime), "--start_date", getStartTime(), "--end_date", getEndTime(), - "--var_table", getPythonScriptPath(t1hDownloadProperties.getLookupFileName()) + "--var_table",t1hDownloadProperties.getLookupFileName() ); ExecutePyUtils.executePythonProcess(processBuilder, "文件合并"); @@ -159,7 +161,7 @@ public class DownloadT1hJob { private String[] buildPythonCommand(String element, String baseTime, String forecastHours) { return new String[]{ t1hDownloadProperties.getPythonPath(), - getPythonScriptPath(t1hDownloadProperties.getDownloadT1hPy()), + t1hDownloadProperties.getDownloadT1hPy(), "--base-time", baseTime, "--element", element, "--output-dir", getFullPath(t1hDownloadProperties.getT1hPath(), baseTime), @@ -187,7 +189,7 @@ public class DownloadT1hJob { } public List readFolderFiles(String baseTime) { - String folderPath = getFullPath(systemStorageProperties.getT1h(), baseTime); + String folderPath = getFullPath(systemStorageProperties.getT1hDataPath(), baseTime); try (Stream paths = Files.list(Paths.get(folderPath))) { return paths.filter(Files::isRegularFile) .filter(Files::isRegularFile) @@ -203,16 +205,11 @@ public class DownloadT1hJob { private WeatherData extractFileInfo(File file, String baseTime) { WeatherData data = new WeatherData(); data.setFileName(file.getName()); - data.setFileSize((double) file.length() / 1024 / 1024); // MB data.setFileExt(getFileExtension(file.getName())); data.setFilePath(file.getAbsolutePath()); data.setDataSource(WeatherDataSourceEnum.T1H.getKey()); data.setDataStartTime(parseStartTimeFromFileName(file.getName())); - data.setCreateTime(new Date()); - data.setMd5Value(calculateMD5(file.getAbsolutePath())); - data.setShareTotal(1); data.setTimeBatch(baseTime); - data.setCreateTime(new Date()); return data; } @@ -246,14 +243,7 @@ public class DownloadT1hJob { * 获取完整路径 */ private String getFullPath(String relativePath, String baseTime) { - return systemStorageProperties.getRootPath() + File.separator + relativePath + File.separator + baseTime; - } - - /** - * 获取Python脚本路径 - */ - private String getPythonScriptPath(String scriptName) { - return systemStorageProperties.getRootPath() + File.separator + scriptName; + return relativePath + File.separator + baseTime; } /** diff --git a/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherDataService.java b/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherDataService.java index 3017e65..ce6adb7 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherDataService.java +++ b/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherDataService.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.service.IService; import org.jeecg.common.system.query.PageRequest; import org.jeecg.modules.base.entity.WeatherData; +import org.jeecg.modules.base.entity.WeatherLinkedDataLog; import org.jeecg.vo.*; import java.time.LocalDate; import java.time.LocalDateTime; @@ -12,7 +13,7 @@ import java.util.List; public interface WeatherDataService extends IService { WeatherResultVO getWeatherData(Integer dataType, Integer weatherType, String timeBatch, LocalDateTime startTime, int hour); - WeatherResultVO getWeatherDataPreview(String weatherId, Integer weatherType); + WeatherResultVO getWeatherDataPreview(Integer weatherId, Integer weatherType); WindDataLineVO getDataLine(Integer dataType, String timeBatch, LocalDateTime startTime, LocalDateTime endTime, double longitude, double latitude); List getWindRose(Integer dataType, String timeBatch, LocalDateTime startTime, LocalDateTime endTime,double longitude, double latitude); @@ -27,24 +28,11 @@ public interface WeatherDataService extends IService { */ IPage page(PageRequest pageRequest,String fileName, String fileExt, String dataSource, LocalDate startDate, LocalDate endDate); - /** - * 验证文件是否存在 - * @param fileMD5Value 文件唯一MD5值 - * @return - */ - FileExistVo verifyFileExist(String fileMD5Value); - - /** - * 上传文件 - * @param fileVo - */ - FileUploadResultVo uploadFile(FileVo fileVo); - /** * 删除气象数据 * @param ids */ - void delete(List ids); + void delete(List ids); /** * 处理静态气象数据入库接口，比上传快 @@ -52,8 +40,16 @@ public interface WeatherDataService extends IService { void handleStaticDataToDB(String path,Integer dataSource); /** - * 保存模型预测好的文件信息到WeatherData - * @param weatherData + * 查询关联气象数据日志 + * @return */ - void saveFileInfo(WeatherData weatherData); + List getLinkedDataLog(); + + /** + * 关联气象数据 + * @param dataType + * @param startDate + * @param endDate + */ + void linkedData(Integer dataType,LocalDate startDate, LocalDate endDate); } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherTaskService.java b/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherTaskService.java index b30477e..e8ac6b9 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherTaskService.java +++ b/jeecg-module-weather/src/main/java/org/jeecg/service/WeatherTaskService.java @@ -35,7 +35,7 @@ public interface WeatherTaskService extends IService { * @param id * @return */ - WeatherTask getById(String id); + WeatherTask getById(Integer id); /** * 修改天气预报预测任务 @@ -47,46 +47,18 @@ public interface WeatherTaskService extends IService { * 删除预测任务 * @param ids */ - void delete(List ids); + void delete(List ids); /** * 运行任务 * @param id */ - void runTask(String id); + void runTask(Integer id); /** * 获取任务运行日志 * @param taskId * @return */ - List getTaskLog(String taskId); - - /** - * 保存任务过程日志 - * @param log - */ - void saveLog(WeatherTaskLog log); - - /** - * 修改任务运行状态 - * @param taskId - * @param taskStatus - */ - void updateTaskStatus(String taskId,Integer taskStatus); - - /** - * 根据任务id删除任务日志 - * @param taskId - */ - void deleteTaskLog(String taskId); - - /** - * 修改任务状态为结束(完成或异常停止) - * @param taskId - * @param timeConsuming - * @param taskStatus - */ - void updateTaskStatusToCompleted(String taskId, Double timeConsuming,Integer taskStatus); - + List getTaskLog(Integer taskId); } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherDataServiceImpl.java b/jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherDataServiceImpl.java index aba7f17..43cb543 100644 --- a/jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherDataServiceImpl.java +++ b/jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherDataServiceImpl.java @@ -5,39 +5,30 @@ import cn.hutool.core.io.FileUtil; import cn.hutool.core.util.StrUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.core.toolkit.StringPool; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; -import org.jeecg.common.constant.WeatherPrefixConstants; import org.jeecg.common.constant.WeatherStepConstants; -import org.jeecg.common.constant.WeatherSuffixConstants; import org.jeecg.common.constant.enums.*; import org.jeecg.common.exception.JeecgBootException; import org.jeecg.common.exception.JeecgFileUploadException; import org.jeecg.common.properties.SystemStorageProperties; import org.jeecg.common.system.query.PageRequest; +import org.jeecg.common.util.Grib2TimeReader; import org.jeecg.common.util.NcUtil; import org.jeecg.modules.base.entity.WeatherData; +import org.jeecg.modules.base.entity.WeatherLinkedDataLog; import org.jeecg.modules.base.mapper.WeatherDataMapper; +import org.jeecg.modules.base.mapper.WeatherLinkedDataLogMapper; import org.jeecg.service.WeatherDataService; import org.jeecg.utils.WindRoseDataGenerator; import org.jeecg.vo.*; -import org.springframework.beans.BeanUtils; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.stereotype.Service; -import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; -import org.springframework.web.multipart.MultipartFile; import ucar.nc2.NetcdfFile; import java.io.*; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -49,7 +40,6 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; - import static org.jeecg.common.constant.LatLonSizeConstants.*; @Slf4j @@ -57,10 +47,9 @@ import static org.jeecg.common.constant.LatLonSizeConstants.*; @RequiredArgsConstructor public class WeatherDataServiceImpl extends ServiceImpl implements WeatherDataService { - private final SystemStorageProperties systemStorageProperties; - private final DataSourceTransactionManager transactionManager; - private final TransactionDefinition transactionDefinition; private final WeatherDataMapper weatherDataMapper; + private final WeatherLinkedDataLogMapper weatherLinkedDataLogMapper; + private final SystemStorageProperties systemStorageProperties; /** * 根据类型和小时数获取天气数据 @@ -104,7 +93,7 @@ public class WeatherDataServiceImpl extends ServiceImpl iPage = new Page<>(pageRequest.getPageNum(),pageRequest.getPageSize()); return this.baseMapper.selectPage(iPage, queryWrapper); } - /** - * 验证文件是否存在 - * - * @param fileMD5Value 文件唯一MD5值 - * @return - */ - @Override - public FileExistVo verifyFileExist(String fileMD5Value) { - LambdaQueryWrapper wrapper = new LambdaQueryWrapper<>(); - wrapper.eq(WeatherData::getMd5Value, fileMD5Value); - - FileExistVo fileExist = new FileExistVo(); - WeatherData weatherData = this.baseMapper.selectOne(wrapper); - if(Objects.nonNull(weatherData)) { - fileExist.setExist(true); - fileExist.setFileId(weatherData.getId()); - fileExist.setShareTotal(weatherData.getShareTotal()); - fileExist.setShareIndex(weatherData.getShareIndex()); - } - return fileExist; - } - - /** - * 上传文件 - * - * @param fileVo - */ - @Override - public FileUploadResultVo uploadFile(FileVo fileVo) { - final TransactionStatus transactionStatus = this.transactionManager.getTransaction(transactionDefinition); - //文件保存路径 - String storagePath = null; - //数据ID - String id= null; - //上传成功总体标记 - boolean flag = true; - try{ - MultipartFile file = fileVo.getFile(); - //文件保存路径 - storagePath = this.getFileStoragePath(fileVo.getDataSource(), fileVo.getFileName()); - fileVo.setFilePath(storagePath); - //如果上传策略是分片则加分片后缀 - if(fileVo.isFileShare()) { - storagePath += StringPool.DOT; - storagePath += fileVo.getShareIndex(); - } - //保存文件 - File shareFile = new File(storagePath.toString()); - if(!shareFile.getParentFile().exists()) { - shareFile.getParentFile().setWritable(true); - shareFile.getParentFile().mkdirs(); - } - file.transferTo(shareFile); - - //保存文件信息入库 - FileExistVo fileExist = this.verifyFileExist(fileVo.getMd5Value()); - if(fileExist.isExist()) { - WeatherData queryResult = this.baseMapper.selectById(fileExist.getFileId()); - queryResult.setShareIndex(fileVo.getShareIndex()); - this.baseMapper.updateById(queryResult); - id = queryResult.getId(); - }else { - WeatherData weatherData = new WeatherData(); - BeanUtils.copyProperties(fileVo, weatherData); - this.baseMapper.insert(weatherData); - id = weatherData.getId(); - } - this.transactionManager.commit(transactionStatus); - }catch (Exception e) { - flag = false; - e.printStackTrace(); - } - - if(fileVo.isFileShare()) { - if(fileVo.getShareIndex() == (fileVo.getShareTotal()-1)) { - flag = this.merge(fileVo); - }else { - //未合并之前需先是false，不然后面就开始处理气象文件数据开始时间和计算文件大小 - flag = false; - } - } - - if(flag) { - final TransactionStatus updateTransactionStatus = this.transactionManager.getTransaction(transactionDefinition); - try{ - //处理气象文件数据开始时间和计算文件大小 - WeatherData queryResult = this.baseMapper.selectById(id); - File dataFile = new File(storagePath); - if(dataFile.exists() && dataFile.length()>0){ - //获取文件数据开始日期 - String reftime = NcUtil.getReftime(dataFile.getAbsolutePath()); - if(StringUtils.isBlank(reftime)) { - throw new JeecgFileUploadException("解析气象文件起始时间数据异常，此文件可能损坏"); - } - Instant instant = Instant.parse(reftime); - LocalDateTime utcDateTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); - queryResult.setDataStartTime(utcDateTime); - //计算文件大小M - BigDecimal divideVal = new BigDecimal("1024"); - BigDecimal bg = new BigDecimal(dataFile.length()); - BigDecimal fileSize = bg.divide(divideVal).divide(divideVal).setScale(2, RoundingMode.HALF_UP); - queryResult.setFileSize(fileSize.doubleValue()); - //把文件移入新路径 - String newFileDirPath = dataFile.getParentFile().getParent()+File.separator+File.separator; - String newFilePath = newFileDirPath+File.separator+dataFile.getName(); - FileUtil.mkdir(newFileDirPath); - File parentDir = dataFile.getParentFile(); - File[] files = parentDir.listFiles(); - if(ArrayUtils.isNotEmpty(files)) { - for(File file : files) { - File targetFile = new File(newFileDirPath + File.separator + file.getName()); - FileUtil.move(file,targetFile,true); - } - } - //删除临时目录 - parentDir.delete(); - //给原路径变量重新赋值 - storagePath = newFileDirPath; - //修改文件新路径存到数据库 - queryResult.setFilePath(newFilePath); - this.baseMapper.updateById(queryResult); - this.transactionManager.commit(updateTransactionStatus); - } - }catch (Exception e){ - flag = false; - e.printStackTrace(); - } - } - //如果单文件上传失败或者分片上传失败则删除数据 - if((!flag && !fileVo.isFileShare()) || - (!flag && fileVo.isFileShare() && fileVo.getShareIndex() == (fileVo.getShareTotal()-1))) { - String delDirPath = null; - File file = new File(storagePath); - if(file.exists() && file.isFile()){ - delDirPath = file.getParent(); - }else if(file.isDirectory()){ - delDirPath = file.getAbsolutePath(); - } - FileUtil.del(delDirPath); - final TransactionStatus delTransactionStatus = this.transactionManager.getTransaction(transactionDefinition); - this.baseMapper.deleteById(id); - this.transactionManager.commit(delTransactionStatus); - log.error("{}文件上传失败",fileVo.getFileName()); - throw new RuntimeException(fileVo.getFileName()+"文件上传失败"); - } - - FileUploadResultVo result = new FileUploadResultVo(); - result.setCompleted(flag); - result.setId(id); - return result; - } - /** * 删除气象数据 * @@ -486,28 +323,40 @@ public class WeatherDataServiceImpl extends ServiceImpl ids) { + public void delete(List ids) { List weatherDatas = this.baseMapper.selectByIds(ids); if (CollUtil.isNotEmpty(weatherDatas)) { for(WeatherData weatherData : weatherDatas) { //删除气象文件和生成的.gbx9、.ncx2文件 - File dataFile = new File(weatherData.getFilePath()); - if(dataFile.exists()) { - dataFile.delete(); - } - File gbx9File = new File(weatherData.getFilePath()+".gbx9"); - if(gbx9File.exists()) { - gbx9File.delete(); - } - File ncx4File = new File(weatherData.getFilePath()+".ncx4"); - if(ncx4File.exists()) { - ncx4File.delete(); + this.delWeatherDataFile(weatherData.getFilePath()); + if(WeatherDataSourceEnum.PANGU.getKey().equals(weatherData.getDataSource()) || + WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherData.getDataSource())) { + this.delWeatherDataFile(weatherData.getFormatFilePath()); } } this.baseMapper.deleteByIds(ids); } } + /** + * 删除气象文件 + * @param filePath + */ + private void delWeatherDataFile(String filePath) { + File dataFile = new File(filePath); + if(dataFile.exists()) { + dataFile.delete(); + } + File gbx9File = new File(filePath+".gbx9"); + if(gbx9File.exists()) { + gbx9File.delete(); + } + File ncx4File = new File(filePath+".ncx4"); + if(ncx4File.exists()) { + ncx4File.delete(); + } + } + /** * 处理静态气象数据入库接口，比上传快 */ @@ -525,21 +374,15 @@ public class WeatherDataServiceImpl extends ServiceImpl getLinkedDataLog() { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.select(WeatherLinkedDataLog::getCreateTime,WeatherLinkedDataLog::getLogContent); + return weatherLinkedDataLogMapper.selectList(queryWrapper); + } + + /** + * 关联气象数据 + * + * @param dataType + * @param startDate + * @param endDate + */ + @Override + public void linkedData(Integer dataType, LocalDate startDate, LocalDate endDate) { + //先清空表 + weatherLinkedDataLogMapper.delete(null); + String linkedAddr = ""; + if(WeatherDataSourceEnum.NCEP.getKey().equals(dataType)) { + linkedAddr = systemStorageProperties.getNcepDataPath(); + }else if(WeatherDataSourceEnum.FNL.getKey().equals(dataType)) { + linkedAddr = systemStorageProperties.getFnlDataPath(); + }else if(WeatherDataSourceEnum.CRA40.getKey().equals(dataType)) { + linkedAddr = systemStorageProperties.getCra40DataPath(); } - storagePath.append(File.separator); - storagePath.append(fileName.substring(0,fileName.lastIndexOf(StringPool.DOT))); - storagePath.append(File.separator); - storagePath.append(fileName); - return storagePath.toString(); + File dir = new File(linkedAddr); + if (!dir.exists()) { + FileUtil.mkdir(dir); + } + List fileList = FileUtil.loopFiles(dir, new FileFilter() { + @Override + public boolean accept(File file) { + if(StrUtil.endWith(file.getName(), WeatherFileSuffixEnum.GRIB.getValue()) + || StrUtil.endWith(file.getName(), WeatherFileSuffixEnum.GRB2.getValue()) + || StrUtil.endWith(file.getName(), WeatherFileSuffixEnum.GRIB2.getValue())){ + try{ + LocalDate localDate = null; + if(WeatherDataSourceEnum.NCEP.getKey().equals(dataType)) { + String dateTimeStr = file.getName().substring(0, file.getName().indexOf(".")); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); + LocalDateTime dateTime = LocalDateTime.parse(dateTimeStr, formatter); + localDate = dateTime.toLocalDate(); + }else if(WeatherDataSourceEnum.FNL.getKey().equals(dataType)){ + String dateTimeStr = file.getName().split("_")[1]; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + localDate = LocalDate.parse(dateTimeStr, formatter); + }else if(WeatherDataSourceEnum.CRA40.getKey().equals(dataType)){ + String dateTimeStr = file.getName().split("_")[1]; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); + localDate = LocalDate.parse(dateTimeStr, formatter); + } + if(Objects.nonNull(localDate) && !localDate.isBefore(startDate) && !localDate.isAfter(endDate)) { + return true; + } + }catch(Exception e){ + log.error("{}文件命名格式错误，需确认",file.getAbsolutePath()); + WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog(); + linkedLog.setDataType(dataType); + linkedLog.setLogContent(file.getAbsolutePath()+"文件命名格式错误，需确认"); + weatherLinkedDataLogMapper.insert(linkedLog); + } + } + return false; + } + }); + if (CollUtil.isNotEmpty(fileList)) { + for (File file : fileList) { + try { + WeatherData weatherData = new WeatherData(); + weatherData.setFileName(file.getName()); + weatherData.setFileExt(file.getName().substring(file.getName().lastIndexOf(".")+1)); + weatherData.setDataSource(dataType); + weatherData.setFilePath(file.getAbsolutePath()); + //获取文件数据开始日期 + LocalDateTime localDateTime = Grib2TimeReader.readValidTime(file.getAbsolutePath()); + weatherData.setDataStartTime(localDateTime); + this.baseMapper.insert(weatherData); + + WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog(); + linkedLog.setDataType(dataType); + linkedLog.setLogContent(file.getAbsolutePath()+"关联成功"); + weatherLinkedDataLogMapper.insert(linkedLog); + }catch (Exception e){ + log.error("关联{}气象数据文件出现错误，原因为：",file.getAbsolutePath(),e); + WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog(); + linkedLog.setDataType(dataType); + linkedLog.setLogContent(file.getAbsolutePath()+"关联失败，原因为："+e.getMessage()); + weatherLinkedDataLogMapper.insert(linkedLog); + } + } + WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog(); + linkedLog.setDataType(dataType); + linkedLog.setLogContent("关联任务执行完毕！"); + weatherLinkedDataLogMapper.insert(linkedLog); + } + } /** @@ -831,16 +687,6 @@ public class WeatherDataServiceImpl extends ServiceImpl ids) { + public void delete(List ids) { List weatherTasks = this.baseMapper.selectByIds(ids); if(CollUtil.isNotEmpty(weatherTasks)){ for (WeatherTask weatherTask : weatherTasks) { if (Objects.nonNull(weatherTask) && StrUtil.isNotBlank(weatherTask.getInputFile())){ - String inputFileParent = ""; - if (WeatherDataSourceEnum.PANGU.getKey().equals(weatherTask.getPredictionModel())){ - inputFileParent = this.systemStorageProperties.getPanguModelExecPath(); - }else if (WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherTask.getPredictionModel())) { - inputFileParent = this.systemStorageProperties.getGraphcastModelExecPath(); - } - File inputFile = new File(inputFileParent+File.separator+weatherTask.getInputFile()); + File inputFile = new File(weatherTask.getInputFile()); if (inputFile.exists()){ inputFile.delete(); } @@ -246,22 +229,18 @@ public class WeatherTaskServiceImpl extends ServiceImpl getTaskLog(String taskId) { + public List getTaskLog(Integer taskId) { LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); queryWrapper.eq(WeatherTaskLog::getTaskId,taskId); queryWrapper.select(WeatherTaskLog::getCreateTime,WeatherTaskLog::getLogContent); queryWrapper.orderByAsc(WeatherTaskLog::getCreateTime); return this.weatherTaskLogMapper.selectList(queryWrapper); } - - /** - * 保存任务过程日志 - * - * @param log - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void saveLog(WeatherTaskLog log) { - this.weatherTaskLogMapper.insert(log); - } - - /** - * 修改任务运行状态 - * - * @param taskId - * @param taskStatus - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void updateTaskStatus(String taskId, Integer taskStatus) { - WeatherTask weatherTask = this.baseMapper.selectById(taskId); - weatherTask.setTaskStatus(taskStatus); - this.updateById(weatherTask); - } - - /** - * 根据任务id删除任务日志 - * - * @param taskId - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void deleteTaskLog(String taskId) { - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.eq(WeatherTaskLog::getTaskId,taskId); - this.weatherTaskLogMapper.delete(queryWrapper); - } - - /** - * 修改任务状态为结束(完成或异常停止) - * @param taskId - * @param timeConsuming - */ - @Transactional(rollbackFor = RuntimeException.class) - @Override - public void updateTaskStatusToCompleted(String taskId, Double timeConsuming,Integer taskStatus) { - WeatherTask weatherTask = this.baseMapper.selectById(taskId); - weatherTask.setTaskStatus(taskStatus); - weatherTask.setTimeConsuming(timeConsuming); - this.updateById(weatherTask); - } } diff --git a/jeecg-module-weather/src/main/java/org/jeecg/task/GraphcastWeatherTaskExec.java b/jeecg-module-weather/src/main/java/org/jeecg/task/GraphcastWeatherTaskExec.java deleted file mode 100644 index 7bd40e5..0000000 --- a/jeecg-module-weather/src/main/java/org/jeecg/task/GraphcastWeatherTaskExec.java +++ /dev/null @@ -1,410 +0,0 @@ -package org.jeecg.task; - -import cn.hutool.core.date.LocalDateTimeUtil; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; -import org.jeecg.common.constant.enums.WeatherDataSourceEnum; -import org.jeecg.common.constant.enums.WeatherForecastDatasourceEnum; -import org.jeecg.common.constant.enums.WeatherTaskStatusEnum; -import org.jeecg.common.exception.JeecgFileUploadException; -import org.jeecg.common.properties.SystemStorageProperties; -import org.jeecg.common.util.NcUtil; -import org.jeecg.modules.base.entity.WeatherData; -import org.jeecg.modules.base.entity.WeatherTask; -import org.jeecg.modules.base.mapper.WeatherDataMapper; -import org.jeecg.service.WeatherTaskService; - -import java.io.*; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * 气象预测任务执行线程 - */ -public class GraphcastWeatherTaskExec extends Thread{ - - private WeatherTask weatherTask; - private WeatherTaskService weatherTaskService; - private SystemStorageProperties systemStorageProperties; - private WeatherDataMapper weatherDataMapper; - - /** - * 初始化 - */ - public void init(WeatherTask weatherTask, - WeatherTaskService weatherTaskService, - SystemStorageProperties systemStorageProperties, - WeatherDataMapper weatherDataMapper){ - this.weatherTask = weatherTask; - this.weatherTaskService = weatherTaskService; - this.systemStorageProperties = systemStorageProperties; - this.weatherDataMapper = weatherDataMapper; - } - - @Override - public void run() { - this.execute(); - } - - /** - * 执行任务 - */ - public void execute() { - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try{ - //修改任务状态为执行中 - this.weatherTaskService.updateTaskStatus(this.weatherTask.getId(), WeatherTaskStatusEnum.IN_OPERATION.getKey()); - //如果此任务已存在历史日志，先清除 - this.weatherTaskService.deleteTaskLog(this.weatherTask.getId()); - //执行模拟 - this.execSimulation(); - //执行完成 - this.execComplete(stopWatch,WeatherTaskStatusEnum.COMPLETED.getKey(), ""); - }catch (Exception e){ - String taskErrorLog = "任务执行失败，原因："+e.getMessage(); - //执行失败 - this.execComplete(stopWatch,WeatherTaskStatusEnum.FAILURE.getKey(), taskErrorLog); - throw e; - } - } - - /** - * 任务执行完成 - * @param stopWatch - * @param taskStatus - * @param taskErrorLog - */ - private void execComplete(StopWatch stopWatch,Integer taskStatus,String taskErrorLog){ - //添加任务耗时 - stopWatch.stop(); - long seconds = stopWatch.getTime(TimeUnit.SECONDS); - double min = seconds/60D; - BigDecimal bgMin = new BigDecimal(min); - BigDecimal result = bgMin.setScale(2, RoundingMode.HALF_UP); - this.weatherTaskService.updateTaskStatusToCompleted(this.weatherTask.getId(),result.doubleValue(),taskStatus); - if (WeatherTaskStatusEnum.COMPLETED.getKey().equals(taskStatus)){ - String taskCompletedLog = "任务执行完成，耗时："+result+"分钟"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskCompletedLog)); - }else if (WeatherTaskStatusEnum.FAILURE.getKey().equals(taskStatus)){ - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskErrorLog)); - } - } - - /** - * 执行模拟 - */ - private void execSimulation(){ - try { - //输入文件 - String inputFileName = this.weatherTask.getInputFile(); - //预测文件 - String outputFileName = "graphcast_output_"+this.weatherTask.getId()+".grib"; - //定义名称后缀,执行grib_set命令时去除，否则命名会冲突 - String gribFileSuffix = "_not_grib_set"; - //临时目录grib_copy,grib_set,python转换都在这个目录里，最后删除 - String tempDir = systemStorageProperties.getGraphcastModelExecPath()+File.separator+this.weatherTask.getId(); - if(!FileUtil.exist(tempDir)){ - FileUtil.mkdir(tempDir); - } - //预测气象数据 - this.forecast(inputFileName,outputFileName); - //切割文件，6小时一个 - this.splitGrib(outputFileName,gribFileSuffix,tempDir); - //适配flexpart -// this.convertGrib(gribFileSuffix,tempDir); - //保存文件数据入库 - this.saveGribInfoToDB(tempDir,this.getPanguWeatherPath(),gribFileSuffix); - //删除预测文件 -// File inputFile = new File(systemStorageProperties.getGraphcastModelExecPath()+File.separator+inputFileName); -// if (inputFile.exists()){ -// inputFile.delete(); -// } -// //删除输出的总文件 -// File outputFile = new File(systemStorageProperties.getGraphcastModelExecPath()+File.separator+outputFileName); -// if (outputFile.exists()){ -// outputFile.delete(); -// } -// //适配flexpart结束，删除目录 -// if(FileUtil.exist(tempDir)){ -// FileUtil.del(tempDir); -// } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * 盘古预测 - * @param inputFileAddr - * @param outputFileAddr - */ - private void forecast(String inputFileAddr,String outputFileAddr){ - try { - //处理开始日志 - String startLogFormat = ""; - String startTimeFormat = LocalDateTimeUtil.format(this.weatherTask.getStartDate(),"yyyy-MM-dd HH:mm:ss"); - String forecastTime = this.weatherTask.getLeadTime().toString(); - String forecastModel = WeatherDataSourceEnum.GRAPHCAST.getValue(); - if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ - startLogFormat = "任务开始，本次使用预测模型为：%s，预测开始时间为：%s，预测时长为：%s，前置数据采用CDS在线数据"; - }else { - startLogFormat = "任务开始，本次使用预测模型为：%s，预测开始时间为：%s，预测时长为：%s，前置数据采用离线数据："+this.weatherTask.getInputFile(); - } - String startLog = String.format(startLogFormat,forecastModel,startTimeFormat,forecastTime); - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),startLog)); - //处理运行命令 - List command = new ArrayList<>(); - String aiModelsPath = systemStorageProperties.getGraphcastEnvPath()+File.separator+"ai-models"; - - if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ - command.add(aiModelsPath); - command.add("--input"); - command.add("cds"); - command.add("--date"); - command.add(LocalDateTimeUtil.format(this.weatherTask.getStartDate(),"yyyyMMdd")); - command.add("--time"); - command.add(this.weatherTask.getStartTime().toString()); - command.add("--lead-time"); - command.add(this.weatherTask.getLeadTime().toString()); - command.add("--path"); - command.add(outputFileAddr); - command.add("--assets"); - command.add("assets-graphcast"); - command.add("--class od graphcast"); - }else if (this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.LOCATION_FILE.getKey())){ - //离线文件还需处理 - command.add(aiModelsPath); - command.add("--file"); - command.add(inputFileAddr); - command.add("--lead-time"); - command.add(this.weatherTask.getLeadTime().toString()); - command.add("--path"); - command.add(outputFileAddr); - command.add("--assets"); - command.add("assets-graphcast"); - command.add("--class od graphcast"); - } - String execLog = "执行任务命令，开始模拟，命令为："+command; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),execLog)); - ProcessBuilder processBuilder = new ProcessBuilder(command); - processBuilder.directory(new File(systemStorageProperties.getGraphcastModelExecPath())); - processBuilder.redirectErrorStream(true); - Process process = processBuilder.start(); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), "UTF-8")); - //读取输出日志 - String line; - while ((line = reader.readLine()) != null) { - if(StrUtil.isNotBlank(line)){ - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),line)); - } - } - //等待进程结束 - process.waitFor(); - }catch (Exception e){ - throw new RuntimeException("盘古模型预测异常",e); - } - } - - /** - * 分割grib文件 - * @param outputFileAddr - * @param gribFileSuffix - * @param gribCopyTargetDir - */ - private void splitGrib(String outputFileAddr,String gribFileSuffix,String gribCopyTargetDir){ - try { - String handleGribLog = "预测结束，开始处理grib文件,切割为6小时一个文件"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),handleGribLog)); - LocalDateTime startTime = this.weatherTask.getStartDate().atTime(this.weatherTask.getStartTime(), 0, 0); - //grib_copy命令 - String gribCopyCommandPath = systemStorageProperties.getGraphcastEnvPath()+File.separator+"grib_copy"; - //grib_set命令 - String gribSetCommandPath = systemStorageProperties.getGraphcastEnvPath()+File.separator+"grib_set"; - //公用gribProcessBuilder - ProcessBuilder gribProcessBuilder = new ProcessBuilder(); - gribProcessBuilder.directory(new File(systemStorageProperties.getGraphcastModelExecPath())); - //把grib文件切割成每6小时一份 - int step = 6; - int i=this.weatherTask.getStartTime(); - while(i <= this.weatherTask.getLeadTime()){ - String gribCopyFileAddr = gribCopyTargetDir+File.separator+"graphcast_"+LocalDateTimeUtil.format(startTime,"yyyyMMddHH")+gribFileSuffix+".grib"; - //切割grib文件命令 - List gribCopyCommand = new ArrayList<>(); - gribCopyCommand.add(gribCopyCommandPath); - gribCopyCommand.add("-w"); - gribCopyCommand.add("step="+i); - gribCopyCommand.add(outputFileAddr); - gribCopyCommand.add(gribCopyFileAddr); - System.out.println("gribCopyCommand:"+gribCopyCommand); - gribProcessBuilder.command(gribCopyCommand); - Process gribCopyProcess = gribProcessBuilder.start(); - gribCopyProcess.waitFor(); - - //重新设置reftime信息 - String gribSetFileAddr = gribCopyTargetDir+File.separator+"graphcast_"+LocalDateTimeUtil.format(startTime,"yyyyMMddHH")+".grib"; - String date = LocalDateTimeUtil.format(startTime,"yyyyMMdd"); - String time = LocalDateTimeUtil.format(startTime,"HHmm"); - List gribSetCommand = new ArrayList<>(); - gribSetCommand.add(gribSetCommandPath); - gribSetCommand.add("-s"); - gribSetCommand.add("dataDate="+date+",dataTime="+time+",endStep="+0); - gribSetCommand.add(gribCopyFileAddr); - gribSetCommand.add(gribSetFileAddr); - System.out.println("gribSetCommand:"+gribSetCommand); - gribProcessBuilder.command(gribSetCommand); - Process gribSetProcess = gribProcessBuilder.start(); - gribSetProcess.waitFor(); - i+=step; - startTime = startTime.plusHours(step); - } - }catch (Exception e){ - throw new RuntimeException("分割grib文件异常",e); - } - } - - /** - * 转换grib - * @param gribFileSuffix - * @param gribFilePath - */ - private void convertGrib(String gribFileSuffix,String gribFilePath){ - try { - // python3 pangu_reformat.py --ini_dir /export/xe_bk_data/weather/pangu_format_script --output_dir /export/xe_bk_data/weather/pangu2 --input_dir /export/xe_bk_data/weather/pangu2 --start_date 20251120 --end_date 20251120 - //调用python脚本转换flexpart能识别的气象数据文件 - //删除带有_not_grib_set名称的文件，此文件还未设置reftime属性 - List files = FileUtil.loopFiles(gribFilePath, file -> file.getName().contains(gribFileSuffix)); - if(!files.isEmpty()){ - for(File file:files){ - file.delete(); - } - } - String adapterGribLog = "文件切割结束，开始调用python脚本适配flexpart模型"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),adapterGribLog)); - //构建转换命令 - String convertScriptPath = systemStorageProperties.getPanguFormatScriptPath(); - String inputPath = systemStorageProperties.getPanguModelExecPath()+File.separator+this.weatherTask.getId(); - String outPutPath = inputPath; - String startDate = this.weatherTask.getStartDate().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - String endDate = this.weatherTask.getStartDate().atTime(this.weatherTask.getStartTime(), 0, 0).plusHours(this.weatherTask.getLeadTime()).format(DateTimeFormatter.ofPattern("yyyyMMdd")); - List convertCommand = new ArrayList<>(); - convertCommand.add(systemStorageProperties.getFormatScriptPythonEnv()+File.separator+"python3"); - convertCommand.add("pangu_reformat.py"); - convertCommand.add("--ini_dir"); - convertCommand.add(convertScriptPath); - convertCommand.add("--output_dir"); - convertCommand.add(outPutPath); - convertCommand.add("--input_dir"); - convertCommand.add(inputPath); - convertCommand.add("--start_date"); - convertCommand.add(startDate); - convertCommand.add("--end_date"); - convertCommand.add(endDate); - System.out.println("convertCommand:"+convertCommand); - ProcessBuilder adapterGribProcessBuilder = new ProcessBuilder(); - adapterGribProcessBuilder.directory(new File(convertScriptPath)); - adapterGribProcessBuilder.command(convertCommand); - Process adapterGribProcess = adapterGribProcessBuilder.start(); - adapterGribProcess.waitFor(); - }catch (Exception e){ - throw new RuntimeException("适配flexpart，转换grib异常",e); - } - } - - /** - * 保存生成的气象数据存储到数据库 - * @param inputPath - * @param targetPath - */ - private void saveGribInfoToDB(String inputPath,String targetPath,String gribFileSuffix){ - String handleGribLog = "格式转换结束，处理grib文件数据入库"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),handleGribLog)); - - List files = FileUtil.loopFiles(inputPath,file -> file.getName().startsWith("graphcast_") && !file.getName().contains(gribFileSuffix)); - List targetFiles = new ArrayList<>(); - if(!files.isEmpty()){ - for(File file:files){ - File targetFile = new File(targetPath+File.separator+file.getName()); - FileUtil.move(file,targetFile,true); - if(targetFile.exists()){ - targetFiles.add(targetFile); - } - } - for(File targetFile:targetFiles){ - //获取文件数据开始日期 - String reftime = NcUtil.getReftime(targetFile.getAbsolutePath()); - if(StringUtils.isBlank(reftime)) { - throw new JeecgFileUploadException("解析气象文件起始时间数据异常，此文件可能损坏"); - } - //计算文件大小M - BigDecimal divideVal = new BigDecimal("1024"); - BigDecimal bg = new BigDecimal(targetFile.length()); - BigDecimal fileSize = bg.divide(divideVal).divide(divideVal).setScale(2, RoundingMode.HALF_UP); - //处理文件数据开始时间 - Instant instant = Instant.parse(reftime); - LocalDateTime utcDateTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); - //计算文件MD5值 - String md5Val = ""; - try (InputStream is = new FileInputStream(targetFile.getAbsolutePath())) { - md5Val = DigestUtils.md5Hex(is); - }catch (Exception e){ - throw new RuntimeException(targetFile.getName()+"MD5值计算失败"); - } - //校验文件是否存在，存在删除从新新增 - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.eq(WeatherData::getFileName,targetFile.getName()); - WeatherData queryResult = weatherDataMapper.selectOne(queryWrapper); - if(Objects.nonNull(queryResult)){ - weatherDataMapper.deleteById(queryResult.getId()); - } - //构建文件信息 - WeatherData weatherData = new WeatherData(); - weatherData.setFileName(targetFile.getName()); - weatherData.setFileSize(fileSize.doubleValue()); - weatherData.setFileExt(targetFile.getName().substring(targetFile.getName().lastIndexOf(".")+1)); - weatherData.setDataStartTime(utcDateTime); - weatherData.setDataSource(weatherTask.getPredictionModel()); - weatherData.setFilePath(targetFile.getAbsolutePath()); - weatherData.setMd5Value(md5Val); - weatherData.setShareTotal(1); - weatherDataMapper.insert(weatherData); - } - } - } - - /** - * 获取盘古数据存储路径 - * @return - */ - private String getPanguWeatherPath(){ - StringBuilder sb = new StringBuilder(); - sb.append(this.systemStorageProperties.getRootPath()); - sb.append(File.separator); - sb.append(this.systemStorageProperties.getPangu()); - return sb.toString(); - } - - /** - * 获取Graphcast数据存储路径 - * @return - */ - private String getGraphcastWeatherPath(){ - StringBuilder sb = new StringBuilder(); - sb.append(this.systemStorageProperties.getRootPath()); - sb.append(File.separator); - sb.append(this.systemStorageProperties.getGraphcast()); - return sb.toString(); - } -} diff --git a/jeecg-module-weather/src/main/java/org/jeecg/task/PanGuWeatherTaskExec.java b/jeecg-module-weather/src/main/java/org/jeecg/task/PanGuWeatherTaskExec.java deleted file mode 100644 index ca02843..0000000 --- a/jeecg-module-weather/src/main/java/org/jeecg/task/PanGuWeatherTaskExec.java +++ /dev/null @@ -1,438 +0,0 @@ -package org.jeecg.task; - -import cn.hutool.core.date.LocalDateTimeUtil; -import cn.hutool.core.io.FileUtil; -import cn.hutool.core.util.StrUtil; -import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.time.StopWatch; -import org.jeecg.common.constant.enums.WeatherDataSourceEnum; -import org.jeecg.common.constant.enums.WeatherForecastDatasourceEnum; -import org.jeecg.common.constant.enums.WeatherTaskStatusEnum; -import org.jeecg.common.exception.JeecgFileUploadException; -import org.jeecg.common.properties.SystemStorageProperties; -import org.jeecg.common.util.NcUtil; -import org.jeecg.modules.base.entity.WeatherData; -import org.jeecg.modules.base.entity.WeatherTask; -import org.jeecg.modules.base.mapper.WeatherDataMapper; -import org.jeecg.service.WeatherTaskService; - -import java.io.*; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.TimeUnit; - -/** - * 气象预测任务执行线程 - */ -@Slf4j -public class PanGuWeatherTaskExec extends Thread{ - - private WeatherTask weatherTask; - private WeatherTaskService weatherTaskService; - private SystemStorageProperties systemStorageProperties; - private WeatherDataMapper weatherDataMapper; - - /** - * 初始化 - */ - public void init(WeatherTask weatherTask, - WeatherTaskService weatherTaskService, - SystemStorageProperties systemStorageProperties, - WeatherDataMapper weatherDataMapper){ - this.weatherTask = weatherTask; - this.weatherTaskService = weatherTaskService; - this.systemStorageProperties = systemStorageProperties; - this.weatherDataMapper = weatherDataMapper; - } - - @Override - public void run() { - this.execute(); - } - - /** - * 执行任务 - */ - public void execute() { - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - try{ - //修改任务状态为执行中 - this.weatherTaskService.updateTaskStatus(this.weatherTask.getId(), WeatherTaskStatusEnum.IN_OPERATION.getKey()); - //如果此任务已存在历史日志，先清除 - this.weatherTaskService.deleteTaskLog(this.weatherTask.getId()); - //执行模拟 - this.execSimulation(); - //执行完成 - this.execComplete(stopWatch,WeatherTaskStatusEnum.COMPLETED.getKey(), ""); - }catch (Exception e){ - String taskErrorLog = "任务执行失败，原因："+e.getMessage(); - //执行失败 - this.execComplete(stopWatch,WeatherTaskStatusEnum.FAILURE.getKey(), taskErrorLog); - throw e; - } - } - - /** - * 任务执行完成 - * @param stopWatch - * @param taskStatus - * @param taskErrorLog - */ - private void execComplete(StopWatch stopWatch,Integer taskStatus,String taskErrorLog){ - //添加任务耗时 - stopWatch.stop(); - long seconds = stopWatch.getTime(TimeUnit.SECONDS); - double min = seconds/60D; - BigDecimal bgMin = new BigDecimal(min); - BigDecimal result = bgMin.setScale(2, RoundingMode.HALF_UP); - this.weatherTaskService.updateTaskStatusToCompleted(this.weatherTask.getId(),result.doubleValue(),taskStatus); - if (WeatherTaskStatusEnum.COMPLETED.getKey().equals(taskStatus)){ - String taskCompletedLog = "任务执行完成，耗时："+result+"分钟"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskCompletedLog)); - }else if (WeatherTaskStatusEnum.FAILURE.getKey().equals(taskStatus)){ - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),taskErrorLog)); - } - } - - /** - * 执行模拟 - */ - private void execSimulation(){ - try { - //输入文件 - String inputFileName = this.weatherTask.getInputFile(); - //预测文件 - String outputFileName = "panguweather_output_"+this.weatherTask.getId()+".grib"; - //定义名称后缀,执行grib_set命令时去除，否则命名会冲突 - String gribFileSuffix = "_not_grib_set"; - - //临时目录grib_copy,grib_set,python转换的输入都在这个目录里，最后删除 - String inputPath = systemStorageProperties.getPanguModelExecPath()+File.separator+this.weatherTask.getId(); - if(!FileUtil.exist(inputPath)){ - FileUtil.mkdir(inputPath); - } - //python转换的输出在这里，最后删除 - String outputPath = inputPath+File.separator+"formatResult"; - if(!FileUtil.exist(outputPath)){ - FileUtil.mkdir(outputPath); - } - //预测气象数据 - this.forecast(inputFileName,outputFileName); - //切割文件，6小时一个 - this.splitGrib(outputFileName,gribFileSuffix,inputPath); - //适配flexpart - this.convertGrib(gribFileSuffix,inputPath,outputPath); - //保存文件数据入库 - this.saveGribInfoToDB(inputPath,this.getPanguWeatherPath()); - //删除预测文件 - File inputFile = new File(systemStorageProperties.getPanguModelExecPath()+File.separator+inputFileName); - if (inputFile.exists()){ - inputFile.delete(); - } -// 删除输出的总文件 - File outputFile = new File(systemStorageProperties.getPanguModelExecPath()+File.separator+outputFileName); - if (outputFile.exists()){ - outputFile.delete(); - } - //适配flexpart结束，删除目录 - if(FileUtil.exist(inputPath)){ - FileUtil.del(inputPath); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * 盘古预测 - * @param inputFileAddr - * @param outputFileAddr - */ - private void forecast(String inputFileAddr,String outputFileAddr){ - try { - //处理开始日志 - String startLogFormat = ""; - String startTimeFormat = LocalDateTimeUtil.format(this.weatherTask.getStartDate(),"yyyy-MM-dd HH:mm:ss"); - String forecastTime = this.weatherTask.getLeadTime().toString(); - String forecastModel = WeatherDataSourceEnum.PANGU.getValue(); - if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ - startLogFormat = "任务开始，本次使用预测模型为：%s，预测开始时间为：%s，预测时长为：%s，前置数据采用CDS在线数据"; - }else { - startLogFormat = "任务开始，本次使用预测模型为：%s，预测开始时间为：%s，预测时长为：%s，前置数据采用离线数据："+this.weatherTask.getInputFile(); - } - String startLog = String.format(startLogFormat,forecastModel,startTimeFormat,forecastTime); - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),startLog)); - //处理运行命令 - List command = new ArrayList<>(); - String aiModelsPath = systemStorageProperties.getPanguEnvPath()+File.separator+"ai-models"; - - if(this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.CDS.getKey())){ - command.add(aiModelsPath); - command.add("--input"); - command.add("cds"); - command.add("--date"); - command.add(LocalDateTimeUtil.format(this.weatherTask.getStartDate(),"yyyyMMdd")); - command.add("--time"); - command.add(this.weatherTask.getStartTime().toString()); - command.add("--lead-time"); - command.add(this.weatherTask.getLeadTime().toString()); - command.add("--path"); - command.add(outputFileAddr); - command.add("--assets"); - command.add("assets-panguweather"); - command.add("panguweather"); - }else if (this.weatherTask.getDataSources().equals(WeatherForecastDatasourceEnum.LOCATION_FILE.getKey())){ - //离线文件还需处理 - command.add(aiModelsPath); - command.add("--file"); - command.add(inputFileAddr); - command.add("--lead-time"); - command.add(this.weatherTask.getLeadTime().toString()); - command.add("--path"); - command.add(outputFileAddr); - command.add("--assets"); - command.add("assets-panguweather"); - command.add("panguweather"); - } - String execLog = "执行任务命令，开始模拟，命令为："+command; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),execLog)); - ProcessBuilder processBuilder = new ProcessBuilder(command); - processBuilder.directory(new File(systemStorageProperties.getPanguModelExecPath())); - processBuilder.redirectErrorStream(true); - Process process = processBuilder.start(); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), "UTF-8")); - //读取输出日志 - String line; - while ((line = reader.readLine()) != null) { - if(StrUtil.isNotBlank(line)){ - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),line)); - } - } - //等待进程结束 - process.waitFor(); - }catch (Exception e){ - throw new RuntimeException("盘古模型预测异常",e); - } - } - - /** - * 分割grib文件 - * @param outputFileAddr - * @param gribFileSuffix - * @param gribCopyTargetDir - */ - private void splitGrib(String outputFileAddr,String gribFileSuffix,String gribCopyTargetDir){ - try { - String handleGribLog = "预测结束，开始处理grib文件,切割为6小时一个文件"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),handleGribLog)); - LocalDateTime startTime = this.weatherTask.getStartDate().atTime(this.weatherTask.getStartTime(), 0, 0); - //grib_copy命令 - String gribCopyCommandPath = systemStorageProperties.getPanguEnvPath()+File.separator+"grib_copy"; - //grib_set命令 - String gribSetCommandPath = systemStorageProperties.getPanguEnvPath()+File.separator+"grib_set"; - //公用gribProcessBuilder - ProcessBuilder gribProcessBuilder = new ProcessBuilder(); - gribProcessBuilder.directory(new File(systemStorageProperties.getPanguModelExecPath())); - //把grib文件切割成每6小时一份 - int step = 6; - int i=this.weatherTask.getStartTime(); - while(i <= this.weatherTask.getLeadTime()){ - String gribCopyFileAddr = gribCopyTargetDir+File.separator+"panguweather_"+LocalDateTimeUtil.format(startTime,"yyyyMMddHH")+gribFileSuffix+".grib"; - //切割grib文件命令 - List gribCopyCommand = new ArrayList<>(); - gribCopyCommand.add(gribCopyCommandPath); - gribCopyCommand.add("-w"); - gribCopyCommand.add("step="+i); - gribCopyCommand.add(outputFileAddr); - gribCopyCommand.add(gribCopyFileAddr); - gribProcessBuilder.command(gribCopyCommand); - log.info("gribCopyCommand:{}", gribCopyCommand); - Process gribCopyProcess = gribProcessBuilder.start(); - gribCopyProcess.waitFor(); - - //重新设置reftime信息 - String gribSetFileAddr = gribCopyTargetDir+File.separator+"panguweather_"+LocalDateTimeUtil.format(startTime,"yyyyMMddHH")+".grib"; - String date = LocalDateTimeUtil.format(startTime,"yyyyMMdd"); - String time = LocalDateTimeUtil.format(startTime,"HHmm"); - List gribSetCommand = new ArrayList<>(); - gribSetCommand.add(gribSetCommandPath); - gribSetCommand.add("-s"); - gribSetCommand.add("dataDate="+date+",dataTime="+time+",endStep="+0); - gribSetCommand.add(gribCopyFileAddr); - gribSetCommand.add(gribSetFileAddr); - gribProcessBuilder.command(gribSetCommand); - log.info("gribSetCommand:{}", gribSetCommand); - Process gribSetProcess = gribProcessBuilder.start(); - gribSetProcess.waitFor(); - i+=step; - startTime = startTime.plusHours(step); - } - }catch (Exception e){ - throw new RuntimeException("分割grib文件异常",e); - } - } - - /** - * 转换grib - * @param gribFileSuffix - * @param inputPath - * @param outputPath - */ - private void convertGrib(String gribFileSuffix,String inputPath,String outputPath){ - try { - // python3 pangu_reformat.py --ini_dir /export/xe_bk_data/weather/pangu_format_script --output_dir /export/xe_bk_data/weather/pangu2 --input_dir /export/xe_bk_data/weather/pangu2 --start_date 20251120 --end_date 20251120 - //调用python脚本转换flexpart能识别的气象数据文件 - //删除带有_not_grib_set名称的文件，此文件还未设置reftime属性 - List files = FileUtil.loopFiles(inputPath, file -> file.getName().contains(gribFileSuffix)); - if(!files.isEmpty()){ - for(File file:files){ - file.delete(); - } - } - //构建转换命令 - String convertScriptPath = systemStorageProperties.getPanguFormatScriptPath(); - String startDate = this.weatherTask.getStartDate().format(DateTimeFormatter.ofPattern("yyyyMMdd")); - String endDate = this.weatherTask.getStartDate().atTime(this.weatherTask.getStartTime(), 0, 0).plusHours(this.weatherTask.getLeadTime()).format(DateTimeFormatter.ofPattern("yyyyMMdd")); - List convertCommand = new ArrayList<>(); - convertCommand.add(systemStorageProperties.getFormatScriptPythonEnv()+File.separator+"python3"); - convertCommand.add("pangu_reformat.py"); - convertCommand.add("--ini_dir"); - convertCommand.add(convertScriptPath); - convertCommand.add("--output_dir"); - convertCommand.add(outputPath); - convertCommand.add("--input_dir"); - convertCommand.add(inputPath); - convertCommand.add("--start_date"); - convertCommand.add(startDate); - convertCommand.add("--end_date"); - convertCommand.add(endDate); - - String adapterGribLog = "文件切割结束，开始调用python脚本适配flexpart模型，执行命令为："+convertCommand; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),adapterGribLog)); - - ProcessBuilder adapterGribProcessBuilder = new ProcessBuilder(); - adapterGribProcessBuilder.directory(new File(convertScriptPath)); - adapterGribProcessBuilder.command(convertCommand); - Process adapterGribProcess = adapterGribProcessBuilder.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(adapterGribProcess.getInputStream(), "UTF-8")); - //读取输出日志 - String line; - while ((line = reader.readLine()) != null) { - if(StrUtil.isNotBlank(line)){ - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),line)); - } - } - boolean finished = adapterGribProcess.waitFor((5*3600),TimeUnit.SECONDS); - if (!finished) { - // 如果在指定时间内进程没有结束，则强制销毁它 - adapterGribProcess.destroyForcibly(); // 强制终止进程 - String destroyForciblyLog = "python脚本适配flexpart模型进程超时，终止进程"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),destroyForciblyLog)); - } else { - int exitCode = adapterGribProcess.exitValue(); - String progressFinishedLog = "python脚本适配flexpart模型进程结束，exitCode："+exitCode; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),progressFinishedLog)); - - } - }catch (Exception e){ - throw new RuntimeException("适配flexpart，转换grib异常",e); - } - } - - /** - * 保存生成的气象数据存储到数据库 - * @param inputPath - * @param targetPath - */ - private void saveGribInfoToDB(String inputPath,String targetPath){ - String handleGribLog = "格式转换结束，处理grib文件数据入库"; - ProgressQueue.getInstance().offer(new ProgressEvent(this.weatherTask.getId(),handleGribLog)); - - List files = FileUtil.loopFiles(inputPath,file -> file.getName().startsWith("pangu_")); - List targetFiles = new ArrayList<>(); - if(!files.isEmpty()){ - for(File file:files){ - File targetFile = new File(targetPath+File.separator+file.getName()); - FileUtil.move(file,targetFile,true); - if(targetFile.exists()){ - targetFiles.add(targetFile); - } - } - for(File targetFile:targetFiles){ - //获取文件数据开始日期 - String reftime = NcUtil.getReftime(targetFile.getAbsolutePath()); - if(StringUtils.isBlank(reftime)) { - throw new JeecgFileUploadException("解析气象文件起始时间数据异常，此文件可能损坏"); - } - //计算文件大小M - BigDecimal divideVal = new BigDecimal("1024"); - BigDecimal bg = new BigDecimal(targetFile.length()); - BigDecimal fileSize = bg.divide(divideVal).divide(divideVal).setScale(2, RoundingMode.HALF_UP); - //处理文件数据开始时间 - Instant instant = Instant.parse(reftime); - LocalDateTime utcDateTime = LocalDateTime.ofInstant(instant, ZoneId.of("UTC")); - //计算文件MD5值 - String md5Val = ""; - try (InputStream is = new FileInputStream(targetFile.getAbsolutePath())) { - md5Val = DigestUtils.md5Hex(is); - }catch (Exception e){ - throw new RuntimeException(targetFile.getName()+"MD5值计算失败"); - } - //校验文件是否存在，存在删除从新新增 - LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); - queryWrapper.eq(WeatherData::getFileName,targetFile.getName()); - WeatherData queryResult = weatherDataMapper.selectOne(queryWrapper); - if(Objects.nonNull(queryResult)){ - weatherDataMapper.deleteById(queryResult.getId()); - } - //构建文件信息 - WeatherData weatherData = new WeatherData(); - weatherData.setFileName(targetFile.getName()); - weatherData.setFileSize(fileSize.doubleValue()); - weatherData.setFileExt(targetFile.getName().substring(targetFile.getName().lastIndexOf(".")+1)); - weatherData.setDataStartTime(utcDateTime); - weatherData.setDataSource(weatherTask.getPredictionModel()); - weatherData.setFilePath(targetFile.getAbsolutePath()); - weatherData.setMd5Value(md5Val); - weatherData.setShareTotal(1); - weatherDataMapper.insert(weatherData); - } - } - } - - /** - * 获取盘古数据存储路径 - * @return - */ - private String getPanguWeatherPath(){ - StringBuilder sb = new StringBuilder(); - sb.append(this.systemStorageProperties.getRootPath()); - sb.append(File.separator); - sb.append(this.systemStorageProperties.getPangu()); - return sb.toString(); - } - - /** - * 获取Graphcast数据存储路径 - * @return - */ - private String getGraphcastWeatherPath(){ - StringBuilder sb = new StringBuilder(); - sb.append(this.systemStorageProperties.getRootPath()); - sb.append(File.separator); - sb.append(this.systemStorageProperties.getGraphcast()); - return sb.toString(); - } -} diff --git a/jeecg-server-cloud/jeecg-consumer-start/src/main/java/org/jeecg/JeecgConsumerCloudApplication.java b/jeecg-server-cloud/jeecg-consumer-start/src/main/java/org/jeecg/JeecgConsumerCloudApplication.java index 0dddbd6..6f2a88d 100644 --- a/jeecg-server-cloud/jeecg-consumer-start/src/main/java/org/jeecg/JeecgConsumerCloudApplication.java +++ b/jeecg-server-cloud/jeecg-consumer-start/src/main/java/org/jeecg/JeecgConsumerCloudApplication.java @@ -5,7 +5,9 @@ import lombok.extern.slf4j.Slf4j; import org.jeecg.common.base.BaseMap; import org.jeecg.common.constant.GlobalConstants; import org.jeecg.common.util.oConvertUtils; -import org.jeecg.transport.consumer.TranTaskMessageConsumerHandler; +import org.jeecg.rebuild.consumer.RebuildTaskConsumerHandler; +import org.jeecg.transport.consumer.TranTaskConsumerHandler; +import org.jeecg.weather.consumer.WeatherTaskConsumerHandler; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -23,7 +25,9 @@ import java.net.UnknownHostException; public class JeecgConsumerCloudApplication extends SpringBootServletInitializer implements CommandLineRunner { private final RedisTemplate redisTemplate; - private final TranTaskMessageConsumerHandler tranTaskMsgConsumerHandler; + private final TranTaskConsumerHandler tranTaskConsumerHandler; + private final RebuildTaskConsumerHandler rebuildTaskConsumerHandler; + private final WeatherTaskConsumerHandler weatherTaskConsumerHandler; @Override protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { @@ -51,6 +55,8 @@ public class JeecgConsumerCloudApplication extends SpringBootServletInitializer params.put(GlobalConstants.HANDLER_NAME, GlobalConstants.LODER_ROUDER_HANDLER); //刷新网关 redisTemplate.convertAndSend(GlobalConstants.REDIS_TOPIC_NAME, params); - tranTaskMsgConsumerHandler.startConsumerThread(); + tranTaskConsumerHandler.startConsumerThread(); + rebuildTaskConsumerHandler.startConsumerThread(); + weatherTaskConsumerHandler.startConsumerThread(); } } \ No newline at end of file