fix:1.修改天气预报主模块业务，添加关联气象数据和查询关联气象数据日志接口2.修改新增和修改预测任务逻辑

jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherData.java

@@ -76,4 +76,10 @@ public class WeatherData implements Serializable {
      */
     @TableField(value = "time_batch")
     private String timeBatch;
+
+    /**
+     * MD5值
+     */
+    @TableField(value = "md5_value")
+    private String md5Value;
 }

jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/entity/WeatherTask.java

@@ -103,11 +103,6 @@ public class WeatherTask{
     @TableField(exist = false)
     private MultipartFile file;
 
-    /**
-     * 创建人
-     */
-    private String createBy;
-
     /**
      * 创建时间
      */
@@ -115,11 +110,6 @@ public class WeatherTask{
     @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
     private LocalDateTime createTime;
 
-    /**
-     * 更新人
-     */
-    private String updateBy;
-
     /**
      * 更新时间
      */

jeecg-module-event/src/main/java/org/jeecg/wrf/service/impl/WrfServiceImpl.java

@@ -86,19 +86,18 @@ public class WrfServiceImpl extends ServiceImpl<WrfMapper, Wrf> implements WrfSe
 
     public WeatherPathVO getWeatherPaths(Integer weatherDataType,String startTime, String endTime) {
         WeatherPathVO weatherPathVO = new WeatherPathVO();
-        String weatherPath = systemStorageProperties.getRootPath() + File.separator;
         if (WeatherDataSourceEnum.PANGU.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getPangu());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getPanguDataPath());
         } else if (WeatherDataSourceEnum.GRAPHCAST.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getGraphcast());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getGraphcastDataPath());
         } else if (WeatherDataSourceEnum.CRA40.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getCra40());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getCra40DataPath());
         } else if (WeatherDataSourceEnum.NCEP.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getNcep());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getNcepDataPath());
         }else if (WeatherDataSourceEnum.FNL.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getFnl());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getFnlDataPath());
         } else if (WeatherDataSourceEnum.T1H.getKey().equals(weatherDataType)) {
-            weatherPathVO.setWeatherPath(weatherPath + systemStorageProperties.getT1h());
+            weatherPathVO.setWeatherPath(systemStorageProperties.getT1hDataPath());
         }
         // 将String类型的时间转换为LocalDateTime
         LocalDateTime start = LocalDateTime.parse(startTime, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"));

jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherDataController.java

@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import io.swagger.v3.oas.annotations.Operation;
 import jakarta.annotation.Resource;
+import jakarta.validation.constraints.NotNull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.jeecg.common.api.vo.Result;
@@ -14,6 +15,7 @@ import org.jeecg.job.DownloadT1hJob;
 import org.jeecg.modules.base.entity.WeatherData;
 import org.jeecg.modules.base.entity.WeatherLinkedDataLog;
 import org.jeecg.service.WeatherDataService;
+import org.jeecg.service.WeatherLinkedDataLogService;
 import org.springframework.format.annotation.DateTimeFormat;
 import org.springframework.validation.annotation.Validated;
 import org.springframework.web.bind.annotation.*;
@@ -30,6 +32,7 @@ import java.util.stream.Collectors;
 public class WeatherDataController {
 
     private final WeatherDataService weatherDataService;
+    private final WeatherLinkedDataLogService weatherLinkedDataLogService;
 
     @Resource
     private DownloadT1hJob downloadT1hJob;
@@ -147,9 +150,9 @@ public class WeatherDataController {
     @AutoLog(value = "关联气象数据")
     @Operation(summary = "关联气象数据")
     @PutMapping("linkedData")
-    public Result<?> linkedData(Integer dataType,
+    public Result<?> linkedData(@NotNull(message = "数据类型不能为空") Integer dataType,
-                                @DateTimeFormat(iso=DateTimeFormat.ISO.DATE) LocalDate startDate,
+                                @NotNull(message = "开始日期不能为空") @DateTimeFormat(iso=DateTimeFormat.ISO.DATE) LocalDate startDate,
-                                @DateTimeFormat(iso=DateTimeFormat.ISO.DATE)LocalDate endDate){
+                                @NotNull(message = "结束日期不能为空") @DateTimeFormat(iso=DateTimeFormat.ISO.DATE)LocalDate endDate){
         weatherDataService.linkedData(dataType,startDate,endDate);
         return Result.OK();
     }
@@ -157,8 +160,8 @@ public class WeatherDataController {
     @AutoLog(value = "查询关联气象数据日志")
     @Operation(summary = "查询关联气象数据日志")
     @GetMapping("getLinkedDataLog")
-    public Result<?> getLinkedDataLog(){
+    public Result<?> getLinkedDataLog(Integer lastId){
-        List<WeatherLinkedDataLog> result = weatherDataService.getLinkedDataLog();
+        List<WeatherLinkedDataLog> result = weatherLinkedDataLogService.getLinkedDataLog(lastId);
         return Result.OK(result);
     }
 }

jeecg-module-weather/src/main/java/org/jeecg/controller/WeatherTaskController.java

@@ -75,7 +75,7 @@ public class WeatherTaskController {
     @AutoLog(value = "启动任务")
     @Operation(summary = "启动任务")
     @PutMapping("runTask")
-    public Result<?> runTask(@NotBlank(message = "任务ID不能为空") Integer taskId){
+    public Result<?> runTask(@NotNull(message = "任务ID不能为空") Integer taskId){
         weatherTaskService.runTask(taskId);
         return Result.OK();
     }
@@ -83,7 +83,7 @@ public class WeatherTaskController {
     @AutoLog(value = "获取天气预测任务过程日志")
     @Operation(summary = "获取天气预测任务过程日志")
     @GetMapping("getTaskLog")
-    public Result<?> getTaskLog(@NotBlank(message = "预测任务ID不能为空") Integer taskId){
+    public Result<?> getTaskLog(@NotNull(message = "预测任务ID不能为空") Integer taskId){
         return Result.OK(weatherTaskService.getTaskLog(taskId));
     }
 }

jeecg-module-weather/src/main/java/org/jeecg/service/WeatherDataService.java

@@ -38,13 +38,6 @@ public interface WeatherDataService extends IService<WeatherData> {
      * 处理静态气象数据入库接口，比上传快
      */
     void handleStaticDataToDB(String path,Integer dataSource);
-
-    /**
-     * 查询关联气象数据日志
-     * @return
-     */
-    List<WeatherLinkedDataLog> getLinkedDataLog();
-
     /**
      * 关联气象数据
      * @param dataType

jeecg-module-weather/src/main/java/org/jeecg/service/WeatherLinkedDataLogService.java

@@ -0,0 +1,31 @@
+package org.jeecg.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import org.jeecg.modules.base.entity.WeatherLinkedDataLog;
+
+import java.util.List;
+
+/**
+ * grib文件关联数据表
+ */
+public interface WeatherLinkedDataLogService extends IService<WeatherLinkedDataLog> {
+
+    /**
+     * 保存关联过程日志
+     * @param dataType
+     * @param logContent
+     */
+    void create(Integer dataType,String logContent);
+
+    /**
+     * 获取关联过程日志
+     * @param lastId
+     * @return
+     */
+    List<WeatherLinkedDataLog> getLinkedDataLog(Integer lastId);
+
+    /**
+     * 清空表日志
+     */
+    void cleanTable();
+}

jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherDataServiceImpl.java

@@ -9,6 +9,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.jeecg.common.constant.WeatherStepConstants;
 import org.jeecg.common.constant.enums.*;
@@ -21,8 +22,8 @@ import org.jeecg.common.util.NcUtil;
 import org.jeecg.modules.base.entity.WeatherData;
 import org.jeecg.modules.base.entity.WeatherLinkedDataLog;
 import org.jeecg.modules.base.mapper.WeatherDataMapper;
-import org.jeecg.modules.base.mapper.WeatherLinkedDataLogMapper;
 import org.jeecg.service.WeatherDataService;
+import org.jeecg.service.WeatherLinkedDataLogService;
 import org.jeecg.utils.WindRoseDataGenerator;
 import org.jeecg.vo.*;
 import org.springframework.stereotype.Service;
@@ -48,7 +49,7 @@ import static org.jeecg.common.constant.LatLonSizeConstants.*;
 public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, WeatherData> implements WeatherDataService {
 
     private final WeatherDataMapper weatherDataMapper;
-    private final WeatherLinkedDataLogMapper weatherLinkedDataLogMapper;
+    private final WeatherLinkedDataLogService weatherLinkedDataLogService;
     private final SystemStorageProperties systemStorageProperties;
 
     /**
@@ -399,18 +400,6 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
         }
     }
 
-    /**
-     * 查询关联气象数据日志
-     *
-     * @return
-     */
-    @Override
-    public List<WeatherLinkedDataLog> getLinkedDataLog() {
-        LambdaQueryWrapper<WeatherLinkedDataLog> queryWrapper = new LambdaQueryWrapper<>();
-        queryWrapper.select(WeatherLinkedDataLog::getCreateTime,WeatherLinkedDataLog::getLogContent);
-        return weatherLinkedDataLogMapper.selectList(queryWrapper);
-    }
-
     /**
      * 关联气象数据
      *
@@ -421,7 +410,7 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
     @Override
     public void linkedData(Integer dataType, LocalDate startDate, LocalDate endDate) {
         //先清空表
-        weatherLinkedDataLogMapper.delete(null);
+        weatherLinkedDataLogService.cleanTable();
         String linkedAddr = "";
         if(WeatherDataSourceEnum.NCEP.getKey().equals(dataType)) {
             linkedAddr = systemStorageProperties.getNcepDataPath();
@@ -460,11 +449,9 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
                             return true;
                         }
                     }catch(Exception  e){
-                        log.error("{}文件命名格式错误，需确认",file.getAbsolutePath());
+                        String msg = file.getAbsolutePath()+"文件命名格式错误，需确认";
-                        WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog();
+                        log.error(msg,e);
-                        linkedLog.setDataType(dataType);
+                        weatherLinkedDataLogService.create(dataType,msg);
-                        linkedLog.setLogContent(file.getAbsolutePath()+"文件命名格式错误，需确认");
-                        weatherLinkedDataLogMapper.insert(linkedLog);
                     }
                 }
                 return false;
@@ -473,6 +460,15 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
         if (CollUtil.isNotEmpty(fileList)) {
            for (File file : fileList) {
                try {
+                   //如果此文件存在则无需再次新增
+                   String gribFileMD5 = this.getGribFileMD5(file.getAbsolutePath());
+                   LambdaQueryWrapper<WeatherData> queryWrapper = new LambdaQueryWrapper<>();
+                   queryWrapper.eq(WeatherData::getMd5Value, gribFileMD5);
+                   WeatherData queryResult = this.baseMapper.selectOne(queryWrapper);
+                   if (Objects.nonNull(queryResult)) {
+                       weatherLinkedDataLogService.create(dataType,file.getAbsolutePath()+"已存在");
+                       continue;
+                   }
                    WeatherData weatherData = new WeatherData();
                    weatherData.setFileName(file.getName());
                    weatherData.setFileExt(file.getName().substring(file.getName().lastIndexOf(".")+1));
@@ -483,26 +479,27 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
                    weatherData.setDataStartTime(localDateTime);
                    this.baseMapper.insert(weatherData);
 
-                   WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog();
+                   weatherLinkedDataLogService.create(dataType,file.getAbsolutePath()+"关联成功");
-                   linkedLog.setDataType(dataType);
-                   linkedLog.setLogContent(file.getAbsolutePath()+"关联成功");
-                   weatherLinkedDataLogMapper.insert(linkedLog);
                }catch (Exception  e){
                     log.error("关联{}气象数据文件出现错误，原因为：",file.getAbsolutePath(),e);
-                   WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog();
+                   weatherLinkedDataLogService.create(dataType,file.getAbsolutePath()+"关联失败，原因为："+e.getMessage());
-                   linkedLog.setDataType(dataType);
-                   linkedLog.setLogContent(file.getAbsolutePath()+"关联失败，原因为："+e.getMessage());
-                   weatherLinkedDataLogMapper.insert(linkedLog);
                }
            }
-            WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog();
+            weatherLinkedDataLogService.create(dataType,"关联任务执行完毕！");
-            linkedLog.setDataType(dataType);
-            linkedLog.setLogContent("关联任务执行完毕！");
-            weatherLinkedDataLogMapper.insert(linkedLog);
         }
 
     }
 
+    /**
+     * 获取GRIB文件的MD5唯一值
+     */
+    private String getGribFileMD5(String filePath) throws IOException {
+        try (FileInputStream fis = new FileInputStream(filePath)) {
+            // 底层自动采用流式读取，内存占用极低
+            return DigestUtils.md5Hex(fis);
+        }
+    }
+
     /**
      * 通用数据处理方法
      */

jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherLinkedDataLogServiceImpl.java

@@ -0,0 +1,56 @@
+package org.jeecg.service.impl;
+
+import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.jeecg.modules.base.entity.WeatherLinkedDataLog;
+import org.jeecg.modules.base.mapper.WeatherLinkedDataLogMapper;
+import org.jeecg.service.WeatherLinkedDataLogService;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.List;
+import java.util.Objects;
+
+@Service
+public class WeatherLinkedDataLogServiceImpl extends ServiceImpl<WeatherLinkedDataLogMapper, WeatherLinkedDataLog> implements WeatherLinkedDataLogService {
+
+    /**
+     * 保存关联过程日志
+     *
+     * @param dataType
+     * @param logContent
+     */
+    @Transactional(rollbackFor = RuntimeException.class)
+    @Override
+    public void create(Integer dataType, String logContent) {
+        WeatherLinkedDataLog linkedLog = new WeatherLinkedDataLog();
+        linkedLog.setDataType(dataType);
+        linkedLog.setLogContent(logContent);
+        this.baseMapper.insert(linkedLog);
+    }
+
+    /**
+     * 获取关联过程日志
+     *
+     * @param lastId
+     * @return
+     */
+    @Override
+    public List<WeatherLinkedDataLog> getLinkedDataLog(Integer lastId) {
+        LambdaQueryWrapper<WeatherLinkedDataLog> queryWrapper = new LambdaQueryWrapper<>();
+        queryWrapper.gt(Objects.nonNull(lastId),WeatherLinkedDataLog::getId,lastId);
+        queryWrapper.select(WeatherLinkedDataLog::getCreateTime,WeatherLinkedDataLog::getLogContent);
+        queryWrapper.orderByAsc(WeatherLinkedDataLog::getId);
+        queryWrapper.last("LIMIT 10");
+        return this.baseMapper.selectList(queryWrapper);
+    }
+
+    /**
+     * 清空表日志
+     */
+    @Transactional(rollbackFor = RuntimeException.class)
+    @Override
+    public void cleanTable() {
+        this.baseMapper.delete(null);
+    }
+}

jeecg-module-weather/src/main/java/org/jeecg/service/impl/WeatherTaskServiceImpl.java

@@ -20,10 +20,12 @@ import org.jeecg.modules.base.mapper.WeatherDataMapper;
 import org.jeecg.modules.base.mapper.WeatherTaskLogMapper;
 import org.jeecg.modules.base.mapper.WeatherTaskMapper;
 import org.jeecg.service.WeatherTaskService;
+import org.reflections.vfs.Vfs;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 import org.springframework.web.multipart.MultipartFile;
 import java.io.File;
+import java.io.FileFilter;
 import java.io.IOException;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -104,9 +106,10 @@ public class WeatherTaskServiceImpl extends ServiceImpl<WeatherTaskMapper, Weath
                 //文件保存地址
                 File storageFile = new File(this.systemStorageProperties.getForecastFileTmpPath()+File.separator+fileName);
                 //如果不存在则创建
-                if(!FileUtil.exist(storageFile)){
+                if(!FileUtil.exist(storageFile.getParent())){
                     FileUtil.mkdir(storageFile.getParent());
                 }else{
+                    //如果重名直接删除
                     if (storageFile.exists()){
                         storageFile.delete();
                     }
@@ -178,11 +181,26 @@ public class WeatherTaskServiceImpl extends ServiceImpl<WeatherTaskMapper, Weath
                     //文件保存地址
                     File storageFile = new File(this.systemStorageProperties.getForecastFileTmpPath()+File.separator+fileName);
                     //如果不存在则创建
-                    if(!FileUtil.exist(storageFile)){
+                    if(!FileUtil.exist(storageFile.getParent())){
                         FileUtil.mkdir(storageFile.getParent());
                     }else{
+                        //如果重名直接删除
                         if (storageFile.exists()){
                             storageFile.delete();
+                        }else {
+                            //如果文件换了，则找到包含此记录id的，先删除再保存
+                            List<File> files = FileUtil.loopFiles(this.systemStorageProperties.getForecastFileTmpPath(), new FileFilter() {
+                                @Override
+                                public boolean accept(File file) {
+                                    String flag = "_"+weatherTask.getId();
+                                    return file.getName().contains(flag);
+                                }
+                            });
+                            if (CollUtil.isNotEmpty(files)) {
+                                for (File delFile : files) {
+                                    delFile.delete();
+                                }
+                            }
                         }
                     }
                     file.transferTo(storageFile);