fix:1.修改数据同步设置同步位置bug2.添加flexpart nc转gif功能

This commit is contained in:
panbaolin 2026-07-09 17:54:47 +08:00
parent a55343a10e
commit b50dbdb8f7
14 changed files with 242 additions and 57 deletions

View File

@ -128,12 +128,22 @@ public class TransportSimulationProperties {
* 反演脚本路径和T1H气象数据有关系 * 反演脚本路径和T1H气象数据有关系
*/ */
private String t1hBackwardScriptPath; private String t1hBackwardScriptPath;
/** /**
* 正演文件路径 * nc转换gif配置
* 脚本路径
*/
private String gifConvertScriptPath;
/**
* python环境路径
*/
private String pythonEnvPath;
/**
* 正演任务模版文件路径
*/ */
private String forwardTemplatePath; private String forwardTemplatePath;
/** /**
* 反演文件路径 * 反演任务模版文件路径
*/ */
private String backwardTemplatePath; private String backwardTemplatePath;
/** /**

View File

@ -41,7 +41,7 @@ public class WeatherTask{
private String taskName; private String taskName;
/** /**
* 任务状态-1失败0-未开始1-等待中2-执行中3-已完成 * 任务状态-1失败0-未开始1-等待中2-执行中3-已完成,4-检查失败
*/ */
@Null(message = "任务状态必须为空",groups = {InsertGroup.class, UpdateGroup.class}) @Null(message = "任务状态必须为空",groups = {InsertGroup.class, UpdateGroup.class})
@TableField(value = "task_status") @TableField(value = "task_status")

View File

@ -11,7 +11,6 @@
t.task_status as taskStatus, t.task_status as taskStatus,
t.use_met_type as useMetType, t.use_met_type as useMetType,
t.time_consuming as timeConsuming, t.time_consuming as timeConsuming,
t.create_by as createBy,
t.create_time as createTime, t.create_time as createTime,
t.top_task as topTask t.top_task as topTask
from stas_transport_task t from stas_transport_task t
@ -31,11 +30,12 @@
</where> </where>
ORDER BY ORDER BY
CASE t.task_status CASE t.task_status
WHEN 1 THEN 0 <!-- 等待中 --> WHEN -1 THEN 0 <!-- 执行失败 -->
WHEN -1 THEN 1 <!-- 执行失败 --> WHEN 4 THEN 1 <!-- 检查未通过 -->
WHEN 2 THEN 2 WHEN 1 THEN 2 <!-- 等待中 -->
WHEN 0 THEN 3 WHEN 2 THEN 3 <!-- 执行中 -->
ELSE 4 <!-- 其他状态 --> WHEN 3 THEN 4 <!-- 已完成 -->
ELSE 5 <!-- 其他状态 -->
END ASC,t.update_time desc END ASC,t.update_time desc
</select> </select>
</mapper> </mapper>

View File

@ -1,13 +1,16 @@
package org.jeecg.transport.flexparttask; package org.jeecg.transport.flexparttask;
import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.LocalDateTimeUtil; import cn.hutool.core.date.LocalDateTimeUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.StringPool;
import com.jcraft.jsch.*; import com.jcraft.jsch.*;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.logging.log4j.util.Strings; import org.apache.logging.log4j.util.Strings;
import org.jeecg.common.constant.CommonConstant; import org.jeecg.common.constant.CommonConstant;
import org.jeecg.common.constant.enums.TransportTaskModeEnum;
import org.jeecg.common.constant.enums.WeatherDataSourceEnum; import org.jeecg.common.constant.enums.WeatherDataSourceEnum;
import org.jeecg.common.properties.DataFusionProperties; import org.jeecg.common.properties.DataFusionProperties;
import org.jeecg.common.properties.ServerProperties; import org.jeecg.common.properties.ServerProperties;
@ -15,6 +18,8 @@ import org.jeecg.common.properties.SystemStorageProperties;
import org.jeecg.common.properties.TransportSimulationProperties; import org.jeecg.common.properties.TransportSimulationProperties;
import org.jeecg.common.util.RedisUtil; import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.base.entity.TransportTask; import org.jeecg.modules.base.entity.TransportTask;
import org.jeecg.modules.base.entity.TransportTaskBackwardChild;
import org.jeecg.modules.base.entity.TransportTaskForwardSpecies;
import org.jeecg.modules.base.entity.WeatherData; import org.jeecg.modules.base.entity.WeatherData;
import org.jeecg.modules.base.mapper.*; import org.jeecg.modules.base.mapper.*;
import org.jeecg.transport.service.StationDataService; import org.jeecg.transport.service.StationDataService;
@ -23,6 +28,7 @@ import org.jeecg.transport.service.TransportTaskService;
import java.io.*; import java.io.*;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.List; import java.util.List;
@Slf4j @Slf4j
@ -43,6 +49,8 @@ public abstract class AbstractTaskExec extends Thread{
protected TransportTask transportTask; protected TransportTask transportTask;
protected RedisUtil redisUtil; protected RedisUtil redisUtil;
protected boolean taskRunError; protected boolean taskRunError;
protected static String FORWARD="forward";
protected static String BACK_FORWARD="backward";
/** /**
* 初始化 * 初始化
@ -109,13 +117,6 @@ public abstract class AbstractTaskExec extends Thread{
redisUtil.hset(CommonConstant.HOST_TASK_STATE,CommonConstant.TRAN_TASK_STATE_PRE+serverProperties.getHost(),transportTask.getId()); redisUtil.hset(CommonConstant.HOST_TASK_STATE,CommonConstant.TRAN_TASK_STATE_PRE+serverProperties.getHost(),transportTask.getId());
} }
/**
* 任务运行失败取消运行标记
*/
protected void cancelTaskRunFlag(){
redisUtil.hdel(CommonConstant.HOST_TASK_STATE,CommonConstant.TRAN_TASK_STATE_PRE+serverProperties.getHost());
}
/** /**
* 检查气象数据 * 检查气象数据
*/ */
@ -140,6 +141,104 @@ public abstract class AbstractTaskExec extends Thread{
return true; return true;
} }
/**
* 用nc文件数据生成GIF动图
*/
protected void genNCToGif(Session session){
int mode = 0;
List<String> ncPaths = null;
if(TransportTaskModeEnum.FORWARD.getKey().equals(transportTask.getTaskMode())){
mode = TransportTaskModeEnum.FORWARD.getKey();
ncPaths = this.getForwardTaskNCPath();
}else if(TransportTaskModeEnum.BACK_FORWARD.getKey().equals(transportTask.getTaskMode())){
mode = TransportTaskModeEnum.BACK_FORWARD.getKey();
ncPaths = this.getBackForwardTaskNCPath();
}
try{
for(String ncPath : ncPaths){
File ncFile = new File(ncPath);
StringBuilder command = new StringBuilder();
command.append(simulationProperties.getPythonEnvPath());
command.append(StringPool.SPACE);
command.append(simulationProperties.getGifConvertScriptPath());
command.append(" --nc_file ");
command.append(ncPath);
command.append(" --mode ");
command.append(mode);
command.append(" --gif_file ");
command.append(ncFile.getParent());
command.append(File.separator);
command.append(ncFile.getParentFile().getName());
command.append(".gif");
log.info(command.toString());
this.execGenGIFCommand(command.toString(),session);
}
}catch (Exception e){
log.error("nc文件生成gif动图失败",e);
}
}
/**
* 获取反演NC结果文件路径
* @return
*/
private List<String> getBackForwardTaskNCPath(){
LambdaQueryWrapper<TransportTaskBackwardChild> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(TransportTaskBackwardChild::getTaskId,this.transportTask.getId());
queryWrapper.select(TransportTaskBackwardChild::getId, TransportTaskBackwardChild::getStationCode);
List<TransportTaskBackwardChild> transportTaskChildren = this.taskBackwardChildMapper.selectList(queryWrapper);
if(CollUtil.isEmpty(transportTaskChildren)){
throw new RuntimeException("此任务站点信息不存在,请确认任务配置信息");
}
List<String> ncPaths = new ArrayList<>();
for (TransportTaskBackwardChild child : transportTaskChildren){
//拼接nc文件路径
StringBuilder path = new StringBuilder();
path.append(simulationProperties.getOutputPath());
path.append(File.separator);
path.append(transportTask.getTaskName());
path.append(File.separator);
path.append(BACK_FORWARD);
path.append(File.separator);
path.append(child.getStationCode());
path.append(File.separator);
path.append("grid_time_"+ DateUtil.format(transportTask.getEndTime(),"yyyyMMddHHmmss")+".nc");
ncPaths.add(path.toString());
}
return ncPaths;
}
/**
* 获取正演NC结果文件路径
* @return
*/
private List<String> getForwardTaskNCPath(){
LambdaQueryWrapper<TransportTaskForwardSpecies> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(TransportTaskForwardSpecies::getTaskId,this.transportTask.getId());
queryWrapper.select(TransportTaskForwardSpecies::getSpeciesId);
List<TransportTaskForwardSpecies> transportTaskSpecies = this.taskForwardSpeciesMapper.selectList(queryWrapper);
if(CollUtil.isEmpty(transportTaskSpecies)){
throw new RuntimeException("此任务模拟核素信息不存在,请确认任务配置信息");
}
List<String> ncPaths = new ArrayList<>();
for (TransportTaskForwardSpecies species : transportTaskSpecies){
//拼接nc文件路径
StringBuilder path = new StringBuilder();
path.append(simulationProperties.getOutputPath());
path.append(File.separator);
path.append(transportTask.getTaskName());
path.append(File.separator);
path.append(FORWARD);
path.append(File.separator);
path.append(species.getSpeciesId());
path.append(File.separator);
path.append("grid_conc_"+DateUtil.format(transportTask.getStartTime(),"yyyyMMddHHmmss")+".nc");
ncPaths.add(path.toString());
}
return ncPaths;
}
/** /**
* 获取气象数据路径 * 获取气象数据路径
* @param dataSource * @param dataSource
@ -209,9 +308,9 @@ public abstract class AbstractTaskExec extends Thread{
} }
/** /**
* 持续读取日志 * 执行flexpart命令持续读取日志
*/ */
protected void execCommand(String command,Session session){ protected void execFlexpartCommand(String command,Session session){
ChannelExec channel = null; ChannelExec channel = null;
try{ try{
//打开一个执行通道 //打开一个执行通道
@ -241,4 +340,38 @@ public abstract class AbstractTaskExec extends Thread{
} }
} }
} }
/**
* 执行生成GIF图片命令持续读取日志
*/
protected void execGenGIFCommand(String command,Session session){
ChannelExec channel = null;
try{
//打开一个执行通道
channel = (ChannelExec) session.openChannel("exec");
String fullCommand = command + " 2>&1";
channel.setCommand(fullCommand);
// 获取脚本的标准输出流包含错误输出流
InputStream in = channel.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
// 连接通道
channel.connect();
String line;
while ((line = reader.readLine()) != null) {
if(StrUtil.isNotBlank(line)){
if(line.trim().startsWith("ERROR")){
taskRunError = true;
}
ProgressQueue.getInstance().offer(new ProgressEvent(transportTask.getId(),line));
}
}
}catch(JSchException |IOException e){
throw new RuntimeException(e);
}finally {
if (channel != null) {
channel.disconnect();
}
}
}
} }

View File

@ -10,6 +10,7 @@ import org.jeecg.common.constant.enums.TransportTaskStatusEnum;
import org.jeecg.common.constant.enums.WeatherDataSourceEnum; import org.jeecg.common.constant.enums.WeatherDataSourceEnum;
import org.jeecg.modules.base.entity.*; import org.jeecg.modules.base.entity.*;
import org.jeecg.jsch.JSchRemoteRunner; import org.jeecg.jsch.JSchRemoteRunner;
import java.io.File; import java.io.File;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.RoundingMode; import java.math.RoundingMode;
@ -98,13 +99,10 @@ public class BackwardTaskExec extends AbstractTaskExec {
//生成SRS文件 //生成SRS文件
// this.generateSRSFile(); // this.generateSRSFile();
}catch (Exception e){ }catch (Exception e){
String taskErrorLog = "任务执行失败,原因:"+e.getMessage(); super.taskRunError = true;
ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskErrorLog)); String taskErrorLog = "任务执行失败,原因:";
//如果还未执行到flexpartjava业务代码报错修改状态 ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskErrorLog+e.getMessage()));
super.transportTaskService.updateTaskStatus(super.transportTask.getId(),TransportTaskStatusEnum.FAILURE.getValue()); log.error(taskErrorLog,e);
super.cancelTaskRunFlag();
log.error(taskErrorLog);
e.printStackTrace();
}finally { }finally {
//添加任务耗时 //添加任务耗时
stopWatch.stop(); stopWatch.stop();
@ -152,8 +150,7 @@ public class BackwardTaskExec extends AbstractTaskExec {
paramContent.append(super.transportTask.getZ1()).append("\n"); paramContent.append(super.transportTask.getZ1()).append("\n");
paramContent.append(super.transportTask.getZ2()).append("\n"); paramContent.append(super.transportTask.getZ2()).append("\n");
paramContent.append(metDataPath).append("\n"); paramContent.append(metDataPath).append("\n");
paramContent.append(super.simulationProperties.getOutputPath()+ "/" +super.transportTask.getTaskName()).append("\n"); paramContent.append(super.simulationProperties.getOutputPath()+ File.separator+super.transportTask.getTaskName()).append("\n");
// paramContent.append(super.simulationProperties.getOutputPath()+ File.separator+super.transportTask.getTaskName()).append("\n");
paramContent.append(FlexpartSpeciesType.NOT_SPECIES.getId()).append("\n");//反演固定61不显示具体核素只显示Xe paramContent.append(FlexpartSpeciesType.NOT_SPECIES.getId()).append("\n");//反演固定61不显示具体核素只显示Xe
jschRemoteRunner.writeFile(paramConfigPath,paramContent.toString()); jschRemoteRunner.writeFile(paramConfigPath,paramContent.toString());
//处理台站数据文件 //处理台站数据文件
@ -182,7 +179,9 @@ public class BackwardTaskExec extends AbstractTaskExec {
String execScriptMsg = "执行任务脚本,开始模拟,路径为:"+scriptPath; String execScriptMsg = "执行任务脚本,开始模拟,路径为:"+scriptPath;
ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),execScriptMsg)); ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),execScriptMsg));
//执行命令 //执行命令
super.execCommand(scriptPath,jschRemoteRunner.getSession()); super.execFlexpartCommand(scriptPath,jschRemoteRunner.getSession());
//生成gif动图
super.genNCToGif(jschRemoteRunner.getSession());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
}finally { }finally {

View File

@ -117,11 +117,10 @@ public class ForwardTaskExec extends AbstractTaskExec {
// stationDataService,simulationProperties,stationsModValService,transportTaskService); // stationDataService,simulationProperties,stationsModValService,transportTaskService);
// conformityAnalysis.exec(); // conformityAnalysis.exec();
}catch (Exception e){ }catch (Exception e){
String taskErrorLog = "任务执行失败,原因:"+e.getMessage(); super.taskRunError = true;
ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskErrorLog)); String taskErrorLog = "任务执行失败,原因:";
this.transportTaskService.updateTaskStatus(super.transportTask.getId(),TransportTaskStatusEnum.FAILURE.getValue()); ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),taskErrorLog+e.getMessage()));
super.cancelTaskRunFlag(); log.error(taskErrorLog,e);
log.error(taskErrorLog);
}finally { }finally {
//添加任务耗时 //添加任务耗时
stopWatch.stop(); stopWatch.stop();
@ -222,7 +221,7 @@ public class ForwardTaskExec extends AbstractTaskExec {
releaseInfoMap.put(station.getStationCode(),station.getForwardReleaseChild()); releaseInfoMap.put(station.getStationCode(),station.getForwardReleaseChild());
} }
releaseInfoMap .forEach((stationCode,releaseInfoList)->{ releaseInfoMap .forEach((stationCode,releaseInfoList)->{
String stationReleaseFile = super.simulationProperties.getInputSiteHourPath() + "/" + stationCode + ".txt"; String stationReleaseFile = super.simulationProperties.getInputSiteHourPath() + File.separator + stationCode + ".txt";
List<String> stationReleaseInfo = new ArrayList<>(); List<String> stationReleaseInfo = new ArrayList<>();
releaseInfoList.forEach(releaseInfo ->{ releaseInfoList.forEach(releaseInfo ->{
String format = "%s,%s,%s\n"; String format = "%s,%s,%s\n";
@ -243,7 +242,9 @@ public class ForwardTaskExec extends AbstractTaskExec {
String execScriptMsg = "执行任务脚本,开始模拟,路径为:"+scriptPath; String execScriptMsg = "执行任务脚本,开始模拟,路径为:"+scriptPath;
ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),execScriptMsg)); ProgressQueue.getInstance().offer(new ProgressEvent(super.transportTask.getId(),execScriptMsg));
//执行命令 //执行命令
super.execCommand(scriptPath,jschRemoteRunner.getSession()); super.execFlexpartCommand(scriptPath,jschRemoteRunner.getSession());
//生成gif动图
super.genNCToGif(jschRemoteRunner.getSession());
} catch (Exception e) { } catch (Exception e) {
throw new RuntimeException(e); throw new RuntimeException(e);
}finally { }finally {

View File

@ -59,7 +59,7 @@ public class SourceRebuildTaskServiceImpl extends ServiceImpl<SourceRebuildTaskM
queryWrapper.between(SourceRebuildTask::getCreateTime,startDateTime,endDateTime); queryWrapper.between(SourceRebuildTask::getCreateTime,startDateTime,endDateTime);
} }
queryWrapper.like(StringUtils.isNotBlank(taskName),SourceRebuildTask::getTaskName,taskName); queryWrapper.like(StringUtils.isNotBlank(taskName),SourceRebuildTask::getTaskName,taskName);
queryWrapper.orderByDesc(SourceRebuildTask::getCreateTime); queryWrapper.last("ORDER BY CASE task_status WHEN -1 THEN 0 WHEN 4 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 3 WHEN 3 THEN 4 ELSE 5 END ASC,update_time desc");
IPage<SourceRebuildTask> iPage = new Page<>(pageRequest.getPageNum(), pageRequest.getPageSize()); IPage<SourceRebuildTask> iPage = new Page<>(pageRequest.getPageNum(), pageRequest.getPageSize());
return this.page(iPage, queryWrapper); return this.page(iPage, queryWrapper);

View File

@ -29,6 +29,7 @@ public abstract class AbstractSyncDataJob implements Runnable{
protected Connection sourceConn; protected Connection sourceConn;
protected Connection targetConn; protected Connection targetConn;
protected void init(StasTaskConfigMapper taskConfigMapper, protected void init(StasTaskConfigMapper taskConfigMapper,
StasDataSourceMapper dataSourceMapper, StasDataSourceMapper dataSourceMapper,
StasSyncStrategyMapper syncStrategyMapper, StasSyncStrategyMapper syncStrategyMapper,
@ -125,7 +126,6 @@ public abstract class AbstractSyncDataJob implements Runnable{
while (rs.next()) { while (rs.next()) {
for (int i = 1; i <= columnCount; i++) { for (int i = 1; i <= columnCount; i++) {
Object value = rs.getObject(i); Object value = rs.getObject(i);
// 特殊处理Oracle的TIMESTAMP类型到PostgreSQL // 特殊处理Oracle的TIMESTAMP类型到PostgreSQL
if (SourceDataTypeEnum.ORACLE.getKey().equals(this.sourceInfo.getType()) && SourceDataTypeEnum.POSTGRES.getKey().equals(this.targetInfo.getType())) { if (SourceDataTypeEnum.ORACLE.getKey().equals(this.sourceInfo.getType()) && SourceDataTypeEnum.POSTGRES.getKey().equals(this.targetInfo.getType())) {
String columnType = metaData.getColumnTypeName(i); String columnType = metaData.getColumnTypeName(i);
@ -153,6 +153,11 @@ public abstract class AbstractSyncDataJob implements Runnable{
pstmt.executeBatch(); pstmt.executeBatch();
this.targetConn.commit(); this.targetConn.commit();
} }
}catch (SQLException e){
// 回滚数据
this.targetConn.rollback();
log.error("保存数据出现错误,原因为",e);
throw e;
} }
} }
return totalRows; return totalRows;
@ -209,15 +214,10 @@ public abstract class AbstractSyncDataJob implements Runnable{
* *
* @param e the exception * @param e the exception
*/ */
public void uncaughtException(Throwable e) { public void uncaughtException(Exception e) {
String tableName = this.syncStrategy.getSourceOwner()+"."+this.syncStrategy.getTableName(); String tableName = this.syncStrategy.getSourceOwner()+"."+this.syncStrategy.getTableName();
log.error(tableName+"表数据同步出现错误,原因为",e);
String errLog = String.format("%s表数据同步出现错误原因为%s",tableName, e.getMessage()); String errLog = String.format("%s表数据同步出现错误原因为%s",tableName, e.getMessage());
log.error(errLog);
saveSyncLog(errLog); saveSyncLog(errLog);
try {
targetConn.rollback();
} catch (SQLException ex) {
log.error("数据回滚出现错误,原因为:{}",ex.getMessage());
}
} }
} }

View File

@ -80,9 +80,15 @@ public class SyncDataByDateColumnJob extends AbstractSyncDataJob {
//保存表批次同步数量 //保存表批次同步数量
super.saveSyncNum(rowsSynced); super.saveSyncNum(rowsSynced);
//保存同步日志并修改同步位置 //保存同步日志并修改同步位置
super.syncStrategy.setSyncOrigin(sdf.format(currentEnd)); Date finalSyncOrigin = null;
if (rowsSynced > 0) {
finalSyncOrigin = getLocalRealSyncOrigin();
}else {
finalSyncOrigin = currentEnd;
}
super.syncStrategy.setSyncOrigin(sdf.format(finalSyncOrigin));
super.syncStrategyMapper.updateById(super.syncStrategy); super.syncStrategyMapper.updateById(super.syncStrategy);
super.saveSyncLog(String.format("%s表最终同步位置已更新为: %s",tableName, sdf.format(currentEnd))); super.saveSyncLog(String.format("%s表最终同步位置已更新为: %s",tableName, sdf.format(finalSyncOrigin)));
} }
/** /**
@ -100,4 +106,20 @@ public class SyncDataByDateColumnJob extends AbstractSyncDataJob {
} }
throw new SQLException("无法获取日期范围"); throw new SQLException("无法获取日期范围");
} }
/**
* 获取本地oracle数据表的本次同步数据对应字段的实际最新值
*/
private Date getLocalRealSyncOrigin() throws SQLException {
String sql = "MAX(" + this.syncStrategy.getColumnName() + ") as max_val " +
"FROM \"" + this.syncStrategy.getTargetOwner().toUpperCase() + "\".\"" + this.syncStrategy.getTableName() + "\"";
try (Statement stmt = this.targetConn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
return rs.getTimestamp("max_val");
}
}
String errLog = this.syncStrategy.getTargetOwner().toUpperCase() + "\".\"" + this.syncStrategy.getTableName() + "\"";
throw new SQLException("无法获取"+errLog+"最新值");
}
} }

View File

@ -4,6 +4,7 @@ import cn.hutool.core.date.StopWatch;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.jeecg.vo.IdRangeVO; import org.jeecg.vo.IdRangeVO;
import java.sql.*; import java.sql.*;
import java.util.Objects; import java.util.Objects;
@ -83,9 +84,15 @@ public class SyncDataByIncrementColumnJob extends AbstractSyncDataJob {
// 更新同步位置 // 更新同步位置
if (currentEnd > 0) { if (currentEnd > 0) {
super.syncStrategy.setSyncOrigin(String.valueOf(currentEnd)); long finalSyncOrigin;
if (rowsSynced > 0) {
finalSyncOrigin = getLocalRealSyncOrigin();
}else {
finalSyncOrigin = currentEnd;
}
super.syncStrategy.setSyncOrigin(String.valueOf(finalSyncOrigin));
super.syncStrategyMapper.updateById(syncStrategy); super.syncStrategyMapper.updateById(syncStrategy);
super.saveSyncLog(String.format("%s表最终同步位置已更新为: %s",tableName, currentEnd)); super.saveSyncLog(String.format("%s表最终同步位置已更新为: %s",tableName, finalSyncOrigin));
} }
} }
@ -104,4 +111,20 @@ public class SyncDataByIncrementColumnJob extends AbstractSyncDataJob {
} }
throw new SQLException("无法获取ID范围"); throw new SQLException("无法获取ID范围");
} }
/**
* 获取本地oracle数据表的本次同步数据对应字段的实际最新值
*/
private long getLocalRealSyncOrigin() throws SQLException {
String sql = "SELECT MAX(" + this.syncStrategy.getColumnName() + ") as max_val " +
"FROM \"" + this.syncStrategy.getTargetOwner().toUpperCase() + "\".\"" + this.syncStrategy.getTableName() + "\"";
try (Statement stmt = this.targetConn.createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
return rs.getLong("max_val");
}
}
String errLog = this.syncStrategy.getTargetOwner().toUpperCase() + "\".\"" + this.syncStrategy.getTableName() + "\"";
throw new SQLException("无法获取"+errLog+"最新值");
}
} }

View File

@ -120,8 +120,8 @@ public class SyncDataJob implements Job {
* @throws InterruptedException * @throws InterruptedException
*/ */
private void closeThreadPool(String recordId,String taskId) throws InterruptedException { private void closeThreadPool(String recordId,String taskId) throws InterruptedException {
//再次多等待2分钟如果还未全部关闭则强制停止 //再次多等待10分钟如果还未全部关闭则强制停止
if (!threadPoolExecutor.awaitTermination(120, TimeUnit.SECONDS)) { if (!threadPoolExecutor.awaitTermination(600, TimeUnit.SECONDS)) {
List<Runnable> droppedTasks = threadPoolExecutor.shutdownNow(); List<Runnable> droppedTasks = threadPoolExecutor.shutdownNow();
log.warn("超时!执行强制关闭,丢弃了 " + droppedTasks.size() + " 个未执行任务"); log.warn("超时!执行强制关闭,丢弃了 " + droppedTasks.size() + " 个未执行任务");
if (!threadPoolExecutor.awaitTermination(60, TimeUnit.SECONDS)) { if (!threadPoolExecutor.awaitTermination(60, TimeUnit.SECONDS)) {

View File

@ -216,6 +216,7 @@ public class TransportTaskServiceImpl extends ServiceImpl<TransportTaskMapper,Tr
taskBackwardChildMapper.delete(delQueryWrapper); taskBackwardChildMapper.delete(delQueryWrapper);
if(CollUtil.isNotEmpty(transportTask.getBackwardChild())){ if(CollUtil.isNotEmpty(transportTask.getBackwardChild())){
transportTask.getBackwardChild().forEach(transportTaskChild -> { transportTask.getBackwardChild().forEach(transportTaskChild -> {
transportTaskChild.setId(null);
transportTaskChild.setTaskId(transportTask.getId()); transportTaskChild.setTaskId(transportTask.getId());
}); });
taskBackwardChildMapper.insert(transportTask.getBackwardChild()); taskBackwardChildMapper.insert(transportTask.getBackwardChild());
@ -229,6 +230,7 @@ public class TransportTaskServiceImpl extends ServiceImpl<TransportTaskMapper,Tr
delForwardChildQueryWrapper.eq(TransportTaskForwardChild::getTaskId,checkIdResult.getId()); delForwardChildQueryWrapper.eq(TransportTaskForwardChild::getTaskId,checkIdResult.getId());
taskForwardChildMapper.delete(delForwardChildQueryWrapper); taskForwardChildMapper.delete(delForwardChildQueryWrapper);
transportTask.getForwardChild().forEach(transportTaskChild -> { transportTask.getForwardChild().forEach(transportTaskChild -> {
transportTaskChild.setId(null);
transportTaskChild.setTaskId(transportTask.getId()); transportTaskChild.setTaskId(transportTask.getId());
}); });
taskForwardChildMapper.insert(transportTask.getForwardChild()); taskForwardChildMapper.insert(transportTask.getForwardChild());

View File

@ -34,10 +34,8 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.text.DecimalFormat; import java.text.DecimalFormat;
import java.time.Instant;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -313,6 +311,7 @@ public class WeatherDataServiceImpl extends ServiceImpl<WeatherDataMapper, Weath
queryWrapper.like(StringUtils.isNotBlank(fileName),WeatherData::getFileName, fileName); queryWrapper.like(StringUtils.isNotBlank(fileName),WeatherData::getFileName, fileName);
queryWrapper.select(WeatherData::getId,WeatherData::getFileName,WeatherData::getDataSource, queryWrapper.select(WeatherData::getId,WeatherData::getFileName,WeatherData::getDataSource,
WeatherData::getFileExt,WeatherData::getDataStartTime,WeatherData::getFilePath,WeatherData::getTimeBatch); WeatherData::getFileExt,WeatherData::getDataStartTime,WeatherData::getFilePath,WeatherData::getTimeBatch);
queryWrapper.orderByDesc(WeatherData::getDataStartTime);
IPage<WeatherData> iPage = new Page<>(pageRequest.getPageNum(),pageRequest.getPageSize()); IPage<WeatherData> iPage = new Page<>(pageRequest.getPageNum(),pageRequest.getPageSize());
return this.baseMapper.selectPage(iPage, queryWrapper); return this.baseMapper.selectPage(iPage, queryWrapper);
} }

View File

@ -17,16 +17,13 @@ import org.jeecg.common.properties.SystemStorageProperties;
import org.jeecg.common.system.query.PageRequest; import org.jeecg.common.system.query.PageRequest;
import org.jeecg.modules.base.entity.WeatherTask; import org.jeecg.modules.base.entity.WeatherTask;
import org.jeecg.modules.base.entity.WeatherTaskLog; import org.jeecg.modules.base.entity.WeatherTaskLog;
import org.jeecg.modules.base.mapper.WeatherDataMapper;
import org.jeecg.modules.base.mapper.WeatherTaskLogMapper; import org.jeecg.modules.base.mapper.WeatherTaskLogMapper;
import org.jeecg.modules.base.mapper.WeatherTaskMapper; import org.jeecg.modules.base.mapper.WeatherTaskMapper;
import org.jeecg.service.WeatherTaskService; import org.jeecg.service.WeatherTaskService;
import org.reflections.vfs.Vfs;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import java.io.File; import java.io.File;
import java.io.FileFilter;
import java.io.IOException; import java.io.IOException;
import java.time.LocalDate; import java.time.LocalDate;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -43,7 +40,6 @@ public class WeatherTaskServiceImpl extends ServiceImpl<WeatherTaskMapper, Weath
private final WeatherTaskLogMapper weatherTaskLogMapper; private final WeatherTaskLogMapper weatherTaskLogMapper;
private final SystemStorageProperties systemStorageProperties; private final SystemStorageProperties systemStorageProperties;
private final WeatherDataMapper weatherDataMapper;
/** /**
* 分页查询任务列表 * 分页查询任务列表
@ -64,7 +60,7 @@ public class WeatherTaskServiceImpl extends ServiceImpl<WeatherTaskMapper, Weath
queryWrapper.between(WeatherTask::getStartDate,startDateTime,endDateTime); queryWrapper.between(WeatherTask::getStartDate,startDateTime,endDateTime);
} }
queryWrapper.like(StringUtils.isNotBlank(taskName),WeatherTask::getTaskName,taskName); queryWrapper.like(StringUtils.isNotBlank(taskName),WeatherTask::getTaskName,taskName);
queryWrapper.orderByDesc(WeatherTask::getStartDate); queryWrapper.last("ORDER BY CASE task_status WHEN -1 THEN 0 WHEN 4 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 3 WHEN 3 THEN 4 ELSE 5 END ASC,update_time desc");
IPage<WeatherTask> iPage = new Page<>(pageRequest.getPageNum(), pageRequest.getPageSize()); IPage<WeatherTask> iPage = new Page<>(pageRequest.getPageNum(), pageRequest.getPageSize());
return this.page(iPage, queryWrapper); return this.page(iPage, queryWrapper);