diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java index 08670ca5..d64cdbd0 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java @@ -51,6 +51,11 @@ public class TaskProperties implements Serializable { */ private Integer undealDirReceiveNum; + /** + * undeal目录文件分析周期 + */ + private Integer undealFileTimeOut; + /** * filesource目录文件获取周期 */ diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/RedisStreamUtil.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/RedisStreamUtil.java index aba5f03f..3da43cd0 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/RedisStreamUtil.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/util/RedisStreamUtil.java @@ -161,6 +161,15 @@ public class RedisStreamUtil { return redisTemplate.opsForStream().delete(streamKey, recordIds); } + /** + * 根据记录id删除n个消息记录 + * + * @param streamKey + */ + public boolean del(String streamKey){ + return redisTemplate.delete(streamKey); + } + /** * 添加Map消息 * diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/common/util/LogFileUtil.java b/jeecg-module-auto-process/src/main/java/org/jeecg/common/util/LogFileUtil.java new file mode 100644 index 00000000..3bc540cb --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/common/util/LogFileUtil.java @@ -0,0 +1,70 @@ +package org.jeecg.common.util; + +import com.baomidou.mybatisplus.core.toolkit.StringPool; +import org.jeecg.common.properties.SpectrumPathProperties; +import org.jeecg.common.properties.TaskProperties; +import org.jeecg.modules.exception.StationNotFoundException; +import org.jeecg.modules.spectrum.SpectrumServiceQuotes; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +@Component +public class LogFileUtil { + + /** + * 任务属性 + */ + @Autowired + private TaskProperties taskProperties; + /** + * 相关Spring组件引用 + */ + @Autowired + private SpectrumPathProperties spectrumPathProperties; + + + public void errorLog(String spectrumFileName, long millis, Exception e) { + String logFilePath = spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath() + File.separator + "Error"; + File logPath = new File(logFilePath); + if (!logPath.exists()) { + logPath.mkdir(); + } + String logFileName = logFilePath + File.separator + spectrumFileName.replace("PHD", "log"); + File logFile = new File(logFileName); + StringBuffer out = new StringBuffer(); + String nowDate = DateUtils.formatDate(new Date(), "yyyy-MM-dd HH:mm:ss"); + out.append(nowDate+ StringPool.SPACE + "Data Anlyse Error:"); + String warning = ""; + if (e.getClass().equals(StationNotFoundException.class)) { + warning = e.getMessage()+StringPool.SPACE+"timeout:"+(long) Math.floor(millis/1000)+",waittime:"+taskProperties.getUndealFileTimeOut(); + } else { + warning = e.getMessage(); + } + out.append(warning); + out.append(System.lineSeparator()); + out.append(System.lineSeparator()); + out.append(System.lineSeparator()); + FileWriter writer = null; + try { + writer = new FileWriter(logFile, true); + writer.write(out.toString()); + } catch (IOException ex) { + throw new RuntimeException(ex); + } finally { + try { + if (Objects.nonNull(writer)) { + writer.close(); + } + } catch (IOException ex) { + throw new RuntimeException(ex); + } + } + } + +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/FileSourceHandleManager.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/FileSourceHandleManager.java index 833e0269..c1f32776 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/FileSourceHandleManager.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/FileSourceHandleManager.java @@ -1,9 +1,12 @@ package org.jeecg.modules; +import com.baomidou.mybatisplus.core.toolkit.StringPool; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.jeecg.common.properties.TaskProperties; +import org.jeecg.common.util.DateUtils; +import org.jeecg.modules.exception.StationNotFoundException; import org.jeecg.modules.service.BlockConstant; import org.jeecg.modules.file.FileOperation; import org.jeecg.modules.spectrum.AbstractSpectrumHandler; @@ -13,7 +16,13 @@ import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.concurrent.*; /** @@ -137,6 +146,9 @@ public class FileSourceHandleManager{ spectrumHandler.handler(); } }catch (Exception e){ + //生成日志 + spectrumServiceQuotes.getLogFileUtil().errorLog(spectrumFile.getName(), 0, e); + log.error("Parsing the {} file of the filesource directory failed.The reason is {}",spectrumFile.getName(),e.getMessage()); e.printStackTrace(); }finally { diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/UndealHandleManager.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/UndealHandleManager.java index 7110116f..55866552 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/UndealHandleManager.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/UndealHandleManager.java @@ -1,19 +1,32 @@ package org.jeecg.modules; +import com.baomidou.mybatisplus.core.toolkit.StringPool; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.jeecg.common.properties.TaskProperties; +import org.jeecg.common.util.DateUtils; +import org.jeecg.common.util.LogFileUtil; +import org.jeecg.modules.exception.StationNotFoundException; import org.jeecg.modules.service.BlockConstant; import org.jeecg.modules.file.FileOperation; import org.jeecg.modules.spectrum.AbstractSpectrumHandler; import org.jeecg.modules.spectrum.SamplephdSpectrum; import org.jeecg.modules.spectrum.SpectrumServiceQuotes; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.scheduling.concurrent.CustomizableThreadFactory; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.util.Date; import java.util.List; +import java.util.Objects; import java.util.concurrent.*; /** @@ -122,6 +135,14 @@ public class UndealHandleManager{ @Override public void run() { + long currentMillis = System.currentTimeMillis(); + long createMillis = currentMillis; + //判断redis是否包含文件名称 key + if (spectrumServiceQuotes.getRedisStreamUtil().hasKey(spectrumFile.getName())) { + createMillis = (long) spectrumServiceQuotes.getRedisStreamUtil().get(spectrumFile.getName()); + } else { + spectrumServiceQuotes.getRedisStreamUtil().set(spectrumFile.getName(), currentMillis); + } try { //获取文件内容 final String fileContent = FileUtils.readFileToString(spectrumFile,"UTF-8"); @@ -134,12 +155,19 @@ public class UndealHandleManager{ spectrumHandler.handler(); } }catch (Exception e){ + //生成日志 + long millis = currentMillis - createMillis; + spectrumServiceQuotes.getLogFileUtil().errorLog(spectrumFile.getName(), millis, e); log.error("The {} file of the undeal directory fails to be parsed again.The reason is {}",spectrumFile.getName(),e.getMessage()); e.printStackTrace(); }finally { this.taskLatch.countDown(); - //解析成功或者失败都会删除源文件 - spectrumFile.delete(); + //满足undeal文件处理周期时长会删除源文件 + if ((currentMillis - createMillis) >= taskProperties.getUndealFileTimeOut()) { + spectrumServiceQuotes.getRedisStreamUtil().del(spectrumFile.getName()); + //解析成功或者失败都会删除源文件 + spectrumFile.delete(); + } } } } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/StationNotFoundException.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/StationNotFoundException.java new file mode 100644 index 00000000..ffa9707b --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/StationNotFoundException.java @@ -0,0 +1,9 @@ +package org.jeecg.modules.exception; + +public class StationNotFoundException extends Exception { + + public StationNotFoundException(String message) { + super(message); + } + +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/GardsSampleDataMapper.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/GardsSampleDataMapper.java index d63a1c0a..67fc201d 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/GardsSampleDataMapper.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/GardsSampleDataMapper.java @@ -13,8 +13,8 @@ public interface GardsSampleDataMapper extends BaseMapper { @Select(value = "select " + "gsd.SAMPLE_ID as sampleId,gsd.input_file_name as inputFileName " + "from ORIGINAL.GARDS_SAMPLE_AUX gsa inner join ORIGINAL.GARDS_SAMPLE_DATA gsd on gsa.sample_id = gsd.sample_id " + - "where gsa.measurement_id = #{measurementId} and gsd.data_type=#{dataType}") - public List getSampleIdAndInputFileName(@Param("measurementId") String measurementId, @Param("dataType") String dataType); + "where gsa.measurement_id = #{measurementId} and gsd.SAMPLE_TYPE = #{systemType} and gsd.data_type=#{dataType} and TRIM(gsd.SITE_DET_CODE) = #{detectorId}") + public List getSampleIdAndInputFileName(@Param("measurementId") String measurementId, @Param("dataType") String dataType, @Param("systemType") String systemType, String detectorId); @Update(value = "UPDATE ORIGINAL.GARDS_SAMPLE_DATA SET STATUS=#{status} WHERE INPUT_FILE_NAME=#{inputFileName}") public int updateStatus(@Param("status") String status,@Param("inputFileName") String inputFileName); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsSampleDataService.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsSampleDataService.java index 8fc6d37d..65083a27 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsSampleDataService.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsSampleDataService.java @@ -25,7 +25,7 @@ public interface GardsSampleDataService extends IService { * @param dataType * @return */ - public GardsSampleData getSampleIdAndInputFileName(String measurementId,String dataType); + public GardsSampleData getSampleIdAndInputFileName(String measurementId,String dataType, String systemType); /** * 修改能谱处理状态 diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsStationsService.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsStationsService.java index 267e3ac4..f462b7e0 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsStationsService.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/GardsStationsService.java @@ -2,6 +2,7 @@ package org.jeecg.modules.service; import com.baomidou.mybatisplus.extension.service.IService; import org.jeecg.modules.base.entity.configuration.GardsStations; +import org.jeecg.modules.exception.StationNotFoundException; /** * 台站服务 @@ -12,5 +13,5 @@ public interface GardsStationsService extends IService { * 校验台站编码是否存在 * @return */ - GardsStations check(String site_code); + GardsStations check(String site_code) throws StationNotFoundException; } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsSampleDataServiceImpl.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsSampleDataServiceImpl.java index c47689a5..fbe35204 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsSampleDataServiceImpl.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsSampleDataServiceImpl.java @@ -61,8 +61,9 @@ public class GardsSampleDataServiceImpl extends ServiceImpl sampleDatas = this.baseMapper.getSampleIdAndInputFileName(measurementId, dataType); + public GardsSampleData getSampleIdAndInputFileName(String measurementId, String dataType, String systemType) { + String detectorId = measurementId.substring(0, 8); + final List sampleDatas = this.baseMapper.getSampleIdAndInputFileName(measurementId, dataType, systemType, detectorId); if(!CollectionUtils.isEmpty(sampleDatas)){ //如果查询出多条则需要根据inputFileName字段降序排序后返回第一个 final List sortResult = sampleDatas.stream().sorted(Comparator.comparing(GardsSampleData::getInputFileName).reversed()).collect(Collectors.toList()); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsStationsServiceImpl.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsStationsServiceImpl.java index 7a35beed..9d786aac 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsStationsServiceImpl.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsStationsServiceImpl.java @@ -5,11 +5,14 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import lombok.RequiredArgsConstructor; import org.jeecg.modules.base.entity.configuration.GardsStations; +import org.jeecg.modules.exception.StationNotFoundException; import org.jeecg.modules.mapper.GardsStationsMapper; import org.jeecg.modules.service.GardsStationsService; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import java.util.Objects; + @DS("ora") @Service @RequiredArgsConstructor @@ -22,12 +25,14 @@ public class GardsStationsServiceImpl extends ServiceImpl gardsStationsQuery = new LambdaQueryWrapper<>(); gardsStationsQuery.select(GardsStations::getStationId); gardsStationsQuery.eq(GardsStations::getStationCode,site_code); final GardsStations station = this.baseMapper.selectOne(gardsStationsQuery); - Assert.notNull(station,"The station to which this "+site_code+" station code belongs does not exist"); + if (Objects.isNull(station)) { + throw new StationNotFoundException("station_code:"+site_code+"=0"); + } return station; } } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_B_Analysis.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_B_Analysis.java index daca2820..5ac1056b 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_B_Analysis.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_B_Analysis.java @@ -181,8 +181,8 @@ public class Sample_B_Analysis implements BlockConstant { */ private void queryPHDFile() throws FileNotExistException { //查询det和gas能谱文 - this.detSampleData = spectrumServiceQuotes.getSampleDataService().getSampleIdAndInputFileName(this.sampleStruct.detector_bk_measurement_id, DataTypeAbbr.DETBKPHD.getType()); - this.gasSampleData = spectrumServiceQuotes.getSampleDataService().getSampleIdAndInputFileName(this.sampleStruct.gas_bk_measurement_id, DataTypeAbbr.GASBKPHD.getType()); + this.detSampleData = spectrumServiceQuotes.getSampleDataService().getSampleIdAndInputFileName(this.sampleStruct.detector_bk_measurement_id, DataTypeAbbr.DETBKPHD.getType(), this.sampleStruct.system_type); + this.gasSampleData = spectrumServiceQuotes.getSampleDataService().getSampleIdAndInputFileName(this.sampleStruct.gas_bk_measurement_id, DataTypeAbbr.GASBKPHD.getType(), this.sampleStruct.system_type); //如果找不到sample、det、gas谱文件数据则解析失败修改记录状态 if(StringUtils.isEmpty(this.sampleData.getInputFileName()) || Objects.isNull(this.detSampleData) || StringUtils.isEmpty(this.detSampleData.getInputFileName()) || Objects.isNull(this.gasSampleData) || StringUtils.isEmpty(this.gasSampleData.getInputFileName())){ diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java index 25048691..b43e6df9 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java @@ -3,6 +3,7 @@ package org.jeecg.modules.spectrum; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jeecg.common.properties.*; +import org.jeecg.common.util.LogFileUtil; import org.jeecg.common.util.NameStandUtil; import org.jeecg.common.util.RedisStreamUtil; import org.jeecg.modules.datasource.OraDataSourceProperties; @@ -76,6 +77,8 @@ public class SpectrumServiceQuotes { private final NameStandUtil nameStandUtil; + private final LogFileUtil logFileUtil; + /** * 原始库插入数据锁 */