From c4ba5341c336d9c6eacf65521439ba8c96678170 Mon Sep 17 00:00:00 2001 From: panbaolin <123456> Date: Fri, 10 Nov 2023 14:18:25 +0800 Subject: [PATCH] =?UTF-8?q?fix:1.=E5=AE=8C=E6=88=90=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=8A=A5=E5=91=8A=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/email/EmailServiceManager.java | 14 +- .../properties/SpectrumPathProperties.java | 5 + .../common/properties/TaskProperties.java | 5 + .../org/jeecg/modules/StatReportManager.java | 298 ++++++++++++++++++ .../eneity/event/FormatErrorEvent.java | 14 + .../eneity/event/RepeatErrorEvent.java | 14 + .../jeecg/modules/eneity/vo/DBInfoCount.java | 28 ++ .../modules/mapper/StatReportMapper.java | 22 ++ .../modules/mapper/xml/StatReportMapper.xml | 75 +++++ .../modules/service/IStatReportService.java | 20 ++ .../modules/service/ISysMailLogService.java | 11 +- .../impl/GardsDetectorsServiceImpl.java | 1 - .../impl/SpectrumBaseBlockServiceImpl.java | 1 - .../service/impl/StatReportServiceImpl.java | 39 +++ .../service/impl/SysMailLogServiceImpl.java | 17 + .../AbstractS_D_Q_G_SpectrumHandler.java | 10 + .../jeecg/modules/spectrum/AlertSpectrum.java | 10 +- .../spectrum/HealthStatusSpectrum.java | 7 +- .../jeecg/modules/spectrum/MetSpectrum.java | 7 +- .../spectrum/SpectrumParsingActuator.java | 11 +- .../spectrum/SpectrumServiceQuotes.java | 3 + .../jeecg/JeecgAutoProcessApplication.java | 11 +- 22 files changed, 600 insertions(+), 23 deletions(-) create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/StatReportManager.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/FormatErrorEvent.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/RepeatErrorEvent.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/vo/DBInfoCount.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/StatReportMapper.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/xml/StatReportMapper.xml create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/IStatReportService.java create mode 100644 jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/StatReportServiceImpl.java diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java index 723f3004..ab4c8aef 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java @@ -346,12 +346,14 @@ public class EmailServiceManager { * @throws MessagingException */ public void removeMail(@NotNull Message message) throws MessagingException { - try { - message.setFlag(Flags.Flag.DELETED,true); - LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Successful", "DELETEID", message.getSubject(), ""); - } catch (MessagingException e) { - LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Error", "DELETEID", message.getSubject(), ""); - throw e; + synchronized (this){ + try { + message.setFlag(Flags.Flag.DELETED,true); + LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Successful", "DELETEID", message.getSubject(), ""); + } catch (MessagingException e) { + LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Error", "DELETEID", message.getSubject(), ""); + throw e; + } } } diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SpectrumPathProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SpectrumPathProperties.java index 03906ee2..4e44e1e2 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SpectrumPathProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SpectrumPathProperties.java @@ -29,6 +29,11 @@ public class SpectrumPathProperties implements Serializable { */ private String saveFilePath; + /** + * 统计报告存储路径 + */ + private String statisticsPath; + /** * eml格式邮件存储路径 */ diff --git a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java index d64cdbd0..a7594460 100644 --- a/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java +++ b/jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java @@ -71,4 +71,9 @@ public class TaskProperties implements Serializable { */ private String temporaryStoragePath; + /** + * 统计报告执行周期 + */ + private Integer statisticsReportExecCycle; + } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/StatReportManager.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/StatReportManager.java new file mode 100644 index 00000000..dcc9d69b --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/StatReportManager.java @@ -0,0 +1,298 @@ +package org.jeecg.modules; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jeecg.common.constant.StringConstant; +import org.jeecg.common.properties.SpectrumPathProperties; +import org.jeecg.common.properties.TaskProperties; +import org.jeecg.common.util.DateUtils; +import org.jeecg.modules.base.abstracts.AbstractLogOrReport; +import org.jeecg.modules.eneity.event.FormatErrorEvent; +import org.jeecg.modules.eneity.event.RepeatErrorEvent; +import org.jeecg.modules.eneity.vo.DBInfoCount; +import org.jeecg.modules.file.FileOperation; +import org.jeecg.modules.service.IStatReportService; +import org.jeecg.modules.service.ISysMailLogService; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import java.io.File; +import java.io.IOException; +import java.text.ParseException; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 统计报告处理 + * 每24小时生成一份邮件解析统计报告 + */ +@Component +@RequiredArgsConstructor +@Slf4j +public class StatReportManager implements ApplicationListener{ + + /** + * 开始时间 + */ + private Date beginTime = null; + /** + * 结束时间 + */ + private Date endTime = null; + /** + * 格式化错误数量 + */ + private Integer formatErrorNumber = 0; + /** + * 邮件重复数量 + */ + private Integer repeatErrorNumber = 0; + + private boolean isFirst = true; + + private final ISysMailLogService mailLogService; + private final IStatReportService statReportService; + private final SpectrumPathProperties spectrumPathProperties; + private final TaskProperties taskProperties; + + public void start(){ + ReportExecThread reportExecThread = new ReportExecThread(); + reportExecThread.setName("statisticsReportExecThread"); + reportExecThread.start(); + } + + /** + * 统计报告执行线程 + */ + private class ReportExecThread extends Thread{ + + @Override + public void run() { + + for(;;){ + long start = System.currentTimeMillis(); + if(isFirst){ + beginTime = new Date(); + isFirst = false; + }else{ + //生成报告和下面onApplicationEvent函数要加类锁，保障生成报告时不接收新的错误事件 + //否则可能造成错误事件数量大于实际的时间范围内的错误数量 + synchronized(StatReportManager.class){ + Date nowTime = new Date(); + endTime = nowTime; + GeneratingReports report = new GeneratingReports(); + report.genReport(); + //报告生成结束后重新赋予beginTime值，为上次结束时间毫秒加1 + beginTime = new Date(nowTime.getTime()+1); + //生成报告后重新初始化参数 + formatErrorNumber = 0; + repeatErrorNumber = 0; + } + } + long end = System.currentTimeMillis(); + long sleepTime = taskProperties.getStatisticsReportExecCycle() - (end-start); + //如果sleepTime > 0 需要睡眠到指定时间，否则继续下次 + if(sleepTime > 0){ + try { + TimeUnit.MILLISECONDS.sleep(sleepTime); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + } + + /** + * 接收错误事件 + * @param event + */ + @Override + public void onApplicationEvent(ApplicationEvent event) { + synchronized(StatReportManager.class){ + if(event instanceof FormatErrorEvent){ + formatErrorNumber++; + }else if (event instanceof RepeatErrorEvent){ + repeatErrorNumber++; + } + } + } + + /** + * 生成报告 + */ + private class GeneratingReports extends AbstractLogOrReport{ + /** + * 创建时间 + */ + private Date createTime = new Date(); + /** + * 邮件数量 + */ + private long emailNumber = 0; + /** + * 原始库数据数量 + */ + private Integer originalDBNumber = 0; + /** + * 邮件解析详情记录 + */ + private List dbInfoCounts = null; + /** + * 报告内容 + */ + StringBuilder report = new StringBuilder(); + + /** + * 生成报告 + */ + private void genReport(){ + try { + this.countMailNumber(); + this.countDBNumber(); + this.setHeader(); + this.setStatisticsInfo(); + this.setStatisticsErrorFileInfo(); + this.setDBInfo(); + this.storeReport(); + }catch (Exception e){ + e.printStackTrace(); + log.error("Failed to generate analysis report because: {}",e.getMessage()); + } + } + + /** + * 统计邮件数量 + */ + private void countMailNumber() throws ParseException { +// Date d1 = DateUtils.parseDate("2023-11-07 00:00:00","yyyy-MM-dd HH:mm:ss"); +// Date d2 = DateUtils.parseDate("2023-11-07 23:59:59","yyyy-MM-dd HH:mm:ss"); + final long emailNumber = mailLogService.countEmailNumByCreateTimeRange(beginTime,endTime); + this.emailNumber = emailNumber; + } + + /** + * 统计原始库和分析库入库数量 + */ + private void countDBNumber() throws ParseException { +// Date d1 = DateUtils.parseDate("2023-10-12 00:00:00","yyyy-MM-dd HH:mm:ss"); +// Date d2 = DateUtils.parseDate("2023-10-20 23:59:59","yyyy-MM-dd HH:mm:ss"); + final List dbInfoCounts = statReportService.countParsingMailRecords(beginTime,endTime); + if(!CollectionUtils.isEmpty(dbInfoCounts)){ + this.dbInfoCounts = dbInfoCounts; + final int records = dbInfoCounts.stream().mapToInt(DBInfoCount::getOriginalDataNumber).sum(); + this.originalDBNumber = records; + } + } + + /** + * Header块 + */ + private void setHeader(){ + final String title = "%-36s GENERATED STATISTICS REPORT"; + final String createDate = "%-33s Creation Date %s"; + report.append(rowFormat(title,StringConstant.SPACE)); + report.append(System.lineSeparator()); + report.append(rowFormat(createDate,StringConstant.SPACE, DateUtils.formatDate(createTime,"yyyy/MM/dd-HH:mm:ss"))); + report.append(System.lineSeparator()).append(System.lineSeparator()); + } + + /** + * #STATISTICS INFO 块 + */ + private void setStatisticsInfo(){ + final String blockTitle = "#STATISTICS INFO"; + final String beginTimeRow = " Statistics BeginTime: %-11s %s"; + final String endTimeRow = " Statistics EndTime: %-13s %s"; + final String emailNumberRow = " Email Number: %-18s %s"; + final String dbNumberRow = " Original Data Load to DB Number: %s"; + report.append(rowFormat(blockTitle)); + report.append(System.lineSeparator()); + report.append(rowFormat(beginTimeRow,StringConstant.SPACE,DateUtils.formatDate(beginTime,"yyyy/MM/dd-HH:mm:ss"))); + report.append(System.lineSeparator()); + report.append(rowFormat(endTimeRow,StringConstant.SPACE,DateUtils.formatDate(endTime,"yyyy/MM/dd-HH:mm:ss"))); + report.append(System.lineSeparator()); + report.append(rowFormat(emailNumberRow,StringConstant.SPACE,String.valueOf(emailNumber))); + report.append(System.lineSeparator()); + report.append(rowFormat(dbNumberRow,String.valueOf(originalDBNumber))); + report.append(System.lineSeparator()).append(System.lineSeparator()); + } + + /** + * #STATISTICS ERROR FILE 块 + */ + private void setStatisticsErrorFileInfo(){ + final String blockTitle = "#STATISTICS ERROR FILE"; + final String formatErrorRow = " File Format Error: %-15s"; + final String repeatErrorRow = " File Repeat Error: %-15s"; + report.append(rowFormat(blockTitle)); + report.append(System.lineSeparator()); + report.append(rowFormat(formatErrorRow,String.valueOf(formatErrorNumber))); + report.append(System.lineSeparator()); + report.append(rowFormat(repeatErrorRow,String.valueOf(repeatErrorNumber))); + report.append(System.lineSeparator()).append(System.lineSeparator()); + } + + /** + * #LOAD INTO DB INFO 块 + */ + private void setDBInfo(){ + final String blockTitle = "#LOAD INTO DB INFO"; + final String rowTitle = " Station Name %-15s DataType %-12s Original Data Number %s Anlyse Data Number"; + final String repeatErrorRow = " %-28s %-21s %-22s %s"; + report.append(rowFormat(blockTitle)); + report.append(System.lineSeparator()); + report.append(rowFormat(rowTitle,StringConstant.SPACE,StringConstant.SPACE,StringConstant.SPACE)); + report.append(System.lineSeparator()); + if(!CollectionUtils.isEmpty(dbInfoCounts)){ + for(DBInfoCount dbInfoCount : dbInfoCounts){ + report.append(rowFormat(repeatErrorRow,dbInfoCount.getStationName(),this.dataTypeConvert(dbInfoCount.getDataType()),String.valueOf(dbInfoCount.getOriginalDataNumber()),String.valueOf(dbInfoCount.getAnlyseDataNumber()))); + report.append(System.lineSeparator()); + } + } + report.append(System.lineSeparator()); + } + + /** + * 保存报告到savefile/Statistics路径 + * @throws IOException + */ + private void storeReport() throws IOException { + final String statisticsPath = spectrumPathProperties.getStatisticsPath(); + StringBuilder finalPath = new StringBuilder(); + finalPath.append(spectrumPathProperties.getRootPath()); + finalPath.append(File.separator); + finalPath.append(spectrumPathProperties.getSaveFilePath()); + finalPath.append(File.separator); + finalPath.append(spectrumPathProperties.getStatisticsPath()); + finalPath.append(File.separator); + finalPath.append(DateUtils.formatDate(beginTime,"yyyyMMdd-HHmmss")); + finalPath.append(StringConstant.UNDER_LINE); + finalPath.append(DateUtils.formatDate(endTime,"yyyyMMdd-HHmmss")); + finalPath.append(".txt"); + FileOperation.saveOrAppendFile(finalPath.toString(),report.toString(),false); + } + + /** + * 格式化能谱类型 + * @param dataType + * @return + */ + private String dataTypeConvert(String dataType){ + switch (dataType){ + case "S": + return "SAMPLEPHD"; + case "Q": + return "QCPHD"; + case "G": + return "GASBKPHD"; + case "D": + return "DETBKPHD"; + default: + return dataType; + } + } + } +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/FormatErrorEvent.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/FormatErrorEvent.java new file mode 100644 index 00000000..94c66153 --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/FormatErrorEvent.java @@ -0,0 +1,14 @@ +package org.jeecg.modules.eneity.event; + +import org.springframework.context.ApplicationEvent; +import java.io.Serializable; + +/** + * 格式化错误事件 + */ +public class FormatErrorEvent extends ApplicationEvent implements Serializable { + + public FormatErrorEvent() { + super(new Object()); + } +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/RepeatErrorEvent.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/RepeatErrorEvent.java new file mode 100644 index 00000000..9705e2b5 --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/RepeatErrorEvent.java @@ -0,0 +1,14 @@ +package org.jeecg.modules.eneity.event; + +import org.springframework.context.ApplicationEvent; +import java.io.Serializable; + +/** + * 邮件重复错误 + */ +public class RepeatErrorEvent extends ApplicationEvent implements Serializable { + + public RepeatErrorEvent() { + super(new Object()); + } +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/vo/DBInfoCount.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/vo/DBInfoCount.java new file mode 100644 index 00000000..8ded6ab1 --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/vo/DBInfoCount.java @@ -0,0 +1,28 @@ +package org.jeecg.modules.eneity.vo; + +import lombok.Data; + +/** + * 统计报告使用 + * 接收邮件解析后存储到数据库的记录数量信息 + */ +@Data +public class DBInfoCount { + + /** + * 台站名称 + */ + private String stationName; + /** + * 能谱类型 + */ + private String dataType; + /** + * 单封邮件解析后存储到原始库的记录数量 + */ + private Integer originalDataNumber; + /** + * 单封邮件解析后存储到分析库的记录数量 + */ + private Integer anlyseDataNumber; +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/StatReportMapper.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/StatReportMapper.java new file mode 100644 index 00000000..4380ff9d --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/StatReportMapper.java @@ -0,0 +1,22 @@ +package org.jeecg.modules.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; +import org.jeecg.modules.eneity.vo.DBInfoCount; + +import java.util.Date; +import java.util.List; + +/** + * 统计报告Mapper + */ +public interface StatReportMapper extends BaseMapper { + + /** + * 统计邮件解析产生的记录数 + * @param beginDate + * @param endDate + * @return + */ + List countParsingMailRecords(@Param("beginDate") Date beginDate, @Param("endDate") Date endDate); +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/xml/StatReportMapper.xml b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/xml/StatReportMapper.xml new file mode 100644 index 00000000..41cdebc4 --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/xml/StatReportMapper.xml @@ -0,0 +1,75 @@ + + + + + + \ No newline at end of file diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/IStatReportService.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/IStatReportService.java new file mode 100644 index 00000000..cdad4d84 --- /dev/null +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/IStatReportService.java @@ -0,0 +1,20 @@ +package org.jeecg.modules.service; + +import org.jeecg.modules.eneity.vo.DBInfoCount; + +import java.util.Date; +import java.util.List; + +/** + * 统计报告Service + */ +public interface IStatReportService { + + /** + * 统计邮件解析产生的记录数 + * @param beginDate + * @param endDate + * @return + */ + List countParsingMailRecords(Date beginDate, Date endDate); +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/ISysMailLogService.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/ISysMailLogService.java index eafbf93a..feeea361 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/ISysMailLogService.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/ISysMailLogService.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.IService; import org.jeecg.modules.base.entity.postgre.SysEmailLog; import org.jeecg.modules.email.EmailProperties; import javax.mail.Message; +import java.util.Date; /** * 邮件处理日志 @@ -15,5 +16,13 @@ public interface ISysMailLogService extends IService { * @param message * @param email */ - public void create(Message message,EmailProperties email) throws Exception; + void create(Message message,EmailProperties email) throws Exception; + + /** + * 根据时间范围统计邮件数量 + * @param beginTime + * @param endTime + * @return + */ + long countEmailNumByCreateTimeRange(Date beginTime, Date endTime); } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsDetectorsServiceImpl.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsDetectorsServiceImpl.java index b88bb174..1b6b6d8a 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsDetectorsServiceImpl.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsDetectorsServiceImpl.java @@ -37,7 +37,6 @@ public class GardsDetectorsServiceImpl extends ServiceImpl countParsingMailRecords(Date beginDate, Date endDate) { + final List dbInfoCounts = statReportMapper.countParsingMailRecords(beginDate, endDate); + if(CollectionUtils.isEmpty(dbInfoCounts)){ + return Collections.EMPTY_LIST; + } + return dbInfoCounts; + } +} diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SysMailLogServiceImpl.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SysMailLogServiceImpl.java index 310e0fd2..088880f7 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SysMailLogServiceImpl.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SysMailLogServiceImpl.java @@ -1,5 +1,6 @@ package org.jeecg.modules.service.impl; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import org.jeecg.modules.base.entity.postgre.SysEmailLog; import org.jeecg.modules.email.EmailProperties; @@ -9,6 +10,8 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import javax.mail.Message; import javax.mail.internet.MimeUtility; +import java.util.Date; + /** * 邮件处理日志 */ @@ -29,4 +32,18 @@ public class SysMailLogServiceImpl extends ServiceImpl query = new LambdaQueryWrapper<>(); + query.between(SysEmailLog::getCreateTime,beginTime,endTime); + final long count = this.count(query); + return count; + } } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractS_D_Q_G_SpectrumHandler.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractS_D_Q_G_SpectrumHandler.java index 6c61ba3f..e406cec2 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractS_D_Q_G_SpectrumHandler.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractS_D_Q_G_SpectrumHandler.java @@ -11,6 +11,8 @@ import org.jeecg.common.util.LogFileUtil; import org.jeecg.modules.base.entity.original.GardsSampleData; import org.jeecg.modules.base.enums.SampleStatus; import org.jeecg.modules.config.datasource.DataSourceSwitcher; +import org.jeecg.modules.eneity.event.FormatErrorEvent; +import org.jeecg.modules.eneity.event.RepeatErrorEvent; import org.jeecg.modules.exception.AcquisitionBlockException; import org.jeecg.modules.exception.FileRepeatException; import org.jeecg.modules.exception.HeaderBlockException; @@ -72,6 +74,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa */ protected void checkHeaderBlock(){ if(this.mailContent.indexOf("#Header") == -1){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Error", "GETIDHEADER", "", ""); throw new HeaderBlockException("header data error"); } @@ -94,6 +98,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa protected void parseingEmail() throws Exception { final EnergySpectrumStruct sourceData = EnergySpectrumHandler.getSourceData(super.spectrumFile.getAbsolutePath()); if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.data_type)){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Error", "GETIDBODY", "", ""); throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath()); } @@ -136,6 +142,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa newFileName.append(StringUtils.replace(this.sourceData.acquisition_start_time.substring(0,this.sourceData.acquisition_start_time.lastIndexOf(":")),":","")); newFileName.append(super.spectrumServiceQuotes.getNameStandUtil().GetSuffix(super.currDataType.getType(),this.sourceData.spectrum_quantity,String.valueOf(this.sourceData.acquisition_live_time))); if(!super.spectrumFile.exists()){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist"); } super.spectrumFile = FileUtil.rename(super.spectrumFile, newFileName.toString(), true); @@ -169,6 +177,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa this.endIntoDatabaseTime = new Date(); //设置文件重复标记为true this.parsingProcessLog.setFileRepeat(true); + //发送文件重复错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new RepeatErrorEvent()); throw new FileRepeatException("file repeat"); } DataSourceSwitcher.switchToOracle(); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java index f05b8449..c8263812 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java @@ -6,6 +6,8 @@ import org.jeecg.common.properties.SpectrumPathProperties; import org.jeecg.common.util.DateUtils; import org.jeecg.modules.base.entity.original.GardsAlertData; import org.jeecg.modules.base.enums.DataType; +import org.jeecg.modules.eneity.event.FormatErrorEvent; +import org.jeecg.modules.eneity.event.RepeatErrorEvent; import org.jeecg.modules.exception.FileRepeatException; import org.jeecg.modules.exception.PHD_ReadException; import org.jeecg.modules.file.FileOperation; @@ -91,7 +93,9 @@ public class AlertSpectrum extends AbstractSpectrumHandler{ @Override protected void parseingEmail() throws Exception { final AlertSpectrumStruct sourceData = EnergySpectrumHandler.getAlertSourceData(super.spectrumFile.getAbsolutePath()); - if(Objects.isNull(sourceData)){ + if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new RepeatErrorEvent()); throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath()); } this.sourceData = sourceData; @@ -130,6 +134,8 @@ public class AlertSpectrum extends AbstractSpectrumHandler{ newFileName.append(StringUtils.replace(this.sourceData.time,":","")); newFileName.append(super.currDataType.getSuffix()); if(!super.spectrumFile.exists()){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist"); } super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true); @@ -146,6 +152,8 @@ public class AlertSpectrum extends AbstractSpectrumHandler{ this.alertData = alertData; this.endIntoDatabaseTime = new Date(); this.isFileRepeat = true; + //发送文件重复错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new FileRepeatException("file repeat"); } this.alertData = super.spectrumServiceQuotes.getAlertSpectrumService().create(this.sourceData, super.spectrumFileRelativePath); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java index edfc7112..e0ef95f2 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java @@ -6,6 +6,7 @@ import org.jeecg.common.properties.SpectrumPathProperties; import org.jeecg.common.util.DateUtils; import org.jeecg.modules.base.entity.original.GardsSohData; import org.jeecg.modules.base.enums.DataType; +import org.jeecg.modules.eneity.event.FormatErrorEvent; import org.jeecg.modules.exception.AirSamplerFlowException; import org.jeecg.modules.exception.PHD_ReadException; import org.jeecg.modules.file.FileOperation; @@ -86,7 +87,9 @@ public class HealthStatusSpectrum extends AbstractSpectrumHandler{ @Override protected void parseingEmail() throws Exception { final SOHSpectrumStruct sourceData = EnergySpectrumHandler.getSOHSourceData(super.spectrumFile.getAbsolutePath()); - if(Objects.isNull(sourceData)){ + if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath()); } if(sourceData.af_record_count <= 0 || sourceData.af_start_date.size() < 0 || sourceData.af_start_time.size() < 0){ @@ -128,6 +131,8 @@ public class HealthStatusSpectrum extends AbstractSpectrumHandler{ newFileName.append(StringUtils.replace(this.sourceData.start_time,":","")); newFileName.append(super.currDataType.getSuffix()); if(!super.spectrumFile.exists()){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist"); } super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java index def4166c..5feae8b9 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java @@ -6,6 +6,7 @@ import org.jeecg.common.properties.SpectrumPathProperties; import org.jeecg.common.util.DateUtils; import org.jeecg.modules.base.entity.original.GardsMetData; import org.jeecg.modules.base.enums.DataType; +import org.jeecg.modules.eneity.event.FormatErrorEvent; import org.jeecg.modules.file.FileOperation; import org.jeecg.modules.native_jni.EnergySpectrumHandler; import org.jeecg.modules.native_jni.struct.MetSpectrumStruct; @@ -83,7 +84,9 @@ public class MetSpectrum extends AbstractSpectrumHandler{ @Override protected void parseingEmail() throws Exception{ final MetSpectrumStruct sourceData = EnergySpectrumHandler.getMetSourceData(super.spectrumFile.getAbsolutePath()); - if(Objects.isNull(sourceData)){ + if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new RuntimeException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath()); } this.sourceData = sourceData; @@ -122,6 +125,8 @@ public class MetSpectrum extends AbstractSpectrumHandler{ newFileName.append(StringUtils.replace(this.sourceData.met_start_time.get(0),":","")); newFileName.append(super.currDataType.getSuffix()); if(!super.spectrumFile.exists()){ + //发送格式化错误事件，后续统计报告使用 + spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent()); throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist"); } super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true); diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java index 064eec92..c8ac63c1 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java @@ -85,6 +85,11 @@ public class SpectrumParsingActuator implements Runnable{ String receiveTime = null; StringBuilder finalFileName = new StringBuilder(); try { + //所有邮件都需以.eml格式存储到eml文件夹中 + downloadEmailToEmlDir(); + //保存邮件日志到PG数据库 + this.spectrumServiceQuotes.getMailLogService().create(message,emailProperties); + subject = MimeUtility.decodeText(message.getSubject()); sendTime = DateUtils.formatDate(message.getSentDate(),"yyyy-MM-dd HH:mm:ss"); receiveTime = DateUtils.formatDate(message.getReceivedDate(),"yyyy-MM-dd HH:mm:ss"); @@ -92,8 +97,6 @@ public class SpectrumParsingActuator implements Runnable{ emailServiceManager.getMailContent(message,mailContent); //读取文件内容成功后写入日志 LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(),"Get", emailProperties, "Successful", "GETALLID", "", ""); - //所有邮件都需以.eml格式存储到eml文件夹中 - downloadEmailToEmlDir(); //读取文件内容成功后写入日志 LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(),"Get", emailProperties, "Successful", "GETIDEML", subject, ""); //判断是否是IMS2.0协议文件 @@ -102,8 +105,6 @@ public class SpectrumParsingActuator implements Runnable{ spectrumHandler.init(mailContent.toString(),spectrumServiceQuotes,finalFileName); final boolean matchResult = spectrumHandler.saveEmailToLocal(); if(matchResult){ - //保存邮件解析日志到PG数据库 - this.spectrumServiceQuotes.getMailLogService().create(message,emailProperties); //开始解析 spectrumHandler.handler(); }else{ @@ -122,13 +123,13 @@ public class SpectrumParsingActuator implements Runnable{ log.error("This email failed to parse. The email subject is: {}, sent on: {}, received on: {}, and the reason for the failure is: {}",subject,sendTime,receiveTime,e.getMessage()); e.printStackTrace(); }finally { + this.taskLatch.countDown(); //删除邮箱中已处理过的邮件 try { emailServiceManager.removeMail(message); } catch (MessagingException e) { e.printStackTrace(); } - this.taskLatch.countDown(); LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Successful", "DONE", "", ""); } } diff --git a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java index b43e6df9..7c3d01ad 100644 --- a/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java +++ b/jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java @@ -8,6 +8,7 @@ import org.jeecg.common.util.NameStandUtil; import org.jeecg.common.util.RedisStreamUtil; import org.jeecg.modules.datasource.OraDataSourceProperties; import org.jeecg.modules.service.*; +import org.springframework.context.ApplicationContext; import org.springframework.core.io.ResourceLoader; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.stereotype.Component; @@ -79,6 +80,8 @@ public class SpectrumServiceQuotes { private final LogFileUtil logFileUtil; + private final ApplicationContext applicationContext; + /** * 原始库插入数据锁 */ diff --git a/jeecg-server-cloud/armd-auto-process-start/src/main/java/org/jeecg/JeecgAutoProcessApplication.java b/jeecg-server-cloud/armd-auto-process-start/src/main/java/org/jeecg/JeecgAutoProcessApplication.java index f8406508..9d7d32ba 100644 --- a/jeecg-server-cloud/armd-auto-process-start/src/main/java/org/jeecg/JeecgAutoProcessApplication.java +++ b/jeecg-server-cloud/armd-auto-process-start/src/main/java/org/jeecg/JeecgAutoProcessApplication.java @@ -8,10 +8,7 @@ import org.jeecg.common.properties.SpectrumPathProperties; import org.jeecg.common.properties.TaskProperties; import org.jeecg.common.util.DateUtils; import org.jeecg.common.util.oConvertUtils; -import org.jeecg.modules.AutoProcessManager; -import org.jeecg.modules.DelFileManager; -import org.jeecg.modules.FileSourceHandleManager; -import org.jeecg.modules.UndealHandleManager; +import org.jeecg.modules.*; import org.jeecg.modules.email.EmailReceivePolicy; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; @@ -46,7 +43,7 @@ public class JeecgAutoProcessApplication extends SpringBootServletInitializer im private final UndealHandleManager undealHandleManager; private final FileSourceHandleManager fileSourceHandleManager; private final DelFileManager delFileManager; - private final Object ftpOpierationLock = new Object(); + private final StatReportManager statReportManager; @Override protected SpringApplicationBuilder configure(SpringApplicationBuilder application) { @@ -81,11 +78,13 @@ public class JeecgAutoProcessApplication extends SpringBootServletInitializer im checkTempStorageDirectory(); //校验存储目录是否存在，不存在则创建，存在无操作 checkStorageDirectory(); - autoProcessManager.start(systemStartupTime); +// autoProcessManager.start(systemStartupTime); undealHandleManager.start(); fileSourceHandleManager.start(); // 删除过期的文件 delFileManager.start(); + //统计报告执行线程 + statReportManager.start(); } /**