xiaoguangbin/AnalysisSystemForRadionuclide
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:1.完成统计报告功能

Browse Source
This commit is contained in:
panbaolin 2023-11-10 14:18:25 +08:00
parent 6a51ac30fc
commit c4ba5341c3
22 changed files with 600 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java
View File

@ -346,12 +346,14 @@ public class EmailServiceManager {
* @throws MessagingException
*/
public void removeMail(@NotNull Message message) throws MessagingException {
try {
message.setFlag(Flags.Flag.DELETED,true);
LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Successful", "DELETEID", message.getSubject(), "");
} catch (MessagingException e) {
LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Error", "DELETEID", message.getSubject(), "");
throw e;
synchronized (this){
try {
message.setFlag(Flags.Flag.DELETED,true);
LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Successful", "DELETEID", message.getSubject(), "");
} catch (MessagingException e) {
LogFileUtil.emailLog(spectrumPathProperties.getRootPath() + File.separator + spectrumPathProperties.getLogPath(), "Get", null, "Error", "DELETEID", message.getSubject(), "");
throw e;
}
}
}

5
jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/SpectrumPathProperties.java
View File

@ -29,6 +29,11 @@ public class SpectrumPathProperties implements Serializable {
*/
private String saveFilePath;
/**
* 统计报告存储路径
*/
private String statisticsPath;
/**
* eml格式邮件存储路径
*/

5
jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/TaskProperties.java
View File

@ -71,4 +71,9 @@ public class TaskProperties implements Serializable {
*/
private String temporaryStoragePath;
/**
* 统计报告执行周期
*/
private Integer statisticsReportExecCycle;
}

298
jeecg-module-auto-process/src/main/java/org/jeecg/modules/StatReportManager.java Normal file
View File

@ -0,0 +1,298 @@
package org.jeecg.modules;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jeecg.common.constant.StringConstant;
import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.properties.TaskProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.base.abstracts.AbstractLogOrReport;
import org.jeecg.modules.eneity.event.FormatErrorEvent;
import org.jeecg.modules.eneity.event.RepeatErrorEvent;
import org.jeecg.modules.eneity.vo.DBInfoCount;
import org.jeecg.modules.file.FileOperation;
import org.jeecg.modules.service.IStatReportService;
import org.jeecg.modules.service.ISysMailLogService;
import org.springframework.context.ApplicationEvent;
import org.springframework.context.ApplicationListener;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* 统计报告处理
* 每24小时生成一份邮件解析统计报告
*/
@Component
@RequiredArgsConstructor
@Slf4j
public class StatReportManager implements ApplicationListener<ApplicationEvent>{
/**
* 开始时间
*/
private Date beginTime = null;
/**
* 结束时间
*/
private Date endTime = null;
/**
* 格式化错误数量
*/
private Integer formatErrorNumber = 0;
/**
* 邮件重复数量
*/
private Integer repeatErrorNumber = 0;
private boolean isFirst = true;
private final ISysMailLogService mailLogService;
private final IStatReportService statReportService;
private final SpectrumPathProperties spectrumPathProperties;
private final TaskProperties taskProperties;
public void start(){
ReportExecThread reportExecThread = new ReportExecThread();
reportExecThread.setName("statisticsReportExecThread");
reportExecThread.start();
}
/**
* 统计报告执行线程
*/
private class ReportExecThread extends Thread{
@Override
public void run() {
for(;;){
long start = System.currentTimeMillis();
if(isFirst){
beginTime = new Date();
isFirst = false;
}else{
//生成报告和下面onApplicationEvent函数要加类锁，保障生成报告时不接收新的错误事件
//否则可能造成错误事件数量大于实际的时间范围内的错误数量
synchronized(StatReportManager.class){
Date nowTime = new Date();
endTime = nowTime;
GeneratingReports report = new GeneratingReports();
report.genReport();
//报告生成结束后重新赋予beginTime值，为上次结束时间毫秒加1
beginTime = new Date(nowTime.getTime()+1);
//生成报告后重新初始化参数
formatErrorNumber = 0;
repeatErrorNumber = 0;
}
}
long end = System.currentTimeMillis();
long sleepTime = taskProperties.getStatisticsReportExecCycle() - (end-start);
//如果sleepTime > 0 需要睡眠到指定时间，否则继续下次
if(sleepTime > 0){
try {
TimeUnit.MILLISECONDS.sleep(sleepTime);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
}
/**
* 接收错误事件
* @param event
*/
@Override
public void onApplicationEvent(ApplicationEvent event) {
synchronized(StatReportManager.class){
if(event instanceof FormatErrorEvent){
formatErrorNumber++;
}else if (event instanceof RepeatErrorEvent){
repeatErrorNumber++;
}
}
}
/**
* 生成报告
*/
private class GeneratingReports extends AbstractLogOrReport{
/**
* 创建时间
*/
private Date createTime = new Date();
/**
* 邮件数量
*/
private long emailNumber = 0;
/**
* 原始库数据数量
*/
private Integer originalDBNumber = 0;
/**
* 邮件解析详情记录
*/
private List<DBInfoCount> dbInfoCounts = null;
/**
* 报告内容
*/
StringBuilder report = new StringBuilder();
/**
* 生成报告
*/
private void genReport(){
try {
this.countMailNumber();
this.countDBNumber();
this.setHeader();
this.setStatisticsInfo();
this.setStatisticsErrorFileInfo();
this.setDBInfo();
this.storeReport();
}catch (Exception e){
e.printStackTrace();
log.error("Failed to generate analysis report because: {}",e.getMessage());
}
}
/**
* 统计邮件数量
*/
private void countMailNumber() throws ParseException {
// Date d1 = DateUtils.parseDate("2023-11-07 00:00:00","yyyy-MM-dd HH:mm:ss");
// Date d2 = DateUtils.parseDate("2023-11-07 23:59:59","yyyy-MM-dd HH:mm:ss");
final long emailNumber = mailLogService.countEmailNumByCreateTimeRange(beginTime,endTime);
this.emailNumber = emailNumber;
}
/**
* 统计原始库和分析库入库数量
*/
private void countDBNumber() throws ParseException {
// Date d1 = DateUtils.parseDate("2023-10-12 00:00:00","yyyy-MM-dd HH:mm:ss");
// Date d2 = DateUtils.parseDate("2023-10-20 23:59:59","yyyy-MM-dd HH:mm:ss");
final List<DBInfoCount> dbInfoCounts = statReportService.countParsingMailRecords(beginTime,endTime);
if(!CollectionUtils.isEmpty(dbInfoCounts)){
this.dbInfoCounts = dbInfoCounts;
final int records = dbInfoCounts.stream().mapToInt(DBInfoCount::getOriginalDataNumber).sum();
this.originalDBNumber = records;
}
}
/**
* Header块
*/
private void setHeader(){
final String title = "%-36s GENERATED STATISTICS REPORT";
final String createDate = "%-33s Creation Date %s";
report.append(rowFormat(title,StringConstant.SPACE));
report.append(System.lineSeparator());
report.append(rowFormat(createDate,StringConstant.SPACE, DateUtils.formatDate(createTime,"yyyy/MM/dd-HH:mm:ss")));
report.append(System.lineSeparator()).append(System.lineSeparator());
}
/**
* #STATISTICS INFO 块
*/
private void setStatisticsInfo(){
final String blockTitle = "#STATISTICS INFO";
final String beginTimeRow = " Statistics BeginTime: %-11s %s";
final String endTimeRow = " Statistics EndTime: %-13s %s";
final String emailNumberRow = " Email Number: %-18s %s";
final String dbNumberRow = " Original Data Load to DB Number: %s";
report.append(rowFormat(blockTitle));
report.append(System.lineSeparator());
report.append(rowFormat(beginTimeRow,StringConstant.SPACE,DateUtils.formatDate(beginTime,"yyyy/MM/dd-HH:mm:ss")));
report.append(System.lineSeparator());
report.append(rowFormat(endTimeRow,StringConstant.SPACE,DateUtils.formatDate(endTime,"yyyy/MM/dd-HH:mm:ss")));
report.append(System.lineSeparator());
report.append(rowFormat(emailNumberRow,StringConstant.SPACE,String.valueOf(emailNumber)));
report.append(System.lineSeparator());
report.append(rowFormat(dbNumberRow,String.valueOf(originalDBNumber)));
report.append(System.lineSeparator()).append(System.lineSeparator());
}
/**
* #STATISTICS ERROR FILE 块
*/
private void setStatisticsErrorFileInfo(){
final String blockTitle = "#STATISTICS ERROR FILE";
final String formatErrorRow = " File Format Error: %-15s";
final String repeatErrorRow = " File Repeat Error: %-15s";
report.append(rowFormat(blockTitle));
report.append(System.lineSeparator());
report.append(rowFormat(formatErrorRow,String.valueOf(formatErrorNumber)));
report.append(System.lineSeparator());
report.append(rowFormat(repeatErrorRow,String.valueOf(repeatErrorNumber)));
report.append(System.lineSeparator()).append(System.lineSeparator());
}
/**
* #LOAD INTO DB INFO 块
*/
private void setDBInfo(){
final String blockTitle = "#LOAD INTO DB INFO";
final String rowTitle = " Station Name %-15s DataType %-12s Original Data Number %s Anlyse Data Number";
final String repeatErrorRow = " %-28s %-21s %-22s %s";
report.append(rowFormat(blockTitle));
report.append(System.lineSeparator());
report.append(rowFormat(rowTitle,StringConstant.SPACE,StringConstant.SPACE,StringConstant.SPACE));
report.append(System.lineSeparator());
if(!CollectionUtils.isEmpty(dbInfoCounts)){
for(DBInfoCount dbInfoCount : dbInfoCounts){
report.append(rowFormat(repeatErrorRow,dbInfoCount.getStationName(),this.dataTypeConvert(dbInfoCount.getDataType()),String.valueOf(dbInfoCount.getOriginalDataNumber()),String.valueOf(dbInfoCount.getAnlyseDataNumber())));
report.append(System.lineSeparator());
}
}
report.append(System.lineSeparator());
}
/**
* 保存报告到savefile/Statistics路径
* @throws IOException
*/
private void storeReport() throws IOException {
final String statisticsPath = spectrumPathProperties.getStatisticsPath();
StringBuilder finalPath = new StringBuilder();
finalPath.append(spectrumPathProperties.getRootPath());
finalPath.append(File.separator);
finalPath.append(spectrumPathProperties.getSaveFilePath());
finalPath.append(File.separator);
finalPath.append(spectrumPathProperties.getStatisticsPath());
finalPath.append(File.separator);
finalPath.append(DateUtils.formatDate(beginTime,"yyyyMMdd-HHmmss"));
finalPath.append(StringConstant.UNDER_LINE);
finalPath.append(DateUtils.formatDate(endTime,"yyyyMMdd-HHmmss"));
finalPath.append(".txt");
FileOperation.saveOrAppendFile(finalPath.toString(),report.toString(),false);
}
/**
* 格式化能谱类型
* @param dataType
* @return
*/
private String dataTypeConvert(String dataType){
switch (dataType){
case "S":
return "SAMPLEPHD";
case "Q":
return "QCPHD";
case "G":
return "GASBKPHD";
case "D":
return "DETBKPHD";
default:
return dataType;
}
}
}
}

14
jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/FormatErrorEvent.java Normal file
View File

@ -0,0 +1,14 @@
package org.jeecg.modules.eneity.event;
import org.springframework.context.ApplicationEvent;
import java.io.Serializable;
/**
* 格式化错误事件
*/
public class FormatErrorEvent extends ApplicationEvent implements Serializable {
public FormatErrorEvent() {
super(new Object());
}
}

14
jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/event/RepeatErrorEvent.java Normal file
View File

@ -0,0 +1,14 @@
package org.jeecg.modules.eneity.event;
import org.springframework.context.ApplicationEvent;
import java.io.Serializable;
/**
* 邮件重复错误
*/
public class RepeatErrorEvent extends ApplicationEvent implements Serializable {
public RepeatErrorEvent() {
super(new Object());
}
}

28
jeecg-module-auto-process/src/main/java/org/jeecg/modules/eneity/vo/DBInfoCount.java Normal file
View File

@ -0,0 +1,28 @@
package org.jeecg.modules.eneity.vo;
import lombok.Data;
/**
* 统计报告使用
* 接收邮件解析后存储到数据库的记录数量信息
*/
@Data
public class DBInfoCount {
/**
* 台站名称
*/
private String stationName;
/**
* 能谱类型
*/
private String dataType;
/**
* 单封邮件解析后存储到原始库的记录数量
*/
private Integer originalDataNumber;
/**
* 单封邮件解析后存储到分析库的记录数量
*/
private Integer anlyseDataNumber;
}

22
jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/StatReportMapper.java Normal file
View File

@ -0,0 +1,22 @@
package org.jeecg.modules.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.jeecg.modules.eneity.vo.DBInfoCount;
import java.util.Date;
import java.util.List;
/**
* 统计报告Mapper
*/
public interface StatReportMapper extends BaseMapper {
/**
* 统计邮件解析产生的记录数
* @param beginDate
* @param endDate
* @return
*/
List<DBInfoCount> countParsingMailRecords(@Param("beginDate") Date beginDate, @Param("endDate") Date endDate);
}

75
jeecg-module-auto-process/src/main/java/org/jeecg/modules/mapper/xml/StatReportMapper.xml Normal file
View File

@ -0,0 +1,75 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="org.jeecg.modules.mapper.StatReportMapper">
<select id="countParsingMailRecords" resultType="org.jeecg.modules.eneity.vo.DBInfoCount">
select
r.stationName as stationName,
r.dataType as dataType,
r.originalDataNumber as originalDataNumber,
r.anlyseDataNumber as anlyseDataNumber
from (
select
substr(t.site_det_code,1,5) as stationName,
t.data_type as dataType,
(
1+
(select count(*) from original.GARDS_ROI_LIMITS grl where grl.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SAMPLE_AUX gsa where gsa.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SAMPLE_CERT gsc where gsc.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SAMPLE_CERT_LINE gscl where gscl.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SAMPLE_DESCRIPTION gsd where gsd.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SAMPLE_RATIOS gsr where gsr.sample_id = t.sample_id)+
(select count(*) from original.GARDS_SPECTRUM gs where gs.sample_id = t.sample_id)+
(select count(*) from original.GARDS_TOTAL_EFFICIENCY_PAIRS gtep where gtep.sample_id = t.sample_id)+
(select count(*) from original.GARDS_HISTOGRAM gh where gh.sample_id = t.sample_id)+
(select count(*) from original.GARDS_CALIBRATION_PAIRS_ORIG gcp where gcp.sample_id = t.sample_id)+
(select count(*) from original.GARDS_BG_EFFICIENCY_PAIRS gbep where gbep.sample_id = t.sample_id)
) as originalDataNumber,
(
(select count(*) from rnauto.GARDS_ANALYSES ga where ga.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_CALIBRATION gc where gc.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_CALIBRATION_PAIRS gcp where gcp.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_NUCL_IDED gn where gn.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_NUCL_LINES_IDED gnli where gnli.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_ROI_CHANNELS grc where grc.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_ROI_RESULTS grr where grr.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_XE_RESULTS gxr where gxr.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_PEAKS gp where gp.sample_id = t.sample_id)+
(select count(*) from rnauto.GARDS_QC_CHECK gqc where gqc.sample_id = t.sample_id)
) as anlyseDataNumber,
t.moddate
from original.GARDS_SAMPLE_DATA t
where t.moddate between #{beginDate} and #{endDate}
union all
select
min(t.station_code) as stationName,
'MET' as dataType,
count(*) as originalDataNumber,
0 as anlyseDataNumber,
min(t.moddate) as moddate
from original.GARDS_MET_DATA t
where t.moddate between #{beginDate} and #{endDate}
group by t.input_file_name
union all
select
min(t.station_code) as stationName,
'SOH' as dataType,
count(*) as originalDataNumber,
0 as anlyseDataNumber,
min(t.moddate) as moddate
from original.GARDS_SOH_DATA t
where t.moddate between #{beginDate} and #{endDate}
group by t.input_file_name
union all
select
t.station_code as stationName,
'ALERT' as dataType,
1 as originalDataNumber,
0 as anlyseDataNumber,
t.time as moddate
from original.GARDS_ALERT_DATA t
where t.time between #{beginDate} and #{endDate}
) r order by r.moddate asc
</select>
</mapper>

20
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/IStatReportService.java Normal file
View File

@ -0,0 +1,20 @@
package org.jeecg.modules.service;
import org.jeecg.modules.eneity.vo.DBInfoCount;
import java.util.Date;
import java.util.List;
/**
* 统计报告Service
*/
public interface IStatReportService {
/**
* 统计邮件解析产生的记录数
* @param beginDate
* @param endDate
* @return
*/
List<DBInfoCount> countParsingMailRecords(Date beginDate, Date endDate);
}

11
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/ISysMailLogService.java
View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.IService;
import org.jeecg.modules.base.entity.postgre.SysEmailLog;
import org.jeecg.modules.email.EmailProperties;
import javax.mail.Message;
import java.util.Date;
/**
* 邮件处理日志
@ -15,5 +16,13 @@ public interface ISysMailLogService extends IService<SysEmailLog> {
* @param message
* @param email
*/
public void create(Message message,EmailProperties email) throws Exception;
void create(Message message,EmailProperties email) throws Exception;
/**
* 根据时间范围统计邮件数量
* @param beginTime
* @param endTime
* @return
*/
long countEmailNumByCreateTimeRange(Date beginTime, Date endTime);
}

1
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/GardsDetectorsServiceImpl.java
View File

@ -37,7 +37,6 @@ public class GardsDetectorsServiceImpl extends ServiceImpl<GardsDetectorsMapper,
detector.setDetectorId(format.formatDetectorCodeToId(detectorCode));
detector.setDetectorCode(detectorCode);
detector.setStatus(DetectorsStatus.OPERATING.getStatus());
detector.setModdate(new Date());
this.baseMapper.insert(detector);
return detector;
}

1
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SpectrumBaseBlockServiceImpl.java
View File

@ -93,7 +93,6 @@ public class SpectrumBaseBlockServiceImpl implements ISpectrumBaseBlockService {
gardsSampleData.setAcquisitionLiveSec(struct.acquisition_live_time);
gardsSampleData.setQuantity(struct.air_volume);
gardsSampleData.setStatus(status);
gardsSampleData.setModdate(new Date());
this.sampleDataMapper.insert(gardsSampleData);

39
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/StatReportServiceImpl.java Normal file
View File

@ -0,0 +1,39 @@
package org.jeecg.modules.service.impl;
import com.baomidou.dynamic.datasource.annotation.DS;
import lombok.RequiredArgsConstructor;
import org.jeecg.modules.eneity.vo.DBInfoCount;
import org.jeecg.modules.mapper.StatReportMapper;
import org.jeecg.modules.service.IStatReportService;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.Collections;
import java.util.Date;
import java.util.List;
/**
* 统计报告Service
*/
@DS("ora")
@Service
@RequiredArgsConstructor
public class StatReportServiceImpl implements IStatReportService {
private final StatReportMapper statReportMapper;
/**
* 统计邮件解析产生的记录数
* @param beginDate
* @param endDate
* @return
*/
@Override
public List<DBInfoCount> countParsingMailRecords(Date beginDate, Date endDate) {
final List<DBInfoCount> dbInfoCounts = statReportMapper.countParsingMailRecords(beginDate, endDate);
if(CollectionUtils.isEmpty(dbInfoCounts)){
return Collections.EMPTY_LIST;
}
return dbInfoCounts;
}
}

17
jeecg-module-auto-process/src/main/java/org/jeecg/modules/service/impl/SysMailLogServiceImpl.java
View File

@ -1,5 +1,6 @@
package org.jeecg.modules.service.impl;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.jeecg.modules.base.entity.postgre.SysEmailLog;
import org.jeecg.modules.email.EmailProperties;
@ -9,6 +10,8 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.mail.Message;
import javax.mail.internet.MimeUtility;
import java.util.Date;
/**
* 邮件处理日志
*/
@ -29,4 +32,18 @@ public class SysMailLogServiceImpl extends ServiceImpl<SysMailLogMapper, SysEmai
mailLog.setReceiveTime(message.getReceivedDate());
this.save(mailLog);
}
/**
* 根据时间范围统计邮件数量
* @param beginTime
* @param endTime
* @return
*/
@Override
public long countEmailNumByCreateTimeRange(Date beginTime, Date endTime) {
LambdaQueryWrapper<SysEmailLog> query = new LambdaQueryWrapper<>();
query.between(SysEmailLog::getCreateTime,beginTime,endTime);
final long count = this.count(query);
return count;
}
}

10
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractS_D_Q_G_SpectrumHandler.java
View File

@ -11,6 +11,8 @@ import org.jeecg.common.util.LogFileUtil;
import org.jeecg.modules.base.entity.original.GardsSampleData;
import org.jeecg.modules.base.enums.SampleStatus;
import org.jeecg.modules.config.datasource.DataSourceSwitcher;
import org.jeecg.modules.eneity.event.FormatErrorEvent;
import org.jeecg.modules.eneity.event.RepeatErrorEvent;
import org.jeecg.modules.exception.AcquisitionBlockException;
import org.jeecg.modules.exception.FileRepeatException;
import org.jeecg.modules.exception.HeaderBlockException;
@ -72,6 +74,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa
*/
protected void checkHeaderBlock(){
if(this.mailContent.indexOf("#Header") == -1){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Error", "GETIDHEADER", "", "");
throw new HeaderBlockException("header data error");
}
@ -94,6 +98,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa
protected void parseingEmail() throws Exception {
final EnergySpectrumStruct sourceData = EnergySpectrumHandler.getSourceData(super.spectrumFile.getAbsolutePath());
if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.data_type)){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Error", "GETIDBODY", "", "");
throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath());
}
@ -136,6 +142,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa
newFileName.append(StringUtils.replace(this.sourceData.acquisition_start_time.substring(0,this.sourceData.acquisition_start_time.lastIndexOf(":")),":",""));
newFileName.append(super.spectrumServiceQuotes.getNameStandUtil().GetSuffix(super.currDataType.getType(),this.sourceData.spectrum_quantity,String.valueOf(this.sourceData.acquisition_live_time)));
if(!super.spectrumFile.exists()){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist");
}
super.spectrumFile = FileUtil.rename(super.spectrumFile, newFileName.toString(), true);
@ -169,6 +177,8 @@ public abstract class AbstractS_D_Q_G_SpectrumHandler extends AbstractSpectrumHa
this.endIntoDatabaseTime = new Date();
//设置文件重复标记为true
this.parsingProcessLog.setFileRepeat(true);
//发送文件重复错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new RepeatErrorEvent());
throw new FileRepeatException("file repeat");
}
DataSourceSwitcher.switchToOracle();

10
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java
View File

@ -6,6 +6,8 @@ import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.base.entity.original.GardsAlertData;
import org.jeecg.modules.base.enums.DataType;
import org.jeecg.modules.eneity.event.FormatErrorEvent;
import org.jeecg.modules.eneity.event.RepeatErrorEvent;
import org.jeecg.modules.exception.FileRepeatException;
import org.jeecg.modules.exception.PHD_ReadException;
import org.jeecg.modules.file.FileOperation;
@ -91,7 +93,9 @@ public class AlertSpectrum extends AbstractSpectrumHandler{
@Override
protected void parseingEmail() throws Exception {
final AlertSpectrumStruct sourceData = EnergySpectrumHandler.getAlertSourceData(super.spectrumFile.getAbsolutePath());
if(Objects.isNull(sourceData)){
if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new RepeatErrorEvent());
throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath());
}
this.sourceData = sourceData;
@ -130,6 +134,8 @@ public class AlertSpectrum extends AbstractSpectrumHandler{
newFileName.append(StringUtils.replace(this.sourceData.time,":",""));
newFileName.append(super.currDataType.getSuffix());
if(!super.spectrumFile.exists()){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist");
}
super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true);
@ -146,6 +152,8 @@ public class AlertSpectrum extends AbstractSpectrumHandler{
this.alertData = alertData;
this.endIntoDatabaseTime = new Date();
this.isFileRepeat = true;
//发送文件重复错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new FileRepeatException("file repeat");
}
this.alertData = super.spectrumServiceQuotes.getAlertSpectrumService().create(this.sourceData, super.spectrumFileRelativePath);

7
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java
View File

@ -6,6 +6,7 @@ import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.base.entity.original.GardsSohData;
import org.jeecg.modules.base.enums.DataType;
import org.jeecg.modules.eneity.event.FormatErrorEvent;
import org.jeecg.modules.exception.AirSamplerFlowException;
import org.jeecg.modules.exception.PHD_ReadException;
import org.jeecg.modules.file.FileOperation;
@ -86,7 +87,9 @@ public class HealthStatusSpectrum extends AbstractSpectrumHandler{
@Override
protected void parseingEmail() throws Exception {
final SOHSpectrumStruct sourceData = EnergySpectrumHandler.getSOHSourceData(super.spectrumFile.getAbsolutePath());
if(Objects.isNull(sourceData)){
if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new PHD_ReadException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath());
}
if(sourceData.af_record_count <= 0 || sourceData.af_start_date.size() < 0 || sourceData.af_start_time.size() < 0){
@ -128,6 +131,8 @@ public class HealthStatusSpectrum extends AbstractSpectrumHandler{
newFileName.append(StringUtils.replace(this.sourceData.start_time,":",""));
newFileName.append(super.currDataType.getSuffix());
if(!super.spectrumFile.exists()){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist");
}
super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true);

7
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java
View File

@ -6,6 +6,7 @@ import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.modules.base.entity.original.GardsMetData;
import org.jeecg.modules.base.enums.DataType;
import org.jeecg.modules.eneity.event.FormatErrorEvent;
import org.jeecg.modules.file.FileOperation;
import org.jeecg.modules.native_jni.EnergySpectrumHandler;
import org.jeecg.modules.native_jni.struct.MetSpectrumStruct;
@ -83,7 +84,9 @@ public class MetSpectrum extends AbstractSpectrumHandler{
@Override
protected void parseingEmail() throws Exception{
final MetSpectrumStruct sourceData = EnergySpectrumHandler.getMetSourceData(super.spectrumFile.getAbsolutePath());
if(Objects.isNull(sourceData)){
if(Objects.isNull(sourceData) || StringUtils.isBlank(sourceData.station_code)){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new RuntimeException("THE PHDFile has some blocks can't be read:"+super.spectrumFile.getAbsolutePath());
}
this.sourceData = sourceData;
@ -122,6 +125,8 @@ public class MetSpectrum extends AbstractSpectrumHandler{
newFileName.append(StringUtils.replace(this.sourceData.met_start_time.get(0),":",""));
newFileName.append(super.currDataType.getSuffix());
if(!super.spectrumFile.exists()){
//发送格式化错误事件，后续统计报告使用
spectrumServiceQuotes.getApplicationContext().publishEvent(new FormatErrorEvent());
throw new FileNotFoundException(super.spectrumFile.getAbsolutePath()+" does not exist");
}
super.spectrumFile = FileOperation.rename(super.spectrumFile,newFileName.toString(),true);

11
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java
View File

@ -85,6 +85,11 @@ public class SpectrumParsingActuator implements Runnable{
String receiveTime = null;
StringBuilder finalFileName = new StringBuilder();
try {
//所有邮件都需以.eml格式存储到eml文件夹中
downloadEmailToEmlDir();
//保存邮件日志到PG数据库
this.spectrumServiceQuotes.getMailLogService().create(message,emailProperties);
subject = MimeUtility.decodeText(message.getSubject());
sendTime = DateUtils.formatDate(message.getSentDate(),"yyyy-MM-dd HH:mm:ss");
receiveTime = DateUtils.formatDate(message.getReceivedDate(),"yyyy-MM-dd HH:mm:ss");
@ -92,8 +97,6 @@ public class SpectrumParsingActuator implements Runnable{
emailServiceManager.getMailContent(message,mailContent);
//读取文件内容成功后写入日志
LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(),"Get", emailProperties, "Successful", "GETALLID", "", "");
//所有邮件都需以.eml格式存储到eml文件夹中
downloadEmailToEmlDir();
//读取文件内容成功后写入日志
LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(),"Get", emailProperties, "Successful", "GETIDEML", subject, "");
//判断是否是IMS2.0协议文件
@ -102,8 +105,6 @@ public class SpectrumParsingActuator implements Runnable{
spectrumHandler.init(mailContent.toString(),spectrumServiceQuotes,finalFileName);
final boolean matchResult = spectrumHandler.saveEmailToLocal();
if(matchResult){
//保存邮件解析日志到PG数据库
this.spectrumServiceQuotes.getMailLogService().create(message,emailProperties);
//开始解析
spectrumHandler.handler();
}else{
@ -122,13 +123,13 @@ public class SpectrumParsingActuator implements Runnable{
log.error("This email failed to parse. The email subject is: {}, sent on: {}, received on: {}, and the reason for the failure is: {}",subject,sendTime,receiveTime,e.getMessage());
e.printStackTrace();
}finally {
this.taskLatch.countDown();
//删除邮箱中已处理过的邮件
try {
emailServiceManager.removeMail(message);
} catch (MessagingException e) {
e.printStackTrace();
}
this.taskLatch.countDown();
LogFileUtil.emailLog(spectrumServiceQuotes.getSpectrumPathProperties().getRootPath() + File.separator + spectrumServiceQuotes.getSpectrumPathProperties().getLogPath(), "Get", null, "Successful", "DONE", "", "");
}
}

3
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java
View File

@ -8,6 +8,7 @@ import org.jeecg.common.util.NameStandUtil;
import org.jeecg.common.util.RedisStreamUtil;
import org.jeecg.modules.datasource.OraDataSourceProperties;
import org.jeecg.modules.service.*;
import org.springframework.context.ApplicationContext;
import org.springframework.core.io.ResourceLoader;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.stereotype.Component;
@ -79,6 +80,8 @@ public class SpectrumServiceQuotes {
private final LogFileUtil logFileUtil;
private final ApplicationContext applicationContext;
/**
* 原始库插入数据锁
*/

11
jeecg-server-cloud/armd-auto-process-start/src/main/java/org/jeecg/JeecgAutoProcessApplication.java
View File

@ -8,10 +8,7 @@ import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.properties.TaskProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.common.util.oConvertUtils;
import org.jeecg.modules.AutoProcessManager;
import org.jeecg.modules.DelFileManager;
import org.jeecg.modules.FileSourceHandleManager;
import org.jeecg.modules.UndealHandleManager;
import org.jeecg.modules.*;
import org.jeecg.modules.email.EmailReceivePolicy;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
@ -46,7 +43,7 @@ public class JeecgAutoProcessApplication extends SpringBootServletInitializer im
private final UndealHandleManager undealHandleManager;
private final FileSourceHandleManager fileSourceHandleManager;
private final DelFileManager delFileManager;
private final Object ftpOpierationLock = new Object();
private final StatReportManager statReportManager;
@Override
protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
@ -81,11 +78,13 @@ public class JeecgAutoProcessApplication extends SpringBootServletInitializer im
checkTempStorageDirectory();
//校验存储目录是否存在，不存在则创建，存在无操作
checkStorageDirectory();
autoProcessManager.start(systemStartupTime);
// autoProcessManager.start(systemStartupTime);
undealHandleManager.start();
fileSourceHandleManager.start();
// 删除过期的文件
delFileManager.start();
//统计报告执行线程
statReportManager.start();
}
/**