xiaoguangbin/AnalysisSystemForRadionuclide
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

自动处理程序增加自动处理分析库表唯一约束异常的判断，以及违反唯一约束性错误日志内容书写，违反唯一约束性文件移动逻辑

Browse Source
This commit is contained in:
qiaoqinzheng 2024-01-12 14:42:16 +08:00
parent d02bd8127d
commit f44ba359d9
21 changed files with 175 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
jeecg-boot-base-core/src/main/java/org/jeecg/common/constant/RedisConstant.java
View File

@ -41,5 +41,8 @@ public interface RedisConstant {
String PREFIX_TEMPLATE = "Template:"; // 消息模板
String UNDEAL_FILE = "Undeal:";
/**
* 删除失败邮件KEY
*/
String DELETE_FAIL_EMAILS = "delete_fail_emails";
}

35
jeecg-boot-base-core/src/main/java/org/jeecg/common/email/EmailServiceManager.java
View File

@ -8,14 +8,16 @@ import com.sun.mail.smtp.SMTPAddressFailedException;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jeecg.common.api.dto.message.MessageDTO;
import org.jeecg.common.constant.RedisConstant;
import org.jeecg.common.constant.StringConstant;
import org.jeecg.common.constant.SymbolConstant;
import org.jeecg.common.email.emuns.MailContentType;
import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.DateUtils;
import org.jeecg.common.util.Md5Util;
import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.base.entity.postgre.SysEmail;
import org.jetbrains.annotations.NotNull;
import javax.mail.*;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
@ -55,6 +57,8 @@ public class EmailServiceManager {
/** 收件箱 */
private Folder folder = null;
private RedisUtil redisUtil;
@NotNull
public static EmailServiceManager getInstance(){
return new EmailServiceManager();
@ -72,12 +76,15 @@ public class EmailServiceManager {
* 初始化邮件服务管理器
* @param email 邮件属性
*/
public void init(SysEmail email,Integer receiveNum,String temporaryStoragePath,Date systemStartupTime, SpectrumPathProperties pathProperties){
public void init(SysEmail email, Integer receiveNum, String temporaryStoragePath,
Date systemStartupTime, SpectrumPathProperties pathProperties,
RedisUtil redisUtil){
this.email = email;
this.receiveNum = receiveNum;
this.temporaryStoragePath = temporaryStoragePath;
this.systemStartupTime = systemStartupTime;
this.spectrumPathProperties = pathProperties;
this.redisUtil = redisUtil;
}
/**
@ -569,6 +576,8 @@ public class EmailServiceManager {
} catch (MessagingException | UnsupportedEncodingException e) {
status = EmailLogManager.STATUS_ERROR;
log.error("Email deletion failed, the subject of the email is :{}, the reason is :{}.",subject,e.getMessage());
String emlName = subject+StringConstant.UNDER_LINE+DateUtils.formatDate(receivedDate,"yyyy-MM-dd HH:mm:ss");
redisUtil.hset(RedisConstant.DELETE_FAIL_EMAILS,Md5Util.md5Encode(emlName,"UTF-8"),emlName);
e.printStackTrace();
}finally {
EmailLogEvent removeEvent = new EmailLogEvent(EmailLogManager.GS_TYPE_GET,status,EmailLogManager.DELETEID,subject,receivedDate);
@ -594,4 +603,26 @@ public class EmailServiceManager {
e.printStackTrace();
}
}
/**
* 校验邮件
* 若此次获取的邮件是上次删除失败的邮件直接删除
* @param message
*/
public boolean check(Message message){
boolean exist = false;
try {
String subject = MimeUtility.decodeText(message.getSubject());
Date receivedDate = message.getReceivedDate();
String emlName = subject+StringConstant.UNDER_LINE+DateUtils.formatDate(receivedDate,"yyyy-MM-dd HH:mm:ss");
exist = redisUtil.hHasKey(RedisConstant.DELETE_FAIL_EMAILS,Md5Util.md5Encode(emlName,"UTF-8"));
if(exist){
message.setFlag(Flags.Flag.DELETED,true);
redisUtil.hdel(RedisConstant.DELETE_FAIL_EMAILS,Md5Util.md5Encode(emlName,"UTF-8"));
}
} catch (MessagingException | UnsupportedEncodingException e) {
return exist;
}
return exist;
}
}

2
jeecg-boot-base-core/src/main/java/org/jeecg/common/properties/MaximumPoolSizeProperties.java
View File

@ -11,4 +11,6 @@ public class MaximumPoolSizeProperties {
private Integer station;
private Integer auto;
}

17
jeecg-module-auto-process/src/main/java/org/jeecg/modules/EmailParsingActuator.java
View File

@ -6,7 +6,6 @@ import org.jeecg.common.email.EmailLogManager;
import org.jeecg.common.email.EmailServiceManager;
import org.jeecg.common.properties.TaskProperties;
import org.jeecg.modules.email.EmailProperties;
import org.jeecg.modules.eneity.event.SpectrumLog;
import org.jeecg.modules.spectrum.EmailCounter;
import org.jeecg.modules.spectrum.SpectrumLogManager;
import org.jeecg.modules.spectrum.SpectrumParsingActuator;
@ -39,7 +38,7 @@ public class EmailParsingActuator extends Thread{
this.systemStartupTime = systemStartupTime;
//获取机器可用核心数
int systemCores = Runtime.getRuntime().availableProcessors();
int systemCores = spectrumServiceQuotes.getMaximumPoolSizeProperties().getAuto();
int maximumPoolSize = taskProperties.getReceiveNum() > systemCores?taskProperties.getReceiveNum():systemCores;
//初始化线程池
@ -60,10 +59,18 @@ public class EmailParsingActuator extends Thread{
}
long start = System.currentTimeMillis();
final EmailServiceManager emailServiceManager = EmailServiceManager.getInstance();
emailServiceManager.init(this.emailProperties,this.taskProperties.getReceiveNum(),this.taskProperties.getTemporaryStoragePath(),this.systemStartupTime, spectrumServiceQuotes.getSpectrumPathProperties());
emailServiceManager.init(this.emailProperties,this.taskProperties.getReceiveNum(),this.taskProperties.getTemporaryStoragePath(),
this.systemStartupTime, spectrumServiceQuotes.getSpectrumPathProperties(),spectrumServiceQuotes.getRedisUtil());
try {
final Message[] messages = emailServiceManager.receiveMail();
Message[] messages = emailServiceManager.receiveMail();
if(ArrayUtils.isNotEmpty(messages)){
//检验获取的邮件是否在之前删除失败列表中，若在直接调用邮件API删除，并且此次数组里元素也删除
for(int i=messages.length-1;i>=0;i--){
final boolean exist = emailServiceManager.check(messages[i]);
if(exist){
messages = ArrayUtils.remove(messages,i);
}
}
CountDownLatch taskLatch = new CountDownLatch(messages.length);
for(Message message : messages){
SpectrumParsingActuator spectrumParsingActuator = new SpectrumParsingActuator();
@ -98,6 +105,4 @@ public class EmailParsingActuator extends Thread{
}
}
}
}

3
jeecg-module-auto-process/src/main/java/org/jeecg/modules/enums/ErrorType.java
View File

@ -7,7 +7,8 @@ public enum ErrorType {
STATION_ERROR("station_code:%s=0"),
FILE_REPEAT("file repeat"),
GAS_OR_DET_ERROR("gas or det file is no exist or is error"),
AIR_SAMPLER_FLOW_ERROR("this is no ariSamplerFlow data");
AIR_SAMPLER_FLOW_ERROR("this is no ariSamplerFlow data"),
INSERT_ERROR("The sample_id：%s has been reported missing in the database");
private String content;

19
jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/AnalyseException.java Normal file
View File

@ -0,0 +1,19 @@
package org.jeecg.modules.exception;
import lombok.Getter;
public class AnalyseException extends Exception{
@Getter
private boolean isDuplicateKeyException = false;
public AnalyseException(String message) {
super(message);
}
public AnalyseException(String message, boolean isDuplicateKeyException) {
super(message);
this.isDuplicateKeyException = isDuplicateKeyException;
}
}

7
jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/BAnalyseException.java
View File

@ -3,7 +3,7 @@ package org.jeecg.modules.exception;
/**
* B谱分析异常
*/
public class BAnalyseException extends Exception{
public class BAnalyseException extends AnalyseException{
/**
* Constructs a new exception with the specified detail message. The
@ -16,4 +16,9 @@ public class BAnalyseException extends Exception{
public BAnalyseException(String message) {
super(message);
}
public BAnalyseException(String message, boolean isDuplicateKeyException) {
super(message,isDuplicateKeyException);
}
}

7
jeecg-module-auto-process/src/main/java/org/jeecg/modules/exception/GAnalyseException.java
View File

@ -3,7 +3,7 @@ package org.jeecg.modules.exception;
/**
* B谱分析异常
*/
public class GAnalyseException extends Exception{
public class GAnalyseException extends AnalyseException{
/**
* Constructs a new exception with the specified detail message. The
@ -16,4 +16,9 @@ public class GAnalyseException extends Exception{
public GAnalyseException(String message) {
super(message);
}
public GAnalyseException(String message, boolean isDuplicateKeyException) {
super(message,isDuplicateKeyException);
}
}

20
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AbstractSpectrumHandler.java
View File

@ -16,6 +16,7 @@ import org.jeecg.modules.eneity.event.SpectrumErrorEvent;
import org.jeecg.modules.eneity.event.SpectrumLog;
import org.jeecg.modules.enums.ErrorType;
import org.jeecg.modules.enums.SpectrumSource;
import org.jeecg.modules.exception.AnalyseException;
import org.jeecg.modules.exception.FileRepeatException;
import org.jeecg.modules.exception.HeaderBlockException;
import org.jeecg.modules.file.FileOperation;
@ -272,9 +273,26 @@ public abstract class AbstractSpectrumHandler extends AbstractChain {
final String rootPath = spectrumServiceQuotes.getSpectrumPathProperties().getRootPath();
final String undealPath = spectrumServiceQuotes.getSpectrumPathProperties().getUndealPath();
final String finalPath = rootPath+File.separator+undealPath;
FileOperation.moveFile(spectrumFile,finalPath,true);
//判断文件是否在savefile下已经保存过
final String savefileName = spectrumServiceQuotes.getSpectrumPathProperties().getSaveFilePath().substring(1);
//若文件已经在savefile了进行复制
if(spectrumFile.getAbsolutePath().contains(savefileName)){
FileOperation.copyFile(spectrumFile,finalPath,true);
}else {
//若文件不在savefile 则判断异常是否属于违反唯一约束性
if (e instanceof AnalyseException) {
AnalyseException exception = (AnalyseException) e;
if (exception.isDuplicateKeyException()) {
this.spectrumFile.delete();
}
} else {
//如果文件在savefile中没有 并且 不属于违反唯一约束性的异常 将文件移动到undeal
FileOperation.moveFile(spectrumFile,finalPath,true);
}
}
}
} catch (IOException ex) {
log.error("An error occurred during the process of processing the failed parsing file. The file is: {}, and the reason is: {}",this.spectrumFile.getAbsolutePath(),e.getMessage());
ex.printStackTrace();
}
}else if(SpectrumSource.FROM_FILE_SOURCE.getSourceType().equals(spectrumSource) && (e instanceof FileRepeatException)){

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/AlertSpectrum.java
View File

@ -81,10 +81,10 @@ public class AlertSpectrum extends AbstractSpectrumHandler{
this.parseingEmail();
//修改能谱文件名称
this.updateSpectrumFileName();
//结构体数据入库
this.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
this.handlerOriginalData();
//若本次文件来自于undel目录，解析成功则删除
deleteIfFromUndelFile();
}catch (Exception e){

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/DetbkphdSpectrum.java
View File

@ -41,10 +41,10 @@ public class DetbkphdSpectrum extends AbstractS_D_Q_G_SpectrumHandler {
super.readFileLabel();
//修改能谱文件名称
super.updateSpectrumFileName();
//结构体数据入库
super.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
super.handlerOriginalData();
//修改状态为解析完成
super.status = SampleStatus.COMPLETE.getValue();
super.updateStatus();

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/GasbkphdSpectrum.java
View File

@ -43,10 +43,10 @@ public class GasbkphdSpectrum extends AbstractS_D_Q_G_SpectrumHandler {
super.readFileLabel();
//修改能谱文件名称
super.updateSpectrumFileName();
//结构体数据入库
super.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
super.handlerOriginalData();
//修改状态为解析完成
super.status = SampleStatus.COMPLETE.getValue();
super.updateStatus();

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/HealthStatusSpectrum.java
View File

@ -80,10 +80,10 @@ public class HealthStatusSpectrum extends AbstractSpectrumHandler{
this.parseingEmail();
//修改能谱文件名称
this.updateSpectrumFileName();
//结构体数据入库
this.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
this.handlerOriginalData();
//把流程日志保存到日志目录
this.saveLogToLogDir();
//若本次文件来自于undel目录，解析成功则删除

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/MetSpectrum.java
View File

@ -71,10 +71,10 @@ public class MetSpectrum extends AbstractSpectrumHandler{
this.parseingEmail();
//修改能谱文件名称
this.updateSpectrumFileName();
//结构体数据入库
this.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
this.handlerOriginalData();
//把流程日志保存到日志目录
this.saveLogToLogDir();
//若本次文件来自于undel目录，解析成功则删除

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/QcphdSpectrum.java
View File

@ -42,10 +42,10 @@ public class QcphdSpectrum extends AbstractS_D_Q_G_SpectrumHandler {
super.readFileLabel();
//修改能谱文件名称
super.updateSpectrumFileName();
//结构体数据入库
super.handlerOriginalData();
//保存PHD文件到savefile
super.saveFileToSavefile();
//结构体数据入库
super.handlerOriginalData();
//修改状态为解析完成
super.status = SampleStatus.COMPLETE.getValue();
super.updateStatus();

7
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_B_Analysis.java
View File

@ -23,6 +23,7 @@ import org.jeecg.modules.native_jni.EnergySpectrumHandler;
import org.jeecg.modules.native_jni.struct.BgAnalyseResult;
import org.jeecg.modules.native_jni.struct.EnergySpectrumStruct;
import org.jeecg.modules.service.BlockConstant;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.transaction.TransactionStatus;
import java.io.*;
import java.time.Instant;
@ -170,7 +171,11 @@ public class Sample_B_Analysis implements BlockConstant {
pushToRedis();
}catch (Exception e){
analyseFail = true;
throw new BAnalyseException(e.getMessage());
if (e instanceof DuplicateKeyException) {
throw new BAnalyseException(e.getMessage(), true);
} else {
throw new BAnalyseException(e.getMessage());
}
}finally {
this.endAnalysisTime = new Date();
//如果分析成功并且analyses对象不为空

15
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/Sample_G_Analysis.java
View File

@ -3,7 +3,6 @@ package org.jeecg.modules.spectrum;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
@ -20,6 +19,7 @@ import org.jeecg.common.constant.enums.SpectrumSystemType;
import org.jeecg.common.properties.ParameterProperties;
import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.*;
import org.jeecg.modules.ErrorLogManager;
import org.jeecg.modules.base.bizVo.AttributeItemVo;
import org.jeecg.modules.base.dto.*;
import org.jeecg.modules.base.entity.original.GardsSampleData;
@ -28,11 +28,14 @@ import org.jeecg.modules.base.enums.DSType;
import org.jeecg.modules.base.enums.MiddleDataType;
import org.jeecg.modules.base.enums.SpectrumType;
import org.jeecg.modules.config.datasource.DataSourceSwitcher;
import org.jeecg.modules.eneity.event.SpectrumErrorEvent;
import org.jeecg.modules.entity.vo.*;
import org.jeecg.modules.enums.ErrorType;
import org.jeecg.modules.exception.GAnalyseException;
import org.jeecg.modules.file.FileOperation;
import org.jeecg.modules.native_jni.struct.EnergySpectrumStruct;
import org.jeecgframework.core.util.ApplicationContextUtil;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.transaction.TransactionStatus;
import org.w3c.dom.*;
import org.xml.sax.SAXException;
@ -43,13 +46,13 @@ import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.sql.SQLIntegrityConstraintViolationException;
import java.text.ParseException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.*;
import static org.jeecg.modules.service.BlockConstant.XE_131m;
@Data
@Slf4j
@ -124,7 +127,7 @@ public class Sample_G_Analysis {
this.dataType = energySpectrumStruct.data_type;
this.sampleInputFilename = sampleData.getInputFileName();
this.sampleFilename = StringUtils.substring(sampleData.getInputFileName(),
sampleData.getInputFileName().lastIndexOf((StringConstant.SLASH)+1));
sampleData.getInputFileName().lastIndexOf((StringConstant.SLASH))+1);
}
public void analysis() throws GAnalyseException{
@ -194,7 +197,11 @@ public class Sample_G_Analysis {
}catch (Exception e){
e.printStackTrace();
log.error("Sample_G_Analysis", e);
throw new GAnalyseException("Sample Analyse Error at "+DateUtils.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"));
if (e instanceof DuplicateKeyException) {
throw new GAnalyseException("Sample Analyse Error at "+DateUtils.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"), true);
} else {
throw new GAnalyseException("Sample Analyse Error at "+DateUtils.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss"));
}
}
log.info("Gamma自动处理分析--End");
}

40
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SamplephdSpectrum.java
View File

@ -1,8 +1,18 @@
package org.jeecg.modules.spectrum;
import org.jeecg.modules.ErrorLogManager;
import org.jeecg.modules.base.enums.DataType;
import org.jeecg.modules.base.enums.SampleStatus;
import org.jeecg.modules.base.enums.SystemType;
import org.jeecg.modules.eneity.event.SpectrumErrorEvent;
import org.jeecg.modules.enums.ErrorType;
import org.jeecg.modules.enums.SpectrumSource;
import org.jeecg.modules.exception.AnalyseException;
import org.jeecg.modules.exception.GAnalyseException;
import org.springframework.dao.DuplicateKeyException;
import java.sql.SQLIntegrityConstraintViolationException;
import java.util.Date;
import java.util.Objects;
/**
@ -42,12 +52,20 @@ public class SamplephdSpectrum extends AbstractS_D_Q_G_SpectrumHandler {
super.readFileLabel();
//修改能谱文件名称
super.updateSpectrumFileName();
//保存PHD文件到savefile
//如果文件来自于邮箱或filesource则在存储原始库之前需移动到savefile，便于和input_file_name字段对应
if(!SpectrumSource.FORM_FILE_UNDEL.getSourceType().equals(super.spectrumSource)){
super.saveFileToSavefile();
}
//结构体数据入库
super.handlerOriginalData();
//进行B、G(P)谱分析
this.autoAnalysis();
//保存PHD文件到savefile
super.saveFileToSavefile();
//如果文件来自于undel必须到分析结束后才能移动到savefile，否则可能导致本次处理再次失败然后就移动到savefile了
if(SpectrumSource.FORM_FILE_UNDEL.getSourceType().equals(super.spectrumSource)){
super.saveFileToSavefile();
}
//修改状态为解析完成
super.status = SampleStatus.COMPLETE.getValue();
super.updateStatus();
@ -56,12 +74,22 @@ public class SamplephdSpectrum extends AbstractS_D_Q_G_SpectrumHandler {
}catch (Exception e){
//异常返回文件名称用于报错日志
super.returnFileName.append(super.spectrumFile.getName());
//修改状态为解析失败
super.status = SampleStatus.FAIL.getValue();
super.updateStatus();
//处理解析失败的文件
super.handleParseingFailFile(e);
if (e instanceof AnalyseException) {
AnalyseException exception = (AnalyseException) e;
if (exception.isDuplicateKeyException()) {
ErrorLogManager.getInstance().write(new SpectrumErrorEvent(new Date(), ErrorType.INSERT_ERROR, super.spectrumFile.getName(), String.valueOf(this.sampleData.getSampleId())));
}
//处理解析失败的文件
super.handleParseingFailFile(e);
} else {
//修改状态为解析失败
super.status = SampleStatus.FAIL.getValue();
super.updateStatus();
//处理解析失败的文件
super.handleParseingFailFile(e);
}
throw e;
}finally {
if(Objects.nonNull(this.parsingProcessLog)){

10
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumLogManager.java
View File

@ -65,10 +65,12 @@ public class SpectrumLogManager {
* 保存所有日志
*/
public void saveAllLog(){
if(!execLogMap.isEmpty()){
execLogMap.forEach((k,v)->{
this.saveLog(k);
});
synchronized (execLogMap){
if(!execLogMap.isEmpty()){
execLogMap.forEach((k,v)->{
this.saveLog(k);
});
}
}
}
}

1
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumParsingActuator.java
View File

@ -25,7 +25,6 @@ public class SpectrumParsingActuator implements Runnable{
private final static String MSG_TYPE = "MSG_TYPE DATA";
private final static String EMAIL_STOP = "STOP";
/**
* 邮件对象
*/

4
jeecg-module-auto-process/src/main/java/org/jeecg/modules/spectrum/SpectrumServiceQuotes.java
View File

@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor;
import org.jeecg.common.properties.*;
import org.jeecg.common.util.NameStandUtil;
import org.jeecg.common.util.RedisStreamUtil;
import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.datasource.OraDataSourceProperties;
import org.jeecg.modules.service.*;
import org.springframework.context.ApplicationContext;
@ -81,6 +82,9 @@ public class SpectrumServiceQuotes {
private final ApplicationContext applicationContext;
private final RedisUtil redisUtil;
private final MaximumPoolSizeProperties maximumPoolSizeProperties;
/**
* 原始库插入数据锁
*/