xiaoguangbin/AnalysisSystemForRadionuclide
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat：清理数据和文件

Browse Source
This commit is contained in:
nieziyan 2023-11-03 19:14:22 +08:00
parent b9890950b5
commit 430ada5d23
8 changed files with 239 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
jeecg-boot-base-core/src/main/java/org/jeecg/common/util/FTPUtil.java
View File

@ -1,6 +1,8 @@
package org.jeecg.common.util;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
@ -20,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@ -292,6 +295,34 @@ public class FTPUtil {
}
}
/*
* 批量删除FTP文件 返回删除失败的文件路径
* */
public List<String> removeFiles(List<String> paths){
List<String> failList = new ArrayList<>();
if (CollUtil.isEmpty(paths))
return failList;
// 连接FTP服务
final FTPClient ftpClient = this.LoginFTP();
for (String path : paths) {
try {
if (StrUtil.isBlank(path)) continue;
boolean success = ftpClient.deleteFile(path);
if (!success) failList.add(path);
} catch (IOException e) {
failList.add(path);
e.printStackTrace();
}
}
// 关闭FTP连接
try {
if (ObjectUtil.isNotNull(ftpClient)) ftpClient.disconnect();
} catch (IOException e) {
e.printStackTrace();
}
return failList;
}
public static boolean createDirs(FTPClient ftp , String path) throws IOException {
/* 该部分为逐级创建 */
String[] split = path.split(SymbolConstant.SINGLE_SLASH);

11
jeecg-boot-base-core/src/main/java/org/jeecg/modules/base/dto/OwnerDto.java Normal file
View File

@ -0,0 +1,11 @@
package org.jeecg.modules.base.dto;
import lombok.Data;
@Data
public class OwnerDto {
private String owner;
private String tableName;
}

7
jeecg-module-abnormal-alarm/src/main/java/org/jeecg/modules/service/impl/SysDatabaseServiceImpl.java
View File

@ -257,6 +257,11 @@ public class SysDatabaseServiceImpl extends ServiceImpl<SysDatabaseMapper, SysDa
}
private List<DBInfo> dbInfoMY(String dataBase){
return baseMapper.dbInfoMY(dataBase);
// 切换数据源
List<DBInfo> dbInfos = baseMapper.dbInfoMY(dataBase);
// 清除数据源
return dbInfos;
}
}

10
jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/system/controller/GardsSampleDataController.java
View File

@ -9,10 +9,7 @@ import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.system.entity.GardsSampleDataSystem;
import org.jeecg.modules.system.service.IGardsSampleDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.Map;
@ -49,8 +46,9 @@ public class GardsSampleDataController {
@DeleteMapping("deleteById")
@ApiOperation(value = "删除DATA_BASE数据", notes = "删除DATA_BASE数据")
public Result deleteById(Integer sampleId){
return gardsSampleDataService.deleteById(sampleId);
public Result<?> deleteById(@RequestParam Integer sampleId, boolean sampleData,
boolean rnAuto, boolean rnMan){
return gardsSampleDataService.deleteById(sampleId, sampleData, rnAuto, rnMan);
}
}

13
jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/system/mapper/GardsSampleDataMapper.java
View File

@ -1,7 +1,20 @@
package org.jeecg.modules.system.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param;
import org.jeecg.modules.base.dto.OwnerDto;
import org.jeecg.modules.base.entity.rnauto.GardsAnalyses;
import org.jeecg.modules.system.entity.GardsSampleDataSystem;
import java.util.List;
public interface GardsSampleDataMapper extends BaseMapper<GardsSampleDataSystem> {
List<OwnerDto> containSampleId(String filed);
void delTables(@Param("tableNames") List<String> tableNames,
@Param("sampleId") Integer sampleId);
GardsAnalyses getAnalysis(@Param("sampleId") Integer sampleId,
@Param("owner") String owner);
}

31
jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/system/mapper/xml/GardsSampleDataMapper.xml Normal file
View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="org.jeecg.modules.system.mapper.GardsSampleDataMapper">
<delete id="delTables">
<foreach collection="tableNames" item="tableName" index="index">
DELETE FROM ${tableName} WHERE SAMPLE_ID = #{sampleId}
</foreach>
</delete>
<select id="containSampleId" resultType="org.jeecg.modules.base.dto.OwnerDto">
SELECT
OWNER,
TABLE_NAME AS tableName
FROM
DBA_TAB_COLUMNS
WHERE
COLUMN_NAME = #{filed}
</select>
<select id="getAnalysis" resultType="org.jeecg.modules.base.entity.rnauto.GardsAnalyses">
SELECT
BASELINE_PATH AS baselinePath,
LC_PATH AS lcPath,
SCAC_PATH AS scacPath,
LOG_PATH AS logPath,
REPORT_PAHT AS reportPath
FROM
${owner}.GARDS_ANALYSES
WHERE
SAMPLE_ID = #{sampleId}
</select>
</mapper>

4
jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/system/service/IGardsSampleDataService.java
View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
import org.jeecg.common.api.QueryRequest;
import org.jeecg.common.api.vo.Result;
import org.jeecg.modules.base.entity.rnauto.GardsAnalyses;
import org.jeecg.modules.system.entity.GardsSampleDataSystem;
public interface IGardsSampleDataService extends IService<GardsSampleDataSystem> {
@ -21,6 +22,7 @@ public interface IGardsSampleDataService extends IService<GardsSampleDataSystem>
* @param sampleId
* @return
*/
Result deleteById(Integer sampleId);
Result<?> deleteById(Integer sampleId, boolean sampleData, boolean rnAuto, boolean rnMan);
GardsSampleDataSystem getOne(Integer sampleId);
}

150
jeecg-module-system/jeecg-system-biz/src/main/java/org/jeecg/modules/system/service/impl/GardsSampleDataServiceImpl.java
View File

@ -1,5 +1,10 @@
package org.jeecg.modules.system.service.impl;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.dynamic.datasource.annotation.DS;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
@ -9,23 +14,44 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.jeecg.common.api.QueryRequest;
import org.jeecg.common.api.vo.Result;
import org.jeecg.common.constant.enums.FileTypeEnum;
import org.jeecg.common.properties.SpectrumPathProperties;
import org.jeecg.common.util.FTPUtil;
import org.jeecg.common.util.RedisUtil;
import org.jeecg.modules.base.dto.OwnerDto;
import org.jeecg.modules.base.entity.rnauto.GardsAnalyses;
import org.jeecg.modules.system.entity.GardsSampleDataSystem;
import org.jeecg.modules.system.mapper.GardsSampleDataMapper;
import org.jeecg.modules.system.service.IGardsSampleDataService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.stereotype.Service;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.TransactionStatus;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.interceptor.TransactionAspectSupport;
import org.springframework.transaction.support.DefaultTransactionDefinition;
import java.util.HashMap;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
@Service("gardsSampleDataService")
@DS("ora")
public class GardsSampleDataServiceImpl extends ServiceImpl<GardsSampleDataMapper, GardsSampleDataSystem> implements IGardsSampleDataService {
@Autowired
private RedisUtil redisUtil;
@Autowired
private FTPUtil ftpUtil;
@Autowired
private SpectrumPathProperties pathProperties;
@Autowired
private PlatformTransactionManager transactionManager;
@Override
public Result<IPage<GardsSampleDataSystem>> findPage(QueryRequest queryRequest, GardsSampleDataSystem gardsSampleData) {
//查询全部台站信息
@ -62,14 +88,118 @@ public class GardsSampleDataServiceImpl extends ServiceImpl<GardsSampleDataMappe
}
@Override
@Transactional
public Result deleteById(Integer sampleId) {
Result result = new Result();
LambdaQueryWrapper<GardsSampleDataSystem> queryWrapper = new LambdaQueryWrapper<>();
queryWrapper.eq(GardsSampleDataSystem::getSampleId, sampleId);
this.baseMapper.delete(queryWrapper);
result.success("Successfully deleted");
return result;
public Result<?> deleteById(Integer sampleId, boolean sampleData,
boolean rnAuto, boolean rnMan) {
String ftpRootPath = ftpUtil.getFtpRootPath();
String savePath = ftpRootPath + pathProperties.getSaveFilePath();
String logPath = ftpRootPath + pathProperties.getLogPath();
/* 删除数据库数据 */
// 过滤掉多余的表
String ORIGINAL = "ORIGINAL";String RNAUTO = "RNAUTO";String RNMAN = "RNMAN";
List<String> suitable = ListUtil.toList(ORIGINAL, RNAUTO, RNMAN);
List<OwnerDto> ownerDtos = baseMapper.containSampleId("SAMPLE_ID").stream()
.filter(owner -> CollUtil.contains(suitable, owner.getOwner()))
.collect(Collectors.toList());
String DOT = StrUtil.DOT;
// 手动控制事务
TransactionDefinition txDef = new DefaultTransactionDefinition();
TransactionStatus txStatus = transactionManager.getTransaction(txDef);
try {
List<String> needDel = new ArrayList<>();
if (sampleData){
// 收集所有表名
List<String> allTables = ownerDtos.stream()
.map(owner -> owner.getOwner() + DOT + owner.getTableName())
.collect(Collectors.toList());
// 删除表数据
if (CollUtil.isNotEmpty(allTables))
baseMapper.delTables(allTables, sampleId);
// 收集待删除文件路径
needDel.add(samplePath(savePath, sampleId)); // 原始谱文件
needDel.addAll(manOrAutoPath(savePath, logPath, sampleId, RNMAN)); // 人工交互文件
needDel.addAll(manOrAutoPath(savePath, logPath, sampleId, RNAUTO)); // 自动处理文件
}
else {
if (rnAuto){
// 收集自动处理库所有表名
List<String> autoTables = ownerDtos.stream()
.filter(owner -> StrUtil.equals(owner.getOwner(), RNAUTO))
.map(owner -> owner.getOwner() + DOT + owner.getTableName())
.collect(Collectors.toList());
// 删除表数据
if (CollUtil.isNotEmpty(autoTables))
baseMapper.delTables(autoTables, sampleId);
// 收集待删除文件路径
needDel.addAll(manOrAutoPath(savePath, logPath, sampleId, RNAUTO)); // 自动处理文件
}
if (rnMan){
// 收集人工交互库所有表名
List<String> manTables = ownerDtos.stream()
.filter(owner -> StrUtil.equals(owner.getOwner(), RNMAN))
.map(owner -> owner.getOwner() + DOT + owner.getTableName())
.collect(Collectors.toList());
// 删除表数据
if (CollUtil.isNotEmpty(manTables))
baseMapper.delTables(manTables, sampleId);
// 收集待删除文件路径
needDel.addAll(manOrAutoPath(savePath, logPath, sampleId, RNMAN)); // 人工交互文件
}
}
transactionManager.commit(txStatus);
needDel = needDel.stream().filter(StrUtil::isNotBlank).collect(Collectors.toList());
if (CollUtil.isEmpty(needDel))
return Result.OK("Data cleaning is complete. No files need to be cleaned!");
// 删除FTP文件
List<String> failList = ftpUtil.removeFiles(needDel);
if (CollUtil.isNotEmpty(failList))
return Result.error("Data clearing is complete, but file clearing fails!", failList);
return Result.OK("Data and file cleanup complete!");
}catch (Exception e){
transactionManager.rollback(txStatus);
e.printStackTrace();
return Result.error("Data deletion is abnormal, The file deletion operation has not been performed!");
}
}
@Override
public GardsSampleDataSystem getOne(Integer sampleId) {
LambdaQueryWrapper<GardsSampleDataSystem> wrapper = new LambdaQueryWrapper<>();
wrapper.eq(GardsSampleDataSystem::getSampleId, sampleId);
return Optional.ofNullable(getOne(wrapper))
.orElse(new GardsSampleDataSystem());
}
private String samplePath(String savePath,
Integer sampleId){
GardsSampleDataSystem sampleData = getOne(sampleId);
String inputFileName = sampleData.getInputFileName();
if (StrUtil.isBlank(inputFileName))
return null;
return savePath + inputFileName;
}
private List<String> manOrAutoPath(String savePath,
String logPath,
Integer sampleId,
String owner){
List<String> fileList = new ArrayList<>();
GardsAnalyses analysisMan = baseMapper.getAnalysis(sampleId, owner);
if (ObjectUtil.isNull(analysisMan))
return fileList;
String baselinePath = analysisMan.getBaselinePath();
if (StrUtil.isNotBlank(baselinePath))
fileList.add(savePath + baselinePath);
String lcPath = analysisMan.getLcPath();
if (StrUtil.isNotBlank(lcPath))
fileList.add(savePath + lcPath);
String scacPath = analysisMan.getScacPath();
if (StrUtil.isNotBlank(scacPath))
fileList.add(savePath + scacPath);
if (StrUtil.isNotBlank(analysisMan.getLogPath()))
fileList.add(logPath + analysisMan.getLogPath());
String reportPath = analysisMan.getReportPath();
if (StrUtil.isNotBlank(reportPath))
fileList.add(savePath + reportPath + FileTypeEnum.txt.getType());
return fileList;
}
}