From ceb16887f250be8ae7c5dd7cd17d4d62356247da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=B5=B7?= Date: Mon, 2 Mar 2026 21:39:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=B2=92=E5=AD=90=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=97=B6=E9=97=B4=E6=8E=92=E5=BA=8F=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/DataProcessWorkPool.cpp | 264 ++++++++++++++++++++++++++-- src/DataProcessWorkPool.h | 21 ++- src/MainWindow.cpp | 6 + src/MeasureAnalysisProjectModel.cpp | 10 ++ src/MeasureAnalysisProjectModel.h | 1 + src/MeasureAnalysisTree.cpp | 5 + src/MeasureAnalysisTree.h | 1 + src/main.cpp | 2 +- 8 files changed, 288 insertions(+), 22 deletions(-) diff --git a/src/DataProcessWorkPool.cpp b/src/DataProcessWorkPool.cpp index 82c633e..c26e642 100644 --- a/src/DataProcessWorkPool.cpp +++ b/src/DataProcessWorkPool.cpp @@ -1,19 +1,25 @@ #include "DataProcessWorkPool.h" #include #include -#include -#include "csv.h" #include "MeasureAnalysisProjectModel.h" #include "OutputInfoDefine.h" +#include "csv.h" +#include +#include +#include +#include +#include +#include +#include using namespace DataProcessWorkPool; using namespace io; -void EveryChannelParticleDataTask::SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename) +void ParticleDataTask::SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename) { this->_all_channel_particle_data_filename = all_channel_particle_data_filename; } -void EveryChannelParticleDataTask::SetFinishedNotifier(QObject* finished_notifier, const char* finished_process, const QString& project_name) +void ParticleDataTask::SetFinishedNotifier(QObject* finished_notifier, const char* finished_process, const QString& project_name) { this->_finished_notifier = finished_notifier; this->_finished_notifier_process = finished_process; @@ -21,28 +27,28 @@ void EveryChannelParticleDataTask::SetFinishedNotifier(QObject* finished_notifie } -const QString& EveryChannelParticleDataTask::GetAllChannelParticleDataFilename() const +const QString& ParticleDataTask::GetAllChannelParticleDataFilename() const { return this->_all_channel_particle_data_filename; } -const QString& EveryChannelParticleDataTask::GetProjectName() const +const QString& ParticleDataTask::GetProjectName() const { return this->_project_name; } -const char* EveryChannelParticleDataTask::GetFinishedNotifierProcess() const +const char* ParticleDataTask::GetFinishedNotifierProcess() const { return this->_finished_notifier_process; } -QObject* EveryChannelParticleDataTask::GetFinishedNotifier() const +QObject* ParticleDataTask::GetFinishedNotifier() const { return this->_finished_notifier; } -bool EveryChannelParticleDataTask::IsValidSetWorkParameters() const +bool ParticleDataTask::IsValidSetWorkParameters() const { return (!GetAllChannelParticleDataFilename().isEmpty()) && (GetFinishedNotifier() != nullptr) && @@ -50,12 +56,12 @@ bool EveryChannelParticleDataTask::IsValidSetWorkParameters() const (GetFinishedNotifierProcess() != nullptr); } -void EveryChannelParticleDataTask::StartTask() +void ParticleDataTask::StartTask() { QThreadPool::globalInstance()->start(this); } -void EveryChannelParticleDataTask::run() +void ParticleDataTask::run() { if (!IsValidSetWorkParameters()) { return; @@ -80,7 +86,7 @@ const QString& EveryChannelParticleDataSeparateTask::GetResultDataDir() const bool EveryChannelParticleDataSeparateTask::IsValidSetWorkParameters() const { - return (!GetResultDataDir().isEmpty()) && EveryChannelParticleDataTask::IsValidSetWorkParameters(); + return (!GetResultDataDir().isEmpty()) && ParticleDataTask::IsValidSetWorkParameters(); } bool EveryChannelParticleDataSeparateTask::processEveryChannelParticleData() @@ -101,8 +107,15 @@ bool EveryChannelParticleDataSeparateTask::processEveryChannelParticleData() std::string address_str = QString(QStringLiteral(u"道址")).toStdString(); std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString(); - io::CSVReader<4> reader(all_channel_particle_data_filename.toStdString()); - reader.read_header(io::ignore_no_column, board_id_str, channel_id_str, address_str, time_str); + // 使用更灵活的方式处理CSV文件，忽略额外列 + io::CSVReader< + 4, + io::trim_chars<' ', '\t'>, + io::double_quote_escape<',', '"'>, + io::throw_on_overflow, + io::empty_line_comment + > reader(all_channel_particle_data_filename.toStdString()); + reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str); uint board_id; uint channel_id; uint address; @@ -177,7 +190,7 @@ bool EveryChannelParticleCountDataTask::IsValidSetWorkParameters() const { return (!GetAllChannelCountResultDir().isEmpty()) && (!GetEveryChannelCountResultDir().isEmpty()) && - EveryChannelParticleDataTask::IsValidSetWorkParameters(); + ParticleDataTask::IsValidSetWorkParameters(); } bool EveryChannelParticleCountDataTask::processEveryChannelParticleData() @@ -208,8 +221,15 @@ bool EveryChannelParticleCountDataTask::processEveryChannelParticleData() std::string address_str = QString(QStringLiteral(u"道址")).toStdString(); std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString(); - io::CSVReader<4> reader(all_channel_particle_data_filename.toStdString()); - reader.read_header(io::ignore_no_column, board_id_str, channel_id_str, address_str, time_str); + // 使用更灵活的方式处理CSV文件，忽略额外列 + io::CSVReader< + 4, + io::trim_chars<' ', '\t'>, + io::double_quote_escape<',', '"'>, + io::throw_on_overflow, + io::empty_line_comment + > reader(all_channel_particle_data_filename.toStdString()); + reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str); uint board_id; uint channel_id; uint address; @@ -300,3 +320,213 @@ bool EveryChannelParticleCountDataTask::processEveryChannelParticleData() return ret_ok; } + +//////////////////////////////////////////////////////////////////////////////////// + +void ParticleDataSortTask::SetSortedResultDir(const QString& sorted_result_dir) +{ + this->_sorted_result_dir = sorted_result_dir; +} + +const QString& ParticleDataSortTask::GetSortedResultDir() const +{ + return this->_sorted_result_dir; +} + +bool ParticleDataSortTask::IsValidSetWorkParameters() const +{ + return (!GetSortedResultDir().isEmpty()) && ParticleDataTask::IsValidSetWorkParameters(); +} + +struct CsvRow { + uint board_id; + uint channel_id; + uint address; + unsigned long long time; + size_t chunk_index; + + bool operator<(const CsvRow& other) const { + return time > other.time; + } +}; + +std::vector splitFile(const std::string& inputFile, size_t chunkSize) { + std::vector chunks; + + try { + std::string board_id_str = QString(QStringLiteral(u"板卡号")).toStdString(); + std::string channel_id_str = QString(QStringLiteral(u"通道号")).toStdString(); + std::string address_str = QString(QStringLiteral(u"道址")).toStdString(); + std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString(); + // Use csv.h to read the file + io::CSVReader<4, io::trim_chars<' ', '\t'>, io::double_quote_escape<',', '"'>, io::throw_on_overflow, io::empty_line_comment> reader(inputFile); + reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str); + + int chunkIndex = 0; + + while (true) { + std::vector rows; + size_t currentSize = 0; + + uint board_id; + uint channel_id; + uint address; + unsigned long long time; + + while (reader.read_row(board_id, channel_id, address, time)) { + CsvRow row; + row.board_id = board_id; + row.channel_id = channel_id; + row.address = address; + row.time = time; + + // Estimate row size + currentSize += std::to_string(board_id).size() + + std::to_string(channel_id).size() + + std::to_string(address).size() + + std::to_string(time).size() + 4; // +4 for commas + + if (currentSize > chunkSize && !rows.empty()) { + break; + } + + rows.push_back(row); + } + + if (rows.empty()) break; + + std::sort(rows.begin(), rows.end(), [](const CsvRow& a, const CsvRow& b) { + return a.time < b.time; + }); + + std::string chunkFile = inputFile + ".chunk" + std::to_string(chunkIndex); + std::ofstream outFile(chunkFile); + for (const auto& row : rows) { + outFile << row.board_id << "," << row.channel_id << "," << row.address << "," << row.time << "\n"; + } + outFile.close(); + + chunks.push_back(chunkFile); + chunkIndex++; + } + } catch (const std::exception& e) { + // Handle exception + } + + return chunks; +} + +void mergeChunks(const std::vector& chunks, const std::string& outputFile) { + std::vector>> chunkReaders; + std::priority_queue minHeap; + + for (const auto& chunk : chunks) { + auto reader = std::make_unique>(chunk); + chunkReaders.push_back(std::move(reader)); + } + + for (size_t i = 0; i < chunkReaders.size(); i++) { + uint board_id; + uint channel_id; + uint address; + unsigned long long time; + + if (chunkReaders[i]->read_row(board_id, channel_id, address, time)) { + CsvRow row; + row.board_id = board_id; + row.channel_id = channel_id; + row.address = address; + row.time = time; + row.chunk_index = i; + minHeap.push(row); + } + } + + std::string board_id_str = QString(QStringLiteral(u"板卡号")).toStdString(); + std::string channel_id_str = QString(QStringLiteral(u"通道号")).toStdString(); + std::string address_str = QString(QStringLiteral(u"道址")).toStdString(); + std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString(); + std::ofstream outFile(outputFile); + outFile << board_id_str << "," << channel_id_str << "," << address_str << "," << time_str << "\n"; + + while (!minHeap.empty()) { + CsvRow current = minHeap.top(); + minHeap.pop(); + + outFile << current.board_id << "," << current.channel_id << "," << current.address << "," << current.time << "\n"; + + size_t chunk_index = current.chunk_index; + if (chunkReaders[chunk_index]) { + uint board_id; + uint channel_id; + uint address; + unsigned long long time; + + if (chunkReaders[chunk_index]->read_row(board_id, channel_id, address, time)) { + CsvRow row; + row.board_id = board_id; + row.channel_id = channel_id; + row.address = address; + row.time = time; + row.chunk_index = chunk_index; + minHeap.push(row); + } else { + chunkReaders[chunk_index].reset(); + } + } + } + + outFile.close(); + + for (const auto& chunk : chunks) { + std::remove(chunk.c_str()); + } +} + +bool ParticleDataSortTask::processEveryChannelParticleData() +{ + bool ret_ok = true; + const QString& sorted_result_dir = GetSortedResultDir(); + QDir sorted_result_output_dir(sorted_result_dir); + sorted_result_output_dir.mkpath(sorted_result_dir); + + const QString& all_channel_particle_data_filename = GetAllChannelParticleDataFilename(); + QString sorted_output_filename = sorted_result_output_dir.filePath("SortedParticleData.csv"); + + try { + const size_t CHUNK_SIZE = 100 * 1024 * 1024; // 100MB chunks + std::vector chunks = splitFile(all_channel_particle_data_filename.toStdString(), CHUNK_SIZE); + + if (chunks.empty()) { + std::ifstream inFile(all_channel_particle_data_filename.toStdString()); + std::ofstream outFile(sorted_output_filename.toStdString()); + std::string line; + while (std::getline(inFile, line)) { + outFile << line << "\n"; + } + inFile.close(); + outFile.close(); + } else { + mergeChunks(chunks, sorted_output_filename.toStdString()); + } + + } catch (const std::exception& e) { + QString error = QString(QStringLiteral(u"处理%1异常:%2")).arg(all_channel_particle_data_filename).arg(e.what()); + LOG_ERROR(error) + ret_ok = false; + } catch (...) { + QString error = QString(QStringLiteral(u"处理%1未知异常.")).arg(all_channel_particle_data_filename); + LOG_ERROR(error) + ret_ok = false; + } + + const QString& project_name = GetProjectName(); + MeasureAnalysisProjectModel* project_model = MeasureAnalysisProjectModelList::GetProjectModel(project_name); + if (project_model == nullptr) { + ret_ok = false; + } else { + project_model->SetSortedParticleDataFilename(sorted_output_filename); + } + + return ret_ok; +} diff --git a/src/DataProcessWorkPool.h b/src/DataProcessWorkPool.h index 58a38e1..3a9aa65 100644 --- a/src/DataProcessWorkPool.h +++ b/src/DataProcessWorkPool.h @@ -7,7 +7,7 @@ namespace DataProcessWorkPool { - class EveryChannelParticleDataTask : public QRunnable + class ParticleDataTask : public QRunnable { public: void SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename); @@ -33,7 +33,7 @@ namespace DataProcessWorkPool QString _project_name; }; - class EveryChannelParticleDataSeparateTask : public EveryChannelParticleDataTask + class EveryChannelParticleDataSeparateTask : public ParticleDataTask { public: void SetResultDataDir(const QString& result_data_dir); @@ -46,7 +46,7 @@ namespace DataProcessWorkPool QString _result_data_dir; }; - class EveryChannelParticleCountDataTask : public EveryChannelParticleDataTask + class EveryChannelParticleCountDataTask : public ParticleDataTask { public: void SetAllChannelCountResultDir(const QString& dir_path); @@ -61,6 +61,19 @@ namespace DataProcessWorkPool QString _all_ch_count_dir; QString _every_ch_count_dir; }; + + class ParticleDataSortTask : public ParticleDataTask + { + public: + void SetSortedResultDir(const QString& sorted_result_dir); + const QString& GetSortedResultDir() const; + + virtual bool IsValidSetWorkParameters() const; + private: + virtual bool processEveryChannelParticleData() override; + private: + QString _sorted_result_dir; + }; } -#endif // DATAPROCESSWORKPOOL_H +#endif // DATAPROCESSWORKPOOL_H \ No newline at end of file diff --git a/src/MainWindow.cpp b/src/MainWindow.cpp index efd54a4..4635d63 100644 --- a/src/MainWindow.cpp +++ b/src/MainWindow.cpp @@ -152,6 +152,12 @@ void MainWindow::initAction() // separate_task->SetFinishedNotifier(this->_tree_measure_analysis, "onFinishedSeparateEveryChannelParticleData", project_name); // separate_task->StartTask(); + auto separate_task = new DataProcessWorkPool::ParticleDataSortTask; + separate_task->SetAllChannelParticleDataFilename(project_model->GetAllChannelParticleDataFilename()); + separate_task->SetSortedResultDir(project_model->GetProjectDir()); + separate_task->SetFinishedNotifier(this->_tree_measure_analysis, "onFinishedParticleSortData", project_name); + separate_task->StartTask(); + const QString& all_ch_count_dir = project_model->GetProjectDir(); const QString& every_ch_count_dir = QDir(project_model->GetProjectDir()).filePath("EveryChannelParticleCountData"); auto count_task = new DataProcessWorkPool::EveryChannelParticleCountDataTask; diff --git a/src/MeasureAnalysisProjectModel.cpp b/src/MeasureAnalysisProjectModel.cpp index 4277b3e..9b66a99 100644 --- a/src/MeasureAnalysisProjectModel.cpp +++ b/src/MeasureAnalysisProjectModel.cpp @@ -60,6 +60,11 @@ void MeasureAnalysisProjectModel::SetAllChannelParticleDataFilename(const QStrin this->_all_channel_particle_data_filename = filename; } +void MeasureAnalysisProjectModel::SetSortedParticleDataFilename(const QString &filename) +{ + this->_sorted_particle_data_filename = filename; +} + void MeasureAnalysisProjectModel::SetChannelParticleDataFilename(uint channel, const QString& filename) { this->_channel_particle_data_filename_list[channel] = filename; @@ -140,6 +145,11 @@ const QString& MeasureAnalysisProjectModel::GetAllChannelParticleDataFilename() return this->_all_channel_particle_data_filename; } +const QString &MeasureAnalysisProjectModel::GetSortParticleDataFilename() const +{ + return this->_sorted_particle_data_filename; +} + const QMap& MeasureAnalysisProjectModel::GetChannelParticleDataFilenameList() const { return this->_channel_particle_data_filename_list; diff --git a/src/MeasureAnalysisProjectModel.h b/src/MeasureAnalysisProjectModel.h index bf60003..b16b98e 100644 --- a/src/MeasureAnalysisProjectModel.h +++ b/src/MeasureAnalysisProjectModel.h @@ -29,6 +29,7 @@ public: void SetEneryScaleFilename(const QString& filename); void SetEfficiencyScaleFilename(const QString& filename); void SetAllChannelParticleDataFilename(const QString& filename); + void SetSortedParticleDataFilename(const QString& filename); void SetChannelParticleDataFilename(uint channel, const QString& filename); void SetChannelParticleCountDataFilename(uint channel, const QString& filename); void SetAllChannelParticleTotalCountDataFilename(const QString& filename); diff --git a/src/MeasureAnalysisTree.cpp b/src/MeasureAnalysisTree.cpp index 1ba47fc..01c0e22 100644 --- a/src/MeasureAnalysisTree.cpp +++ b/src/MeasureAnalysisTree.cpp @@ -176,6 +176,11 @@ void TreeWidget::AddProjectModel(MeasureAnalysisProjectModel* model) } } +void TreeWidget::onFinishedParticleSortData(const QString &project_name) +{ + +} + /* void TreeWidget::onFinishedSeparateEveryChannelParticleData(const QString& project_name) { diff --git a/src/MeasureAnalysisTree.h b/src/MeasureAnalysisTree.h index a64007a..e5da2d8 100644 --- a/src/MeasureAnalysisTree.h +++ b/src/MeasureAnalysisTree.h @@ -25,6 +25,7 @@ public: private slots: // void onFinishedSeparateEveryChannelParticleData(const QString& project_name); + void onFinishedParticleSortData(const QString& project_name); void onFinishedParticleCountData(const QString& project_name); signals: diff --git a/src/main.cpp b/src/main.cpp index 71c620b..91b749a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -61,5 +61,5 @@ int main(int argc, char *argv[]) MainWindow w; w.showMaximized(); - return app.exec(); + return app.exec(); }