添加粒子数据时间排序处理

src/DataProcessWorkPool.cpp

@@ -1,19 +1,25 @@
 #include "DataProcessWorkPool.h"
 #include <QThreadPool>
 #include <QDir>
-#include <fstream>
-#include "csv.h"
 #include "MeasureAnalysisProjectModel.h"
 #include "OutputInfoDefine.h"
+#include "csv.h"
+#include <fstream>
+#include <vector>
+#include <algorithm>
+#include <queue>
+#include <memory>
+#include <string>
+#include <sstream>
 
 using namespace DataProcessWorkPool;
 using namespace io;
 
-void EveryChannelParticleDataTask::SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename)
+void ParticleDataTask::SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename)
 {
     this->_all_channel_particle_data_filename = all_channel_particle_data_filename;
 }
-void EveryChannelParticleDataTask::SetFinishedNotifier(QObject* finished_notifier, const char* finished_process, const QString& project_name)
+void ParticleDataTask::SetFinishedNotifier(QObject* finished_notifier, const char* finished_process, const QString& project_name)
 {
     this->_finished_notifier = finished_notifier;
     this->_finished_notifier_process = finished_process;
@@ -21,28 +27,28 @@ void EveryChannelParticleDataTask::SetFinishedNotifier(QObject* finished_notifie
 }
 
 
-const QString& EveryChannelParticleDataTask::GetAllChannelParticleDataFilename() const
+const QString& ParticleDataTask::GetAllChannelParticleDataFilename() const
 {
     return this->_all_channel_particle_data_filename;
 }
 
 
-const QString& EveryChannelParticleDataTask::GetProjectName() const
+const QString& ParticleDataTask::GetProjectName() const
 {
     return this->_project_name;
 }
 
-const char* EveryChannelParticleDataTask::GetFinishedNotifierProcess() const
+const char* ParticleDataTask::GetFinishedNotifierProcess() const
 {
     return this->_finished_notifier_process;
 }
 
-QObject* EveryChannelParticleDataTask::GetFinishedNotifier() const
+QObject* ParticleDataTask::GetFinishedNotifier() const
 {
     return this->_finished_notifier;
 }
 
-bool EveryChannelParticleDataTask::IsValidSetWorkParameters() const
+bool ParticleDataTask::IsValidSetWorkParameters() const
 {
     return (!GetAllChannelParticleDataFilename().isEmpty()) &&
            (GetFinishedNotifier() != nullptr) &&
@@ -50,12 +56,12 @@ bool EveryChannelParticleDataTask::IsValidSetWorkParameters() const
            (GetFinishedNotifierProcess() != nullptr);
 }
 
-void EveryChannelParticleDataTask::StartTask()
+void ParticleDataTask::StartTask()
 {
     QThreadPool::globalInstance()->start(this);
 }
 
-void EveryChannelParticleDataTask::run()
+void ParticleDataTask::run()
 {
     if (!IsValidSetWorkParameters()) {
         return;
@@ -80,7 +86,7 @@ const QString& EveryChannelParticleDataSeparateTask::GetResultDataDir() const
 
 bool EveryChannelParticleDataSeparateTask::IsValidSetWorkParameters() const
 {
-    return (!GetResultDataDir().isEmpty()) && EveryChannelParticleDataTask::IsValidSetWorkParameters();
+    return (!GetResultDataDir().isEmpty()) && ParticleDataTask::IsValidSetWorkParameters();
 }
 
 bool EveryChannelParticleDataSeparateTask::processEveryChannelParticleData()
@@ -101,8 +107,15 @@ bool EveryChannelParticleDataSeparateTask::processEveryChannelParticleData()
         std::string address_str = QString(QStringLiteral(u"道址")).toStdString();
         std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString();
 
-        io::CSVReader<4> reader(all_channel_particle_data_filename.toStdString());
+        // 使用更灵活的方式处理CSV文件，忽略额外列
-        reader.read_header(io::ignore_no_column, board_id_str, channel_id_str, address_str, time_str);
+        io::CSVReader<
+            4,
+            io::trim_chars<' ', '\t'>,
+            io::double_quote_escape<',', '"'>,
+            io::throw_on_overflow,
+            io::empty_line_comment
+            > reader(all_channel_particle_data_filename.toStdString());
+        reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str);
         uint board_id;
         uint channel_id;
         uint address;
@@ -177,7 +190,7 @@ bool EveryChannelParticleCountDataTask::IsValidSetWorkParameters() const
 {
     return (!GetAllChannelCountResultDir().isEmpty()) && 
            (!GetEveryChannelCountResultDir().isEmpty()) && 
-           EveryChannelParticleDataTask::IsValidSetWorkParameters();
+           ParticleDataTask::IsValidSetWorkParameters();
 }
 
 bool EveryChannelParticleCountDataTask::processEveryChannelParticleData()
@@ -208,8 +221,15 @@ bool EveryChannelParticleCountDataTask::processEveryChannelParticleData()
         std::string address_str = QString(QStringLiteral(u"道址")).toStdString();
         std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString();
         
-        io::CSVReader<4> reader(all_channel_particle_data_filename.toStdString());
+        // 使用更灵活的方式处理CSV文件，忽略额外列
-        reader.read_header(io::ignore_no_column, board_id_str, channel_id_str, address_str, time_str);
+        io::CSVReader<
+            4,
+            io::trim_chars<' ', '\t'>,
+            io::double_quote_escape<',', '"'>,
+            io::throw_on_overflow,
+            io::empty_line_comment
+            > reader(all_channel_particle_data_filename.toStdString());
+        reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str);
         uint board_id;
         uint channel_id;
         uint address;
@@ -300,3 +320,213 @@ bool EveryChannelParticleCountDataTask::processEveryChannelParticleData()
 
     return ret_ok;
 }
+
+////////////////////////////////////////////////////////////////////////////////////
+
+void ParticleDataSortTask::SetSortedResultDir(const QString& sorted_result_dir)
+{
+    this->_sorted_result_dir = sorted_result_dir;
+}
+
+const QString& ParticleDataSortTask::GetSortedResultDir() const
+{
+    return this->_sorted_result_dir;
+}
+
+bool ParticleDataSortTask::IsValidSetWorkParameters() const
+{
+    return (!GetSortedResultDir().isEmpty()) && ParticleDataTask::IsValidSetWorkParameters();
+}
+
+struct CsvRow {
+    uint board_id;
+    uint channel_id;
+    uint address;
+    unsigned long long time;
+    size_t chunk_index;
+
+    bool operator<(const CsvRow& other) const {
+        return time > other.time;
+    }
+};
+
+std::vector<std::string> splitFile(const std::string& inputFile, size_t chunkSize) {
+    std::vector<std::string> chunks;
+    
+    try {
+        std::string board_id_str = QString(QStringLiteral(u"板卡号")).toStdString();
+        std::string channel_id_str = QString(QStringLiteral(u"通道号")).toStdString();
+        std::string address_str = QString(QStringLiteral(u"道址")).toStdString();
+        std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString();
+        // Use csv.h to read the file
+        io::CSVReader<4, io::trim_chars<' ', '\t'>, io::double_quote_escape<',', '"'>, io::throw_on_overflow, io::empty_line_comment> reader(inputFile);
+        reader.read_header(io::ignore_extra_column, board_id_str, channel_id_str, address_str, time_str);
+        
+        int chunkIndex = 0;
+        
+        while (true) {
+            std::vector<CsvRow> rows;
+            size_t currentSize = 0;
+            
+            uint board_id;
+            uint channel_id;
+            uint address;
+            unsigned long long time;
+            
+            while (reader.read_row(board_id, channel_id, address, time)) {
+                CsvRow row;
+                row.board_id = board_id;
+                row.channel_id = channel_id;
+                row.address = address;
+                row.time = time;
+                
+                // Estimate row size
+                currentSize += std::to_string(board_id).size() + 
+                              std::to_string(channel_id).size() + 
+                              std::to_string(address).size() + 
+                              std::to_string(time).size() + 4; // +4 for commas
+                
+                if (currentSize > chunkSize && !rows.empty()) {
+                    break;
+                }
+                
+                rows.push_back(row);
+            }
+            
+            if (rows.empty()) break;
+            
+            std::sort(rows.begin(), rows.end(), [](const CsvRow& a, const CsvRow& b) {
+                return a.time < b.time;
+            });
+            
+            std::string chunkFile = inputFile + ".chunk" + std::to_string(chunkIndex);
+            std::ofstream outFile(chunkFile);
+            for (const auto& row : rows) {
+                outFile << row.board_id << "," << row.channel_id << "," << row.address << "," << row.time << "\n";
+            }
+            outFile.close();
+            
+            chunks.push_back(chunkFile);
+            chunkIndex++;
+        }
+    } catch (const std::exception& e) {
+        // Handle exception
+    }
+    
+    return chunks;
+}
+
+void mergeChunks(const std::vector<std::string>& chunks, const std::string& outputFile) {
+    std::vector<std::unique_ptr<io::CSVReader<4>>> chunkReaders;
+    std::priority_queue<CsvRow> minHeap;
+
+    for (const auto& chunk : chunks) {
+        auto reader = std::make_unique<io::CSVReader<4>>(chunk);
+        chunkReaders.push_back(std::move(reader));
+    }
+    
+    for (size_t i = 0; i < chunkReaders.size(); i++) {
+        uint board_id;
+        uint channel_id;
+        uint address;
+        unsigned long long time;
+        
+        if (chunkReaders[i]->read_row(board_id, channel_id, address, time)) {
+            CsvRow row;
+            row.board_id = board_id;
+            row.channel_id = channel_id;
+            row.address = address;
+            row.time = time;
+            row.chunk_index = i;
+            minHeap.push(row);
+        }
+    }
+    
+    std::string board_id_str = QString(QStringLiteral(u"板卡号")).toStdString();
+    std::string channel_id_str = QString(QStringLiteral(u"通道号")).toStdString();
+    std::string address_str = QString(QStringLiteral(u"道址")).toStdString();
+    std::string time_str = QString(QStringLiteral(u"时间计数")).toStdString();
+    std::ofstream outFile(outputFile);
+    outFile << board_id_str << "," << channel_id_str << "," << address_str << "," << time_str << "\n";
+    
+    while (!minHeap.empty()) {
+        CsvRow current = minHeap.top();
+        minHeap.pop();
+
+        outFile << current.board_id << "," << current.channel_id << "," << current.address << "," << current.time << "\n";
+        
+        size_t chunk_index = current.chunk_index;
+        if (chunkReaders[chunk_index]) {
+            uint board_id;
+            uint channel_id;
+            uint address;
+            unsigned long long time;
+            
+            if (chunkReaders[chunk_index]->read_row(board_id, channel_id, address, time)) {
+                CsvRow row;
+                row.board_id = board_id;
+                row.channel_id = channel_id;
+                row.address = address;
+                row.time = time;
+                row.chunk_index = chunk_index;
+                minHeap.push(row);
+            } else {
+                chunkReaders[chunk_index].reset();
+            }
+        }
+    }
+    
+    outFile.close();
+
+    for (const auto& chunk : chunks) {
+        std::remove(chunk.c_str());
+    }
+}
+
+bool ParticleDataSortTask::processEveryChannelParticleData()
+{
+    bool ret_ok = true;
+    const QString& sorted_result_dir = GetSortedResultDir();
+    QDir sorted_result_output_dir(sorted_result_dir);
+    sorted_result_output_dir.mkpath(sorted_result_dir);
+
+    const QString& all_channel_particle_data_filename = GetAllChannelParticleDataFilename();
+    QString sorted_output_filename = sorted_result_output_dir.filePath("SortedParticleData.csv");
+
+    try {
+        const size_t CHUNK_SIZE = 100 * 1024 * 1024; // 100MB chunks
+        std::vector<std::string> chunks = splitFile(all_channel_particle_data_filename.toStdString(), CHUNK_SIZE);
+        
+        if (chunks.empty()) {
+            std::ifstream inFile(all_channel_particle_data_filename.toStdString());
+            std::ofstream outFile(sorted_output_filename.toStdString());
+            std::string line;
+            while (std::getline(inFile, line)) {
+                outFile << line << "\n";
+            }
+            inFile.close();
+            outFile.close();
+        } else {
+            mergeChunks(chunks, sorted_output_filename.toStdString());
+        }
+        
+    } catch (const std::exception& e) {
+        QString error = QString(QStringLiteral(u"处理%1异常:%2")).arg(all_channel_particle_data_filename).arg(e.what());
+        LOG_ERROR(error)
+        ret_ok = false;
+    } catch (...) {
+        QString error = QString(QStringLiteral(u"处理%1未知异常.")).arg(all_channel_particle_data_filename);
+        LOG_ERROR(error)
+        ret_ok = false;
+    }
+
+    const QString& project_name = GetProjectName();
+    MeasureAnalysisProjectModel* project_model = MeasureAnalysisProjectModelList::GetProjectModel(project_name);
+    if (project_model == nullptr) {
+        ret_ok = false;
+    } else {
+        project_model->SetSortedParticleDataFilename(sorted_output_filename);
+    }
+
+    return ret_ok;
+}

src/DataProcessWorkPool.h

@@ -7,7 +7,7 @@
 
 namespace DataProcessWorkPool
 {
-    class EveryChannelParticleDataTask : public QRunnable
+    class ParticleDataTask : public QRunnable
     {
     public:
         void SetAllChannelParticleDataFilename(const QString& all_channel_particle_data_filename);
@@ -33,7 +33,7 @@ namespace DataProcessWorkPool
         QString _project_name;
     };
     
-    class EveryChannelParticleDataSeparateTask : public EveryChannelParticleDataTask
+    class EveryChannelParticleDataSeparateTask : public ParticleDataTask
     {
     public:
         void SetResultDataDir(const QString& result_data_dir);
@@ -46,7 +46,7 @@ namespace DataProcessWorkPool
         QString _result_data_dir;
     };
 
-    class EveryChannelParticleCountDataTask : public EveryChannelParticleDataTask
+    class EveryChannelParticleCountDataTask : public ParticleDataTask
     {
     public:
         void SetAllChannelCountResultDir(const QString& dir_path);
@@ -61,6 +61,19 @@ namespace DataProcessWorkPool
         QString _all_ch_count_dir;
         QString _every_ch_count_dir;
     };
+
+    class ParticleDataSortTask : public ParticleDataTask
+    {
+    public:
+        void SetSortedResultDir(const QString& sorted_result_dir);
+        const QString& GetSortedResultDir() const;
+
+        virtual bool IsValidSetWorkParameters() const;
+    private:
+        virtual bool processEveryChannelParticleData() override;
+    private:
+        QString _sorted_result_dir;
+    };
 }
 
 #endif // DATAPROCESSWORKPOOL_H

src/MainWindow.cpp

@@ -152,6 +152,12 @@ void MainWindow::initAction()
                 // separate_task->SetFinishedNotifier(this->_tree_measure_analysis, "onFinishedSeparateEveryChannelParticleData", project_name);
                 // separate_task->StartTask();
 
+                auto separate_task = new DataProcessWorkPool::ParticleDataSortTask;
+                separate_task->SetAllChannelParticleDataFilename(project_model->GetAllChannelParticleDataFilename());
+                separate_task->SetSortedResultDir(project_model->GetProjectDir());
+                separate_task->SetFinishedNotifier(this->_tree_measure_analysis, "onFinishedParticleSortData", project_name);
+                separate_task->StartTask();
+
                 const QString& all_ch_count_dir = project_model->GetProjectDir();
                 const QString& every_ch_count_dir = QDir(project_model->GetProjectDir()).filePath("EveryChannelParticleCountData");
                 auto count_task = new DataProcessWorkPool::EveryChannelParticleCountDataTask;

src/MeasureAnalysisProjectModel.cpp

@@ -60,6 +60,11 @@ void MeasureAnalysisProjectModel::SetAllChannelParticleDataFilename(const QStrin
     this->_all_channel_particle_data_filename = filename;
 }
 
+void MeasureAnalysisProjectModel::SetSortedParticleDataFilename(const QString &filename)
+{
+    this->_sorted_particle_data_filename = filename;
+}
+
 void MeasureAnalysisProjectModel::SetChannelParticleDataFilename(uint channel, const QString& filename) 
 {
     this->_channel_particle_data_filename_list[channel] = filename;
@@ -140,6 +145,11 @@ const QString& MeasureAnalysisProjectModel::GetAllChannelParticleDataFilename()
     return this->_all_channel_particle_data_filename;
 }
 
+const QString &MeasureAnalysisProjectModel::GetSortParticleDataFilename() const
+{
+    return this->_sorted_particle_data_filename;
+}
+
 const QMap<uint, QString>& MeasureAnalysisProjectModel::GetChannelParticleDataFilenameList() const
 {
     return this->_channel_particle_data_filename_list;

src/MeasureAnalysisProjectModel.h

@@ -29,6 +29,7 @@ public:
     void SetEneryScaleFilename(const QString& filename);
     void SetEfficiencyScaleFilename(const QString& filename);
     void SetAllChannelParticleDataFilename(const QString& filename);
+    void SetSortedParticleDataFilename(const QString& filename);
     void SetChannelParticleDataFilename(uint channel, const QString& filename);
     void SetChannelParticleCountDataFilename(uint channel, const QString& filename);
     void SetAllChannelParticleTotalCountDataFilename(const QString& filename);

src/MeasureAnalysisTree.cpp

@@ -176,6 +176,11 @@ void TreeWidget::AddProjectModel(MeasureAnalysisProjectModel* model)
     }
 }
 
+void TreeWidget::onFinishedParticleSortData(const QString &project_name)
+{
+
+}
+
 /*
 void TreeWidget::onFinishedSeparateEveryChannelParticleData(const QString& project_name)
 {

src/MeasureAnalysisTree.h

@@ -25,6 +25,7 @@ public:
 
 private slots:
     // void onFinishedSeparateEveryChannelParticleData(const QString& project_name);
+    void onFinishedParticleSortData(const QString& project_name);
     void onFinishedParticleCountData(const QString& project_name);
 
 signals: