Work on ZS encoding / decoding + unrelated commits from my dev branch that give conflicts if not merged together

DataFormats/Detectors/TPC/include/DataFormatsTPC/Constants.h

@@ -33,6 +33,9 @@ class Constants
 #else
   static constexpr int MAXGLOBALPADROW = 152; // Correct number of pad rows in Run 3
 #endif
+
+  // number of LHC bunch crossings per TPC time bin (40 MHz / 5 MHz)
+  static constexpr int LHCBCPERTIMEBIN = 8;
 };
 } // namespace tpc
 } // namespace o2

Detectors/Raw/CMakeLists.txt

@@ -11,49 +11,45 @@
 o2_add_library(DetectorsRaw
                SOURCES src/RawFileReader.cxx
                        src/RawFileWriter.cxx
-	       	       src/SimpleRawReader.cxx
-	               src/HBFUtils.cxx
-	               src/RDHUtils.cxx
+                       src/SimpleRawReader.cxx
+                       src/HBFUtils.cxx
+                       src/RDHUtils.cxx
                PUBLIC_LINK_LIBRARIES FairRoot::Base
-		                     O2::Headers
-		                     O2::CommonDataFormat  
+                                     O2::Headers
+                                     O2::CommonDataFormat
                                      O2::DetectorsCommonDataFormats
-				     O2::Framework
-				     FairMQ::FairMQ)
+                                     O2::Framework
+                                     FairMQ::FairMQ)
 
 o2_target_root_dictionary(DetectorsRaw
                           HEADERS include/DetectorsRaw/RawFileReader.h
-                                  include/DetectorsRaw/RawFileWriter.h
-			          include/DetectorsRaw/SimpleRawReader.h
-				  include/DetectorsRaw/HBFUtils.h
-				  include/DetectorsRaw/RDHUtils.h)
+                          include/DetectorsRaw/RawFileWriter.h
+                          include/DetectorsRaw/SimpleRawReader.h
+                          include/DetectorsRaw/HBFUtils.h
+                          include/DetectorsRaw/RDHUtils.h)
 
-
-
-o2_add_executable(file-check 
+o2_add_executable(file-check
                   COMPONENT_NAME raw
                   SOURCES src/rawfileCheck.cxx
                   PUBLIC_LINK_LIBRARIES O2::DetectorsRaw
-                                        Boost::program_options)
+                  Boost::program_options)
 
 o2_add_executable(file-reader-workflow
                   COMPONENT_NAME raw
                   SOURCES src/rawfile-reader-workflow.cxx
-     		          src/RawFileReaderWorkflow.cxx
+                  src/RawFileReaderWorkflow.cxx
                   PUBLIC_LINK_LIBRARIES O2::DetectorsRaw)
 
-
 o2_add_test(HBFUtils
             PUBLIC_LINK_LIBRARIES O2::DetectorsRaw
                                   O2::Steer
             SOURCES test/testHBFUtils.cxx
             COMPONENT_NAME raw
             LABELS raw)
-	  
+
 o2_add_test(RawReaderWriter
             PUBLIC_LINK_LIBRARIES O2::DetectorsRaw
                                   O2::Steer
             SOURCES test/testRawReaderWriter.cxx
             COMPONENT_NAME raw
             LABELS raw)
-

Detectors/TPC/simulation/CMakeLists.txt

@@ -41,6 +41,11 @@ o2_target_root_dictionary(TPCSimulation
                                   include/TPCSimulation/SAMPAProcessing.h
                                   include/TPCSimulation/SpaceCharge.h)
 
+o2_add_executable(digits-to-rawzs
+                  COMPONENT_NAME tpc
+                  PUBLIC_LINK_LIBRARIES O2::TPCBase O2::SimulationDataFormat O2::GPUTracking O2::DetectorsRaw
+                  SOURCES run/convertDigitsToRawZS.cxx)
+
 o2_data_file(COPY files DESTINATION Detectors/TPC)
 o2_data_file(COPY data  DESTINATION Detectors/TPC/simulation)
 

Detectors/TPC/simulation/run/convertDigitsToRawZS.cxx

@@ -0,0 +1,200 @@
+// Copyright CERN and copyright holders of ALICE O2. This software is
+// distributed under the terms of the GNU General Public License v3 (GPL
+// Version 3), copied verbatim in the file "COPYING".
+//
+// See http://alice-o2.web.cern.ch/license for full licensing information.
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+/// \file convertDigitsToRawZS.cxx
+/// \author Jens Wiechula (Jens.Wiechula@ikf.uni-frankfurt.de)
+
+#include <boost/program_options.hpp>
+
+#include <string_view>
+#include <memory>
+#include <vector>
+#include <fmt/format.h>
+
+#include "TFile.h"
+#include "TTree.h"
+#include "TROOT.h"
+
+#include "GPUO2Interface.h"
+#include "GPUReconstructionConvert.h"
+#include "GPUHostDataTypes.h"
+#include "GPUParam.h"
+#include "Digit.h"
+
+#include "DetectorsRaw/RawFileWriter.h"
+#include "SimulationDataFormat/MCCompLabel.h"
+#include "TPCBase/Digit.h"
+#include "TPCBase/Sector.h"
+#include "DataFormatsTPC/ZeroSuppression.h"
+#include "DataFormatsTPC/Helpers.h"
+#include "DetectorsRaw/HBFUtils.h"
+
+namespace bpo = boost::program_options;
+
+using namespace o2::tpc;
+using namespace o2::gpu;
+using o2::MCCompLabel;
+
+constexpr static size_t NSectors = o2::tpc::Sector::MAXSECTOR;
+constexpr static size_t NEndpoints = o2::gpu::GPUTrackingInOutZS::NENDPOINTS;
+using DigitArray = std::array<gsl::span<const o2::tpc::Digit>, Sector::MAXSECTOR>;
+using MCLabelContainer = o2::dataformats::MCTruthContainer<MCCompLabel>;
+
+struct ProcessAttributes {
+  std::unique_ptr<unsigned long long int[]> zsoutput;
+  std::vector<unsigned int> sizes;
+  MCLabelContainer mctruthArray;
+  std::unique_ptr<o2::gpu::GPUReconstructionConvert> zsEncoder;
+  std::vector<int> inputIds;
+  bool zs12bit = true;
+  bool verify = false;
+  int verbosity = 1;
+};
+
+void convert(DigitArray& inputDigits, ProcessAttributes* processAttributes, o2::raw::RawFileWriter& writer);
+#include "DetectorsRaw/HBFUtils.h"
+void convertDigitsToZSfinal(std::string_view digitsFile, std::string_view outputPath)
+{
+
+  // ===| open file and get tree |==============================================
+  std::unique_ptr<TFile> o2simDigits(TFile::Open(digitsFile.data()));
+  auto treeSim = (TTree*)o2simDigits->Get("o2sim");
+
+  gROOT->cd();
+
+  // ===| set up branch addresses |=============================================
+  MCLabelContainer* vLabelContainers[Sector::MAXSECTOR];             // label container per sector
+  std::vector<Digit>* vDigitsPerSectorCollection[Sector::MAXSECTOR]; // container that keeps Digits per sector
+
+  for (int iSec = 0; iSec < Sector::MAXSECTOR; ++iSec) {
+    vDigitsPerSectorCollection[iSec] = nullptr;
+    treeSim->SetBranchAddress(TString::Format("TPCDigit_%d", iSec), &vDigitsPerSectorCollection[iSec]);
+
+    vLabelContainers[iSec] = nullptr;
+    treeSim->SetBranchAddress(TString::Format("TPCDigitMCTruth_%d", iSec), &vLabelContainers[iSec]);
+  }
+
+  DigitArray inputDigits;
+  ProcessAttributes attr;
+
+  // raw data output
+  o2::raw::RawFileWriter writer;
+
+  const unsigned int defaultLink = 15;
+
+  // set up raw writer
+  std::string outDir{outputPath};
+  if (outDir.empty()) {
+    outDir = "./";
+  }
+  if (outDir.back() != '/') {
+    outDir += '/';
+  }
+  for (unsigned int i = 0; i < NSectors; i++) {
+    for (unsigned int j = 0; j < NEndpoints; j++) {
+      const unsigned int cruInSector = j / 2;
+      const unsigned int cruID = i * 10 + cruInSector;
+      const unsigned int feeid = (cruID << 7) | ((j & 1) << 6) | (defaultLink & 0x3F);
+      writer.registerLink(feeid, cruID, defaultLink, j % 2, fmt::format("{}cru{}.raw", outDir, cruID));
+    }
+  }
+  for (Long64_t ievent = 0; ievent < treeSim->GetEntries(); ++ievent) {
+    treeSim->GetEntry(ievent);
+
+    for (int iSec = 0; iSec < Sector::MAXSECTOR; ++iSec) {
+      inputDigits[iSec] = *vDigitsPerSectorCollection[iSec]; //????
+    }
+    convert(inputDigits, &attr, writer);
+  }
+  // for further use we write the configuration file for the output
+  writer.writeConfFile("TPC", "RAWDATA", fmt::format("{}tpcraw.cfg", outDir));
+}
+
+void convert(DigitArray& inputDigits, ProcessAttributes* processAttributes, o2::raw::RawFileWriter& writer)
+{
+  auto& zsEncoder = processAttributes->zsEncoder;
+  const auto verify = processAttributes->verify;
+  const auto zs12bit = processAttributes->zs12bit;
+  GPUParam _GPUParam;
+  _GPUParam.SetDefaults(5.00668);
+  const GPUParam mGPUParam = _GPUParam;
+
+  std::vector<deprecated::PackedDigit> gpuDigits[NSectors];
+  GPUTrackingInOutDigits gpuDigitsMap;
+
+  //convert to GPU digits
+  const float zsThreshold = 0;
+  for (int i = 0; i < NSectors; i++) {
+    const auto& d = inputDigits[i];
+    gpuDigits[i].reserve(d.size());
+    for (int j = 0; j < d.size(); j++) {
+      if (d[j].getChargeFloat() >= zsThreshold) {
+        gpuDigits[i].emplace_back(
+          deprecated::PackedDigit{
+            d[j].getChargeFloat(),
+            (Timestamp)d[j].getTimeStamp(),
+            (Pad)d[j].getPad(),
+            (Row)d[j].getRow()});
+      }
+    }
+
+    gpuDigitsMap.tpcDigits[i] = gpuDigits[i].data();
+    gpuDigitsMap.nTPCDigits[i] = gpuDigits[i].size();
+  }
+
+  const GPUTrackingInOutDigits gpuDigitsMap2 = std::move(gpuDigitsMap);
+  o2::InteractionRecord ir = o2::raw::HBFUtils::Instance().getFirstIR();
+
+  zsEncoder->RunZSEncoder(&gpuDigitsMap, nullptr, nullptr, &writer, &ir, mGPUParam, zs12bit, verify);
+}
+
+int main(int argc, char** argv)
+{
+  bpo::variables_map vm;
+  bpo::options_description opt_general("Usage:\n  " + std::string(argv[0]) +
+                                       " <cmds/options>\n"
+                                       "  Tool will convert simulation digits to raw zero suppressed data\n"
+                                       "Commands / Options");
+  bpo::options_description opt_hidden("");
+  bpo::options_description opt_all;
+  bpo::positional_options_description opt_pos;
+
+  try {
+    auto add_option = opt_general.add_options();
+    add_option("help,h", "Print this help message");
+    add_option("verbose,v", bpo::value<uint32_t>()->default_value(0), "Select verbosity level [0 = no output]");
+    add_option("input-file,i", bpo::value<std::string>()->required(), "Specifies input file.");
+    add_option("output-dir,o", bpo::value<std::string>()->default_value("./"), "Specify output directory");
+
+    opt_all.add(opt_general).add(opt_hidden);
+    bpo::store(bpo::command_line_parser(argc, argv).options(opt_all).positional(opt_pos).run(), vm);
+
+    if (vm.count("help") || argc == 1) {
+      std::cout << opt_general << std::endl;
+      exit(0);
+    }
+
+    bpo::notify(vm);
+  } catch (bpo::error& e) {
+    std::cerr << "ERROR: " << e.what() << std::endl
+              << std::endl;
+    std::cerr << opt_general << std::endl;
+    exit(1);
+  } catch (std::exception& e) {
+    std::cerr << e.what() << ", application will now exit" << std::endl;
+    exit(2);
+  }
+
+  convertDigitsToZSfinal(
+    vm["input-file"].as<std::string>(),
+    vm["output-dir"].as<std::string>());
+
+  return 0;
+}

Detectors/TPC/workflow/include/TPCWorkflow/CATrackerSpec.h

@@ -22,7 +22,7 @@ namespace tpc
 
 /// create a processor spec
 /// read simulated TPC clusters from file and publish
-framework::DataProcessorSpec getCATrackerSpec(bool processMC, bool caClusterer, std::vector<int> const& inputIds);
+framework::DataProcessorSpec getCATrackerSpec(bool processMC, bool caClusterer, bool zsDecoder, std::vector<int> const& inputIds);
 
 } // end namespace tpc
 } // end namespace o2

Detectors/TPC/workflow/include/TPCWorkflow/RecoWorkflow.h

@@ -28,13 +28,14 @@ namespace tpc
 namespace reco_workflow
 {
 /// define input and output types of the workflow
-enum struct InputType { Digitizer, // directly read digits from channel {TPC:DIGITS}
-                        Digits,    // read digits from file
-                        Raw,       // read hardware clusters in raw page format from file
-                        Clusters,  // read native clusters from file
+enum struct InputType { Digitizer,        // directly read digits from channel {TPC:DIGITS}
+                        Digits,           // read digits from file
+                        ClustersHardware, // read hardware clusters in raw page format from file
+                        Clusters,         // read native clusters from file
+                        ZSRaw,
 };
 enum struct OutputType { Digits,
-                         Raw,
+                         ClustersHardware,
                          Clusters,
                          Tracks,
                          DisableWriter,