Changes in TPC residuals validation

The meaning of the scdcalib.writeUnfiltered (default: false) ConfigurableParam has changed.
If set to true, tracks that do not pass the validateTrack check will no longer be discarded
but will instead be stored in the standard unbinned residuals data.
The int8_t TrackData::filterFlag field provides the status of the validation check:
-1: no validation check was requested; 0: validation passed; >0: reason for
failing validation (see the validateTrack method).

A new ConfigurableParam, scdcalib.writeValidationData (default: false), has been added.
If set to true, the TrackValidationData object, together with the corresponding TrackData,
will be written to a dedicated debug tree, "valdata", in "track_interpolation_dbg.root"
(or "track_interpolation_dbg_<lane>.root" when multiple lanes are used).

If a track passes validation but some residuals are flagged as bad, the new ConfigurableParam
scdcalib.keepRejectedResiduals (default: false) allows these residuals to be written to the
unbinned residuals tree with the bool UnbinnedResid::rejected flag set
(previously, such residuals were discarded).

Author: shahor02

Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCInterpolationSpec.cxx

@@ -32,6 +32,7 @@
 #include "SpacePoints/SpacePointsCalibConfParam.h"
 #include "Framework/ConfigParamRegistry.h"
 #include "Framework/ControlService.h"
+#include "Framework/DeviceSpec.h"
 
 using namespace o2::framework;
 using namespace o2::globaltracking;
@@ -55,6 +56,9 @@ void TPCInterpolationDPL::init(InitContext& ic)
   if (mProcessSeeds && mSources != mSourcesMap) {
     LOG(fatal) << "process-seeds option is not compatible with using different track sources for vDrift and map extraction";
   }
+  int lane = ic.services().get<const o2::framework::DeviceSpec>().inputTimesliceId;
+  int maxLanes = ic.services().get<const o2::framework::DeviceSpec>().maxInputTimeslices;
+  mInterpolation.setLane(lane, maxLanes);
 }
 
 void TPCInterpolationDPL::updateTimeDependentParams(ProcessingContext& pc)
@@ -136,13 +140,6 @@ void TPCInterpolationDPL::run(ProcessingContext& pc)
   mInterpolation.process();
   mTimer.Stop();
   LOGF(info, "TPC interpolation timing: Cpu: %.3e Real: %.3e s", mTimer.CpuTime(), mTimer.RealTime());
-  if (SpacePointsCalibConfParam::Instance().writeUnfiltered) {
-    // these are the residuals and tracks before outlier rejection; they are not used in production
-    pc.outputs().snapshot(Output{"GLO", "TPCINT_RES", 0}, mInterpolation.getClusterResidualsUnfiltered());
-    if (mSendTrackData) {
-      pc.outputs().snapshot(Output{"GLO", "TPCINT_TRK", 0}, mInterpolation.getReferenceTracksUnfiltered());
-    }
-  }
   pc.outputs().snapshot(Output{"GLO", "UNBINNEDRES", 0}, mInterpolation.getClusterResiduals());
   pc.outputs().snapshot(Output{"GLO", "DETINFORES", 0}, mInterpolation.getClusterResidualsDetInfo());
   pc.outputs().snapshot(Output{"GLO", "TRKREFS", 0}, mInterpolation.getTrackDataCompact());
@@ -157,6 +154,7 @@ void TPCInterpolationDPL::run(ProcessingContext& pc)
 
 void TPCInterpolationDPL::endOfStream(EndOfStreamContext& ec)
 {
+  mInterpolation.finalize();
   LOGF(info, "TPC residuals extraction total timing: Cpu: %.3e Real: %.3e s in %d slots",
        mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
 }
@@ -183,12 +181,6 @@ DataProcessorSpec getTPCInterpolationSpec(GTrackID::mask_t srcCls, GTrackID::mas
                                                               dataRequest->inputs,
                                                               true);
   o2::tpc::VDriftHelper::requestCCDBInputs(dataRequest->inputs);
-  if (SpacePointsCalibConfParam::Instance().writeUnfiltered) {
-    outputs.emplace_back("GLO", "TPCINT_TRK", 0, Lifetime::Timeframe);
-    if (sendTrackData) {
-      outputs.emplace_back("GLO", "TPCINT_RES", 0, Lifetime::Timeframe);
-    }
-  }
   outputs.emplace_back("GLO", "UNBINNEDRES", 0, Lifetime::Timeframe);
   outputs.emplace_back("GLO", "DETINFORES", 0, Lifetime::Timeframe);
   outputs.emplace_back("GLO", "TRKREFS", 0, Lifetime::Timeframe);

Detectors/GlobalTrackingWorkflow/tpcinterpolationworkflow/src/TPCResidualWriterSpec.cxx

@@ -35,8 +35,6 @@ DataProcessorSpec getTPCResidualWriterSpec(bool writeTrackData, bool debugOutput
   return MakeRootTreeWriterSpec("tpc-residuals-writer",
                                 "o2residuals_tpc.root",
                                 "residualsTPC",
-                                BranchDefinition<std::vector<TrackData>>{InputSpec{"tracksUnfiltered", "GLO", "TPCINT_TRK", 0}, "tracksUnfiltered", ((writeUnfiltered && writeTrackData) ? 1 : 0)},
-                                BranchDefinition<std::vector<TPCClusterResiduals>>{InputSpec{"residualsUnfiltered", "GLO", "TPCINT_RES", 0}, "residualsUnfiltered", (writeUnfiltered ? 1 : 0)},
                                 BranchDefinition<std::vector<UnbinnedResid>>{InputSpec{"residuals", "GLO", "UNBINNEDRES"}, "residuals"},
                                 BranchDefinition<std::vector<DetInfoResid>>{InputSpec{"detInfo", "GLO", "DETINFORES"}, "detInfo"},
                                 BranchDefinition<std::vector<TrackDataCompact>>{InputSpec{"trackRefs", "GLO", "TRKREFS"}, "trackRefs"},

Detectors/TPC/calibration/SpacePoints/include/SpacePoints/SpacePointsCalibConfParam.h

@@ -63,7 +63,8 @@ struct SpacePointsCalibConfParam : public o2::conf::ConfigurableParamHelper<Spac
 
   // parameters for outlier rejection
   bool skipOutlierFiltering{false};      ///< if set, the outlier filtering will not be applied at all
-  bool writeUnfiltered{false};           ///< if set, all residuals and track parameters will be aggregated and dumped additionally without outlier rejection
+  bool writeUnfiltered{false};           ///< if set, tracks failing validation will be kept with filterFlag>0 giving the reason of rejection
+  bool keepRejectedResiduals{false};     ///< if set, keep rejected residuals setting rejected flag
   int nMALong{15};                       ///< number of points to be used for moving average (long range)
   int nMAShort{3};                       ///< number of points to be used for estimation of distance from local line (short range)
   float maxRejFrac{.15f};                ///< if the fraction of rejected clusters of a track is higher, the full track is invalidated
@@ -74,6 +75,7 @@ struct SpacePointsCalibConfParam : public o2::conf::ConfigurableParamHelper<Spac
   float maxDevHelixZ = .3f;              ///< max deviation in Z for clusters wrt helix fit
   int minNumberOfAcceptedResiduals = 30; ///< min number of accepted residuals for
   float maxStdDevMA = 25.f;              ///< max cluster std. deviation (Y^2 + Z^2) wrt moving average to accept
+  bool writeValidationData = false;      ///< write track validation data for debugging
 
   // settings for voxel residuals extraction
   bool isBfieldZero = false;           ///< for B=0 we set the radial distortions to zero and don't fit dy vs tgSlp