Skip to content

Commit 68f5eb5

Browse files
committed
WIP QC-1251 possible solution for postprocessing without accessing DB
This proposes "late tasks", which can take as input merged MOs and any QOs within the same workflow. See JIRA for more deliberations about why it's probably the best approach. Definitely not finished, I'm just saving the progress and making it available for early review.
1 parent ece9f30 commit 68f5eb5

File tree

2 files changed

+208
-0
lines changed

2 files changed

+208
-0
lines changed

Framework/basic-latetask.json

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"qc": {
3+
"config": {
4+
"database": {
5+
"implementation": "CCDB",
6+
"host": "ccdb-test.cern.ch:8080"
7+
},
8+
"Activity": {
9+
"number": "42",
10+
"type": "NONE"
11+
},
12+
"monitoring": {
13+
"url": "infologger:///debug?qc"
14+
},
15+
"consul": {
16+
"url": ""
17+
},
18+
"conditionDB": {
19+
"url": "ccdb-test.cern.ch:8080"
20+
},
21+
"infologger": {
22+
"filterDiscardDebug": "false",
23+
"filterDiscardLevel": "12"
24+
},
25+
"bookkeeping": {
26+
"url": ""
27+
}
28+
},
29+
"tasks": {
30+
"QcTask": {
31+
"active": "true",
32+
"className": "o2::quality_control_modules::skeleton::SkeletonTask",
33+
"moduleName": "QcSkeleton",
34+
"detectorName": "TST",
35+
"cycleDurations": [
36+
{"cycleDurationSeconds": 10, "validitySeconds": 35},
37+
{"cycleDurationSeconds": 12, "validitySeconds": 1}
38+
],
39+
"dataSource": {
40+
"type": "dataSamplingPolicy",
41+
"name": "tst-raw"
42+
}
43+
}
44+
},
45+
"checks": {
46+
"QcCheck": {
47+
"active": "true",
48+
"className": "o2::quality_control_modules::skeleton::SkeletonCheck",
49+
"moduleName": "QcSkeleton",
50+
"policy": "OnAny",
51+
"detectorName": "TST",
52+
"dataSource": [{
53+
"type": "Task",
54+
"name": "QcTask",
55+
"MOs": ["example"]
56+
}]
57+
}
58+
},
59+
"lateTasks": {
60+
"QcTaskLate": {
61+
"active": "true",
62+
"className": "o2::quality_control::core::GeneralPurposeLateTask",
63+
"moduleName": "QualityControl",
64+
"detectorName": "TST",
65+
"policy": "OnAll", "": "just like Checks and Aggregators, late tasks support update policies",
66+
"dataSources": [{ "" : "provides the late task with an MO",
67+
"type": "Task",
68+
"name": "QcTask",
69+
"MOs": ["example"]
70+
}, {
71+
"type": "TaskReduced", "": "provides the late task with an object reduced to a few statistics",
72+
"name": "QcTask",
73+
"MOs": ["example"],
74+
"reducerName": "o2::quality_control_modules::common::TH1Reducer",
75+
"moduleName": "QcCommon"
76+
}
77+
],
78+
"TODO": "how to describe expected plots?"
79+
}
80+
}
81+
},
82+
"dataSamplingPolicies": [
83+
{
84+
"id": "tst-raw",
85+
"active": "true",
86+
"machines": [],
87+
"query": "data:TST/RAWDATA/0",
88+
"samplingConditions": [
89+
{
90+
"condition": "random",
91+
"fraction": "0.1",
92+
"seed": "1234"
93+
}
94+
]
95+
}
96+
]
97+
}
Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
///
13+
/// \file LateTaskInterface.h
14+
/// \author Piotr Konopka
15+
///
16+
17+
#ifndef LATETASKINTERFACE_H
18+
#define LATETASKINTERFACE_H
19+
20+
#include <memory>
21+
// O2
22+
#include <Framework/InitContext.h>
23+
#include <Framework/ProcessingContext.h>
24+
// QC
25+
#include "QualityControl/Activity.h"
26+
#include "QualityControl/ObjectsManager.h"
27+
#include "QualityControl/UserCodeInterface.h"
28+
#include "QualityControl/MonitorObject.h"
29+
#include "QualityControl/QualityObject.h"
30+
31+
namespace o2::monitoring
32+
{
33+
class Monitoring;
34+
}
35+
36+
//namespace o2::globaltracking
37+
//{
38+
//struct DataRequest;
39+
//}
40+
41+
namespace o2::framework
42+
{
43+
struct ConcreteDataMatcher;
44+
}
45+
46+
namespace o2::quality_control::core
47+
{
48+
49+
/// \brief Skeleton of a late QC task.
50+
///
51+
/// Purely abstract class defining the skeleton and the common interface of a late QC task.
52+
/// It is therefore the parent class of any late QC task.
53+
/// It is responsible for the instantiation, modification and destruction of the TObjects that are published.
54+
///
55+
/// Late tasks can process any output of a Task, Check or Aggregator and produce new MonitorObjects.
56+
/// In a multinode setup, they always run on remote (QC) nodes, so they can access merged MonitorObjects and any
57+
/// QualityObjects. Thus, it is not possible to run late Tasks on FLPs or EPNs.
58+
/// In async QC, late Tasks can be used in combination with a QCDB reader (not implemented yet) to perform
59+
/// any trends or correlations on series of objects which are available only in the QCDB.
60+
///
61+
/// TODO: one could even consider allowing to feed late tasks with output of Reductors.
62+
/// It could be an opportunity to refactor them as well (and rename them to Reducers, which sounds more like English).
63+
///
64+
/// \author Piotr Konopka
65+
class LateTaskInterface : public UserCodeInterface
66+
{
67+
public:
68+
/// \brief Constructor
69+
/// Can't be used when dynamically loading the class with ROOT.
70+
/// @param objectsManager
71+
explicit LateTaskInterface(ObjectsManager* objectsManager);
72+
73+
/// \brief Default constructor
74+
LateTaskInterface() = default;
75+
76+
/// \brief Destructor
77+
virtual ~LateTaskInterface() noexcept = default;
78+
/// Copy constructor
79+
LateTaskInterface(const LateTaskInterface& other) = default;
80+
/// Move constructor
81+
LateTaskInterface(LateTaskInterface&& other) noexcept = default;
82+
/// Copy assignment operator
83+
LateTaskInterface& operator=(const LateTaskInterface& other) = default;
84+
/// Move assignment operator
85+
LateTaskInterface& operator=(LateTaskInterface&& other) /* noexcept */ = default; // error with gcc if noexcept
86+
87+
// Definition of the methods for the template method pattern
88+
virtual void initialize(o2::framework::InitContext& ctx) = 0;
89+
virtual void startOfActivity(const Activity& activity) = 0;
90+
virtual void process(std::map<std::string, std::shared_ptr<const core::MonitorObject>>& moMap, std::map<std::string, std::shared_ptr<const core::QualityObject>>& qoMap) = 0;
91+
virtual void endOfActivity(const Activity& activity) = 0;
92+
virtual void reset() = 0;
93+
94+
/// \brief Called each time mCustomParameters is updated.
95+
virtual void configure() override;
96+
97+
// Setters and getters
98+
void setObjectsManager(std::shared_ptr<ObjectsManager> objectsManager);
99+
void setMonitoring(const std::shared_ptr<o2::monitoring::Monitoring>& mMonitoring);
100+
101+
protected:
102+
std::shared_ptr<ObjectsManager> getObjectsManager();
103+
std::shared_ptr<o2::monitoring::Monitoring> mMonitoring;
104+
105+
private:
106+
std::shared_ptr<ObjectsManager> mObjectsManager;
107+
};
108+
109+
} // namespace o2::quality_control::core
110+
111+
#endif // LATETASKINTERFACE_H

0 commit comments

Comments
 (0)