Project
Loading...
Searching...
No Matches
Plugin.cxx
Go to the documentation of this file.
1// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3// All rights not expressly granted are reserved.
4//
5// This software is distributed under the terms of the GNU General Public
6// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7//
8// In applying this license CERN does not waive the privileges and immunities
9// granted to it by virtue of its status as an Intergovernmental Organization
10// or submit itself to any jurisdiction.
11#include "Framework/Plugins.h"
17#include "Framework/Signpost.h"
19#include "AODWriterHelpers.h"
20#include <TFile.h>
21#include <TMap.h>
22#include <TGrid.h>
23#include <TObjString.h>
24#include <TString.h>
25#include <fmt/format.h>
26#include <memory>
27
28O2_DECLARE_DYNAMIC_LOG(analysis_support);
29
31 EDataType type;
32 char suffix[3];
33 int size;
34};
35
42
49
56
57using namespace o2::framework;
60 {
62 .name = "analysis-run-summary",
63 .init = [](ServiceRegistryRef ref, DeviceState&, fair::mq::ProgOptions&) -> ServiceHandle {
64 return ServiceHandle{TypeIdHelpers::uniqueId<RunSummary>(), nullptr, ServiceKind::Serial, "analysis-run-summary"};
65 },
66 .summaryHandling = [](ServiceMetricsInfo const& info) {
67 LOGP(info, "## Analysis Run Summary ##");
69 for (size_t mi = 0; mi < info.deviceMetricsInfos.size(); ++mi) {
70 DeviceMetricsInfo &metrics = info.deviceMetricsInfos[mi];
71 for (size_t li = 0; li < metrics.metricLabels.size(); ++li) {
72 MetricLabel const&label = metrics.metricLabels[li];
73 if (strcmp(label.label, "aod-file-read-info") != 0) {
74 continue;
75 }
76 MetricInfo const&metric = metrics.metrics[li];
77 auto &files = metrics.stringMetrics[metric.storeIdx];
78 if (metric.filledMetrics) {
79 LOGP(info, "### Files read stats ###");
80 }
81 for (size_t fi = 0; fi < metric.filledMetrics; ++fi) {
82 LOGP(info, "{}", files[fi % files.size()].data);
83 }
84 }
85 for (size_t li = 0; li < metrics.metricLabels.size(); ++li) {
86 MetricLabel const&label = metrics.metricLabels[li];
87 if (strcmp(label.label, "aod-file-open-info") != 0) {
88 continue;
89 }
90 MetricInfo const&metric = metrics.metrics[li];
91 auto &files = metrics.stringMetrics[metric.storeIdx];
92 if (metric.filledMetrics) {
93 LOGP(info, "### Files opened stats ###");
94 }
95 std::string lastFileRead;
96 for (size_t fi = 0; fi < metric.filledMetrics; ++fi) {
97 lastFileRead = files[fi % files.size()].data;
98 }
99 if (lastFileRead.empty() == false) {
100 LOGP(info, "Last file opened: {}", lastFileRead);
101 }
102 }
103 } },
104 .kind = ServiceKind::Serial};
105 }
106};
107
108std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
109{
110 std::vector<std::string> r;
111 TList* keyList = f->GetListOfKeys();
112 // We should handle two cases, one where the list of tables in a TDirectory,
113 // the other one where the dataframe number is just a prefix
114 std::string first = "";
115
116 for (auto key : *keyList) {
117 if (!std::string_view(key->GetName()).starts_with("DF_") && !std::string_view(key->GetName()).starts_with("/DF_")) {
118 continue;
119 }
120 auto* d = (TDirectory*)f->GetObjectChecked(key->GetName(), TClass::GetClass("TDirectory"));
121 // Objects are in a folder, list it.
122 if (d) {
123 TList* branchList = d->GetListOfKeys();
124 for (auto b : *branchList) {
125 r.emplace_back(b->GetName());
126 }
127 break;
128 }
129
130#if __has_include(<ROOT/RFieldBase.hxx>)
131 void* v = f->GetObjectChecked(key->GetName(), TClass::GetClass("ROOT::RNTuple"));
132#else
133 void* v = f->GetObjectChecked(key->GetName(), TClass::GetClass("ROOT::Experimental::RNTuple"));
134#endif
135 if (v) {
136 std::string s = key->GetName();
137 size_t pos = s.find('-');
138 // Check if '-' is found
139 // Skip metaData and parentFiles
140 if (pos == std::string::npos) {
141 continue;
142 }
143 std::string t = s.substr(pos + 1);
144 // If we find a duplicate table name, it means we are in the next DF and we can stop.
145 if (t == first) {
146 break;
147 }
148 if (first.empty()) {
149 first = t;
150 }
151 // Create a new string starting after the '-'
152 r.emplace_back(t);
153 }
154 }
155 return r;
156}
157
158auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
159{
160 // Get the metadata, if any
161 auto m = (TMap*)currentFile->Get("metaData");
162 if (!m) {
163 return {};
164 }
165 std::vector<ConfigParamSpec> results;
166 auto it = m->MakeIterator();
167
168 // Serialise metadata into a ; separated string with : separating key and value
169 bool first = true;
170 while (auto obj = it->Next()) {
171 if (first) {
172 LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
173 first = false;
174 }
175 auto objString = (TObjString*)m->GetValue(obj);
176 LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
177 std::string key = "aod-metadata-" + std::string(obj->GetName());
178 char const* value = strdup(objString->String());
179 results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
180 }
181
182 return results;
183}
184
187 {
188 return new ConfigDiscovery{
189 .init = []() {},
190 .discover = [](ConfigParamRegistry& registry, int argc, char** argv) -> std::vector<ConfigParamSpec> {
191 auto filename = registry.get<std::string>("aod-file");
192 if (filename.empty()) {
193 return {};
194 }
195 if (filename.at(0) == '@') {
196 filename.erase(0, 1);
197 // read the text file and set filename to the contents of the first line
198 std::ifstream file(filename);
199 if (!file.is_open()) {
200 LOGP(fatal, "Couldn't open file \"{}\"!", filename);
201 }
202 std::getline(file, filename);
203 file.close();
204 }
205 if (filename.rfind("alien://", 0) == 0 && !gGrid) {
206 TGrid::Connect("alien://");
207 }
208 LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
209 std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
210 if (currentFile.get() == nullptr) {
211 LOGP(fatal, "Couldn't open file \"{}\"!", filename);
212 }
213 std::vector<ConfigParamSpec> results = readMetadata(currentFile);
214 const bool metaDataEmpty = results.empty();
215 auto tables = getListOfTables(currentFile);
216 if (tables.empty() == false) {
217 results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
218 }
219
220 // Found metadata already in the main file.
221 if (!metaDataEmpty) {
222 results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
223 return results;
224 }
225
226 if (!registry.isSet("aod-parent-access-level") || registry.get<int>("aod-parent-access-level") == 0) {
227 LOGP(info, "No metadata found in file \"{}\" and parent level 0 prevents further lookup.", filename);
228 results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
229 return results;
230 }
231
232 // Lets try in parent file.
233 auto parentFiles = (TMap*)currentFile->Get("parentFiles");
234 if (!parentFiles) {
235 LOGP(info, "No metadata found in file \"{}\"", filename);
236 results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
237 return results;
238 }
239 LOGP(info, "No metadata found in file \"{}\", checking in its parents.", filename);
240 for (auto* p : *parentFiles) {
241 std::string parentFilename = ((TPair*)p)->Value()->GetName();
242 // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
243 // a workflow option (because the metadata itself is potentially changing the topology).
244 if (registry.isSet("aod-parent-base-path-replacement")) {
245 auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
246 auto pos = parentFileReplacement.find(';');
247 if (pos == std::string::npos) {
248 throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
249 }
250 auto from = parentFileReplacement.substr(0, pos);
251 auto to = parentFileReplacement.substr(pos + 1);
252 pos = parentFilename.find(from);
253 if (pos != std::string::npos) {
254 parentFilename.replace(pos, from.length(), to);
255 }
256 }
257
258 if (parentFilename.starts_with("alien://") && !gGrid) {
259 TGrid::Connect("alien://");
260 }
261
262 std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
263 if (parentFile.get() == nullptr) {
264 LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
265 }
266 results = readMetadata(parentFile);
267 // Found metadata already in the main file.
268 if (!results.empty()) {
269 auto tables = getListOfTables(parentFile);
270 if (tables.empty() == false) {
271 results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
272 }
273 results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
274 return results;
275 }
276 LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
277 break;
278 }
279 results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
280 return results;
281 }};
282 }
283};
284
std::vector< std::string > getListOfTables(std::unique_ptr< TFile > &f)
Definition Plugin.cxx:108
auto readMetadata(std::unique_ptr< TFile > &currentFile) -> std::vector< ConfigParamSpec >
Definition Plugin.cxx:158
#define DEFINE_DPL_PLUGIN_INSTANCE(NAME, KIND)
Definition Plugins.h:112
#define DEFINE_DPL_PLUGINS_END
Definition Plugins.h:115
#define DEFINE_DPL_PLUGINS_BEGIN
Definition Plugins.h:107
uint16_t pos
Definition RawData.h:3
#define O2_DECLARE_DYNAMIC_LOG(name)
Definition Signpost.h:489
StringRef key
const GLfloat * m
Definition glcorearb.h:4066
const GLdouble * v
Definition glcorearb.h:832
GLsizei GLenum const void GLuint GLsizei GLfloat * metrics
Definition glcorearb.h:5500
GLdouble f
Definition glcorearb.h:310
GLboolean GLboolean GLboolean b
Definition glcorearb.h:1233
GLsizei const GLfloat * value
Definition glcorearb.h:819
GLuint GLsizei const GLchar * label
Definition glcorearb.h:2519
GLboolean r
Definition glcorearb.h:1233
GLint ref
Definition glcorearb.h:291
Defining PrimaryVertex explicitly as messageable.
Definition TFIDInfo.h:20
std::string filename()
ConfigDiscovery * create() override
Definition Plugin.cxx:186
o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const &config) override
Definition Plugin.cxx:37
o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const &config) override
Definition Plugin.cxx:44
o2::framework::AlgorithmSpec create(o2::framework::ConfigContext const &config) override
Definition Plugin.cxx:51
EDataType type
Definition Plugin.cxx:31
char suffix[3]
Definition Plugin.cxx:32
o2::framework::ServiceSpec * create() final
Definition Plugin.cxx:59
Helper class for an algorithm which is loaded as a plugin.
Running state information of a given device.
Definition DeviceState.h:34
std::string name
Name of the service.
static AlgorithmSpec rootFileReaderCallback(ConfigContext const &context)
static AlgorithmSpec getOutputObjHistWriter(ConfigContext const &context)
static AlgorithmSpec getOutputTTreeWriter(ConfigContext const &context)