From 7953ff4107f47d6500e7dd19b8b30943bc12b71f Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 13 Nov 2019 16:23:14 +0100 Subject: [PATCH 001/112] WIP: import the new DQMStore header. And implementation stubs as well. Needs lots of adaptions. --- DQMServices/Core/interface/DQMEDAnalyzer.h | 7 +- DQMServices/Core/interface/DQMStore.h | 1039 +++---- DQMServices/Core/interface/MonitorElement.h | 12 +- DQMServices/Core/src/DQMService.cc | 61 +- DQMServices/Core/src/DQMStore.cc | 2736 +++---------------- DQMServices/Core/src/MonitorElement.cc | 170 -- 6 files changed, 877 insertions(+), 3148 deletions(-) diff --git a/DQMServices/Core/interface/DQMEDAnalyzer.h b/DQMServices/Core/interface/DQMEDAnalyzer.h index 054c82b016ad8..55d0d4248e7e2 100644 --- a/DQMServices/Core/interface/DQMEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMEDAnalyzer.h @@ -10,8 +10,7 @@ #include "DataFormats/Histograms/interface/DQMToken.h" /** - * The standard DQM module base. For now, this is the same as DQMOneLumiEDAnalyzer, - * but they can (and will) diverge in the future. + * The standard DQM module base. */ class DQMEDAnalyzer : public edm::one::EDProducer()->cloneLumiHistograms(lumi.run(), lumi.luminosityBlock(), this->moduleDescription().id()); + edm::Service()->leaveLumi(lumi.run(), lumi.luminosityBlock(), this->moduleDescription().id()); lumi.emplace(lumiToken_); } @@ -57,7 +56,7 @@ class DQMEDAnalyzer : public edm::one::EDProducer()->cloneRunHistograms(run.run(), this->moduleDescription().id()); + edm::Service()->leaveLumi(run.run(), 0, this->moduleDescription().id()); run.emplace(runToken_); } diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 4078465d66855..8876716b3f0f7 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -1,628 +1,483 @@ #ifndef DQMServices_Core_DQMStore_h #define DQMServices_Core_DQMStore_h -#if __GNUC__ && !defined DQM_DEPRECATED -//#define DQM_DEPRECATED __attribute__((deprecated)) -#define DQM_DEPRECATED -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "DQMServices/Core/interface/DQMDefinitions.h" #include "DQMServices/Core/interface/MonitorElement.h" -namespace edm { - class DQMHttpSource; - class ParameterSet; - class ActivityRegistry; - class GlobalContext; -} // namespace edm -namespace lat { - class Regexp; -} -namespace dqmstorepb { - class ROOTFilePB; - class ROOTFilePB_Histo; -} // namespace dqmstorepb - -class TFile; -class TBufferFile; -class TObject; -class TH1; -class TObjString; -class TH1F; -class TH1S; -class TH1D; -class TH2F; -class TH2S; -class TH2D; -class TH3F; -class TProfile; -class TProfile2D; -class TNamed; - -// forward-declare all our friends -class DQMService; -class DQMNet; -class DQMArchiver; -class DQMStoreExample; // for get{All,Matching}Contents -- sole user of this method! -class DQMRootOutputModule; -class DQMRootSource; -class DQMFileSaver; -class MEtoEDMConverter; - -namespace dqm::dqmstoreimpl { - // dqmstoreimpl is a different namespace from dqm::impl, so we can globally - // change the ME subtype here - typedef dqm::legacy::MonitorElement MonitorElement; - - class DQMStore { - public: - // legacy exposes the biggest API and implicitly converts to reco and - // harvesting, with reduced APIs. - - enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; - enum OpenRunDirs { KeepRunDirs, StripRunDirs }; - - class IBooker { - public: - friend class DQMStore; - - MonitorElement* bookInt(TString const& name); - MonitorElement* bookFloat(TString const& name); - MonitorElement* bookString(TString const& name, TString const& value); - MonitorElement* book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX); - MonitorElement* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize); - MonitorElement* book1D(TString const& name, TH1F* object); - MonitorElement* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX); - MonitorElement* book1S(TString const& name, TH1S* object); - MonitorElement* book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX); - MonitorElement* book1DD(TString const& name, TH1D* object); - MonitorElement* book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - MonitorElement* book2D(TString const& name, TH2F* object); - MonitorElement* book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - MonitorElement* book2S(TString const& name, TH2S* object); - MonitorElement* book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2DD(TString const& name, TH2D* object); - MonitorElement* book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ); - MonitorElement* book3D(TString const& name, TH3F* object); - MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int nchY, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(TString const& name, TProfile* object); - MonitorElement* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option = "s"); - MonitorElement* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ, - char const* option = "s"); - MonitorElement* bookProfile2D(TString const& name, TProfile2D* object); +// TODO: Remove at some point: +#define TRACE(msg) \ + std::cout << "TRACE: " << __FILE__ << ":" << __LINE__ << "(" << __FUNCTION__ << ") " << msg << std::endl; +#define TRACE_ TRACE(""); +namespace dqm { + namespace implementation { + // The common implementation to change folders + class NavigatorBase { + public: void cd(); void cd(std::string const& dir); + // This is the only method that is allowed to change cwd_ value void setCurrentFolder(std::string const& fullpath); void goUp(); std::string const& pwd(); - IBooker() = delete; - IBooker(IBooker const&) = delete; - - private: - explicit IBooker(DQMStore* store) noexcept : owner_{store} { assert(store); } + protected: + NavigatorBase(){}; + std::string cwd_ = ""; + }; + } - // Embedded classes do not natively own a pointer to the embedding - // class. We therefore need to store a pointer to the main - // DQMStore instance (owner_). - DQMStore* owner_; - }; // IBooker + namespace legacy { - class IGetter { + // The basic IBooker is a virtual interface that returns the common base + // class of MEs (legacy). That justifies it being in the legacy namespace. + class IBooker : public dqm::implementation::NavigatorBase { public: - friend class DQMStore; + virtual MonitorElement* bookInt(TString const& name) = 0; + virtual MonitorElement* bookFloat(TString const& name) = 0; + virtual MonitorElement* bookString(TString const& name, TString const& value) = 0; + virtual MonitorElement* book1D( + TString const& name, TString const& title, int const nchX, double const lowX, double const highX) = 0; + virtual MonitorElement* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) = 0; + virtual MonitorElement* book1D(TString const& name, TH1F* object) = 0; + virtual MonitorElement* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) = 0; + virtual MonitorElement* book1S(TString const& name, TH1S* object) = 0; + virtual MonitorElement* book1DD( + TString const& name, TString const& title, int nchX, double lowX, double highX) = 0; + virtual MonitorElement* book1DD(TString const& name, TH1D* object) = 0; + virtual MonitorElement* book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) = 0; + virtual MonitorElement* book2D(TString const& name, + TString const& title, + int nchX, + float const* xbinsize, + int nchY, + float const* ybinsize) = 0; + virtual MonitorElement* book2D(TString const& name, TH2F* object) = 0; + virtual MonitorElement* book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) = 0; + virtual MonitorElement* book2S(TString const& name, + TString const& title, + int nchX, + float const* xbinsize, + int nchY, + float const* ybinsize) = 0; + virtual MonitorElement* book2S(TString const& name, TH2S* object) = 0; + virtual MonitorElement* book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) = 0; + virtual MonitorElement* book2DD(TString const& name, TH2D* object) = 0; + virtual MonitorElement* book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ) = 0; + virtual MonitorElement* book3D(TString const& name, TH3F* object) = 0; + virtual MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + int nchY, + double lowY, + double highY, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + double lowY, + double highY, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile(TString const& name, TProfile* object) = 0; + virtual MonitorElement* bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + double lowZ, + double highZ, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ, + char const* option = "s") = 0; + virtual MonitorElement* bookProfile2D(TString const& name, TProfile2D* object) = 0; + + virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope) = 0; + + virtual ~IBooker(); + + protected: + IBooker(); + }; + class IGetter : public dqm::implementation::NavigatorBase { + public: + // TODO: review and possibly rename the all methods below: + // get MEs that are direct children of full path `path` + virtual std::vector getContents(std::string const& path) const = 0; + // not clear what this is good for. + DQM_DEPRECATED + virtual void getContents(std::vector& into, bool showContents = true) const = 0; + + // get all elements below full path `path` + // we have to discuss semantics here -- are run/lumi ever used? + virtual std::vector getAllContents(std::string const& path) const = 0; + DQM_DEPRECATED + virtual std::vector getAllContents(std::string const& path, + uint32_t runNumber, + uint32_t lumi) const = 0; + // TODO: rename to reflect the fact that it requires full path + // return ME identified by full path `path`, or nullptr + virtual MonitorElement* get(std::string const& fullpath) const = 0; - // for the supported syntaxes, see the declarations of DQMStore::getContents - template - std::vector getContents(Args&&... args) { - return owner_->getContents(std::forward(args)...); - } + // same as get, throws an exception if histogram not found + // Deprecated simply because it is barely used. + DQM_DEPRECATED + virtual MonitorElement* getElement(std::string const& path) const = 0; - std::vector getAllContents(std::string const& path, uint32_t runNumber = 0, uint32_t lumi = 0); - MonitorElement* get(std::string const& path); + // return sub-directories of current directory + virtual std::vector getSubdirs() const = 0; + // return element names of direct children of current directory + virtual std::vector getMEs() const = 0; + // returns whether there are objects at full path `path` + virtual bool dirExists(std::string const& path) const = 0; - // same as get, throws an exception if histogram not found - MonitorElement* getElement(std::string const& path); + virtual ~IGetter(); - std::vector getSubdirs(); - std::vector getMEs(); - bool containsAnyMonitorable(std::string const& path); - bool dirExists(std::string const& path); - void cd(); - void cd(std::string const& dir); - void setCurrentFolder(std::string const& fullpath); + protected: + IGetter(); + }; - IGetter() = delete; - IGetter(IGetter const&) = delete; - - private: - explicit IGetter(DQMStore* store) noexcept : owner_{store} { assert(store); } - - // Embedded classes do not natively own a pointer to the embedding - // class. We therefore need to store a pointer to the main - // DQMStore instance (owner_). - DQMStore* owner_; - }; //IGetter - - // Template function to be used inside each DQM Modules' lambda - // functions to book MonitorElements into the DQMStore. The function - // calls whatever user-supplied code via the function f. The latter - // is passed the instance of the IBooker class (owned by the *only* - // DQMStore instance), that is capable of booking MonitorElements - // into the DQMStore via a public API. The central mutex is acquired - // *before* invoking and automatically released upon returns. - template - void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi) { - std::lock_guard guard(book_mutex_); - /* Set the run number and module id only if multithreading is enabled */ - if (enableMultiThread_) { - run_ = run; - moduleId_ = moduleId; - canSaveByLumi_ = canSaveByLumi; - } - IBooker booker{this}; - f(booker); - - /* Reset the run number and module id only if multithreading is enabled */ - if (enableMultiThread_) { - run_ = 0; - moduleId_ = 0; - canSaveByLumi_ = false; - } - } - - template - void meBookerGetter(iFunc f) { - IBooker booker{this}; - IGetter getter{this}; - f(booker, getter); - } - - //------------------------------------------------------------------------- - // ---------------------- Constructors ------------------------------------ - DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&); - DQMStore(edm::ParameterSet const& pset); - ~DQMStore(); - - //------------------------------------------------------------------------- - void setVerbose(unsigned level); - - // ---------------------- public navigation ------------------------------- - std::string const& pwd() const; - void cd(); - void cd(std::string const& subdir); - void setCurrentFolder(std::string const& fullpath); - void goUp(); - - bool dirExists(std::string const& path) const; - - // Conversion class to allow specifications of TString const&, - // std::string const&, and char const* for the booking functions. - // Ideally, this will be replaced with std::string_view (what to do - // about TString?) whenever we move to C++17. - class char_string { + } // namespace legacy + // this namespace is for internal use only. + namespace implementation { + // this provides a templated implementation of the DQMStore. The operations it + // does are rather always the same; the only thing that changes are the return + // types. We keep IBooker and IGetter separate, just in case. DQMStore simply + // multi-inherits them for now. + // We will instantiate this for reco MEs and harvesting MEs, and maybe for + // legacy as well. + + template + class IBooker : public dqm::legacy::IBooker { public: - char_string(TString const& str) : data_{str.Data()} {} - char_string(char const* str) : data_{str} {} - char_string(std::string const& str) : data_{str} {} - operator std::string const&() const { return data_; } - operator char const*() const { return data_.c_str(); } - - private: - std::string data_; - }; + virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); - //------------------------------------------------------------------------- - // ---------------------- public ME booking ------------------------------- - MonitorElement* bookInt(char_string const& name); - MonitorElement* bookFloat(char_string const& name); - MonitorElement* bookString(char_string const& name, char_string const& value); - MonitorElement* book1D( - char_string const& name, char_string const& title, int const nchX, double const lowX, double const highX); - MonitorElement* book1D(char_string const& name, char_string const& title, int nchX, float const* xbinsize); - MonitorElement* book1D(char_string const& name, TH1F* h); - MonitorElement* book1S(char_string const& name, char_string const& title, int nchX, double lowX, double highX); - MonitorElement* book1S(char_string const& name, TH1S* h); - MonitorElement* book1DD(char_string const& name, char_string const& title, int nchX, double lowX, double highX); - MonitorElement* book1DD(char_string const& name, TH1D* h); - MonitorElement* book2D(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2D(char_string const& name, - char_string const& title, - int nchX, - float const* xbinsize, - int nchY, - float const* ybinsize); - MonitorElement* book2D(char_string const& name, TH2F* h); - MonitorElement* book2S(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2S(char_string const& name, - char_string const& title, - int nchX, - float const* xbinsize, - int nchY, - float const* ybinsize); - MonitorElement* book2S(char_string const& name, TH2S* h); - MonitorElement* book2DD(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - MonitorElement* book2DD(char_string const& name, TH2D* h); - MonitorElement* book3D(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ); - MonitorElement* book3D(char_string const& name, TH3F* h); - MonitorElement* bookProfile(char_string const& name, - char_string const& title, + virtual ME* bookInt(TString const& name); + virtual ME* bookFloat(TString const& name); + virtual ME* bookString(TString const& name, TString const& value); + virtual ME* book1D( + TString const& name, TString const& title, int const nchX, double const lowX, double const highX); + virtual ME* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize); + virtual ME* book1D(TString const& name, TH1F* object); + virtual ME* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX); + virtual ME* book1S(TString const& name, TH1S* object); + virtual ME* book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX); + virtual ME* book1DD(TString const& name, TH1D* object); + virtual ME* book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY); + virtual ME* book2D( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); + virtual ME* book2D(TString const& name, TH2F* object); + virtual ME* book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY); + virtual ME* book2S( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); + virtual ME* book2S(TString const& name, TH2S* object); + virtual ME* book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY); + virtual ME* book2DD(TString const& name, TH2D* object); + virtual ME* book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ); + virtual ME* book3D(TString const& name, TH3F* object); + virtual ME* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + char const* option = "s"); + virtual ME* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option = "s"); + virtual ME* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + int nchY, + double lowY, + double highY, + char const* option = "s"); + virtual ME* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + double lowY, + double highY, + char const* option = "s"); + virtual ME* bookProfile(TString const& name, TProfile* object); + virtual ME* bookProfile2D(TString const& name, + TString const& title, int nchX, double lowX, double highX, int nchY, double lowY, double highY, + double lowZ, + double highZ, char const* option = "s"); - MonitorElement* bookProfile(char_string const& name, - char_string const& title, + virtual ME* bookProfile2D(TString const& name, + TString const& title, int nchX, double lowX, double highX, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(char_string const& name, - char_string const& title, - int nchX, - double const* xbinsize, int nchY, double lowY, double highY, + int nchZ, + double lowZ, + double highZ, char const* option = "s"); - MonitorElement* bookProfile(char_string const& name, - char_string const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option = "s"); - MonitorElement* bookProfile(char_string const& name, TProfile* h); - MonitorElement* bookProfile2D(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option = "s"); - MonitorElement* bookProfile2D(char_string const& name, - char_string const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ, - char const* option = "s"); - MonitorElement* bookProfile2D(char_string const& name, TProfile2D* h); - - //------------------------------------------------------------------------- - // ---------------------- public ME getters ------------------------------- - std::vector getSubdirs() const; - std::vector getMEs() const; - bool containsAnyMonitorable(std::string const& path) const; - - MonitorElement* get(std::string const& path) const; - std::vector getContents(std::string const& path) const; - void getContents(std::vector& into, bool showContents = true) const; - - // ------------------------------------------------------------------------ - // ---------------------- public I/O -------------------------------------- - void save(std::string const& filename, - std::string const& path = "", - std::string const& pattern = "", - std::string const& rewrite = "", - uint32_t run = 0, - uint32_t lumi = 0, - SaveReferenceTag ref = SaveWithReference, - int minStatus = dqm::qstatus::STATUS_OK, - std::string const& fileupdate = "RECREATE"); - bool open(std::string const& filename, - bool overwrite = false, - std::string const& path = "", - std::string const& prepend = "", - OpenRunDirs stripdirs = KeepRunDirs, - bool fileMustExist = true); - bool load(std::string const& filename, OpenRunDirs stripdirs = StripRunDirs, bool fileMustExist = true); - bool mtEnabled() { return enableMultiThread_; }; - - public: - // ------------------------------------------------------------------------- - // ---------------------- Public print methods ----------------------------- - void showDirStructure() const; - - // ---------------------- Public check options ----------------------------- - bool isCollate() const; - - private: - // ---------------- Navigation ----------------------- - bool cdInto(std::string const& path) const; - - // ------------------- Reference ME ------------------------------- - bool isCollateME(MonitorElement* me) const; - - // ------------------- Private "getters" ------------------------------ - bool readFilePB(std::string const& filename, - bool overwrite = false, - std::string const& path = "", - std::string const& prepend = "", - OpenRunDirs stripdirs = StripRunDirs, - bool fileMustExist = true); - bool readFile(std::string const& filename, - bool overwrite = false, - std::string const& path = "", - std::string const& prepend = "", - OpenRunDirs stripdirs = StripRunDirs, - bool fileMustExist = true); - void makeDirectory(std::string const& path); - unsigned int readDirectory(TFile* file, - bool overwrite, - std::string const& path, - std::string const& prepend, - std::string const& curdir, - OpenRunDirs stripdirs); - - MonitorElement* findObject( - uint32_t run, uint32_t lumi, uint32_t moduleId, std::string const& dir, std::string const& name) const; - - void get_info(dqmstorepb::ROOTFilePB_Histo const&, std::string& dirname, std::string& objname, TObject** obj); - - public: - std::vector getAllContents(std::string const& path, - uint32_t runNumber = 0, - uint32_t lumi = 0) const; - std::vector getMatchingContents(std::string const& pattern, - lat::Regexp::Syntax syntaxType = lat::Regexp::Wildcard) const; - - // lumisection based histograms manipulations - void cloneLumiHistograms(uint32_t run, uint32_t lumi, uint32_t moduleId); - void cloneRunHistograms(uint32_t run, uint32_t moduleId); - - void deleteUnusedLumiHistograms(uint32_t run, uint32_t lumi); - - DQMStore(DQMStore const&) = delete; - DQMStore& operator=(DQMStore const&) = delete; - - private: - // ---------------- Miscellaneous ----------------------------- - void initializeFrom(const edm::ParameterSet&); - void reset(); - void forceReset(); - void postGlobalBeginLumi(const edm::GlobalContext&); - - bool extract(TObject* obj, std::string const& dir, bool overwrite, bool collateHistograms); - TObject* extractNextObject(TBufferFile&) const; - - // ---------------------- Booking ------------------------------------ - MonitorElement* initialise(MonitorElement* me, std::string const& path); - MonitorElement* book_(std::string const& dir, std::string const& name, char const* context); - template - MonitorElement* book_(std::string const& dir, - std::string const& name, - char const* context, - MonitorElement::Kind const kind, - HISTO* h, - COLLATE collate); - - MonitorElement* bookInt_(std::string const& dir, std::string const& name); - MonitorElement* bookFloat_(std::string const& dir, std::string const& name); - MonitorElement* bookString_(std::string const& dir, std::string const& name, std::string const& value); - MonitorElement* book1D_(std::string const& dir, std::string const& name, TH1F* h); - MonitorElement* book1S_(std::string const& dir, std::string const& name, TH1S* h); - MonitorElement* book1DD_(std::string const& dir, std::string const& name, TH1D* h); - MonitorElement* book2D_(std::string const& dir, std::string const& name, TH2F* h); - MonitorElement* book2S_(std::string const& dir, std::string const& name, TH2S* h); - MonitorElement* book2DD_(std::string const& dir, std::string const& name, TH2D* h); - MonitorElement* book3D_(std::string const& dir, std::string const& name, TH3F* h); - MonitorElement* bookProfile_(std::string const& dir, std::string const& name, TProfile* h); - MonitorElement* bookProfile2D_(std::string const& dir, std::string const& name, TProfile2D* h); - - static bool checkBinningMatches(MonitorElement* me, TH1* h, unsigned verbose); - - static void collate1D(MonitorElement* me, TH1F* h, unsigned verbose); - static void collate1S(MonitorElement* me, TH1S* h, unsigned verbose); - static void collate1DD(MonitorElement* me, TH1D* h, unsigned verbose); - static void collate2D(MonitorElement* me, TH2F* h, unsigned verbose); - static void collate2S(MonitorElement* me, TH2S* h, unsigned verbose); - static void collate2DD(MonitorElement* me, TH2D* h, unsigned verbose); - static void collate3D(MonitorElement* me, TH3F* h, unsigned verbose); - static void collateProfile(MonitorElement* me, TProfile* h, unsigned verbose); - static void collateProfile2D(MonitorElement* me, TProfile2D* h, unsigned verbose); - - void print_trace(std::string const& dir, std::string const& name); - - //------------------------------------------------------------------------------- - //------------------------------------------------------------------------------- - using MEMap = std::set; - - // ------------------------ private I/O helpers ------------------------------ - unsigned verbose_{1}; - unsigned verboseQT_{1}; - bool reset_{false}; - double scaleFlag_; - bool collateHistograms_{false}; - bool enableMultiThread_{false}; - bool LSbasedMode_; - bool forceResetOnBeginLumi_{false}; - std::string readSelectedDirectory_{}; - uint32_t run_{}; - uint32_t moduleId_{}; - // set to true in the transaction if module supports per-lumi saving. - bool canSaveByLumi_{false}; - // set to true in configuration if per-lumi saving is requested. - bool doSaveByLumi_{false}; - std::unique_ptr stream_{nullptr}; - - std::string pwd_{}; - MEMap data_; - std::set dirs_; - - std::mutex book_mutex_; - - friend DQMService; - friend DQMNet; - friend DQMArchiver; - friend DQMStoreExample; // for get{All,Matching}Contents -- sole user of this method! - friend DQMRootOutputModule; - friend DQMRootSource; - friend DQMFileSaver; - friend MEtoEDMConverter; - }; - -} // namespace dqm::dqmstoreimpl - -// These will become distinct classes in the future. -namespace dqm::legacy { - typedef dqm::dqmstoreimpl::DQMStore DQMStore; -} -namespace dqm::reco { - typedef dqm::dqmstoreimpl::DQMStore DQMStore; -} -namespace dqm::harvesting { - typedef dqm::dqmstoreimpl::DQMStore DQMStore; -} - -#endif // DQMServices_Core_DQMStore_h + virtual ME* bookProfile2D(TString const& name, TProfile2D* object); + + virtual ~IBooker(){}; + + protected: + IBooker(STORE* store); + ME* bookME(TString const& name, MonitorElementData::Kind kind, TH1* object); + + STORE* store_; + MonitorElementData::Scope scope_; + }; + + template + class IGetter : public dqm::legacy::IGetter { + public: + // TODO: while we can have covariant return types for individual ME*, it seems we can't for the vectors. + virtual std::vector getContents(std::string const& path) const; + virtual void getContents(std::vector& into, bool showContents = true) const; + + virtual std::vector getAllContents(std::string const& path) const; + DQM_DEPRECATED + virtual std::vector getAllContents(std::string const& path, + uint32_t runNumber, + uint32_t lumi) const; + virtual ME* get(std::string const& fullpath) const; + + DQM_DEPRECATED + virtual ME* getElement(std::string const& path) const; + + virtual std::vector getSubdirs() const; + virtual std::vector getMEs() const; + virtual bool dirExists(std::string const& path) const; + + virtual ~IGetter(){}; + + protected: + IGetter(STORE* store); + + STORE* store_; + }; + + template + class DQMStore : public IGetter>, public IBooker> { + public: + + // TODO: There are no references any more. we should gt rid of these. + enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; + enum OpenRunDirs { KeepRunDirs, StripRunDirs }; + + DQMStore(); + ~DQMStore(); + + // ------------------------------------------------------------------------ + // ---------------------- public I/O -------------------------------------- + // TODO: these need some cleanup, though in general we want to keep the + // functionality. Maybe move it to a different class, and change the (rather + // few) usages. + void save(std::string const& filename, + std::string const& path = "", + std::string const& pattern = "", + std::string const& rewrite = "", + uint32_t run = 0, + uint32_t lumi = 0, + SaveReferenceTag ref = SaveWithReference, + int minStatus = dqm::qstatus::STATUS_OK, + std::string const& fileupdate = "RECREATE"); + void savePB(std::string const& filename, std::string const& path = "", uint32_t run = 0, uint32_t lumi = 0); + bool open(std::string const& filename, + bool overwrite = false, + std::string const& path = "", + std::string const& prepend = "", + OpenRunDirs stripdirs = KeepRunDirs, + bool fileMustExist = true); + bool load(std::string const& filename, OpenRunDirs stripdirs = StripRunDirs, bool fileMustExist = true); + + DQM_DEPRECATED + bool mtEnabled() { assert(!"NIY"); } + + DQM_DEPRECATED + void showDirStructure() const; + + // TODO: getting API not part of IGetter. + DQM_DEPRECATED + std::vector getMatchingContents(std::string const& pattern) const; + + DQMStore(DQMStore const&) = delete; + DQMStore& operator=(DQMStore const&) = delete; + + // ------------------------------------------------------------------------ + // ------------ IBooker/IGetter overrides to prevent ambiguity ------------ + virtual void cd() { + this->IBooker>::cd(); + this->IGetter>::cd(); + } + virtual void cd(std::string const& dir) { + this->IBooker>::cd(dir); + this->IGetter>::cd(dir); + } + virtual void setCurrentFolder(std::string const& fullpath) { + this->IBooker>::setCurrentFolder(fullpath); + this->IGetter>::setCurrentFolder(fullpath); + } + virtual void goUp() { + this->IBooker>::goUp(); + this->IGetter>::goUp(); + } + std::string const& pwd() { return this->IBooker>::pwd(); } + + public: + // internal -- figure out better protection. + template + void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi) {}; + template + void meBookerGetter(iFunc f) {}; + + // Make a ME owned by this DQMStore. Will return a pointer to a ME owned + // by this DQMStore: either an existing ME matching the key of `me` or + // a newly added one. + // Will take ownership of the ROOT object in `me`, deleting it if not + // needed. + ME* putME(std::unique_ptr&& me); + // Log a backtrace on booking. + void printTrace(std::string const& message); + // Prepare MEs for the next lumisection. This will create per-lumi copies + // if ther previous lumi has not yet finished and recycle reusable MEs if + // booking left any. + // enterLumi is idempotent; it can be called at any time to update lumi + // ranges in the MEs. This may be needed in harvesting, where MEs can be + // booked at any time. + void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID); + // Turn the MEs associated with t, run, lumi into a global ME. + void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleIDi); + + // Clone data including the underlying ROOT object (calls ->Clone()). + static MonitorElementData* cloneMonitorElementData(MonitorElementData const* input); + + // TODO: Make this section private. localmes_ and inputs_ should be friends with IGetter. + public: + }; + } // namespace implementation + + // Since we still use a single, edm::Serivce instance of a DQMStore, these are all the same. + namespace legacy { + class DQMStore : public dqm::implementation::DQMStore { + public: + typedef dqm::legacy::IBooker IBooker; + typedef dqm::legacy::IGetter IGetter; + }; + } // namespace legacy + namespace reco { + typedef dqm::legacy::DQMStore DQMStore; + } // namespace reco + namespace harvesting { + typedef dqm::legacy::DQMStore DQMStore; + } // namespace harvesting +} // namespace dqm + +#endif diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 8a5e21745d7ec..f5f370987c20c 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -41,9 +41,6 @@ namespace dqm::dqmstoreimpl { class DQMStore; } -// tag for a special constructor, see below -struct MonitorElementNoCloneTag {}; - namespace dqm::impl { using dqmmutex = tbb::spin_mutex; @@ -162,18 +159,11 @@ namespace dqm::impl { TH1 *refvalue_; //< Soft reference if any. std::vector qreports_; //< QReports associated to this object. - MonitorElement *initialise(Kind kind); - MonitorElement *initialise(Kind kind, TH1 *rootobj); - MonitorElement *initialise(Kind kind, const std::string &value); void globalize() { data_.moduleId = 0; } void setLumi(uint32_t ls) { data_.lumi = ls; } public: - MonitorElement(); - MonitorElement(const std::string *path, const std::string &name); - MonitorElement(const std::string *path, const std::string &name, uint32_t run, uint32_t moduleId); - MonitorElement(const MonitorElement &, MonitorElementNoCloneTag); - MonitorElement(const MonitorElement &); + MonitorElement(MonitorElementData* data, bool is_owned, bool is_readonly) {}; MonitorElement &operator=(const MonitorElement &) = delete; MonitorElement &operator=(MonitorElement &&) = delete; virtual ~MonitorElement(); diff --git a/DQMServices/Core/src/DQMService.cc b/DQMServices/Core/src/DQMService.cc index e9fa4ddf1f5fb..2e6dff4f917e5 100644 --- a/DQMServices/Core/src/DQMService.cc +++ b/DQMServices/Core/src/DQMService.cc @@ -78,65 +78,9 @@ void DQMService::flushStandalone() { // Lock the network layer so we can modify the data. net_->lock(); bool updated = false; + + // TODO: re-implement sending MEs. - // Find updated contents and update the network cache. - DQMStore::MEMap::iterator i, e; - net_->reserveLocalSpace(store_->data_.size()); - for (i = store_->data_.begin(), e = store_->data_.end(); i != e; ++i) { - const MonitorElement &me = *i; - fullpath.clear(); - fullpath += *me.data_.dirname; - if (!me.data_.dirname->empty()) - fullpath += '/'; - fullpath += me.data_.objname; - - if (filter_ && filter_->search(fullpath) < 0) - continue; - - seen.insert(fullpath); - if (!me.wasUpdated()) - continue; - - o.lastreq = 0; - o.hash = DQMNet::dqmhash(fullpath.c_str(), fullpath.size()); - o.flags = me.data_.flags; - o.tag = me.data_.tag; - o.version = version; - o.dirname = me.data_.dirname; - o.objname = me.data_.objname; - assert(o.rawdata.empty()); - assert(o.scalar.empty()); - assert(o.qdata.empty()); - - // Pack object and reference, scalar and quality data. - switch (me.kind()) { - case MonitorElement::Kind::INT: - case MonitorElement::Kind::REAL: - case MonitorElement::Kind::STRING: - me.packScalarData(o.scalar, ""); - break; - - default: { - TBufferFile buffer(TBufferFile::kWrite); - buffer.WriteObject(me.getRootObject()); - if (me.reference_) - buffer.WriteObject(me.reference_); - else - buffer.WriteObjectAny(nullptr, nullptr); - o.rawdata.resize(buffer.Length()); - memcpy(&o.rawdata[0], buffer.Buffer(), buffer.Length()); - DQMNet::packQualityData(o.qdata, me.data_.qreports); - break; - } - } - - // Update. - net_->updateLocalObject(o); - DQMNet::DataBlob().swap(o.rawdata); - std::string().swap(o.scalar); - std::string().swap(o.qdata); - updated = true; - } // Find removed contents and clear the network cache. if (net_->removeLocalExcept(seen)) @@ -150,7 +94,6 @@ void DQMService::flushStandalone() { net_->sendLocalChanges(); } - store_->reset(); lastFlush_ = lat::Time::current().ns() * 1e-9; } void DQMService::flush(edm::StreamContext const &sc) { diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index a8fbdf13a21aa..0a09f0e4461eb 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -1,2374 +1,486 @@ -#include "DQMServices/Core/interface/Standalone.h" +// silence deprecation warnings for the DQMStore itself. +#define DQM_DEPRECATED #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/interface/LegacyIOHelper.h" -#include "DQMServices/Core/interface/QReport.h" -#include "DQMServices/Core/src/ROOTFilePB.pb.h" -#include "DQMServices/Core/src/DQMError.h" -#include "classlib/utils/RegexpMatch.h" -#include "classlib/utils/Regexp.h" -#include "classlib/utils/StringOps.h" -#include -#include -#include -#include "TFile.h" -#include "TROOT.h" -#include "TKey.h" -#include "TClass.h" -#include "TSystem.h" -#include "TBufferFile.h" -#include -#include - -#include -#include -#include -#include -#include -#include - -/** @var DQMStore::verbose_ - Universal verbose flag for DQM. */ - -/** @var DQMStore::verboseQT_ - Verbose flag for xml-based QTests. */ - -/** @var DQMStore::reset_ - - Flag used to print out a warning when calling quality tests. - twice without having called reset() in between; to be reset in - DQMOldReceiver::runQualityTests. */ - -/** @var DQMStore::collateHistograms_ */ - -/** @var DQMStore::readSelectedDirectory_ - If non-empty, read from file only selected directory. */ - -/** @var DQMStore::pwd_ - Current directory. */ - -/** @var DQMStore::qtests_. - All the quality tests. */ - -/** @var DQMStore::qalgos_ - Set of all the available quality test algorithms. */ - -namespace dqm::dqmstoreimpl { - - ////////////////////////////////////////////////////////////////////// - /// name of global monitoring folder (containing all sources subdirectories) - std::string const s_monitorDirName{"DQMData"}; - std::string const s_referenceDirName{"Reference"}; - std::string const s_collateDirName{"Collate"}; - std::string const s_safe{"/ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-+=_()# "}; - - lat::Regexp const s_rxmeval{"^<(.*)>(i|f|s|e|t|qr)=(.*)$"}; - lat::Regexp const s_rxmeqr1{"^st:(\\d+):([-+e.\\d]+):([^:]*):(.*)$"}; - lat::Regexp const s_rxmeqr2{"^st\\.(\\d+)\\.(.*)$"}; - lat::Regexp const s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*"}; - lat::Regexp const s_rxself{"^[^()]*DQMStore::.*"}; - lat::Regexp const s_rxpbfile{".*\\.pb$"}; - - std::string const empty_str{}; - - ////////////////////////////////////////////////////////////////////// - /// Check whether the @a path is a subdirectory of @a ofdir. Returns - /// true both for an exact match and any nested subdirectory. - bool isSubdirectory(std::string const& ofdir, std::string const& path) { - return (ofdir.empty() || (path.size() >= ofdir.size() && path.compare(0, ofdir.size(), ofdir) == 0 && - (path.size() == ofdir.size() || path[ofdir.size()] == '/'))); - } - - void cleanTrailingSlashes(std::string const& path, std::string& clean, std::string const*& cleaned) { - clean.clear(); - cleaned = &path; - - size_t len = path.size(); - for (; len > 0 && path[len - 1] == '/'; --len) - ; - - if (len != path.size()) { - clean = path.substr(0, len); - cleaned = &clean; - } - } - - void splitPath(std::string& dir, std::string& name, std::string const& path) { - size_t slash = path.rfind('/'); - if (slash != std::string::npos) { - dir.append(path, 0, slash); - name.append(path, slash + 1, std::string::npos); - } else - name = path; - } - - void mergePath(std::string& path, std::string const& dir, std::string const& name) { - path.reserve(dir.size() + name.size() + 2); - path += dir; - if (!path.empty()) - path += '/'; - path += name; - } - - //IBooker methods - MonitorElement* DQMStore::IBooker::bookInt(TString const& name) { return owner_->bookInt(name); } - - MonitorElement* DQMStore::IBooker::bookFloat(TString const& name) { return owner_->bookFloat(name); } - - MonitorElement* DQMStore::IBooker::bookString(TString const& name, TString const& value) { - return owner_->bookString(name, value); - } - - MonitorElement* DQMStore::IBooker::book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { - return owner_->book1D(name, title, nchX, lowX, highX); - } - - MonitorElement* DQMStore::IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { - return owner_->book1D(name, title, nchX, xbinsize); - }; - - MonitorElement* DQMStore::IBooker::book1D(TString const& name, TH1F* object) { return owner_->book1D(name, object); } - - MonitorElement* DQMStore::IBooker::book1S( - TString const& name, TString const& title, int nchX, double lowX, double highX) { - return owner_->book1S(name, title, nchX, lowX, highX); - } - - MonitorElement* DQMStore::IBooker::book1S(TString const& name, TH1S* object) { return owner_->book1S(name, object); } - - MonitorElement* DQMStore::IBooker::book1DD( - TString const& name, TString const& title, int nchX, double lowX, double highX) { - return owner_->book1DD(name, title, nchX, lowX, highX); - } - - MonitorElement* DQMStore::IBooker::book1DD(TString const& name, TH1D* object) { - return owner_->book1DD(name, object); - } - - MonitorElement* DQMStore::IBooker::book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - return owner_->book2D(name, title, nchX, lowX, highX, nchY, lowY, highY); - } - - MonitorElement* DQMStore::IBooker::book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - return owner_->book2D(name, title, nchX, xbinsize, nchY, ybinsize); - } - - MonitorElement* DQMStore::IBooker::book2D(TString const& name, TH2F* object) { return owner_->book2D(name, object); } - - MonitorElement* DQMStore::IBooker::book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - return owner_->book2S(name, title, nchX, lowX, highX, nchY, lowY, highY); - } - - MonitorElement* DQMStore::IBooker::book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - return owner_->book2S(name, title, nchX, xbinsize, nchY, ybinsize); - } - - MonitorElement* DQMStore::IBooker::book2S(TString const& name, TH2S* object) { return owner_->book2S(name, object); } - - MonitorElement* DQMStore::IBooker::book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - return owner_->book2DD(name, title, nchX, lowX, highX, nchY, lowY, highY); - } - - MonitorElement* DQMStore::IBooker::book2DD(TString const& name, TH2D* object) { - return owner_->book2DD(name, object); - } - - MonitorElement* DQMStore::IBooker::book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ) { - return owner_->book3D(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); - } - - MonitorElement* DQMStore::IBooker::book3D(TString const& name, TH3F* object) { return owner_->book3D(name, object); } - - MonitorElement* DQMStore::IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - char const* option) { - return owner_->bookProfile(name, title, nchX, lowX, highX, nchY, lowY, highY, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option) { - return owner_->bookProfile(name, title, nchX, lowX, highX, lowY, highY, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int nchY, - double lowY, - double highY, - char const* option) { - return owner_->bookProfile(name, title, nchX, xbinsize, nchY, lowY, highY, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option) { - return owner_->bookProfile(name, title, nchX, xbinsize, lowY, highY, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile(TString const& name, TProfile* object) { - return owner_->bookProfile(name, object); - } - - MonitorElement* DQMStore::IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option) { - return owner_->bookProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ, - char const* option) { - return owner_->bookProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ, option); - } - - MonitorElement* DQMStore::IBooker::bookProfile2D(TString const& name, TProfile2D* object) { - return owner_->bookProfile2D(name, object); - } - - void DQMStore::IBooker::cd() { owner_->cd(); } - - void DQMStore::IBooker::cd(std::string const& dir) { owner_->cd(dir); } - - void DQMStore::IBooker::setCurrentFolder(std::string const& fullpath) { owner_->setCurrentFolder(fullpath); } - - void DQMStore::IGetter::cd() { owner_->cd(); } - - void DQMStore::IGetter::cd(std::string const& dir) { owner_->cd(dir); } - - void DQMStore::IGetter::setCurrentFolder(std::string const& fullpath) { owner_->setCurrentFolder(fullpath); } - - void DQMStore::IBooker::goUp() { owner_->goUp(); } - - std::string const& DQMStore::IBooker::pwd() { return owner_->pwd(); } - - //IGetter methods - std::vector DQMStore::IGetter::getAllContents(std::string const& path, - uint32_t const run /* = 0 */, - uint32_t const lumi /* = 0 */) { - return owner_->getAllContents(path, run, lumi); - } - - MonitorElement* DQMStore::IGetter::get(std::string const& path) { return owner_->get(path); } - - MonitorElement* DQMStore::IGetter::getElement(std::string const& path) { - MonitorElement* ptr = this->get(path); - if (ptr == nullptr) { - std::stringstream msg; - msg << "DQM object not found"; - - msg << ": " << path; - - // can't use cms::Exception inside DQMStore - throw std::out_of_range(msg.str()); - } - return ptr; - } - - std::vector DQMStore::IGetter::getSubdirs() { return owner_->getSubdirs(); } - - std::vector DQMStore::IGetter::getMEs() { return owner_->getMEs(); } - - bool DQMStore::IGetter::containsAnyMonitorable(std::string const& path) { - return owner_->containsAnyMonitorable(path); - } - - bool DQMStore::IGetter::dirExists(std::string const& path) { return owner_->dirExists(path); } - - ////////////////////////////////////////////////////////////////////// - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry& ar) : DQMStore{pset} { - ar.preallocateSignal_.connect([this](edm::service::SystemBounds const& iBounds) { - if (iBounds.maxNumberOfStreams() > 1) { - enableMultiThread_ = true; +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include +#include +#include + +#include +#include + + +namespace dqm { + + namespace legacy { + IBooker::IBooker() {} + IBooker::~IBooker() {} + IGetter::IGetter() {} + IGetter::~IGetter() {} + + } // namespace legacy + + namespace implementation { + std::string const& NavigatorBase::pwd() { return cwd_; } + void NavigatorBase::cd() { setCurrentFolder(""); } + void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(cwd_ + dir); } + void NavigatorBase::goUp() { cd(".."); } + void NavigatorBase::setCurrentFolder(std::string const& fullpath) { + MonitorElementData::Path path; + path.set(fullpath, MonitorElementData::Path::Type::DIR); + assert(this); + cwd_ = path.getDirname(); + } + + template + IBooker::IBooker(STORE* store) { + store_ = store; + scope_ = MonitorElementData::Scope::RUN; + } + + template + MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) { + auto oldscope = scope_; + scope_ = newscope; + return oldscope; + } + + template + ME* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { + MonitorElementData* data = new MonitorElementData(); + MonitorElementData::Key key; + key.kind_ = kind; + std::string fullpath = pwd() + std::string(name.View()); + key.path_.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = scope_; + data->key_ = key; + { + //MonitorElementData::Value::Access value(data->value_); + //value.object = std::unique_ptr(object); } - }); - if (pset.getUntrackedParameter("forceResetOnBeginRun", false)) { - ar.watchPostSourceRun([this](edm::RunIndex) { forceReset(); }); - } - if (pset.getUntrackedParameter("forceResetOnBeginLumi", false) && enableMultiThread_ == false) { -#if !WITHOUT_CMS_FRAMEWORK - forceResetOnBeginLumi_ = true; - ar.watchPreSourceLumi([this](edm::LuminosityBlockIndex) { forceReset(); }); -#endif - } - ar.watchPostGlobalBeginLumi(this, &DQMStore::postGlobalBeginLumi); - } - - DQMStore::DQMStore(edm::ParameterSet const& pset) { initializeFrom(pset); } - - DQMStore::~DQMStore() {} - - void DQMStore::initializeFrom(edm::ParameterSet const& pset) { - makeDirectory(""); - reset(); - - // set steerable parameters - verbose_ = pset.getUntrackedParameter("verbose", 0); - if (verbose_ > 0) - std::cout << "DQMStore: verbosity set to " << verbose_ << std::endl; - - verboseQT_ = pset.getUntrackedParameter("verboseQT", 0); - if (verbose_ > 0) - std::cout << "DQMStore: QTest verbosity set to " << verboseQT_ << std::endl; - - collateHistograms_ = pset.getUntrackedParameter("collateHistograms", false); - if (collateHistograms_) - std::cout << "DQMStore: histogram collation is enabled\n"; - - enableMultiThread_ = pset.getUntrackedParameter("enableMultiThread", false); - if (enableMultiThread_) - std::cout << "DQMStore: MultiThread option is enabled\n"; - - LSbasedMode_ = pset.getUntrackedParameter("LSbasedMode", false); - if (LSbasedMode_) - std::cout << "DQMStore: LSbasedMode option is enabled\n"; - - doSaveByLumi_ = pset.getUntrackedParameter("saveByLumi", false); - if (doSaveByLumi_) - std::cout << "DQMStore: saveByLumi option is enabled\n"; - - std::string ref = pset.getUntrackedParameter("referenceFileName", ""); - if (!ref.empty()) { - std::cout << "DQMStore: using reference file '" << ref << "'\n"; - readFile(ref, true, "", s_referenceDirName, StripRunDirs, false); - } - - scaleFlag_ = pset.getUntrackedParameter("ScalingFlag", 0.0); - if (verbose_ > 0) - std::cout << "DQMStore: Scaling Flag set to " << scaleFlag_ << std::endl; - } - - /* Generic method to do a backtrace and print it to stdout. It is - customised to properly get the routine that called the booking of the - histograms, which, following the usual stack, is at position 4. The - name of the calling function is properly demangled and the original - shared library including this function is also printed. For a more - detailed explanation of the routines involved, see here: - http://www.gnu.org/software/libc/manual/html_node/Backtraces.html - http://gcc.gnu.org/onlinedocs/libstdc++/manual/ext_demangling.html.*/ - - void DQMStore::print_trace(std::string const& dir, std::string const& name) { - // the access to the member stream_ is implicitely protected against - // concurrency problems because the print_trace method is always called behind - // a lock (see bookTransaction). - if (!stream_) - stream_ = std::make_unique("histogramBookingBT.log"); - - void* array[10]; - size_t size; - char** strings; - int r = 0; - lat::RegexpMatch m; - m.reset(); - - size = backtrace(array, 10); - strings = backtrace_symbols(array, size); - - size_t level = 1; - char* demangled = nullptr; - for (; level < size; ++level) { - if (!s_rxtrace.match(strings[level], 0, 0, &m)) - continue; - demangled = abi::__cxa_demangle(m.matchString(strings[level], 2).c_str(), nullptr, nullptr, &r); - if (!demangled) - continue; - if (!s_rxself.match(demangled, 0, 0)) - break; - free(demangled); - demangled = nullptr; - } - if (demangled != nullptr) { - *stream_ << "\"" << dir << "/" << name << "\" " << (r ? m.matchString(strings[level], 2) : demangled) << " " - << m.matchString(strings[level], 1) << "\n"; - free(demangled); - } else { - *stream_ << "Skipping " << dir << "/" << name << " with stack size " << size << "\n"; + std::unique_ptr me = std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); + assert(me); + ME* me_ptr = store_->putME(std::move(me)); + assert(me_ptr); + return me_ptr; } - /* In this case print the full stack trace, up to main or to the - * maximum stack size, i.e. 10. */ - if (verbose_ > 4 || demangled == nullptr) { - size_t i; - m.reset(); - - for (i = 0; i < size; ++i) - if (s_rxtrace.match(strings[i], 0, 0, &m)) { - char* demangled = abi::__cxa_demangle(m.matchString(strings[i], 2).c_str(), nullptr, nullptr, &r); - *stream_ << "\t\t" << i << "/" << size << " " << (r ? m.matchString(strings[i], 2) : demangled) << " " - << m.matchString(strings[i], 1) << std::endl; - free(demangled); - } - } - free(strings); - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /// set verbose level (0 turns all non-error messages off) - void DQMStore::setVerbose(unsigned /* level */) {} - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /// return pathname of current directory - std::string const& DQMStore::pwd() const { return pwd_; } - - /// go to top directory (ie. root) - void DQMStore::cd() { setCurrentFolder(""); } - - /// cd to subdirectory (if there) - void DQMStore::cd(std::string const& subdir) { - std::string clean; - std::string const* cleaned = nullptr; - cleanTrailingSlashes(subdir, clean, cleaned); - - if (!dirExists(*cleaned)) - raiseDQMError("DQMStore", "Cannot 'cd' into non-existent directory '%s'", cleaned->c_str()); - - setCurrentFolder(*cleaned); - } - - /// set the last directory in fullpath as the current directory(create if needed); - /// to be invoked by user to specify directories for monitoring objects - /// before booking; - /// commands book1D (etc) & removeElement(name) imply elements in this directory!; - void DQMStore::setCurrentFolder(std::string const& fullpath) { - std::string clean; - std::string const* cleaned = nullptr; - cleanTrailingSlashes(fullpath, clean, cleaned); - makeDirectory(*cleaned); - pwd_ = *cleaned; - } - - /// equivalent to "cd .." - void DQMStore::goUp() { - size_t pos = pwd_.rfind('/'); - if (pos == std::string::npos) - setCurrentFolder(""); - else - setCurrentFolder(pwd_.substr(0, pos)); - } - - // ------------------------------------------------------------------- - /// get folder corresponding to inpath wrt to root (create subdirs if - /// necessary) - void DQMStore::makeDirectory(std::string const& path) { - std::string prev; - std::string subdir; - std::string name; - prev.reserve(path.size()); - subdir.reserve(path.size()); - name.reserve(path.size()); - size_t prevname = 0; - size_t slash = 0; - - while (true) { - // Create this subdirectory component. - subdir.clear(); - subdir.append(path, 0, slash); - name.clear(); - name.append(subdir, prevname, std::string::npos); - if (!prev.empty() && findObject(0, 0, 0, prev, name)) - raiseDQMError("DQMStore", - "Attempt to create subdirectory '%s'" - " which already exists as a monitor element", - subdir.c_str()); - - if (!dirs_.count(subdir)) - dirs_.insert(subdir); - - // Stop if we've reached the end (including possibly a trailing slash). - if (slash + 1 >= path.size()) - break; - - // Find the next slash, making sure we progress. If reach the end, - // process the last path component; the next loop round will terminate. - prevname = slash ? slash + 1 : slash; - prev = subdir; - if ((slash = path.find('/', ++slash)) == std::string::npos) - slash = path.size(); - } - } - - /// true if directory exists - bool DQMStore::dirExists(std::string const& path) const { return dirs_.count(path) > 0; } - - // //==================================================== - // // Global-histogram booking - // MonitorElement* - // DQMStore::bookInt(char_string const& name) - // { - // return bookInt(0, 0, pwd_, name); - // } - - // MonitorElement* - // DQMStore::bookFloat(char_string const& name) - // { - // return bookFloat(0, 0, pwd_, name); - // } - - // MonitorElement* - // DQMStore::bookString(char_string const& name, - // char_string const& value) - // { - // return bookString(0, 0, pwd_, name, value); - // } - - // MonitorElement* - // DQMStore::book1D(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX) - // { - // return book1D(0, 0, pwd_, name, title, nchX, lowX, highX); - // } - - // MonitorElement* - // DQMStore::book1D(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize) - // { - // return book1D(0, 0, pwd_, name, title, nchX, xbinsize); - // } - - // MonitorElement* - // DQMStore::book1D(char_string const& name, TH1F* h) - // { - // return book1D(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book1S(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX) - // { - // return book1S(0, 0, pwd_, name, title, nchX, lowX, highX); - // } - - // MonitorElement* - // DQMStore::book1S(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize) - // { - // return book1S(0, 0, pwd_, name, title, nchX, xbinsize); - // } - - // MonitorElement* - // DQMStore::book1S(char_string const& name, TH1S* h) - // { - // return book1S(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book1DD(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX) - // { - // return book1DD(0, 0, pwd_, name, title, nchX, lowX, highX); - // } - - // MonitorElement* - // DQMStore::book1DD(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize) - // { - // return book1DD(0, 0, pwd_, name, title, nchX, xbinsize); - // } - - // MonitorElement* - // DQMStore::book1DD(char_string const& name, TH1D* h) - // { - // return book1DD(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book2D(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY) - // { - // return book2D(0, 0, pwd_, name, title, nchX, lowX, highX, nchY, lowY, highY); - // } - - // MonitorElement* - // DQMStore::book2D(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize, - // int const nchY, float const* ybinsize) - // { - // return book2D(0, 0, pwd_, name, title, nchX, xbinsize, nchY, ybinsize); - // } - - // MonitorElement* - // DQMStore::book2D(char_string const& name, TH2F* h) - // { - // return book2D(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book2S(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY) - // { - // return book2S(0, 0, pwd_, name, title, nchX, lowX, highX, nchY, lowY, highY); - // } - - // MonitorElement* - // DQMStore::book2S(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize, - // int const nchY, float const* ybinsize) - // { - // return book2S(0, 0, pwd_, name, title, nchX, xbinsize, nchY, ybinsize); - // } - - // MonitorElement* - // DQMStore::book2S(char_string const& name, TH2S* h) - // { - // return book2S(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book2DD(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY) - // { - // return book2DD(0, 0, pwd_, name, title, nchX, lowX, highX, nchY, lowY, highY); - // } - - // MonitorElement* - // DQMStore::book2DD(char_string const& name, - // char_string const& title, - // int const nchX, float const* xbinsize, - // int const nchY, float const* ybinsize) - // { - // return book2DD(0, 0, pwd_, name, title, nchX, xbinsize, nchY, ybinsize); - // } - - // MonitorElement* - // DQMStore::book2DD(char_string const& name, TH2D* h) - // { - // return book2DD(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::book3D(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY, - // int const nchZ, double const lowZ, double const highZ) - // { - // return book3D(0, 0, pwd_, name, title, - // nchX, lowX, highX, - // nchY, lowY, highY, - // nchZ, lowZ, highZ); - // } - - // MonitorElement* - // DQMStore::book3D(char_string const& name, TH3F* h) - // { - // return book3D(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::bookProfile(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY, - // char const* option) - // { - // return bookProfile(0, 0, pwd_, name, title, nchX, lowX, highX, nchY, lowY, highY, option); - // } - - // MonitorElement* - // DQMStore::bookProfile(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // double const lowY, double const highY, - // char const* option) - // { - // return bookProfile(0, 0, pwd_, name, title, nchX, lowX, highX, lowY, highY, option); - // } - - // MonitorElement* - // DQMStore::bookProfile(char_string const& name, - // char_string const& title, - // int const nchX, double const* xbinsize, - // int const nchY, double const lowY, double const highY, - // char const* option) - // { - // return bookProfile(0, 0, pwd_, name, title, nchX, xbinsize, nchY, lowY, highY, option); - // } - - // MonitorElement* - // DQMStore::bookProfile(char_string const& name, - // char_string const& title, - // int const nchX, double const* xbinsize, - // double const lowY, double const highY, - // char const* option) - // { - // return bookProfile(0, 0, pwd_, name, title, nchX, xbinsize, lowY, highY, option); - // } - - // MonitorElement* - // DQMStore::bookProfile(char_string const& name, TProfile* h) - // { - // return bookProfile(0, 0, pwd_, name, h); - // } - - // MonitorElement* - // DQMStore::bookProfile2D(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY, - // int const nchZ, double const lowZ, double const highZ, - // char const* option) - // { - // return bookProfile2D(0, 0, pwd_, name, title, - // nchX, lowX, highX, - // nchY, lowY, highY, - // nchZ, lowZ, highZ, option); - // } - - // MonitorElement* - // DQMStore::bookProfile2D(char_string const& name, - // char_string const& title, - // int const nchX, double const lowX, double const highX, - // int const nchY, double const lowY, double const highY, - // double const lowZ, double const highZ, - // char const* option) - // { - // return bookProfile2D(0, 0, pwd_, name, title, - // nchX, lowX, highX, - // nchY, lowY, highY, - // lowZ, highZ, option); - // } - - // MonitorElement* - // DQMStore::bookProfile2D(char_string const& name, TProfile2D* h) - // { - // return bookProfile2D(0, 0, pwd_, name, h); - // } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - template - MonitorElement* DQMStore::book_(std::string const& dir, - std::string const& name, - char const* context, - MonitorElement::Kind const kind, - HISTO* h, - COLLATE collate) { - assert(name.find('/') == std::string::npos); - if (verbose_ > 3) - print_trace(dir, name); - std::string path; - mergePath(path, dir, name); - - // Put us in charge of h. - h->SetDirectory(nullptr); - - // Check if the request monitor element already exists. - MonitorElement* me = findObject(run_, 0, moduleId_, dir, name); - if (me) { - if (collateHistograms_) { - collate(me, h, verbose_); - delete h; - return me; - } else { - if (verbose_ > 1) - std::cout << "DQMStore: " << context << ": monitor element '" << path << "' already exists, collating" - << std::endl; - me->Reset(); - collate(me, h, verbose_); - delete h; - return me; - } - } else { - // Create and initialise core object. - assert(dirs_.count(dir)); - MonitorElement proto(&*dirs_.find(dir), name, run_, moduleId_); - if (doSaveByLumi_ && canSaveByLumi_) { - // for legacy (not DQMEDAnalyzer) this is not save. - proto.setLumiFlag(); // default to per-lumi mode for all non-legacy MEs. - } - me = (MonitorElement*)const_cast(*data_.insert(std::move(proto)).first) - .initialise((MonitorElement::Kind)kind, h); - - // If we just booked a (plain) MonitorElement, and there is a reference - // MonitorElement with the same name, link the two together. - // The other direction is handled by the extract method. - std::string refdir; - refdir.reserve(s_referenceDirName.size() + dir.size() + 1); - refdir += s_referenceDirName; - refdir += '/'; - refdir += dir; - MonitorElement* referenceME = findObject(0, 0, 0, refdir, name); - if (referenceME) { - // We have booked a new MonitorElement with a specific dir and name. - // Then, if we can find the corresponding MonitorElement in the reference - // dir we assign the object_ of the reference MonitorElement to the - // reference_ property of our new MonitorElement. - me->data_.flags |= DQMNet::DQM_PROP_HAS_REFERENCE; - me->reference_ = referenceME->getTH1(); - } - - // Return the monitor element. - return me; - } - } - - MonitorElement* DQMStore::book_(std::string const& dir, std::string const& name, char const* context) { - assert(name.find('/') == std::string::npos); - if (verbose_ > 3) - print_trace(dir, name); - - // Check if the request monitor element already exists. - if (MonitorElement* me = findObject(run_, 0, moduleId_, dir, name)) { - if (verbose_ > 1) { - std::string path; - mergePath(path, dir, name); - - std::cout << "DQMStore: " << context << ": monitor element '" << path << "' already exists, resetting" - << std::endl; - } - me->Reset(); - return me; - } else { - // Create it and return for initialisation. - assert(dirs_.count(dir)); - MonitorElement proto(&*dirs_.find(dir), name, run_, moduleId_); - // this is used only for Int/String/Float. We don't save these by lumi by - // default, since we can't merge them properly. - return &const_cast(*data_.insert(std::move(proto)).first); - } - } - - // ------------------------------------------------------------------- - /// Book int. - MonitorElement* DQMStore::bookInt_(std::string const& dir, std::string const& name) { - if (collateHistograms_) { - if (MonitorElement* me = findObject(run_, 0, moduleId_, dir, name)) { - me->Fill(0); - return me; - } - } - return (MonitorElement*)book_(dir, name, "bookInt")->initialise(MonitorElement::Kind::INT); - } - - /// Book int. - MonitorElement* DQMStore::bookInt(char_string const& name) { return bookInt_(pwd_, name); } - - // ------------------------------------------------------------------- - /// Book float. - MonitorElement* DQMStore::bookFloat_(std::string const& dir, std::string const& name) { - if (collateHistograms_) { - if (MonitorElement* me = findObject(run_, 0, moduleId_, dir, name)) { - me->Fill(0.); - return me; - } - } - return (MonitorElement*)book_(dir, name, "bookFloat")->initialise(MonitorElement::Kind::REAL); - } - - /// Book float. - MonitorElement* DQMStore::bookFloat(char_string const& name) { return bookFloat_(pwd_, name); } - - // ------------------------------------------------------------------- - /// Book string. - MonitorElement* DQMStore::bookString_(std::string const& dir, std::string const& name, std::string const& value) { - if (collateHistograms_) { - if (MonitorElement* me = findObject(run_, 0, moduleId_, dir, name)) - return me; + template + ME* DQMStore::putME(std::unique_ptr&& me) { + //TODO + return nullptr; } - return (MonitorElement*)book_(dir, name, "bookString")->initialise(MonitorElement::Kind::STRING, value); - } - - /// Book string. - MonitorElement* DQMStore::bookString(char_string const& name, char_string const& value) { - return bookString_(pwd_, name, value); - } - - // ------------------------------------------------------------------- - /// Book 1D histogram based on TH1F. - MonitorElement* DQMStore::book1D_(std::string const& dir, std::string const& name, TH1F* h) { - return book_(dir, name, "book1D", MonitorElement::Kind::TH1F, h, collate1D); - } - - /// Book 1D histogram based on TH1S. - MonitorElement* DQMStore::book1S_(std::string const& dir, std::string const& name, TH1S* h) { - return book_(dir, name, "book1S", MonitorElement::Kind::TH1S, h, collate1S); - } - - /// Book 1D histogram based on TH1D. - MonitorElement* DQMStore::book1DD_(std::string const& dir, std::string const& name, TH1D* h) { - return book_(dir, name, "book1DD", MonitorElement::Kind::TH1D, h, collate1DD); - } - - /// Book 1D histogram. - MonitorElement* DQMStore::book1D( - char_string const& name, char_string const& title, int const nchX, double const lowX, double const highX) { - return book1D_(pwd_, name, new TH1F(name, title, nchX, lowX, highX)); - } - - /// Book 1S histogram. - MonitorElement* DQMStore::book1S( - char_string const& name, char_string const& title, int const nchX, double const lowX, double const highX) { - return book1S_(pwd_, name, new TH1S(name, title, nchX, lowX, highX)); - } - - /// Book 1S histogram. - MonitorElement* DQMStore::book1DD( - char_string const& name, char_string const& title, int const nchX, double const lowX, double const highX) { - return book1DD_(pwd_, name, new TH1D(name, title, nchX, lowX, highX)); - } - - /// Book 1D variable bin histogram. - MonitorElement* DQMStore::book1D(char_string const& name, - char_string const& title, - int const nchX, - const float* xbinsize) { - return book1D_(pwd_, name, new TH1F(name, title, nchX, xbinsize)); - } - - /// Book 1D histogram by cloning an existing histogram. - MonitorElement* DQMStore::book1D(char_string const& name, TH1F* source) { - return book1D_(pwd_, name, static_cast(source->Clone(name))); - } - - /// Book 1S histogram by cloning an existing histogram. - MonitorElement* DQMStore::book1S(char_string const& name, TH1S* source) { - return book1S_(pwd_, name, static_cast(source->Clone(name))); - } - - /// Book 1D double histogram by cloning an existing histogram. - MonitorElement* DQMStore::book1DD(char_string const& name, TH1D* source) { - return book1DD_(pwd_, name, static_cast(source->Clone(name))); - } - - // ------------------------------------------------------------------- - /// Book 2D histogram based on TH2F. - MonitorElement* DQMStore::book2D_(std::string const& dir, std::string const& name, TH2F* h) { - return book_(dir, name, "book2D", MonitorElement::Kind::TH2F, h, collate2D); - } - - /// Book 2D histogram based on TH2S. - MonitorElement* DQMStore::book2S_(std::string const& dir, std::string const& name, TH2S* h) { - return book_(dir, name, "book2S", MonitorElement::Kind::TH2S, h, collate2S); - } - - /// Book 2D histogram based on TH2D. - MonitorElement* DQMStore::book2DD_(std::string const& dir, std::string const& name, TH2D* h) { - return book_(dir, name, "book2DD", MonitorElement::Kind::TH2D, h, collate2DD); - } - /// Book 2D histogram. - MonitorElement* DQMStore::book2D(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY) { - return book2D_(pwd_, name, new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY)); - } + template + void DQMStore::printTrace(std::string const& message) { + edm::LogWarning("DQMStoreBooking").log([&]( auto& logger) { + std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; + std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; - /// Book 2S histogram. - MonitorElement* DQMStore::book2S(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY) { - return book2S_(pwd_, name, new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY)); - } + void* array[10]; + size_t size; + char** strings; + int demangle_status = 0; + std::vector clean_trace; - /// Book 2D histogram. - MonitorElement* DQMStore::book2DD(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY) { - return book2DD_(pwd_, name, new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY)); - } + // glibc/libgcc backtrace functionality, declared in execinfo.h. + size = backtrace(array, 10); + strings = backtrace_symbols(array, size); - /// Book 2D variable bin histogram. - MonitorElement* DQMStore::book2D(char_string const& name, - char_string const& title, - int const nchX, - const float* xbinsize, - int const nchY, - const float* ybinsize) { - return book2D_(pwd_, name, new TH2F(name, title, nchX, xbinsize, nchY, ybinsize)); - } + size_t level = 1; + char* demangled = nullptr; + for (; level < size; ++level) { + std::cmatch match; + bool ok = std::regex_match(strings[level], match, s_rxtrace); - /// Book 2S variable bin histogram. - MonitorElement* DQMStore::book2S(char_string const& name, - char_string const& title, - int const nchX, - const float* xbinsize, - int const nchY, - const float* ybinsize) { - return book2S_(pwd_, name, new TH2S(name, title, nchX, xbinsize, nchY, ybinsize)); - } - - /// Book 2D histogram by cloning an existing histogram. - MonitorElement* DQMStore::book2D(char_string const& name, TH2F* source) { - return book2D_(pwd_, name, static_cast(source->Clone(name))); - } - - /// Book 2DS histogram by cloning an existing histogram. - MonitorElement* DQMStore::book2S(char_string const& name, TH2S* source) { - return book2S_(pwd_, name, static_cast(source->Clone(name))); - } - - /// Book 2DS histogram by cloning an existing histogram. - MonitorElement* DQMStore::book2DD(char_string const& name, TH2D* source) { - return book2DD_(pwd_, name, static_cast(source->Clone(name))); - } + if (!ok) { + edm::LogWarning("DQMStoreBacktrace") << "failed match" << level << strings[level]; + continue; + } - // ------------------------------------------------------------------- - /// Book 3D histogram based on TH3F. - MonitorElement* DQMStore::book3D_(std::string const& dir, std::string const& name, TH3F* h) { - return book_(dir, name, "book3D", MonitorElement::Kind::TH3F, h, collate3D); - } + if (match[2].length() == 0) { + // no symbol, ignore. + continue; + } - /// Book 3D histogram. - MonitorElement* DQMStore::book3D(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY, - int const nchZ, - double const lowZ, - double const highZ) { - return book3D_(pwd_, name, new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ)); - } + // demangle name to human readable form + demangled = abi::__cxa_demangle(std::string(match[2]).c_str(), nullptr, nullptr, &demangle_status); + if (!demangled || demangle_status != 0) { + edm::LogWarning("DQMStoreBacktrace") << "failed demangle! status " << demangle_status << " on " << match[2]; + continue; + } - /// Book 3D histogram by cloning an existing histogram. - MonitorElement* DQMStore::book3D(char_string const& name, TH3F* source) { - return book3D_(pwd_, name, static_cast(source->Clone(name))); - } + if (std::regex_match(demangled, s_rxself)) { + // ignore framework/internal methods + free(demangled); + demangled = nullptr; + continue; + } else { + // keep the demangled name and the address. + // The address can be resolved to a line number in gdb attached to + // the process, using `list *0x`, but it can only be done in + // the running process and we can"t easily do it in this code. + clean_trace.push_back(std::string(demangled) + std::string(match[3])); + free(demangled); + demangled = nullptr; + } + } - // ------------------------------------------------------------------- - /// Book profile histogram based on TProfile. - MonitorElement* DQMStore::bookProfile_(std::string const& dir, std::string const& name, TProfile* h) { - return book_(dir, name, "bookProfile", MonitorElement::Kind::TPROFILE, h, collateProfile); - } + if (clean_trace.size() > 0) { + logger << message << " at "; + for (auto const& s : clean_trace) { + logger << s << "; "; + } + } else { + logger << message << " : failed to collect stack trace."; + } - /// Book profile. Option is one of: " ", "s" (default), "i", "G" (see - /// TProfile::BuildOptions). The number of channels in Y is - /// disregarded in a profile plot. - MonitorElement* DQMStore::bookProfile(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, + free(strings); + }); + } + + + template + ME* IBooker::bookInt(TString const& name) { + return bookME(name, MonitorElementData::Kind::INT, nullptr); + } + template + ME* IBooker::bookFloat(TString const& name) { + return bookME(name, MonitorElementData::Kind::INT, nullptr); + } + template + ME* IBooker::bookString(TString const& name, TString const& value) { + return bookME(name, MonitorElementData::Kind::INT, nullptr); + } + template + ME* IBooker::book1D( + TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { + auto th1 = new TH1F(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + template + ME* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { + auto th1 = new TH1F(name, title, nchX, xbinsize); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + template + ME* IBooker::book1D(TString const& name, TH1F* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + template + ME* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { + auto th1 = new TH1S(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1S, th1); + } + template + ME* IBooker::book1S(TString const& name, TH1S* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1S, th1); + } + template + ME* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { + auto th1 = new TH1D(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1D, th1); + } + template + ME* IBooker::book1DD(TString const& name, TH1D* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1D, th1); + } + template + ME* IBooker::book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + template + ME* IBooker::book2D( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { + auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + template + ME* IBooker::book2D(TString const& name, TH2F* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + template + ME* IBooker::book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + template + ME* IBooker::book2S( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { + auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + template + ME* IBooker::book2S(TString const& name, TH2S* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + template + ME* IBooker::book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2D, th2); + } + template + ME* IBooker::book2DD(TString const& name, TH2D* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2D, th2); + } + template + ME* IBooker::book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ) { + auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); + return bookME(name, MonitorElementData::Kind::TH3F, th3); + } + template + ME* IBooker::book3D(TString const& name, TH3F* object) { + auto th3 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH3F, th3); + } + template + ME* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, int /* nchY */, - double const lowY, - double const highY, - char const* option /* = "s" */) { - return bookProfile_(pwd_, name, new TProfile(name, title, nchX, lowX, highX, lowY, highY, option)); - } - - /// Book profile. Option is one of: " ", "s" (default), "i", "G" (see - /// TProfile::BuildOptions). The number of channels in Y is - /// disregarded in a profile plot. - MonitorElement* DQMStore::bookProfile(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - double const lowY, - double const highY, - char const* option /* = "s" */) { - return bookProfile_(pwd_, name, new TProfile(name, title, nchX, lowX, highX, lowY, highY, option)); - } - - /// Book variable bin profile. Option is one of: " ", "s" (default), "i", "G" (see - /// TProfile::BuildOptions). The number of channels in Y is - /// disregarded in a profile plot. - MonitorElement* DQMStore::bookProfile(char_string const& name, - char_string const& title, - int const nchX, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + template + ME* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + template + ME* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, double const* xbinsize, int /* nchY */, - double const lowY, - double const highY, - char const* option /* = "s" */) { - return bookProfile_(pwd_, name, new TProfile(name, title, nchX, xbinsize, lowY, highY, option)); - } - - /// Book variable bin profile. Option is one of: " ", "s" (default), "i", "G" (see - /// TProfile::BuildOptions). The number of channels in Y is - /// disregarded in a profile plot. - MonitorElement* DQMStore::bookProfile(char_string const& name, - char_string const& title, - int const nchX, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + template + ME* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, double const* xbinsize, - double const lowY, - double const highY, - char const* option /* = "s" */) { - return bookProfile_(pwd_, name, new TProfile(name, title, nchX, xbinsize, lowY, highY, option)); - } - - /// Book TProfile by cloning an existing profile. - MonitorElement* DQMStore::bookProfile(char_string const& name, TProfile* source) { - return bookProfile_(pwd_, name, static_cast(source->Clone(name))); - } - - // ------------------------------------------------------------------- - /// Book 2D profile histogram based on TProfile2D. - MonitorElement* DQMStore::bookProfile2D_(std::string const& dir, std::string const& name, TProfile2D* h) { - return book_(dir, name, "bookProfile2D", MonitorElement::Kind::TPROFILE2D, h, collateProfile2D); - } - - /// Book 2-D profile. Option is one of: " ", "s" (default), "i", "G" - /// (see TProfile2D::BuildOptions). The number of channels in Z is - /// disregarded in a 2-D profile. - MonitorElement* DQMStore::bookProfile2D(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + template + ME* IBooker::bookProfile(TString const& name, TProfile* object) { + auto tprofile = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + template + ME* IBooker::bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + double lowZ, + double highZ, + char const* option) { + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); + } + template + ME* IBooker::bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, int /* nchZ */, - double const lowZ, - double const highZ, - char const* option /* = "s" */) { - return bookProfile2D_( - pwd_, name, new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option)); - } - - /// Book 2-D profile. Option is one of: " ", "s" (default), "i", "G" - /// (see TProfile2D::BuildOptions). The number of channels in Z is - /// disregarded in a 2-D profile. - MonitorElement* DQMStore::bookProfile2D(char_string const& name, - char_string const& title, - int const nchX, - double const lowX, - double const highX, - int const nchY, - double const lowY, - double const highY, - double const lowZ, - double const highZ, - char const* option /* = "s" */) { - return bookProfile2D_( - pwd_, name, new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option)); - } - - /// Book TProfile2D by cloning an existing profile. - MonitorElement* DQMStore::bookProfile2D(char_string const& name, TProfile2D* source) { - return bookProfile2D_(pwd_, name, static_cast(source->Clone(name))); - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - bool DQMStore::checkBinningMatches(MonitorElement* me, TH1* h, unsigned const verbose) { - if (me->getTH1()->GetNbinsX() != h->GetNbinsX() || me->getTH1()->GetNbinsY() != h->GetNbinsY() || - me->getTH1()->GetNbinsZ() != h->GetNbinsZ() || - me->getTH1()->GetXaxis()->GetXmin() != h->GetXaxis()->GetXmin() || - me->getTH1()->GetYaxis()->GetXmin() != h->GetYaxis()->GetXmin() || - me->getTH1()->GetZaxis()->GetXmin() != h->GetZaxis()->GetXmin() || - me->getTH1()->GetXaxis()->GetXmax() != h->GetXaxis()->GetXmax() || - me->getTH1()->GetYaxis()->GetXmax() != h->GetYaxis()->GetXmax() || - me->getTH1()->GetZaxis()->GetXmax() != h->GetZaxis()->GetXmax() || - !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetXaxis(), (TAxis*)h->GetXaxis()) || - !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetYaxis(), (TAxis*)h->GetYaxis()) || - !MonitorElement::CheckBinLabels((TAxis*)me->getTH1()->GetZaxis(), (TAxis*)h->GetZaxis())) { - if (verbose > 0) - std::cout << "*** DQMStore: WARNING:" - << "checkBinningMatches: different binning - cannot add object '" << h->GetName() << "' of type " - << h->IsA()->GetName() << " to existing ME: '" << me->getFullname() << "'\n"; - return false; - } - return true; - } - - void DQMStore::collate1D(MonitorElement* me, TH1F* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH1F()->Add(h); - } - - void DQMStore::collate1S(MonitorElement* me, TH1S* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH1S()->Add(h); - } - - void DQMStore::collate1DD(MonitorElement* me, TH1D* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH1D()->Add(h); - } - - void DQMStore::collate2D(MonitorElement* me, TH2F* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH2F()->Add(h); - } - - void DQMStore::collate2S(MonitorElement* me, TH2S* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH2S()->Add(h); - } - - void DQMStore::collate2DD(MonitorElement* me, TH2D* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH2D()->Add(h); - } - - void DQMStore::collate3D(MonitorElement* me, TH3F* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) - me->getTH3F()->Add(h); - } - - void DQMStore::collateProfile(MonitorElement* me, TProfile* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) { - TProfile* meh = me->getTProfile(); - me->addProfiles(h, meh, meh, 1, 1); - } - } - - void DQMStore::collateProfile2D(MonitorElement* me, TProfile2D* h, unsigned const verbose) { - if (checkBinningMatches(me, h, verbose)) { - TProfile2D* meh = me->getTProfile2D(); - me->addProfiles(h, meh, meh, 1, 1); - } - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /// get list of subdirectories of current directory - std::vector DQMStore::getSubdirs() const { - std::vector result; - auto e = dirs_.end(); - auto i = dirs_.find(pwd_); - - // If we didn't find current directory, the tree is empty, so quit. - if (i == e) - return result; - - // Skip the current directory and then start looking for immediate - // subdirectories in the dirs_ list. Stop when we are no longer in - // (direct or indirect) subdirectories of pwd_. Note that we don't - // "know" which order the set will sort A/B, A/B/C and A/D. - while (++i != e && isSubdirectory(pwd_, *i)) - if (i->find('/', pwd_.size() + 1) == std::string::npos) - result.push_back(*i); - - return result; - } - - /// get list of (non-dir) MEs of current directory - std::vector DQMStore::getMEs() const { - MonitorElement proto(&pwd_, std::string()); - std::vector result; - auto e = data_.end(); - auto i = data_.lower_bound(proto); - for (; i != e && isSubdirectory(pwd_, *i->data_.dirname); ++i) - if (pwd_ == *i->data_.dirname) - result.push_back(i->getName()); - - return result; - } - - /// true if directory (or any subfolder at any level below it) contains - /// at least one monitorable element - bool DQMStore::containsAnyMonitorable(std::string const& path) const { - MonitorElement proto(&path, std::string()); - auto e = data_.end(); - auto i = data_.lower_bound(proto); - return (i != e && isSubdirectory(path, *i->data_.dirname)); - } - - /// get ME from full pathname (e.g. "my/long/dir/my_histo") - MonitorElement* DQMStore::get(std::string const& path) const { - std::string dir; - std::string name; - splitPath(dir, name, path); - MonitorElement proto(&dir, name); - auto mepos = data_.find(proto); - return (mepos == data_.end() ? nullptr : const_cast(&*mepos)); - } - - /// get vector with all children of folder - /// (does NOT include contents of subfolders) - std::vector DQMStore::getContents(std::string const& path) const { - std::string clean; - std::string const* cleaned = nullptr; - cleanTrailingSlashes(path, clean, cleaned); - MonitorElement proto(cleaned, std::string()); - - std::vector result; - auto e = data_.end(); - auto i = data_.lower_bound(proto); - for (; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i) - if (*cleaned == *i->data_.dirname) - result.push_back(const_cast(&*i)); - - return result; - } - - /// get contents; - /// return vector of the form :,,; - /// if showContents = false, change form to : - /// (useful for subscription requests; meant to imply "all contents") - void DQMStore::getContents(std::vector& into, bool const showContents /* = true */) const { - into.clear(); - into.reserve(dirs_.size()); - - auto me = data_.end(); - for (auto const& dir : dirs_) { - MonitorElement proto(&dir, std::string()); - auto mi = data_.lower_bound(proto); - auto m = mi; - size_t sz = dir.size() + 2; - size_t nfound = 0; - for (; m != me && isSubdirectory(dir, *m->data_.dirname); ++m) - if (dir == *m->data_.dirname) { - sz += m->data_.objname.size() + 1; - ++nfound; - } - - if (!nfound) - continue; - - auto istr = into.insert(into.end(), std::string()); - - if (showContents) { - istr->reserve(sz); - - *istr += dir; - *istr += ':'; - for (sz = 0; mi != m; ++mi) { - if (dir != *mi->data_.dirname) - continue; - - if (sz > 0) - *istr += ','; - - *istr += mi->data_.objname; - ++sz; - } - } else { - istr->reserve(dir.size() + 2); - *istr += dir; - *istr += ':'; - } - } - } - - /// get MonitorElement in directory - /// (null if MonitorElement does not exist) - MonitorElement* DQMStore::findObject(uint32_t const run, - uint32_t const lumi, - uint32_t const moduleId, - std::string const& dir, - std::string const& name) const { - if (dir.find_first_not_of(s_safe) != std::string::npos) - raiseDQMError("DQMStore", - "Monitor element path name '%s' uses" - " unacceptable characters", - dir.c_str()); - if (name.find_first_not_of(s_safe) != std::string::npos) - raiseDQMError("DQMStore", - "Monitor element path name '%s' uses" - " unacceptable characters", - name.c_str()); - - MonitorElement proto; - proto.data_.dirname = &dir; - proto.data_.objname = name; - proto.data_.run = run; - proto.data_.lumi = lumi; - proto.data_.moduleId = moduleId; - - auto mepos = data_.find(proto); - return (mepos == data_.end() ? nullptr : const_cast(&*mepos)); - } - - /// get vector with children of folder, including all subfolders + their children; - /// must use an exact pathname - std::vector DQMStore::getAllContents(std::string const& path, - uint32_t const run /* = 0 */, - uint32_t const lumi /* = 0 */) const { - std::string clean; - std::string const* cleaned = nullptr; - cleanTrailingSlashes(path, clean, cleaned); - MonitorElement proto(cleaned, std::string(), run, 0); - proto.setLumi(lumi); - - std::vector result; - auto e = data_.end(); - auto i = data_.lower_bound(proto); - for (; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i) { - if (run != 0) { - if (i->data_.run > run // TODO[rovere]: pleonastic? first we encounter local ME of the same run ... - || i->data_.moduleId != 0) - break; - } - if (lumi != 0) { - if (i->data_.lumi > lumi || i->data_.moduleId != 0) - break; - } - if (run != 0 or lumi != 0) { - assert(i->data_.moduleId == 0); - } - result.push_back(const_cast(&*i)); - } - - if (enableMultiThread_) { - //save legacy modules when running MT - i = data_.begin(); - for (; i != e && isSubdirectory(*cleaned, *i->data_.dirname); ++i) { - if (i->data_.run != 0 or i->data_.moduleId != 0) - break; - result.push_back(const_cast(&*i)); - } - } - - return result; - } - - /// get vector with children of folder, including all subfolders + their children; - /// matches names against a wildcard pattern matched against the full ME path - std::vector DQMStore::getMatchingContents( - std::string const& pattern, lat::Regexp::Syntax const syntaxType /* = Wildcard */) const { - lat::Regexp rx; - try { - rx = lat::Regexp(pattern, 0, syntaxType); - rx.study(); - } catch (lat::Error& e) { - raiseDQMError("DQMStore", "Invalid regular expression '%s': %s", pattern.c_str(), e.explain().c_str()); - } - - std::string path; - std::vector result; - for (auto const& me : data_) { - path.clear(); - mergePath(path, *me.data_.dirname, me.data_.objname); - if (rx.match(path)) - result.push_back(const_cast(&me)); + double lowZ, + double highZ, + char const* option) { + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); } - - return result; - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /** Invoke this method after flushing all recently changed monitoring. - Clears updated flag on all recently updated MEs and calls their - Reset() method for those that have resetMe = true. */ - void DQMStore::reset() { - for (auto const& m : data_) { - auto& me = const_cast(m); - if (me.wasUpdated()) { - if (me.resetMe()) - me.Reset(); - me.resetUpdate(); - } + template + ME* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { + auto tprofile = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); } - reset_ = true; - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /** Invoke this method after flushing all recently changed monitoring. - Clears updated flag on all MEs and calls their Reset() method. */ - void DQMStore::forceReset() { - for (auto const& m : data_) { - if (forceResetOnBeginLumi_ && (m.getLumiFlag() == false)) - continue; - auto& me = const_cast(m); - me.Reset(); - me.resetUpdate(); + template + void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID) { + //TODO } - reset_ = true; - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /** Called after all globalBeginLuminosityBlock. - * Reset global per-lumi MEs (or all MEs if LSbasedMode) so that - * they can be reused. - */ - void DQMStore::postGlobalBeginLumi(edm::GlobalContext const& gc) { - static const std::string null_str(""); - - auto const& lumiblock = gc.luminosityBlockID(); - uint32_t run = lumiblock.run(); - - // find the range of non-legacy global MEs for the current run: - // run != 0, lumi == 0 (implicit), stream id == 0, module id == 0 - const MonitorElement begin(&null_str, null_str, run, 0); - const MonitorElement end(&null_str, null_str, run, 1); - auto i = data_.lower_bound(begin); - const auto e = data_.lower_bound(end); - while (i != e) { - auto& me = const_cast(*i++); - // skip per-run MEs - if (not LSbasedMode_ and not me.getLumiFlag()) - continue; - me.Reset(); - me.resetUpdate(); - } - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /** Clone the lumisection-based histograms from the 'global' ones - * (which have lumi = 0) into per-lumi ones (with the lumi number) - * and reset the global ones. - * The per-lumi histograms can be saved by the output modules, and - * will be deleted at the beginninng of the next lumisection. - */ - - void DQMStore::cloneLumiHistograms(uint32_t const run, uint32_t const lumi, uint32_t const moduleId) { - if (verbose_ > 1) { - std::cout << "DQMStore::cloneLumiHistograms - Preparing lumi histograms for run: " << run << ", lumi: " << lumi - << ", module: " << moduleId << std::endl; - } - - // acquire the global lock since this accesses the undelying data structure - std::lock_guard guard(book_mutex_); - - // MEs are sorted by (run, lumi, stream id, module id, directory, name) - // lumi deafults to 0 - // stream id is always 0 - std::string null_str(""); - auto i = data_.lower_bound(MonitorElement(&null_str, null_str, run, moduleId)); - auto e = data_.lower_bound(MonitorElement(&null_str, null_str, run, moduleId + 1)); - // we will later modify data_, so better do two passes. - auto tobehandled = std::vector(); - for (; i != e; ++i) { - tobehandled.push_back(&*i); + template + MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { + //TODO + return nullptr; } - for (auto i : tobehandled) { - // handle only lumisection-based histograms - if (not LSbasedMode_ and not i->getLumiFlag()) - continue; - // clone the lumisection-based histograms - MonitorElement clone{*i}; - clone.globalize(); - clone.setLumi(lumi); - clone.markToDelete(); - data_.insert(std::move(clone)); - - // reset the ME for the next lumisection - const_cast(&*i)->Reset(); + template + void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID) { + //TODO } - } - /** Same as above, but for run histograms. - */ - void DQMStore::cloneRunHistograms(uint32_t const run, uint32_t const moduleId) { - if (verbose_ > 1) { - std::cout << "DQMStore::cloneRunHistograms - Preparing run histograms for run: " << run - << ", module: " << moduleId << std::endl; + template + std::vector IGetter::getContents(std::string const& path) const { + assert(!"NIY"); } - - // acquire the global lock since this accesses the undelying data structure - std::lock_guard guard(book_mutex_); - - // MEs are sorted by (run, lumi, stream id, module id, directory, name) - // lumi deafults to 0 - // stream id is always 0 - std::string null_str(""); - auto i = data_.lower_bound(MonitorElement(&null_str, null_str, run, moduleId)); - auto e = data_.lower_bound(MonitorElement(&null_str, null_str, run, moduleId + 1)); - // we will later modify data_, so better do two passes. - auto tobehandled = std::vector(); - for (; i != e; ++i) { - tobehandled.push_back(&*i); + template + void IGetter::getContents(std::vector& into, bool showContents) const { + assert(!"NIY"); } - for (auto i : tobehandled) { - // handle only non lumisection-based histograms - if (LSbasedMode_ or i->getLumiFlag()) - continue; - // clone the lumisection-based histograms - MonitorElement clone{*i}; - clone.globalize(); - clone.markToDelete(); - data_.insert(std::move(clone)); - // reset the ME for the next lumisection - const_cast(&*i)->Reset(); + template + std::vector IGetter::getAllContents(std::string const& path) const { + assert(!"NIY"); } - } - - /** Delete *global* histograms which are no longer in use. - * Such histograms are created at the end of each lumi and should be - * deleted after the last globalEndLuminosityBlock. - */ - void DQMStore::deleteUnusedLumiHistograms(uint32_t const run, uint32_t const lumi) { - if (!enableMultiThread_) - return; - - std::lock_guard guard(book_mutex_); - - std::string null_str(""); - MonitorElement proto(&null_str, null_str, run, 0); - proto.setLumi(lumi); - - auto e = data_.end(); - auto i = data_.lower_bound(proto); - - while (i != e) { - if (i->data_.moduleId != 0) - break; - if (i->data_.lumi != lumi) - break; - if (i->data_.run != run) - break; - if (not i->markedToDelete()) { - ++i; - continue; - } - - if (verbose_ > 1) { - std::cout << "DQMStore::deleteUnusedLumiHistograms: deleted monitor element '" << *i->data_.dirname << "/" - << i->data_.objname << "'" - << "flags " << i->data_.flags << "\n"; - } - - i = data_.erase(i); + template + std::vector IGetter::getAllContents(std::string const& path, + uint32_t runNumber, + uint32_t lumi) const { + assert(!"NIY"); } - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - /// extract object (TH1F, TH2F, ...) from ; return success flag - /// flag fromRemoteNode indicating if ME arrived from different node - bool DQMStore::extract(TObject* obj, std::string const& dir, bool const overwrite, bool const collateHistograms) { - // NB: Profile histograms inherit from TH*D, checking order matters. - MonitorElement* refcheck = nullptr; - if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = bookProfile_(dir, h->GetName(), (TProfile*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collateProfile(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = bookProfile2D_(dir, h->GetName(), (TProfile2D*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collateProfile2D(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book1D_(dir, h->GetName(), (TH1F*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate1D(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book1S_(dir, h->GetName(), (TH1S*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate1S(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book1DD_(dir, h->GetName(), (TH1D*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate1DD(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book2D_(dir, h->GetName(), (TH2F*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate2D(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book2S_(dir, h->GetName(), (TH2S*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate2S(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book2DD_(dir, h->GetName(), (TH2D*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate2DD(me, h, verbose_); - refcheck = me; - } else if (auto* h = dynamic_cast(obj)) { - MonitorElement* me = findObject(0, 0, 0, dir, h->GetName()); - if (!me) - me = book3D_(dir, h->GetName(), (TH3F*)h->Clone()); - else if (overwrite) - me->copyFrom(h); - else if (isCollateME(me) || collateHistograms) - collate3D(me, h, verbose_); - refcheck = me; - } else if (dynamic_cast(obj)) { - lat::RegexpMatch m; - if (!s_rxmeval.match(obj->GetName(), 0, 0, &m)) { - if (strstr(obj->GetName(), "CMSSW")) { - if (verbose_) - std::cout << "Input file version: " << obj->GetName() << std::endl; - return true; - } else if (strstr(obj->GetName(), "DQMPATCH")) { - if (verbose_) - std::cout << "DQM patch version: " << obj->GetName() << std::endl; - return true; - } else { - std::cout << "*** DQMStore: WARNING: cannot extract object '" << obj->GetName() << "' of type '" - << obj->IsA()->GetName() << "'\n"; - return false; - } - } - - std::string label = m.matchString(obj->GetName(), 1); - std::string kind = m.matchString(obj->GetName(), 2); - std::string value = m.matchString(obj->GetName(), 3); - - if (kind == "i") { - MonitorElement* me = findObject(0, 0, 0, dir, label); - if (!me || overwrite) { - if (!me) - me = bookInt_(dir, label); - me->Fill(atoll(value.c_str())); - } - } else if (kind == "f") { - MonitorElement* me = findObject(0, 0, 0, dir, label); - if (!me || overwrite) { - if (!me) - me = bookFloat_(dir, label); - me->Fill(atof(value.c_str())); - } - } else if (kind == "s") { - MonitorElement* me = findObject(0, 0, 0, dir, label); - if (!me) - me = bookString_(dir, label, value); - else if (overwrite) - me->Fill(value); - } else if (kind == "e") { - MonitorElement* me = findObject(0, 0, 0, dir, label); - if (!me) { - std::cout << "*** DQMStore: WARNING: no monitor element '" << label << "' in directory '" << dir - << "' to be marked as efficiency plot.\n"; - return false; - } - me->setEfficiencyFlag(); - } else if (kind == "t") { - // ignore tags. - } else if (kind == "qr") { - // Handle qreports, but skip them while reading in references. - if (!isSubdirectory(s_referenceDirName, dir)) { - size_t dot = label.find('.'); - if (dot == std::string::npos) { - std::cout << "*** DQMStore: WARNING: quality report label in '" << label - << "' is missing a '.' and cannot be extracted\n"; - return false; - } - std::string mename(label, 0, dot); - std::string qrname(label, dot + 1, std::string::npos); - - m.reset(); - DQMNet::QValue qv; - if (s_rxmeqr1.match(value, 0, 0, &m)) { - qv.code = atoi(m.matchString(value, 1).c_str()); - qv.qtresult = strtod(m.matchString(value, 2).c_str(), nullptr); - qv.message = m.matchString(value, 4); - qv.qtname = qrname; - qv.algorithm = m.matchString(value, 3); - } else if (s_rxmeqr2.match(value, 0, 0, &m)) { - qv.code = atoi(m.matchString(value, 1).c_str()); - qv.qtresult = 0; // unavailable in old format - qv.message = m.matchString(value, 2); - qv.qtname = qrname; - // qv.algorithm unavailable in old format - } else { - std::cout << "*** DQMStore: WARNING: quality test value '" << value << "' is incorrectly formatted\n"; - return false; - } - - MonitorElement* me = findObject(0, 0, 0, dir, mename); - if (!me) { - std::cout << "*** DQMStore: WARNING: no monitor element '" << mename << "' in directory '" << dir - << "' for quality test '" << label << "'\n"; - return false; - } - - QReport* qr_ref; - DQMNet::QValue* qv_ref; - me->getQReport(true, qv.qtname, qr_ref, qv_ref); - *qv_ref = qv; - me->update(); - } - } else { - std::cout << "*** DQMStore: WARNING: cannot extract object '" << obj->GetName() << "' of type '" - << obj->IsA()->GetName() << "'\n"; - return false; - } - } else if (auto* n = dynamic_cast(obj)) { - // For old DQM data. - std::string s; - s.reserve(6 + strlen(n->GetTitle()) + 2 * strlen(n->GetName())); - s += '<'; - s += n->GetName(); - s += '>'; - s += n->GetTitle(); - s += '<'; - s += '/'; - s += n->GetName(); - s += '>'; - TObjString os(s.c_str()); - return extract(&os, dir, overwrite, collateHistograms_); - } else { - std::cout << "*** DQMStore: WARNING: cannot extract object '" << obj->GetName() << "' of type '" - << obj->IsA()->GetName() << "' and with title '" << obj->GetTitle() << "'\n"; - return false; + template + ME* IGetter::get(std::string const& fullpath) const { + assert(!"NIY"); } - // If we just read in a reference MonitorElement, and there is a - // MonitorElement with the same name, link the two together. - // The other direction is handled by the book() method. - if (refcheck && isSubdirectory(s_referenceDirName, dir)) { - std::string mdir(dir, s_referenceDirName.size() + 1, std::string::npos); - if (MonitorElement* master = findObject(0, 0, 0, mdir, obj->GetName())) { - // We have extracted a MonitorElement, and it's located in the reference - // dir. Then we find the corresponding MonitorElement in the - // non-reference dir and assign the object_ of the reference - // MonitorElement to the reference_ property of the corresponding - // non-reference MonitorElement. - master->data_.flags |= DQMNet::DQM_PROP_HAS_REFERENCE; - master->reference_ = refcheck->getTH1(); - } + template + ME* IGetter::getElement(std::string const& path) const { + assert(!"NIY"); } - return true; - } - - /// Use this for saving monitoring objects in ROOT files with dir structure; - /// cd into directory (create first if it doesn't exist); - /// returns success flag - bool DQMStore::cdInto(std::string const& path) const { - assert(!path.empty()); - - // Find the first path component. - size_t start = 0; - size_t end = path.find('/', start); - if (end == std::string::npos) - end = path.size(); - - while (true) { - // Check if this subdirectory component exists. If yes, make sure - // it is actually a subdirectory. Otherwise create or cd into it. - std::string part(path, start, end - start); - TObject* o = gDirectory->Get(part.c_str()); - if (o && !dynamic_cast(o)) - raiseDQMError("DQMStore", - "Attempt to create directory '%s' in a file" - " fails because the part '%s' already exists and is not" - " directory", - path.c_str(), - part.c_str()); - else if (!o) - gDirectory->mkdir(part.c_str()); - - if (!gDirectory->cd(part.c_str())) - raiseDQMError("DQMStore", - "Attempt to create directory '%s' in a file" - " fails because could not cd into subdirectory '%s'", - path.c_str(), - part.c_str()); - - // Stop if we reached the end, ignoring any trailing '/'. - if (end + 1 >= path.size()) - break; - - // Find the next path component. - start = end + 1; - end = path.find('/', start); - if (end == std::string::npos) - end = path.size(); + template + std::vector IGetter::getSubdirs() const { + assert(!"NIY"); } - - return true; - } - - /// save directory with monitoring objects into protobuf file ; - /// if directory="", save full monitoring structure - void DQMStore::save(std::string const& filename, - std::string const& path /* = "" */, - std::string const& pattern /* = "" */, - std::string const& rewrite /* = "" */, - uint32_t const run /* = 0 */, - uint32_t const lumi /* = 0 */, - SaveReferenceTag const ref /* = SaveWithReference */, - int const minStatus /* = dqm::qstatus::STATUS_OK */, - std::string const& fileupdate /* = RECREATE */) { - // This has slightly different semantics compared to the legacy save(), but - // should work for all relevant cases. - // Most legacy users only pass a filename anyways. - LegacyIOHelper h(this); - h.save(filename, run, fileupdate); - } - - /// read ROOT objects from file in directory ; - /// return total # of ROOT objects read - unsigned int DQMStore::readDirectory(TFile* file, - bool const overwrite, - std::string const& onlypath, - std::string const& prepend, - std::string const& curdir, - OpenRunDirs const stripdirs) { - unsigned int ntot = 0; - unsigned int count = 0; - - if (!file->cd(curdir.c_str())) - raiseDQMError("DQMStore", - "Failed to process directory '%s' while" - " reading file '%s'", - curdir.c_str(), - file->GetName()); - - // Figure out current directory name, but strip out the top - // directory into which we dump everything. - std::string dirpart = curdir; - if (dirpart.compare(0, s_monitorDirName.size(), s_monitorDirName) == 0) { - if (dirpart.size() == s_monitorDirName.size()) - dirpart.clear(); - else if (dirpart[s_monitorDirName.size()] == '/') - dirpart.erase(0, s_monitorDirName.size() + 1); + template + std::vector IGetter::getMEs() const { + assert(!"NIY"); } - - // See if we are going to skip this directory. - bool skip = (!onlypath.empty() && !isSubdirectory(onlypath, dirpart)); - - if (prepend == s_collateDirName || prepend == s_referenceDirName || stripdirs == StripRunDirs) { - // Remove Run # and RunSummary dirs - // first look for Run summary, - // if that is found and erased, also erase Run dir - size_t slash = dirpart.find('/'); - size_t pos = dirpart.find("/Run summary"); - if (slash != std::string::npos && pos != std::string::npos) { - dirpart.erase(pos, 12); - - pos = dirpart.find("Run "); - size_t length = dirpart.find('/', pos + 1) - pos + 1; - if (pos != std::string::npos) - dirpart.erase(pos, length); - } + template + bool IGetter::dirExists(std::string const& path) const { + assert(!"NIY"); } - // If we are prepending, add it to the directory name, - // and suppress reading of already existing reference histograms - if (prepend == s_collateDirName || prepend == s_referenceDirName) { - size_t slash = dirpart.find('/'); - // If we are reading reference, skip previous reference. - if (slash == std::string::npos // skip if Reference is toplevel folder, i.e. no slash - && slash + 1 + s_referenceDirName.size() == dirpart.size() && - dirpart.compare(slash + 1, s_referenceDirName.size(), s_referenceDirName) == 0) - return 0; - - slash = dirpart.find('/'); - // Skip reading of EventInfo subdirectory. - if (slash != std::string::npos && slash + 10 == dirpart.size() && - dirpart.compare(slash + 1, 9, "EventInfo") == 0) { - if (verbose_) - std::cout << "DQMStore::readDirectory: skipping '" << dirpart << "'\n"; - return 0; - } - - // Add prefix. - if (dirpart.empty()) - dirpart = prepend; - else - dirpart = prepend + '/' + dirpart; - } else if (!prepend.empty()) { - if (dirpart.empty()) - dirpart = prepend; - else - dirpart = prepend + '/' + dirpart; + template + IGetter::IGetter(STORE* store) { + store_ = store; } - // Loop over the contents of this directory in the file. - // Post-pone string object handling to happen after other - // objects have been read in so we are guaranteed to have - // histograms by the time we read in quality tests and tags. - TKey* key; - TIter next(gDirectory->GetListOfKeys()); - std::list delayed; - while ((key = (TKey*)next())) { - std::unique_ptr obj(key->ReadObj()); - if (dynamic_cast(obj.get())) { - std::string subdir; - subdir.reserve(curdir.size() + strlen(obj->GetName()) + 2); - subdir += curdir; - if (!curdir.empty()) - subdir += '/'; - subdir += obj->GetName(); + template + DQMStore::DQMStore() : IGetter>(this), IBooker>(this) {} + template + DQMStore::~DQMStore() {} - ntot += readDirectory(file, overwrite, onlypath, prepend, subdir, stripdirs); - } else if (skip) - ; - else if (dynamic_cast(obj.get())) { - delayed.push_back(obj.release()); - } else { - if (verbose_ > 2) - std::cout << "DQMStore: reading object '" << obj->GetName() << "' of type '" << obj->IsA()->GetName() - << "' from '" << file->GetName() << "' into '" << dirpart << "'\n"; - makeDirectory(dirpart); - if (extract(obj.get(), dirpart, overwrite, collateHistograms_)) - ++count; - } + template + void DQMStore::save(std::string const& filename, + std::string const& path, + std::string const& pattern, + std::string const& rewrite, + uint32_t run, + uint32_t lumi, + SaveReferenceTag ref, + int minStatus, + std::string const& fileupdate) { + assert(!"NIY"); } - - while (!delayed.empty()) { - if (verbose_ > 2) - std::cout << "DQMStore: reading object '" << delayed.front()->GetName() << "' of type '" - << delayed.front()->IsA()->GetName() << "' from '" << file->GetName() << "' into '" << dirpart - << "'\n"; - - makeDirectory(dirpart); - if (extract(delayed.front(), dirpart, overwrite, collateHistograms_)) - ++count; - - delete delayed.front(); - delayed.pop_front(); + template + void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { + assert(!"NIY"); } - - if (verbose_ > 1) - std::cout << "DQMStore: read " << count << '/' << ntot << " objects from directory '" << dirpart << "'\n"; - - return ntot + count; - } - - /// public open/read root file , and copy MonitorElements; - /// if flag=true, overwrite identical MonitorElements (default: false); - /// if onlypath != "", read only selected directory - /// if prepend !="", prepend string to path - /// note: by default this method keeps the dir structure as in file - /// and does not update monitor element references! - bool DQMStore::open(std::string const& filename, - bool const overwrite /* = false */, - std::string const& onlypath /* ="" */, - std::string const& prepend /* ="" */, - OpenRunDirs const stripdirs /* =KeepRunDirs */, - bool const fileMustExist /* =true */) { - return readFile(filename, overwrite, onlypath, prepend, stripdirs, fileMustExist); - } - - /// public load root file , and copy MonitorElements; - /// overwrite identical MonitorElements (default: true); - /// set DQMStore.collateHistograms to true to sum several files - /// note: by default this method strips off run dir structure - bool DQMStore::load(std::string const& filename, - OpenRunDirs const stripdirs /* =StripRunDirs */, - bool const fileMustExist /* =true */) { - bool overwrite = true; - if (collateHistograms_) - overwrite = false; - if (verbose_) { - std::cout << "DQMStore::load: reading from file '" << filename << "'\n"; - if (collateHistograms_) - std::cout << "DQMStore::load: in collate mode " - << "\n"; - else - std::cout << "DQMStore::load: in overwrite mode " - << "\n"; + template + bool DQMStore::open(std::string const& filename, + bool overwrite, + std::string const& path, + std::string const& prepend, + OpenRunDirs stripdirs, + bool fileMustExist) { + assert(!"NIY"); } - - if (!s_rxpbfile.match(filename, 0, 0)) - return readFile(filename, overwrite, "", "", stripdirs, fileMustExist); - else - return readFilePB(filename, overwrite, "", "", stripdirs, fileMustExist); - } - - /// private readFile , and copy MonitorElements; - /// if flag=true, overwrite identical MonitorElements (default: false); - /// if onlypath != "", read only selected directory - /// if prepend !="", prepend string to path - /// if StripRunDirs is set the run and run summary folders are erased. - bool DQMStore::readFile(std::string const& filename, - bool const overwrite /* = false */, - std::string const& onlypath /* ="" */, - std::string const& prepend /* ="" */, - OpenRunDirs const stripdirs /* =StripRunDirs */, - bool const fileMustExist /* =true */) { - if (verbose_) - std::cout << "DQMStore::readFile: reading from file '" << filename << "'\n"; - - std::unique_ptr f; - - try { - f.reset(TFile::Open(filename.c_str())); - if (!f.get() || f->IsZombie()) - raiseDQMError("DQMStore", "Failed to open file '%s'", filename.c_str()); - } catch (std::exception&) { - if (fileMustExist) - throw; - else { - if (verbose_) - std::cout << "DQMStore::readFile: file '" << filename << "' does not exist, continuing\n"; - return false; - } + template + bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { + assert(!"NIY"); } - unsigned n = readDirectory(f.get(), overwrite, onlypath, prepend, "", stripdirs); - f->Close(); - - for (auto const& me : data_) - const_cast(me).updateQReportStats(); - - if (verbose_) { - std::cout << "DQMStore::open: successfully read " << n << " objects from file '" << filename << "'"; - if (!onlypath.empty()) - std::cout << " from directory '" << onlypath << "'"; - if (!prepend.empty()) - std::cout << " into directory '" << prepend << "'"; - std::cout << std::endl; + template + void DQMStore::showDirStructure() const { + assert(!"NIY"); } - return true; - } - - /** Extract the next serialised ROOT object from @a buf. Returns null - if there are no more objects in the buffer, or a null pointer was - serialised at this location. */ - inline TObject* DQMStore::extractNextObject(TBufferFile& buf) const { - if (buf.Length() == buf.BufferSize()) - return nullptr; - buf.InitMap(); - void* ptr = buf.ReadObjectAny(nullptr); - return reinterpret_cast(ptr); - } - void DQMStore::get_info(dqmstorepb::ROOTFilePB::Histo const& h, - std::string& dirname, - std::string& objname, - TObject** obj) { - size_t slash = h.full_pathname().rfind('/'); - size_t dirpos = (slash == std::string::npos ? 0 : slash); - size_t namepos = (slash == std::string::npos ? 0 : slash + 1); - dirname.assign(h.full_pathname(), 0, dirpos); - objname.assign(h.full_pathname(), namepos, std::string::npos); - TBufferFile buf(TBufferFile::kRead, h.size(), (void*)h.streamed_histo().data(), kFALSE); - buf.Reset(); - *obj = extractNextObject(buf); - if (!*obj) { - raiseDQMError("DQMStore", "Error reading element:'%s'", h.full_pathname().c_str()); + template + std::vector DQMStore::getMatchingContents(std::string const& pattern) const { + assert(!"NIY"); } - } - - bool DQMStore::readFilePB(std::string const& filename, - bool const overwrite /* = false */, - std::string const& onlypath /* ="" */, - std::string const& prepend /* ="" */, - OpenRunDirs const stripdirs /* =StripRunDirs */, - bool const fileMustExist /* =true */) { - using google::protobuf::io::ArrayInputStream; - using google::protobuf::io::CodedInputStream; - using google::protobuf::io::FileInputStream; - using google::protobuf::io::FileOutputStream; - using google::protobuf::io::GzipInputStream; - using google::protobuf::io::GzipOutputStream; - - if (verbose_) - std::cout << "DQMStore::readFile: reading from file '" << filename << "'\n"; - - int filedescriptor; - if ((filedescriptor = ::open(filename.c_str(), O_RDONLY)) == -1) { - if (fileMustExist) - raiseDQMError("DQMStore", "Failed to open file '%s'", filename.c_str()); - else if (verbose_) - std::cout << "DQMStore::readFile: file '" << filename << "' does not exist, continuing\n"; - return false; - } - - dqmstorepb::ROOTFilePB dqmstore_message; - FileInputStream fin(filedescriptor); - GzipInputStream input(&fin); - CodedInputStream input_coded(&input); - input_coded.SetTotalBytesLimit(1024 * 1024 * 1024, -1); - if (!dqmstore_message.ParseFromCodedStream(&input_coded)) { - raiseDQMError("DQMStore", "Fatal parsing file '%s'", filename.c_str()); - return false; - } - ::close(filedescriptor); - - for (int i = 0; i < dqmstore_message.histo_size(); ++i) { - std::string path; - std::string objname; - - TObject* obj = nullptr; - dqmstorepb::ROOTFilePB::Histo const& h = dqmstore_message.histo(i); - get_info(h, path, objname, &obj); - - setCurrentFolder(path); - if (obj) { - /* Before calling the extract() check if histogram exists: - * if it does - flags for the given monitor are already set (and merged) - * else - set the flags after the histogram is created. - */ - MonitorElement* me = findObject(0, 0, 0, path, objname); - - /* Run histograms should be collated and not overwritten, - * Lumi histograms should be overwritten (and collate flag is not checked) - */ - bool overwrite = h.flags() & DQMNet::DQM_PROP_LUMI; - bool collate = !(h.flags() & DQMNet::DQM_PROP_LUMI); - extract(static_cast(obj), path, overwrite, collate); - - if (me == nullptr) { - me = findObject(0, 0, 0, path, objname); - me->data_.flags = h.flags(); - } - - delete obj; - } - } - - cd(); - return true; - } - - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - void DQMStore::showDirStructure() const { - std::vector contents; - getContents(contents); - - std::cout << " ------------------------------------------------------------\n" - << " Directory structure: \n" - << " ------------------------------------------------------------\n"; - std::copy(contents.begin(), contents.end(), std::ostream_iterator(std::cout, "\n")); + } // namespace implementation +} // namespace dqm - std::cout << " ------------------------------------------------------------\n"; - } +template class dqm::implementation::DQMStore; - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - // check if the collate option is active on the DQMStore - bool DQMStore::isCollate() const { return collateHistograms_; } - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////// - // check if the monitor element is in auto-collation folder - bool DQMStore::isCollateME(MonitorElement* me) const { - return me && isSubdirectory(s_collateDirName, *me->data_.dirname); - } +template class dqm::implementation::IGetter>; -} // namespace dqm::dqmstoreimpl +template class dqm::implementation::IBooker>; diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index e30d77be3c836..80245e99fb526 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -40,176 +40,6 @@ namespace dqm::impl { return h; } - MonitorElement *MonitorElement::initialise(Kind kind) { - switch (kind) { - case Kind::INT: - case Kind::REAL: - case Kind::STRING: - case Kind::TH1F: - case Kind::TH1S: - case Kind::TH1D: - case Kind::TH2F: - case Kind::TH2S: - case Kind::TH2D: - case Kind::TH3F: - case Kind::TPROFILE: - case Kind::TPROFILE2D: - data_.flags &= ~DQMNet::DQM_PROP_TYPE_MASK; - data_.flags |= ((int)kind); - break; - - default: - raiseDQMError("MonitorElement", - "cannot initialise monitor element" - " to invalid type %d", - (int)kind); - } - - return this; - } - - MonitorElement *MonitorElement::initialise(Kind kind, TH1 *rootobj) { - initialise(kind); - auto access = this->accessMut(); - switch (kind) { - case Kind::TH1F: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH1S: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH1D: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH2F: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH2S: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH2D: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TH3F: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TPROFILE: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - case Kind::TPROFILE2D: - assert(dynamic_cast(rootobj)); - assert(!reference_ || dynamic_cast(reference_)); - access.value.object_ = std::unique_ptr(rootobj); - break; - - default: - raiseDQMError("MonitorElement", - "cannot initialise monitor element" - " as a root object with type %d", - (int)kind); - } - - if (reference_) - data_.flags |= DQMNet::DQM_PROP_HAS_REFERENCE; - - return this; - } - - MonitorElement *MonitorElement::initialise(Kind kind, const std::string &value) { - initialise(kind); - auto access = this->accessMut(); - if (kind == Kind::STRING) - access.value.scalar_.str = value; - else - raiseDQMError("MonitorElement", - "cannot initialise monitor element" - " as a string with type %d", - (int)kind); - - return this; - } - - MonitorElement::MonitorElement() - : frozen_(nullptr), mutable_(new MutableMonitorElementData()), reference_(nullptr), refvalue_(nullptr) { - data_.version = 0; - data_.dirname = nullptr; - data_.run = 0; - data_.lumi = 0; - data_.streamId = 0; - data_.moduleId = 0; - data_.tag = 0; - data_.flags = ((int)Kind::INVALID) | DQMNet::DQM_PROP_NEW; - } - - MonitorElement::MonitorElement(const std::string *path, const std::string &name) - : frozen_(nullptr), mutable_(new MutableMonitorElementData()), reference_(nullptr), refvalue_(nullptr) { - data_.version = 0; - data_.run = 0; - data_.lumi = 0; - data_.streamId = 0; - data_.moduleId = 0; - data_.dirname = path; - data_.objname = name; - data_.tag = 0; - data_.flags = ((int)Kind::INVALID) | DQMNet::DQM_PROP_NEW; - } - - MonitorElement::MonitorElement(const std::string *path, const std::string &name, uint32_t run, uint32_t moduleId) - : frozen_(nullptr), mutable_(new MutableMonitorElementData()), reference_(nullptr), refvalue_(nullptr) { - data_.version = 0; - data_.run = run; - data_.lumi = 0; - data_.streamId = 0; - data_.moduleId = moduleId; - data_.dirname = path; - data_.objname = name; - data_.tag = 0; - data_.flags = ((int)Kind::INVALID) | DQMNet::DQM_PROP_NEW; - } - - MonitorElement::MonitorElement(const MonitorElement &x, MonitorElementNoCloneTag) - : data_(x.data_), - frozen_(nullptr), - mutable_(new MutableMonitorElementData()), - reference_(x.reference_), - refvalue_(nullptr), - qreports_(x.qreports_) {} - - MonitorElement::MonitorElement(const MonitorElement &x) - : MonitorElement::MonitorElement(x, MonitorElementNoCloneTag()) { - auto access = this->accessMut(); - auto xaccess = x.access(); - if (xaccess.value.object_) - access.value.object_ = std::unique_ptr(static_cast(xaccess.value.object_->Clone())); - access.value.scalar_ = xaccess.value.scalar_; - - if (x.refvalue_) - refvalue_ = static_cast(x.refvalue_->Clone()); - } MonitorElement::~MonitorElement() { // TODO: this is only as long as we use the edm::Service DQMStore. From c7eb529c02acae76a3713bc8edcfdd203bb3e41d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 13 Nov 2019 18:15:07 +0100 Subject: [PATCH 002/112] Some more fixes to make DQM/ compile. DQMGlobalEDAnalyzer might need an import from the new DQMStore branch. --- DQMServices/Core/interface/DQMEDAnalyzer.h | 1 + DQMServices/Core/interface/DQMOneEDAnalyzer.h | 7 +++++-- DQMServices/Core/interface/DQMStore.h | 6 +++++- DQMServices/Core/src/DQMStore.cc | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/DQMServices/Core/interface/DQMEDAnalyzer.h b/DQMServices/Core/interface/DQMEDAnalyzer.h index 55d0d4248e7e2..d8df411160f6a 100644 --- a/DQMServices/Core/interface/DQMEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMEDAnalyzer.h @@ -40,6 +40,7 @@ class DQMEDAnalyzer : public edm::one::EDProducermoduleDescription().id(), this->getCanSaveByLumi()); + edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); } void beginLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const& setup) final {} diff --git a/DQMServices/Core/interface/DQMOneEDAnalyzer.h b/DQMServices/Core/interface/DQMOneEDAnalyzer.h index a3ad4d8fc3d89..e7c3dcc0d578f 100644 --- a/DQMServices/Core/interface/DQMOneEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMOneEDAnalyzer.h @@ -41,13 +41,14 @@ class DQMOneEDAnalyzer run.run(), this->moduleDescription().id(), this->getCanSaveByLumi()); + edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); } void accumulate(edm::Event const& event, edm::EventSetup const& setup) final { analyze(event, setup); } void endRunProduce(edm::Run& run, edm::EventSetup const& setup) final { dqmEndRun(run, setup); - edm::Service()->cloneRunHistograms(run.run(), this->moduleDescription().id()); + edm::Service()->leaveLumi(run.run(), 0, this->moduleDescription().id()); run.emplace(runToken_); } @@ -89,6 +90,8 @@ class DQMOneLumiEDAnalyzer } void beginLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const& setup) final { + edm::Service()->enterLumi( + lumi.run(), lumi.luminosityBlock(), this->moduleDescription().id()); dqmBeginLuminosityBlock(lumi, setup); } @@ -101,7 +104,7 @@ class DQMOneLumiEDAnalyzer void endLuminosityBlockProduce(edm::LuminosityBlock& lumi, edm::EventSetup const& setup) final { dqmEndLuminosityBlock(lumi, setup); // fully qualified name required for... reasons. - edm::Service()->cloneLumiHistograms( + edm::Service()->leaveLumi( lumi.run(), lumi.luminosityBlock(), this->moduleDescription().id()); lumi.emplace(lumiToken_); } diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 8876716b3f0f7..1d9a15b532212 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -2,6 +2,9 @@ #define DQMServices_Core_DQMStore_h #include "DQMServices/Core/interface/MonitorElement.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" + +#include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" // TODO: Remove at some point: #define TRACE(msg) \ @@ -372,7 +375,7 @@ namespace dqm { enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; enum OpenRunDirs { KeepRunDirs, StripRunDirs }; - DQMStore(); + DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&); ~DQMStore(); // ------------------------------------------------------------------------ @@ -470,6 +473,7 @@ namespace dqm { public: typedef dqm::legacy::IBooker IBooker; typedef dqm::legacy::IGetter IGetter; + using dqm::implementation::DQMStore::DQMStore; }; } // namespace legacy namespace reco { diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 0a09f0e4461eb..a875a709caccc 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -429,7 +429,7 @@ namespace dqm { } template - DQMStore::DQMStore() : IGetter>(this), IBooker>(this) {} + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter>(this), IBooker>(this) {} template DQMStore::~DQMStore() {} From 63ab033b6895bcb7f028631b47eba1c7fccc17b1 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 13 Nov 2019 18:24:48 +0100 Subject: [PATCH 003/112] Random changes to make more compile --- DQMServices/Components/plugins/MEtoEDMConverter.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Components/plugins/MEtoEDMConverter.cc b/DQMServices/Components/plugins/MEtoEDMConverter.cc index 1a99e334e6714..7c36299b89cea 100644 --- a/DQMServices/Components/plugins/MEtoEDMConverter.cc +++ b/DQMServices/Components/plugins/MEtoEDMConverter.cc @@ -85,7 +85,7 @@ MEtoEDMConverter::~MEtoEDMConverter() = default; void MEtoEDMConverter::beginJob() { // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor DQMStore* dbe = edm::Service().operator->(); - enableMultiThread_ = dbe->enableMultiThread_; + enableMultiThread_ = false;//dbe->enableMultiThread_; } void MEtoEDMConverter::endJob() {} From 4697c3a67087f8486f30f42ae7bf6009c364444f Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 18 Nov 2019 11:47:53 +0100 Subject: [PATCH 004/112] Code-format. --- DQMServices/Components/plugins/MEtoEDMConverter.cc | 2 +- DQMServices/Core/interface/DQMStore.h | 9 ++++----- DQMServices/Core/interface/MonitorElement.h | 2 +- DQMServices/Core/src/DQMService.cc | 3 +-- DQMServices/Core/src/DQMStore.cc | 12 ++++-------- DQMServices/Core/src/MonitorElement.cc | 1 - 6 files changed, 11 insertions(+), 18 deletions(-) diff --git a/DQMServices/Components/plugins/MEtoEDMConverter.cc b/DQMServices/Components/plugins/MEtoEDMConverter.cc index 7c36299b89cea..0f6b7303628a6 100644 --- a/DQMServices/Components/plugins/MEtoEDMConverter.cc +++ b/DQMServices/Components/plugins/MEtoEDMConverter.cc @@ -85,7 +85,7 @@ MEtoEDMConverter::~MEtoEDMConverter() = default; void MEtoEDMConverter::beginJob() { // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor DQMStore* dbe = edm::Service().operator->(); - enableMultiThread_ = false;//dbe->enableMultiThread_; + enableMultiThread_ = false; //dbe->enableMultiThread_; } void MEtoEDMConverter::endJob() {} diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 1d9a15b532212..c5ff9e8d6f27f 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -27,7 +27,7 @@ namespace dqm { NavigatorBase(){}; std::string cwd_ = ""; }; - } + } // namespace implementation namespace legacy { @@ -370,7 +370,6 @@ namespace dqm { template class DQMStore : public IGetter>, public IBooker> { public: - // TODO: There are no references any more. we should gt rid of these. enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; enum OpenRunDirs { KeepRunDirs, StripRunDirs }; @@ -437,9 +436,9 @@ namespace dqm { public: // internal -- figure out better protection. template - void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi) {}; + void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi){}; template - void meBookerGetter(iFunc f) {}; + void meBookerGetter(iFunc f){}; // Make a ME owned by this DQMStore. Will return a pointer to a ME owned // by this DQMStore: either an existing ME matching the key of `me` or @@ -473,7 +472,7 @@ namespace dqm { public: typedef dqm::legacy::IBooker IBooker; typedef dqm::legacy::IGetter IGetter; - using dqm::implementation::DQMStore::DQMStore; + using dqm::implementation::DQMStore::DQMStore; }; } // namespace legacy namespace reco { diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index f5f370987c20c..295098480f410 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -163,7 +163,7 @@ namespace dqm::impl { void setLumi(uint32_t ls) { data_.lumi = ls; } public: - MonitorElement(MonitorElementData* data, bool is_owned, bool is_readonly) {}; + MonitorElement(MonitorElementData *data, bool is_owned, bool is_readonly){}; MonitorElement &operator=(const MonitorElement &) = delete; MonitorElement &operator=(MonitorElement &&) = delete; virtual ~MonitorElement(); diff --git a/DQMServices/Core/src/DQMService.cc b/DQMServices/Core/src/DQMService.cc index 2e6dff4f917e5..0dc837af5720d 100644 --- a/DQMServices/Core/src/DQMService.cc +++ b/DQMServices/Core/src/DQMService.cc @@ -78,9 +78,8 @@ void DQMService::flushStandalone() { // Lock the network layer so we can modify the data. net_->lock(); bool updated = false; - - // TODO: re-implement sending MEs. + // TODO: re-implement sending MEs. // Find removed contents and clear the network cache. if (net_->removeLocalExcept(seen)) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index a875a709caccc..09d13a2a75eb4 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -9,7 +9,6 @@ #include #include - namespace dqm { namespace legacy { @@ -72,9 +71,9 @@ namespace dqm { return nullptr; } - template + template void DQMStore::printTrace(std::string const& message) { - edm::LogWarning("DQMStoreBooking").log([&]( auto& logger) { + edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; @@ -140,7 +139,6 @@ namespace dqm { }); } - template ME* IBooker::bookInt(TString const& name) { return bookME(name, MonitorElementData::Kind::INT, nullptr); @@ -378,7 +376,6 @@ namespace dqm { //TODO } - template std::vector IGetter::getContents(std::string const& path) const { assert(!"NIY"); @@ -388,7 +385,6 @@ namespace dqm { assert(!"NIY"); } - template std::vector IGetter::getAllContents(std::string const& path) const { assert(!"NIY"); @@ -429,11 +425,11 @@ namespace dqm { } template - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter>(this), IBooker>(this) {} + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) + : IGetter>(this), IBooker>(this) {} template DQMStore::~DQMStore() {} - template void DQMStore::save(std::string const& filename, std::string const& path, diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 80245e99fb526..bd5fbe07d82d7 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -40,7 +40,6 @@ namespace dqm::impl { return h; } - MonitorElement::~MonitorElement() { // TODO: this is only as long as we use the edm::Service DQMStore. delete mutable_; From 56654225130a0fa8d7f8e992a7ef788387a16ce1 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 14 Nov 2019 12:54:22 +0100 Subject: [PATCH 005/112] Remove enableMultiThread and other unsupported stuff ... from the IO modules. Getting the semantics right will be future work. --- .../FwkIO/plugins/DQMRootOutputModule.cc | 9 ++------ DQMServices/FwkIO/plugins/DQMRootSource.cc | 21 +------------------ 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc index 13ed0764b2be7..22d569c485a09 100644 --- a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc +++ b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc @@ -221,7 +221,6 @@ class DQMRootOutputModule : public edm::one::OutputModule<> { ULong64_t m_firstIndex; ULong64_t m_lastIndex; unsigned int m_filterOnRun; - bool m_enableMultiThread; std::string m_fullNameBuffer; std::string* m_fullNameBufferPtr; @@ -284,7 +283,6 @@ DQMRootOutputModule::DQMRootOutputModule(edm::ParameterSet const& pset) m_treeHelpers(kNIndicies, std::shared_ptr()), m_presentHistoryIndex(0), m_filterOnRun(pset.getUntrackedParameter("filterOnRun")), - m_enableMultiThread(false), m_fullNameBufferPtr(&m_fullNameBuffer), m_indicesTree(nullptr) {} @@ -294,9 +292,6 @@ DQMRootOutputModule::DQMRootOutputModule(edm::ParameterSet const& pset) // } void DQMRootOutputModule::beginJob() { - // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor - edm::Service dstore; - m_enableMultiThread = dstore->enableMultiThread_; } DQMRootOutputModule::~DQMRootOutputModule() {} @@ -389,7 +384,7 @@ void DQMRootOutputModule::writeLuminosityBlock(edm::LuminosityBlockForOutput con if (!shouldWrite) return; std::vector items( - dstore->getAllContents("", m_enableMultiThread ? m_run : 0, m_enableMultiThread ? m_lumi : 0)); + dstore->getAllContents("", m_run, m_lumi)); for (std::vector::iterator it = items.begin(), itEnd = items.end(); it != itEnd; ++it) { if ((*it)->getLumiFlag()) { std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); @@ -446,7 +441,7 @@ void DQMRootOutputModule::writeRun(edm::RunForOutput const& iRun) { if (!shouldWrite) return; - std::vector items(dstore->getAllContents("", m_enableMultiThread ? m_run : 0)); + std::vector items(dstore->getAllContents("", m_run, 0)); for (std::vector::iterator it = items.begin(), itEnd = items.end(); it != itEnd; ++it) { if (not(*it)->getLumiFlag()) { std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index a2fd1a762028e..792f64c459f2f 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -543,28 +543,9 @@ void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { assert(runID == runLumiRange.m_run); m_shouldReadMEs = (m_filterOnRun == 0 || (m_filterOnRun != 0 && m_filterOnRun == runID)); - // std::cout <<"readRun_"< store; - std::vector allMEs = (*store).getAllContents(""); - for (auto const& ME : allMEs) { - if (!(*store).isCollate()) - ME->Reset(); - } + // TODO: Worry about resetting depending on Collate option here. } m_lastSeenReducedPHID = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex); m_lastSeenRun = runID; From 454917454745a016d6f6211a20407adf35deb621 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 14 Nov 2019 17:33:21 +0100 Subject: [PATCH 006/112] Two more changes to make everything compile --- .../SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc | 1 - DQMServices/FileIO/plugins/DQMFileSaverBase.cc | 2 -- 2 files changed, 3 deletions(-) diff --git a/CalibTracker/SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc b/CalibTracker/SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc index 332e65e73d3fb..a2a16507f0c66 100644 --- a/CalibTracker/SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc +++ b/CalibTracker/SiStripChannelGain/plugins/SiStripGainFromCalibTree.cc @@ -394,7 +394,6 @@ SiStripGainFromCalibTree::SiStripGainFromCalibTree(const edm::ParameterSet& iCon // Gather DQM Service dbe = edm::Service().operator->(); - dbe->setVerbose(10); //Set the monitoring element tag and store dqm_tag_.reserve(7); diff --git a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc index 5422149d6c3a6..ac2396fe5f38f 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc @@ -86,8 +86,6 @@ void DQMFileSaverBase::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, this->saveLumi(fp); - edm::Service store; - store->deleteUnusedLumiHistograms(store->mtEnabled() ? irun : 0, ilumi); } void DQMFileSaverBase::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { From 8a601d56ab58df476acaa10b31639127cdfd008d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 18 Nov 2019 16:09:12 +0100 Subject: [PATCH 007/112] Include QReport and friends in MEData. --- DQMServices/Core/interface/DQMChannel.h | 35 -------- DQMServices/Core/interface/DQMNet.h | 11 +-- DQMServices/Core/interface/QReport.h | 54 ------------ .../FileIO/plugins/DQMFileSaverBase.cc | 1 - .../FwkIO/plugins/DQMRootOutputModule.cc | 6 +- .../interface/MonitorElementCollection.h | 88 ++++++++++++++++--- 6 files changed, 81 insertions(+), 114 deletions(-) delete mode 100644 DQMServices/Core/interface/DQMChannel.h delete mode 100644 DQMServices/Core/interface/QReport.h diff --git a/DQMServices/Core/interface/DQMChannel.h b/DQMServices/Core/interface/DQMChannel.h deleted file mode 100644 index c8e6b7eedb14b..0000000000000 --- a/DQMServices/Core/interface/DQMChannel.h +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef DQMSERVICES_CORE_DQM_CHANNEL_H -#define DQMSERVICES_CORE_DQM_CHANNEL_H - -struct DQMChannel { - int binx; //< bin # in x-axis (or bin # for 1D histogram) - int biny; //< bin # in y-axis (for 2D or 3D histograms) - int binz; //< bin # in z-axis (for 3D histograms) - float content; //< bin content - float RMS; //< RMS of bin content - - int getBin() { return getBinX(); } - int getBinX() { return binx; } - int getBinY() { return biny; } - int getBinZ() { return binz; } - float getContents() { return content; } - float getRMS() { return RMS; } - - DQMChannel(int bx, int by, int bz, float data, float rms) { - binx = bx; - biny = by; - binz = bz; - content = data; - RMS = rms; - } - - DQMChannel() { - binx = 0; - biny = 0; - binz = 0; - content = 0; - RMS = 0; - } -}; - -#endif // DQMSERVICES_CORE_DQM_CHANNEL_H diff --git a/DQMServices/Core/interface/DQMNet.h b/DQMServices/Core/interface/DQMNet.h index 5d466ff3f0690..d418302821d6d 100644 --- a/DQMServices/Core/interface/DQMNet.h +++ b/DQMServices/Core/interface/DQMNet.h @@ -18,6 +18,9 @@ #include #include +// for definition of QValue +#inlcude "DataFormats/Histograms/interface/MonitorElementCollection.h" + //class DQMStore; class DQMNet { @@ -82,13 +85,7 @@ class DQMNet { using TagList = std::vector; // DEPRECATED using WaitList = std::list; - struct QValue { - int code; - float qtresult; - std::string message; - std::string qtname; - std::string algorithm; - }; + using QValue = MonitorElementData::QValue; struct CoreObject { uint32_t flags; diff --git a/DQMServices/Core/interface/QReport.h b/DQMServices/Core/interface/QReport.h deleted file mode 100644 index 36b700ac1ccb6..0000000000000 --- a/DQMServices/Core/interface/QReport.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef DQMSERVICES_CORE_Q_REPORT_H -#define DQMSERVICES_CORE_Q_REPORT_H - -#include "DQMServices/Core/interface/DQMDefinitions.h" -#include "DQMServices/Core/interface/DQMNet.h" -#include -#include - -class QCriterion; -namespace dqm::impl { - class MonitorElement; -} -namespace dqm::dqmstoreimpl { - class DQMStore; -} -namespace dqm::legacy { - class MonitorElement; -} - -/** Class for reporting results of quality tests for Monitoring Elements */ -class QReport { -public: - /// get test status (see Core/interface/QTestStatus.h) - int getStatus() const { return qvalue_->code; } - - /// get test result i.e. prob value - float getQTresult() const { return qvalue_->qtresult; } - - /// get message attached to test - const std::string &getMessage() const { return qvalue_->message; } - - /// get name of quality test - const std::string &getQRName() const { return qvalue_->qtname; } - - /// get quality test algorithm - const std::string &getAlgorithm() const { return qvalue_->algorithm; } - - /// get vector of channels that failed test - /// (not relevant for all quality tests!) - const std::vector &getBadChannels() const { return badChannels_; } - -private: - friend class QCriterion; - friend class dqm::legacy::MonitorElement; // for running the quality test - friend class dqm::impl::MonitorElement; // for running the quality test - friend class dqm::dqmstoreimpl::DQMStore; // for setting QReport parameters after receiving report - - QReport(DQMNet::QValue *value) : qvalue_(value) {} - - DQMNet::QValue *qvalue_; //< Pointer to the actual data. - std::vector badChannels_; //< Bad channels from QCriterion. -}; - -#endif // DQMSERVICES_CORE_Q_REPORT_H diff --git a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc index ac2396fe5f38f..ff566494594a1 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc @@ -85,7 +85,6 @@ void DQMFileSaverBase::globalEndLuminosityBlock(const edm::LuminosityBlock &iLS, fp.run_ = irun; this->saveLumi(fp); - } void DQMFileSaverBase::globalEndRun(const edm::Run &iRun, const edm::EventSetup &) const { diff --git a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc index 22d569c485a09..c1f0cd997335a 100644 --- a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc +++ b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc @@ -291,8 +291,7 @@ DQMRootOutputModule::DQMRootOutputModule(edm::ParameterSet const& pset) // // do actual copying here; // } -void DQMRootOutputModule::beginJob() { -} +void DQMRootOutputModule::beginJob() {} DQMRootOutputModule::~DQMRootOutputModule() {} @@ -383,8 +382,7 @@ void DQMRootOutputModule::writeLuminosityBlock(edm::LuminosityBlockForOutput con if (!shouldWrite) return; - std::vector items( - dstore->getAllContents("", m_run, m_lumi)); + std::vector items(dstore->getAllContents("", m_run, m_lumi)); for (std::vector::iterator it = items.begin(), itEnd = items.end(); it != itEnd; ++it) { if ((*it)->getLumiFlag()) { std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index a5a587c069784..6c6f5615405c5 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -12,14 +12,6 @@ required fields to represent a ME. The only opration allowed on these objects is merging, which is a important part of the DQM functionality and should be handled by EDM. - Once a MonitorElement enters this product, it is immutable. During event - processing, the ROOT objects need to be protectd by locks. These locks are not - present in this structure: Any potential modification needs to be done as a - copy-on-write and create a new MonitorElement. - The file IO for these objects should still be handled by the DQMIO classes - (DQMRootOutputModule and DQMRootSource), so persistent=false would be ok for - this class. However, if we can get EDM IO cheaply, it could be useful to - handle corner cases like MEtoEDM more cleanly. Usage: This product should only be handled by the DQMStore, which provides access to the MEs inside. The DQMStore will wrap the MonitorElementData in @@ -27,9 +19,9 @@ histograms, depending on the current stage of processing: In the RECO step, only filling is allowed, while in HARVESTING, the same data will be wrapped in a MonitorElement that also allows access to the ROOT objects. - We only use pointers to MonitorElementData, to allow replacing it with a - variable-size templated version later. That could eliminate one level of - indirection in accessing histograms. + + Currently, the product types are not used as products and all data is passed + through the edm::Service. */ // @@ -56,6 +48,76 @@ struct MonitorElementData { std::string str; }; + // Quality test result types. + // These are inherited from DQMNet/old DQMStore, and left unchanged to avoid + // another layer of wrapping. The APIs are used in some places in subsystem + // code, and could be changed, but not removed. + class QReport { + public: + struct QValue { + int code; + float qtresult; + std::string message; + std::string qtname; + std::string algorithm; + }; + struct DQMChannel { + int binx; //< bin # in x-axis (or bin # for 1D histogram) + int biny; //< bin # in y-axis (for 2D or 3D histograms) + int binz; //< bin # in z-axis (for 3D histograms) + float content; //< bin content + float RMS; //< RMS of bin content + + int getBin() { return getBinX(); } + int getBinX() { return binx; } + int getBinY() { return biny; } + int getBinZ() { return binz; } + float getContents() { return content; } + float getRMS() { return RMS; } + + DQMChannel(int bx, int by, int bz, float data, float rms) { + binx = bx; + biny = by; + binz = bz; + content = data; + RMS = rms; + } + + DQMChannel() { + binx = 0; + biny = 0; + binz = 0; + content = 0; + RMS = 0; + } + }; + + /// get test status + int getStatus() const { return qvalue_->code; } + + /// get test result i.e. prob value + float getQTresult() const { return qvalue_->qtresult; } + + /// get message attached to test + const std::string& getMessage() const { return qvalue_->message; } + + /// get name of quality test + const std::string& getQRName() const { return qvalue_->qtname; } + + /// get quality test algorithm + const std::string& getAlgorithm() const { return qvalue_->algorithm; } + + /// get vector of channels that failed test + /// (not relevant for all quality tests!) + const std::vector& getBadChannels() const { return badChannels_; } + + QReport(DQMNet::QValue* value) : qvalue_(value) {} + + private: + DQMNet::QValue* qvalue_; //< Pointer to the actual data. + std::vector badChannels_; //< Bad channels from QCriterion. + }; + // These values are compatible to DQMNet, but DQMNet is not likely to exist // in the future. enum class Kind { @@ -88,6 +150,7 @@ struct MonitorElementData { struct Value { Scalar scalar_; edm::propagate_const> object_; + std::vector qreports_; }; struct Path { @@ -179,8 +242,7 @@ struct MonitorElementData { // For now, no additional (meta-)data is needed apart from the MEs themselves. // The framework will take care of tracking the plugin and LS/run that the MEs // belong to. - -// Only to hold the mergeProduct placeholder for now. +// Unused for now. class MonitorElementCollection { std::vector> data_; From d318e19541d2e0a56e73bd21b0c5ac0b065bb2f4 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 18 Nov 2019 16:12:52 +0100 Subject: [PATCH 008/112] Remove old includes. --- DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc | 1 - DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.cc | 1 - DQM/BeamMonitor/plugins/BeamMonitor.cc | 1 - DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.cc | 1 - DQM/L1TMonitorClient/src/L1EmulatorErrorFlagClient.cc | 1 - DQM/L1TMonitorClient/src/L1TCSCTFClient.cc | 1 - DQM/L1TMonitorClient/src/L1TDTTPGClient.cc | 1 - DQM/L1TMonitorClient/src/L1TEventInfoClient.cc | 1 - DQM/L1TMonitorClient/src/L1TGCTClient.cc | 1 - DQM/L1TMonitorClient/src/L1TOccupancyClient.cc | 1 - DQM/L1TMonitorClient/src/L1TOccupancyClientHistogramService.cc | 1 - DQM/L1TMonitorClient/src/L1TRPCTFClient.cc | 1 - DQM/L1TMonitorClient/src/L1TTestsSummary.cc | 2 -- DQM/SiStripMonitorClient/src/SiStripQualityChecker.cc | 1 - DQM/TrackingMonitorClient/src/TrackingQualityChecker.cc | 1 - DQMOffline/Trigger/plugins/DQMOfflineHLTEventInfoClient.cc | 1 - DQMServices/Core/interface/DQMDefinitions.h | 1 - 17 files changed, 18 deletions(-) diff --git a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc index 55708fd6fbb06..066c61dcb4e62 100644 --- a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc +++ b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc @@ -17,7 +17,6 @@ #include "RecoVertex/BeamSpotProducer/interface/BeamFitter.h" #include "RecoVertex/BeamSpotProducer/interface/PVFitter.h" #include "DQM/BeamMonitor/plugins/AlcaBeamMonitor.h" -#include "DQMServices/Core/interface/QReport.h" #include "FWCore/Framework/interface/Run.h" #include "FWCore/Framework/interface/LuminosityBlock.h" #include "FWCore/Framework/interface/EventSetup.h" diff --git a/DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.cc b/DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.cc index 11161d0fa5605..0ccce86314bc4 100644 --- a/DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.cc +++ b/DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.cc @@ -6,7 +6,6 @@ #include "DQM/BeamMonitor/plugins/AlcaBeamMonitorClient.h" #include -#include "DQMServices/Core/interface/QReport.h" #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/Framework/interface/LuminosityBlock.h" diff --git a/DQM/BeamMonitor/plugins/BeamMonitor.cc b/DQM/BeamMonitor/plugins/BeamMonitor.cc index 0c5ed7f60572c..39363b6b8ac1e 100644 --- a/DQM/BeamMonitor/plugins/BeamMonitor.cc +++ b/DQM/BeamMonitor/plugins/BeamMonitor.cc @@ -15,7 +15,6 @@ V00-03-25 */ #include "DQM/BeamMonitor/plugins/BeamMonitor.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include "DataFormats/TrackCandidate/interface/TrackCandidate.h" diff --git a/DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.cc b/DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.cc index 72d2d57652d26..20a91d4026df3 100644 --- a/DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.cc +++ b/DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.cc @@ -5,7 +5,6 @@ */ #include "DQM/BeamMonitor/plugins/BeamSpotProblemMonitor.h" -#include "DQMServices/Core/interface/QReport.h" #include "DataFormats/BeamSpot/interface/BeamSpot.h" #include "DataFormats/TrackCandidate/interface/TrackCandidate.h" #include "DataFormats/TrackCandidate/interface/TrackCandidateCollection.h" diff --git a/DQM/L1TMonitorClient/src/L1EmulatorErrorFlagClient.cc b/DQM/L1TMonitorClient/src/L1EmulatorErrorFlagClient.cc index 3c9e8264b4df2..3915c56b86be8 100644 --- a/DQM/L1TMonitorClient/src/L1EmulatorErrorFlagClient.cc +++ b/DQM/L1TMonitorClient/src/L1EmulatorErrorFlagClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "TRandom.h" #include diff --git a/DQM/L1TMonitorClient/src/L1TCSCTFClient.cc b/DQM/L1TMonitorClient/src/L1TCSCTFClient.cc index 7f73cca212a8b..1221ed884e31c 100644 --- a/DQM/L1TMonitorClient/src/L1TCSCTFClient.cc +++ b/DQM/L1TMonitorClient/src/L1TCSCTFClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "TRandom.h" using namespace edm; diff --git a/DQM/L1TMonitorClient/src/L1TDTTPGClient.cc b/DQM/L1TMonitorClient/src/L1TDTTPGClient.cc index a99fd5ac45f5a..84628f0045e54 100644 --- a/DQM/L1TMonitorClient/src/L1TDTTPGClient.cc +++ b/DQM/L1TMonitorClient/src/L1TDTTPGClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "TRandom.h" diff --git a/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc b/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc index 48406490e4c7b..de26e3b7f36a1 100644 --- a/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc +++ b/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc @@ -33,7 +33,6 @@ #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/Utilities/interface/Exception.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "DQMServices/Core/interface/DQMDefinitions.h" diff --git a/DQM/L1TMonitorClient/src/L1TGCTClient.cc b/DQM/L1TMonitorClient/src/L1TGCTClient.cc index f6db7cb949383..d43b2bb0096d6 100644 --- a/DQM/L1TMonitorClient/src/L1TGCTClient.cc +++ b/DQM/L1TMonitorClient/src/L1TGCTClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" using namespace edm; diff --git a/DQM/L1TMonitorClient/src/L1TOccupancyClient.cc b/DQM/L1TMonitorClient/src/L1TOccupancyClient.cc index 3230b15b5e99d..edb880fcd5cbd 100644 --- a/DQM/L1TMonitorClient/src/L1TOccupancyClient.cc +++ b/DQM/L1TMonitorClient/src/L1TOccupancyClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include #include diff --git a/DQM/L1TMonitorClient/src/L1TOccupancyClientHistogramService.cc b/DQM/L1TMonitorClient/src/L1TOccupancyClientHistogramService.cc index 1b709ebf5ff73..7396853f0682f 100644 --- a/DQM/L1TMonitorClient/src/L1TOccupancyClientHistogramService.cc +++ b/DQM/L1TMonitorClient/src/L1TOccupancyClientHistogramService.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include #include diff --git a/DQM/L1TMonitorClient/src/L1TRPCTFClient.cc b/DQM/L1TMonitorClient/src/L1TRPCTFClient.cc index 7f169863d9854..a4cf359ea216a 100644 --- a/DQM/L1TMonitorClient/src/L1TRPCTFClient.cc +++ b/DQM/L1TMonitorClient/src/L1TRPCTFClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include "TRandom.h" diff --git a/DQM/L1TMonitorClient/src/L1TTestsSummary.cc b/DQM/L1TMonitorClient/src/L1TTestsSummary.cc index 9e7cce1b57fcd..cacbcee47b97c 100644 --- a/DQM/L1TMonitorClient/src/L1TTestsSummary.cc +++ b/DQM/L1TMonitorClient/src/L1TTestsSummary.cc @@ -5,9 +5,7 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/interface/DQMChannel.h" #include #include #include diff --git a/DQM/SiStripMonitorClient/src/SiStripQualityChecker.cc b/DQM/SiStripMonitorClient/src/SiStripQualityChecker.cc index 173e7177f67cd..0e6496c30254b 100644 --- a/DQM/SiStripMonitorClient/src/SiStripQualityChecker.cc +++ b/DQM/SiStripMonitorClient/src/SiStripQualityChecker.cc @@ -1,7 +1,6 @@ #include "DQM/SiStripMonitorClient/interface/SiStripQualityChecker.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/interface/QReport.h" #include "CalibFormats/SiStripObjects/interface/SiStripDetCabling.h" diff --git a/DQM/TrackingMonitorClient/src/TrackingQualityChecker.cc b/DQM/TrackingMonitorClient/src/TrackingQualityChecker.cc index ec5ea062ac7de..83362071416ae 100644 --- a/DQM/TrackingMonitorClient/src/TrackingQualityChecker.cc +++ b/DQM/TrackingMonitorClient/src/TrackingQualityChecker.cc @@ -1,7 +1,6 @@ #include "DQM/TrackingMonitorClient/interface/TrackingQualityChecker.h" #include "FWCore/ServiceRegistry/interface/Service.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/interface/QReport.h" #include "CalibFormats/SiStripObjects/interface/SiStripDetCabling.h" diff --git a/DQMOffline/Trigger/plugins/DQMOfflineHLTEventInfoClient.cc b/DQMOffline/Trigger/plugins/DQMOfflineHLTEventInfoClient.cc index 292727590bd97..36662571823e7 100644 --- a/DQMOffline/Trigger/plugins/DQMOfflineHLTEventInfoClient.cc +++ b/DQMOffline/Trigger/plugins/DQMOfflineHLTEventInfoClient.cc @@ -5,7 +5,6 @@ #include "FWCore/Framework/interface/ESHandle.h" #include "FWCore/Framework/interface/EventSetup.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" -#include "DQMServices/Core/interface/QReport.h" #include "DQMServices/Core/interface/DQMStore.h" #include #include diff --git a/DQMServices/Core/interface/DQMDefinitions.h b/DQMServices/Core/interface/DQMDefinitions.h index a3d8a8e660981..c2fefb12c1e9e 100644 --- a/DQMServices/Core/interface/DQMDefinitions.h +++ b/DQMServices/Core/interface/DQMDefinitions.h @@ -1,7 +1,6 @@ #ifndef DQMSERVICES_CORE_DQM_DEFINITIONS_H #define DQMSERVICES_CORE_DQM_DEFINITIONS_H -#include "DQMServices/Core/interface/DQMChannel.h" namespace dqm { /** Numeric constants for quality test results. The smaller the From 2b719783bb79f72017353b4ebeea7588caaaf53d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 18 Nov 2019 18:13:10 +0100 Subject: [PATCH 009/112] Restructure more headers. Mostly to get QReports into MEData. More random changes to make things compile. DQMStore.h is now un-templated, since we don't need the templated declarations for now. --- .../src/L1TEventInfoClient.cc | 1 - DQMServices/Core/interface/DQMDefinitions.h | 24 -- DQMServices/Core/interface/DQMNet.h | 6 +- DQMServices/Core/interface/DQMStore.h | 351 +++++---------- DQMServices/Core/interface/MonitorElement.h | 45 +- DQMServices/Core/interface/QTest.h | 40 +- DQMServices/Core/src/DQMNet.cc | 15 +- DQMServices/Core/src/DQMStore.cc | 400 +++++++----------- DQMServices/Core/src/MonitorElement.cc | 74 ++-- .../interface/MonitorElementCollection.h | 6 +- 10 files changed, 350 insertions(+), 612 deletions(-) delete mode 100644 DQMServices/Core/interface/DQMDefinitions.h diff --git a/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc b/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc index de26e3b7f36a1..a0b8cd6f571ab 100644 --- a/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc +++ b/DQM/L1TMonitorClient/src/L1TEventInfoClient.cc @@ -34,7 +34,6 @@ #include "FWCore/Utilities/interface/Exception.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "DQMServices/Core/interface/DQMDefinitions.h" #include #include "TROOT.h" diff --git a/DQMServices/Core/interface/DQMDefinitions.h b/DQMServices/Core/interface/DQMDefinitions.h deleted file mode 100644 index c2fefb12c1e9e..0000000000000 --- a/DQMServices/Core/interface/DQMDefinitions.h +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef DQMSERVICES_CORE_DQM_DEFINITIONS_H -#define DQMSERVICES_CORE_DQM_DEFINITIONS_H - - -namespace dqm { - /** Numeric constants for quality test results. The smaller the - number, the less severe the message. */ - namespace qstatus { - static const int OTHER = 30; //< Anything but 'ok','warning' or 'error'. - static const int DISABLED = 50; //< Test has been disabled. - static const int INVALID = 60; //< Problem preventing test from running. - static const int INSUF_STAT = 70; //< Insufficient statistics. - static const int DID_NOT_RUN = 90; //< Algorithm did not run. - static const int STATUS_OK = 100; //< Test was succesful. - static const int WARNING = 200; //< Test had some problems. - static const int ERROR = 300; //< Test has failed. - } // namespace qstatus - - namespace me_util { - using Channel = DQMChannel; - } -} // namespace dqm - -#endif // DQMSERVICES_CORE_DQM_DEFINITIONS_H diff --git a/DQMServices/Core/interface/DQMNet.h b/DQMServices/Core/interface/DQMNet.h index d418302821d6d..87a85865e4344 100644 --- a/DQMServices/Core/interface/DQMNet.h +++ b/DQMServices/Core/interface/DQMNet.h @@ -19,7 +19,7 @@ #include // for definition of QValue -#inlcude "DataFormats/Histograms/interface/MonitorElementCollection.h" +#include "DataFormats/Histograms/interface/MonitorElementCollection.h" //class DQMStore; @@ -77,16 +77,14 @@ class DQMNet { static const uint32_t MAX_PEER_WAITREQS = 128; struct Peer; - struct QValue; struct WaitObject; + using QValue = MonitorElementData::QReport::QValue; using DataBlob = std::vector; using QReports = std::vector; using TagList = std::vector; // DEPRECATED using WaitList = std::list; - using QValue = MonitorElementData::QValue; - struct CoreObject { uint32_t flags; uint32_t tag; diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index c5ff9e8d6f27f..13474bf4bb357 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -13,6 +13,16 @@ namespace dqm { namespace implementation { + using MonitorElement = dqm::legacy::MonitorElement; + class DQMStore; + + struct MEComparison { + using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. + bool operator()(MonitorElement* left, MonitorElement* right) const { return false; } + bool operator()(MonitorElement* left, MonitorElementData::Path const& right) const { return false; } + bool operator()(MonitorElementData::Path const& left, MonitorElement* right) const { return false; } + }; + // The common implementation to change folders class NavigatorBase { public: @@ -27,26 +37,20 @@ namespace dqm { NavigatorBase(){}; std::string cwd_ = ""; }; - } // namespace implementation - - namespace legacy { - // The basic IBooker is a virtual interface that returns the common base - // class of MEs (legacy). That justifies it being in the legacy namespace. class IBooker : public dqm::implementation::NavigatorBase { public: - virtual MonitorElement* bookInt(TString const& name) = 0; - virtual MonitorElement* bookFloat(TString const& name) = 0; - virtual MonitorElement* bookString(TString const& name, TString const& value) = 0; + virtual MonitorElement* bookInt(TString const& name); + virtual MonitorElement* bookFloat(TString const& name); + virtual MonitorElement* bookString(TString const& name, TString const& value); virtual MonitorElement* book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX) = 0; - virtual MonitorElement* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) = 0; - virtual MonitorElement* book1D(TString const& name, TH1F* object) = 0; - virtual MonitorElement* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) = 0; - virtual MonitorElement* book1S(TString const& name, TH1S* object) = 0; - virtual MonitorElement* book1DD( - TString const& name, TString const& title, int nchX, double lowX, double highX) = 0; - virtual MonitorElement* book1DD(TString const& name, TH1D* object) = 0; + TString const& name, TString const& title, int const nchX, double const lowX, double const highX); + virtual MonitorElement* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize); + virtual MonitorElement* book1D(TString const& name, TH1F* object); + virtual MonitorElement* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX); + virtual MonitorElement* book1S(TString const& name, TH1S* object); + virtual MonitorElement* book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX); + virtual MonitorElement* book1DD(TString const& name, TH1D* object); virtual MonitorElement* book2D(TString const& name, TString const& title, int nchX, @@ -54,14 +58,10 @@ namespace dqm { double highX, int nchY, double lowY, - double highY) = 0; - virtual MonitorElement* book2D(TString const& name, - TString const& title, - int nchX, - float const* xbinsize, - int nchY, - float const* ybinsize) = 0; - virtual MonitorElement* book2D(TString const& name, TH2F* object) = 0; + double highY); + virtual MonitorElement* book2D( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); + virtual MonitorElement* book2D(TString const& name, TH2F* object); virtual MonitorElement* book2S(TString const& name, TString const& title, int nchX, @@ -69,14 +69,10 @@ namespace dqm { double highX, int nchY, double lowY, - double highY) = 0; - virtual MonitorElement* book2S(TString const& name, - TString const& title, - int nchX, - float const* xbinsize, - int nchY, - float const* ybinsize) = 0; - virtual MonitorElement* book2S(TString const& name, TH2S* object) = 0; + double highY); + virtual MonitorElement* book2S( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); + virtual MonitorElement* book2S(TString const& name, TH2S* object); virtual MonitorElement* book2DD(TString const& name, TString const& title, int nchX, @@ -84,8 +80,8 @@ namespace dqm { double highX, int nchY, double lowY, - double highY) = 0; - virtual MonitorElement* book2DD(TString const& name, TH2D* object) = 0; + double highY); + virtual MonitorElement* book2DD(TString const& name, TH2D* object); virtual MonitorElement* book3D(TString const& name, TString const& title, int nchX, @@ -96,8 +92,8 @@ namespace dqm { double highY, int nchZ, double lowZ, - double highZ) = 0; - virtual MonitorElement* book3D(TString const& name, TH3F* object) = 0; + double highZ); + virtual MonitorElement* book3D(TString const& name, TH3F* object); virtual MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -106,7 +102,7 @@ namespace dqm { int nchY, double lowY, double highY, - char const* option = "s") = 0; + char const* option = "s"); virtual MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -114,7 +110,7 @@ namespace dqm { double highX, double lowY, double highY, - char const* option = "s") = 0; + char const* option = "s"); virtual MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -122,15 +118,15 @@ namespace dqm { int nchY, double lowY, double highY, - char const* option = "s") = 0; + char const* option = "s"); virtual MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, double const* xbinsize, double lowY, double highY, - char const* option = "s") = 0; - virtual MonitorElement* bookProfile(TString const& name, TProfile* object) = 0; + char const* option = "s"); + virtual MonitorElement* bookProfile(TString const& name, TProfile* object); virtual MonitorElement* bookProfile2D(TString const& name, TString const& title, int nchX, @@ -141,7 +137,7 @@ namespace dqm { double highY, double lowZ, double highZ, - char const* option = "s") = 0; + char const* option = "s"); virtual MonitorElement* bookProfile2D(TString const& name, TString const& title, int nchX, @@ -153,222 +149,62 @@ namespace dqm { int nchZ, double lowZ, double highZ, - char const* option = "s") = 0; - virtual MonitorElement* bookProfile2D(TString const& name, TProfile2D* object) = 0; + char const* option = "s"); + virtual MonitorElement* bookProfile2D(TString const& name, TProfile2D* object); - virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope) = 0; + virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); virtual ~IBooker(); protected: - IBooker(); + IBooker(DQMStore* store); + MonitorElement* bookME(TString const& name, MonitorElementData::Kind kind, TH1* object); + + DQMStore* store_; + MonitorElementData::Scope scope_; }; + class IGetter : public dqm::implementation::NavigatorBase { public: // TODO: review and possibly rename the all methods below: // get MEs that are direct children of full path `path` - virtual std::vector getContents(std::string const& path) const = 0; + virtual std::vector getContents(std::string const& path) const; // not clear what this is good for. DQM_DEPRECATED - virtual void getContents(std::vector& into, bool showContents = true) const = 0; + virtual void getContents(std::vector& into, bool showContents = true) const; // get all elements below full path `path` // we have to discuss semantics here -- are run/lumi ever used? - virtual std::vector getAllContents(std::string const& path) const = 0; + virtual std::vector getAllContents(std::string const& path) const; DQM_DEPRECATED virtual std::vector getAllContents(std::string const& path, uint32_t runNumber, - uint32_t lumi) const = 0; + uint32_t lumi) const; // TODO: rename to reflect the fact that it requires full path // return ME identified by full path `path`, or nullptr - virtual MonitorElement* get(std::string const& fullpath) const = 0; + virtual MonitorElement* get(std::string const& fullpath) const; // same as get, throws an exception if histogram not found // Deprecated simply because it is barely used. DQM_DEPRECATED - virtual MonitorElement* getElement(std::string const& path) const = 0; + virtual MonitorElement* getElement(std::string const& path) const; // return sub-directories of current directory - virtual std::vector getSubdirs() const = 0; - // return element names of direct children of current directory - virtual std::vector getMEs() const = 0; - // returns whether there are objects at full path `path` - virtual bool dirExists(std::string const& path) const = 0; - - virtual ~IGetter(); - - protected: - IGetter(); - }; - - } // namespace legacy - // this namespace is for internal use only. - namespace implementation { - // this provides a templated implementation of the DQMStore. The operations it - // does are rather always the same; the only thing that changes are the return - // types. We keep IBooker and IGetter separate, just in case. DQMStore simply - // multi-inherits them for now. - // We will instantiate this for reco MEs and harvesting MEs, and maybe for - // legacy as well. - - template - class IBooker : public dqm::legacy::IBooker { - public: - virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); - - virtual ME* bookInt(TString const& name); - virtual ME* bookFloat(TString const& name); - virtual ME* bookString(TString const& name, TString const& value); - virtual ME* book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX); - virtual ME* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize); - virtual ME* book1D(TString const& name, TH1F* object); - virtual ME* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX); - virtual ME* book1S(TString const& name, TH1S* object); - virtual ME* book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX); - virtual ME* book1DD(TString const& name, TH1D* object); - virtual ME* book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual ME* book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - virtual ME* book2D(TString const& name, TH2F* object); - virtual ME* book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual ME* book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - virtual ME* book2S(TString const& name, TH2S* object); - virtual ME* book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual ME* book2DD(TString const& name, TH2D* object); - virtual ME* book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ); - virtual ME* book3D(TString const& name, TH3F* object); - virtual ME* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - char const* option = "s"); - virtual ME* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option = "s"); - virtual ME* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int nchY, - double lowY, - double highY, - char const* option = "s"); - virtual ME* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option = "s"); - virtual ME* bookProfile(TString const& name, TProfile* object); - virtual ME* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option = "s"); - virtual ME* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ, - char const* option = "s"); - virtual ME* bookProfile2D(TString const& name, TProfile2D* object); - - virtual ~IBooker(){}; - - protected: - IBooker(STORE* store); - ME* bookME(TString const& name, MonitorElementData::Kind kind, TH1* object); - - STORE* store_; - MonitorElementData::Scope scope_; - }; - - template - class IGetter : public dqm::legacy::IGetter { - public: - // TODO: while we can have covariant return types for individual ME*, it seems we can't for the vectors. - virtual std::vector getContents(std::string const& path) const; - virtual void getContents(std::vector& into, bool showContents = true) const; - - virtual std::vector getAllContents(std::string const& path) const; - DQM_DEPRECATED - virtual std::vector getAllContents(std::string const& path, - uint32_t runNumber, - uint32_t lumi) const; - virtual ME* get(std::string const& fullpath) const; - - DQM_DEPRECATED - virtual ME* getElement(std::string const& path) const; - virtual std::vector getSubdirs() const; + // return element names of direct children of current directory virtual std::vector getMEs() const; + // returns whether there are objects at full path `path` virtual bool dirExists(std::string const& path) const; - virtual ~IGetter(){}; + virtual ~IGetter(); protected: - IGetter(STORE* store); + IGetter(DQMStore* store); - STORE* store_; + DQMStore* store_; }; - template - class DQMStore : public IGetter>, public IBooker> { + class DQMStore : public IGetter, public IBooker { public: // TODO: There are no references any more. we should gt rid of these. enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; @@ -408,7 +244,7 @@ namespace dqm { // TODO: getting API not part of IGetter. DQM_DEPRECATED - std::vector getMatchingContents(std::string const& pattern) const; + std::vector getMatchingContents(std::string const& pattern) const; DQMStore(DQMStore const&) = delete; DQMStore& operator=(DQMStore const&) = delete; @@ -416,22 +252,22 @@ namespace dqm { // ------------------------------------------------------------------------ // ------------ IBooker/IGetter overrides to prevent ambiguity ------------ virtual void cd() { - this->IBooker>::cd(); - this->IGetter>::cd(); + this->IBooker::cd(); + this->IGetter::cd(); } virtual void cd(std::string const& dir) { - this->IBooker>::cd(dir); - this->IGetter>::cd(dir); + this->IBooker::cd(dir); + this->IGetter::cd(dir); } virtual void setCurrentFolder(std::string const& fullpath) { - this->IBooker>::setCurrentFolder(fullpath); - this->IGetter>::setCurrentFolder(fullpath); + this->IBooker::setCurrentFolder(fullpath); + this->IGetter::setCurrentFolder(fullpath); } virtual void goUp() { - this->IBooker>::goUp(); - this->IGetter>::goUp(); + this->IBooker::goUp(); + this->IGetter::goUp(); } - std::string const& pwd() { return this->IBooker>::pwd(); } + std::string const& pwd() { return this->IBooker::pwd(); } public: // internal -- figure out better protection. @@ -440,39 +276,50 @@ namespace dqm { template void meBookerGetter(iFunc f){}; - // Make a ME owned by this DQMStore. Will return a pointer to a ME owned - // by this DQMStore: either an existing ME matching the key of `me` or - // a newly added one. // Will take ownership of the ROOT object in `me`, deleting it if not // needed. - ME* putME(std::unique_ptr&& me); + MonitorElement* putME(std::unique_ptr&& me); // Log a backtrace on booking. void printTrace(std::string const& message); - // Prepare MEs for the next lumisection. This will create per-lumi copies - // if ther previous lumi has not yet finished and recycle reusable MEs if - // booking left any. - // enterLumi is idempotent; it can be called at any time to update lumi - // ranges in the MEs. This may be needed in harvesting, where MEs can be - // booked at any time. - void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID); - // Turn the MEs associated with t, run, lumi into a global ME. - void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleIDi); + void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); + void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); // Clone data including the underlying ROOT object (calls ->Clone()). static MonitorElementData* cloneMonitorElementData(MonitorElementData const* input); - // TODO: Make this section private. localmes_ and inputs_ should be friends with IGetter. - public: + private: + // TODO: MEComparison need to deref pointers, and allow comparison to + // bare strings, and provide `is_transparent` for heterogeneous lookup. + // The ME objects here are lightweight, all the important stuff is in the + // MEData. However we never handle MEData directly, but always keep it + // wrapped in MEs (created as needed). MEs can share MEData. + // ME objects must never appear more than once in these sets. ME objects + // in localMEs_ cannot be deleted, since the module might hold pointers. + // MEs in globalMEs_ can be deleted/recycled at the end of their scope, + // if there are no MEs left that share the data -- for non-legacy modules + // that should hold by construction, for legacy MEs (moduleID == 0) it + // needs an explicit check. + // MEs can be _protoype MEs_ if their scope is not yet known (after + // booking, after leaveLumi). A prototype is kept if and only if there is + // no other global instance of the same ME. Prototype MEs have + // run = lumi = 0 and scope != JOB. If scope == JOB, a prototype is never + // required. Prototype MEs are reset *before* inserting, so fill calls + // can go into prototype MEs and not be lost. + // Key is (run, lumi), potentially one or both 0 for SCOPE::RUN or SCOPE::JOB + std::map, std::set> + globalMEs_; + // Key is (moduleID [, run]), run is only needed for edm::global. + std::map> localMEs_; }; } // namespace implementation // Since we still use a single, edm::Serivce instance of a DQMStore, these are all the same. namespace legacy { - class DQMStore : public dqm::implementation::DQMStore { + class DQMStore : public dqm::implementation::DQMStore { public: - typedef dqm::legacy::IBooker IBooker; - typedef dqm::legacy::IGetter IGetter; - using dqm::implementation::DQMStore::DQMStore; + typedef dqm::implementation::IBooker IBooker; + typedef dqm::implementation::IGetter IGetter; + using dqm::implementation::DQMStore::DQMStore; }; } // namespace legacy namespace reco { diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 295098480f410..f079a5b48a763 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -7,7 +7,6 @@ #endif #include "DQMServices/Core/interface/DQMNet.h" -#include "DQMServices/Core/interface/QReport.h" #include "DataFormats/Histograms/interface/MonitorElementCollection.h" @@ -36,6 +35,30 @@ #include #include +// TODO: cleaup the usages and remove. +using QReport = MonitorElementData::QReport; +using DQMChannel = MonitorElementData::QReport::DQMChannel; + +// TODO: move to a better location (changing all usages) +namespace dqm { + /** Numeric constants for quality test results. The smaller the + number, the less severe the message. */ + namespace qstatus { + static const int OTHER = 30; //< Anything but 'ok','warning' or 'error'. + static const int DISABLED = 50; //< Test has been disabled. + static const int INVALID = 60; //< Problem preventing test from running. + static const int INSUF_STAT = 70; //< Insufficient statistics. + static const int DID_NOT_RUN = 90; //< Algorithm did not run. + static const int STATUS_OK = 100; //< Test was succesful. + static const int WARNING = 200; //< Test had some problems. + static const int ERROR = 300; //< Test has failed. + } // namespace qstatus + + namespace me_util { + using Channel = DQMChannel; + } +} // namespace dqm + class DQMService; namespace dqm::dqmstoreimpl { class DQMStore; @@ -154,14 +177,6 @@ namespace dqm::impl { // products once all processing is done (logically, this is safe). } - //TODO: to be dropped. - TH1 *reference_; //< Current ROOT reference object. - TH1 *refvalue_; //< Soft reference if any. - std::vector qreports_; //< QReports associated to this object. - - void globalize() { data_.moduleId = 0; } - void setLumi(uint32_t ls) { data_.lumi = ls; } - public: MonitorElement(MonitorElementData *data, bool is_owned, bool is_readonly){}; MonitorElement &operator=(const MonitorElement &) = delete; @@ -303,20 +318,20 @@ namespace dqm::impl { bool isEfficiency() const { return data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT; } /// get QReport corresponding to (null pointer if QReport does not exist) - const QReport *getQReport(const std::string &qtname) const; + const MonitorElementData::QReport *getQReport(const std::string &qtname) const; /// get map of QReports - std::vector getQReports() const; + std::vector getQReports() const; /// access QReport, potentially adding it. - void getQReport(bool create, const std::string &qtname, QReport *&qr, DQMNet::QValue *&qv); + void getQReport(bool create, const std::string &qtname, MonitorElementData::QReport *&qr, DQMNet::QValue *&qv); /// propagate QReport status bits after change void updateQReportStats(); /// get warnings from last set of quality tests - std::vector getQWarnings() const; + std::vector getQWarnings() const; /// get errors from last set of quality tests - std::vector getQErrors() const; + std::vector getQErrors() const; /// from last set of quality tests - std::vector getQOthers() const; + std::vector getQOthers() const; // const and data-independent -- safe virtual int getNbinsX() const; diff --git a/DQMServices/Core/interface/QTest.h b/DQMServices/Core/interface/QTest.h index 73b10ff45eefb..5f14b59a074a3 100644 --- a/DQMServices/Core/interface/QTest.h +++ b/DQMServices/Core/interface/QTest.h @@ -13,40 +13,8 @@ //#include "DQMServices/Core/interface/DQMStore.h" -class ContentsXRange; -using ContentsXRangeROOT = ContentsXRange; -class ContentsYRange; -using ContentsYRangeROOT = ContentsYRange; -class NoisyChannel; -using NoisyChannelROOT = NoisyChannel; -class ContentSigma; -using ContentSigmaROOT = ContentSigma; -class DeadChannel; -using DeadChannelROOT = DeadChannel; -class ContentsWithinExpected; -using ContentsWithinExpectedROOT = ContentsWithinExpected; -class MeanWithinExpected; -using MeanWithinExpectedROOT = MeanWithinExpected; -//class AllContentWithinFixedRange; typedef AllContentWithinFixedRange RuleAllContentWithinFixedRange; typedef AllContentWithinFixedRange AllContentWithinFixedRangeROOT; -//class AllContentWithinFloatingRange; typedef AllContentWithinFloatingRange RuleAllContentWithinFloatingRange; typedef AllContentWithinFloatingRange AllContentWithinFloatingRangeROOT; -class FlatOccupancy1d; -using RuleFlatOccupancy1d = FlatOccupancy1d; -using FlatOccupancy1dROOT = FlatOccupancy1d; -class FixedFlatOccupancy1d; -using RuleFixedFlatOccupancy1d = FixedFlatOccupancy1d; -using FixedFlatOccupancy1dROOT = FixedFlatOccupancy1d; -class CSC01; -using RuleCSC01 = CSC01; -using CSC01ROOT = CSC01; -class AllContentAlongDiagonal; -using RuleAllContentAlongDiagonal = AllContentAlongDiagonal; -using AllContentAlongDiagonalROOT = AllContentAlongDiagonal; -class CompareToMedian; -using CompareToMedianROOT = CompareToMedian; -class CompareLastFilledBin; -using CompareLastFilledBinROOT = CompareLastFilledBin; -class CheckVariance; -using CheckVarianceROOT = CheckVariance; +using DQMChannel = MonitorElementData::QReport::DQMChannel; +using QReport = MonitorElementData::QReport; /** Base class for quality tests run on Monitoring Elements; @@ -65,7 +33,7 @@ class QCriterion { public: typedef dqm::legacy::MonitorElement MonitorElement; - /// get test status (see Core/interface/DQMDefinitions.h) + /// get test status int getStatus() const { return status_; } /// get message attached to test std::string getMessage() const { return message_; } @@ -118,7 +86,7 @@ class QCriterion { qv.qtname = qtname_; qv.algorithm = algoName_; qv.qtresult = prob_; - qr.badChannels_ = getBadChannels(); + qr.setBadChannels(getBadChannels()); return prob_; } diff --git a/DQMServices/Core/src/DQMNet.cc b/DQMServices/Core/src/DQMNet.cc index f236d244eee97..7fab0fac44a5e 100644 --- a/DQMServices/Core/src/DQMNet.cc +++ b/DQMServices/Core/src/DQMNet.cc @@ -1,5 +1,4 @@ #include "DQMServices/Core/interface/DQMNet.h" -#include "DQMServices/Core/interface/DQMDefinitions.h" #include "DQMServices/Core/src/DQMError.h" #include "classlib/iobase/InetServerSocket.h" #include "classlib/iobase/LocalServerSocket.h" @@ -36,6 +35,20 @@ using namespace lat; static const Regexp s_rxmeval("<(.*)>(i|f|s|qr)=(.*)"); +// TODO: Can't include the header file since that leads to ambiguities. +namespace dqm { + namespace qstatus { + static const int OTHER = 30; //< Anything but 'ok','warning' or 'error'. + static const int DISABLED = 50; //< Test has been disabled. + static const int INVALID = 60; //< Problem preventing test from running. + static const int INSUF_STAT = 70; //< Insufficient statistics. + static const int DID_NOT_RUN = 90; //< Algorithm did not run. + static const int STATUS_OK = 100; //< Test was succesful. + static const int WARNING = 200; //< Test had some problems. + static const int ERROR = 300; //< Test has failed. + } // namespace qstatus +} // namespace dqm + ////////////////////////////////////////////////////////////////////// // Generate log prefix. std::ostream &DQMNet::logme() { diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 09d13a2a75eb4..92947b921f108 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -11,14 +11,6 @@ namespace dqm { - namespace legacy { - IBooker::IBooker() {} - IBooker::~IBooker() {} - IGetter::IGetter() {} - IGetter::~IGetter() {} - - } // namespace legacy - namespace implementation { std::string const& NavigatorBase::pwd() { return cwd_; } void NavigatorBase::cd() { setCurrentFolder(""); } @@ -31,21 +23,20 @@ namespace dqm { cwd_ = path.getDirname(); } - template - IBooker::IBooker(STORE* store) { + IBooker::IBooker(DQMStore* store) { store_ = store; scope_ = MonitorElementData::Scope::RUN; } - template - MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) { + IBooker::~IBooker() {} + + MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) { auto oldscope = scope_; scope_ = newscope; return oldscope; } - template - ME* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { + MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { MonitorElementData* data = new MonitorElementData(); MonitorElementData::Key key; key.kind_ = kind; @@ -58,21 +49,20 @@ namespace dqm { //value.object = std::unique_ptr(object); } - std::unique_ptr me = std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); + std::unique_ptr me = + std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); assert(me); - ME* me_ptr = store_->putME(std::move(me)); + MonitorElement* me_ptr = store_->putME(std::move(me)); assert(me_ptr); return me_ptr; } - template - ME* DQMStore::putME(std::unique_ptr&& me) { + MonitorElement* DQMStore::putME(std::unique_ptr&& me) { //TODO return nullptr; } - template - void DQMStore::printTrace(std::string const& message) { + void DQMStore::printTrace(std::string const& message) { edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; @@ -139,344 +129,268 @@ namespace dqm { }); } - template - ME* IBooker::bookInt(TString const& name) { + MonitorElement* IBooker::bookInt(TString const& name) { return bookME(name, MonitorElementData::Kind::INT, nullptr); } - template - ME* IBooker::bookFloat(TString const& name) { + MonitorElement* IBooker::bookFloat(TString const& name) { return bookME(name, MonitorElementData::Kind::INT, nullptr); } - template - ME* IBooker::bookString(TString const& name, TString const& value) { + MonitorElement* IBooker::bookString(TString const& name, TString const& value) { return bookME(name, MonitorElementData::Kind::INT, nullptr); } - template - ME* IBooker::book1D( + MonitorElement* IBooker::book1D( TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { auto th1 = new TH1F(name, title, nchX, lowX, highX); return bookME(name, MonitorElementData::Kind::TH1F, th1); } - template - ME* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { + MonitorElement* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { auto th1 = new TH1F(name, title, nchX, xbinsize); return bookME(name, MonitorElementData::Kind::TH1F, th1); } - template - ME* IBooker::book1D(TString const& name, TH1F* object) { + MonitorElement* IBooker::book1D(TString const& name, TH1F* object) { auto th1 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH1F, th1); } - template - ME* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { + MonitorElement* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { auto th1 = new TH1S(name, title, nchX, lowX, highX); return bookME(name, MonitorElementData::Kind::TH1S, th1); } - template - ME* IBooker::book1S(TString const& name, TH1S* object) { + MonitorElement* IBooker::book1S(TString const& name, TH1S* object) { auto th1 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH1S, th1); } - template - ME* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { + MonitorElement* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { auto th1 = new TH1D(name, title, nchX, lowX, highX); return bookME(name, MonitorElementData::Kind::TH1D, th1); } - template - ME* IBooker::book1DD(TString const& name, TH1D* object) { + MonitorElement* IBooker::book1DD(TString const& name, TH1D* object) { auto th1 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH1D, th1); } - template - ME* IBooker::book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { + MonitorElement* IBooker::book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); return bookME(name, MonitorElementData::Kind::TH2F, th2); } - template - ME* IBooker::book2D( + MonitorElement* IBooker::book2D( TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); return bookME(name, MonitorElementData::Kind::TH2F, th2); } - template - ME* IBooker::book2D(TString const& name, TH2F* object) { + MonitorElement* IBooker::book2D(TString const& name, TH2F* object) { auto th2 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH2F, th2); } - template - ME* IBooker::book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { + MonitorElement* IBooker::book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); return bookME(name, MonitorElementData::Kind::TH2S, th2); } - template - ME* IBooker::book2S( + MonitorElement* IBooker::book2S( TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); return bookME(name, MonitorElementData::Kind::TH2S, th2); } - template - ME* IBooker::book2S(TString const& name, TH2S* object) { + MonitorElement* IBooker::book2S(TString const& name, TH2S* object) { auto th2 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH2S, th2); } - template - ME* IBooker::book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { + MonitorElement* IBooker::book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); return bookME(name, MonitorElementData::Kind::TH2D, th2); } - template - ME* IBooker::book2DD(TString const& name, TH2D* object) { + MonitorElement* IBooker::book2DD(TString const& name, TH2D* object) { auto th2 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH2D, th2); } - template - ME* IBooker::book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ) { + MonitorElement* IBooker::book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ) { auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); return bookME(name, MonitorElementData::Kind::TH3F, th3); } - template - ME* IBooker::book3D(TString const& name, TH3F* object) { + MonitorElement* IBooker::book3D(TString const& name, TH3F* object) { auto th3 = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TH3F, th3); } - template - ME* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int /* nchY */, - double lowY, - double highY, - char const* option) { + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int /* nchY */, + double lowY, + double highY, + char const* option) { auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); } - template - ME* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option) { + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option) { auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); } - template - ME* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int /* nchY */, - double lowY, - double highY, - char const* option) { + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + int /* nchY */, + double lowY, + double highY, + char const* option) { auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); } - template - ME* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option) { + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + double lowY, + double highY, + char const* option) { auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); } - template - ME* IBooker::bookProfile(TString const& name, TProfile* object) { + MonitorElement* IBooker::bookProfile(TString const& name, TProfile* object) { auto tprofile = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); } - template - ME* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option) { + MonitorElement* IBooker::bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + double lowZ, + double highZ, + char const* option) { auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); } - template - ME* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int /* nchZ */, - double lowZ, - double highZ, - char const* option) { + MonitorElement* IBooker::bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int /* nchZ */, + double lowZ, + double highZ, + char const* option) { auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); } - template - ME* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { + MonitorElement* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { auto tprofile = static_cast(object->Clone(name)); return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); } - template - void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID) { + void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { //TODO } - template - MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { + MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { //TODO return nullptr; } - template - void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, unsigned int moduleID) { + void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { //TODO } - template - std::vector IGetter::getContents(std::string const& path) const { - assert(!"NIY"); - } - template - void IGetter::getContents(std::vector& into, bool showContents) const { + std::vector IGetter::getContents(std::string const& path) const { assert(!"NIY"); } + void IGetter::getContents(std::vector& into, bool showContents) const { assert(!"NIY"); } - template - std::vector IGetter::getAllContents(std::string const& path) const { + std::vector IGetter::getAllContents(std::string const& path) const { assert(!"NIY"); } - template - std::vector IGetter::getAllContents(std::string const& path, - uint32_t runNumber, - uint32_t lumi) const { + std::vector IGetter::getAllContents(std::string const& path, + uint32_t runNumber, + uint32_t lumi) const { assert(!"NIY"); } - template - ME* IGetter::get(std::string const& fullpath) const { - assert(!"NIY"); - } + MonitorElement* IGetter::get(std::string const& fullpath) const { assert(!"NIY"); } - template - ME* IGetter::getElement(std::string const& path) const { - assert(!"NIY"); - } + MonitorElement* IGetter::getElement(std::string const& path) const { assert(!"NIY"); } - template - std::vector IGetter::getSubdirs() const { - assert(!"NIY"); - } - template - std::vector IGetter::getMEs() const { - assert(!"NIY"); - } - template - bool IGetter::dirExists(std::string const& path) const { - assert(!"NIY"); - } + std::vector IGetter::getSubdirs() const { assert(!"NIY"); } + std::vector IGetter::getMEs() const { assert(!"NIY"); } + bool IGetter::dirExists(std::string const& path) const { assert(!"NIY"); } - template - IGetter::IGetter(STORE* store) { - store_ = store; - } + IGetter::IGetter(DQMStore* store) { store_ = store; } - template - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) - : IGetter>(this), IBooker>(this) {} - template - DQMStore::~DQMStore() {} - - template - void DQMStore::save(std::string const& filename, - std::string const& path, - std::string const& pattern, - std::string const& rewrite, - uint32_t run, - uint32_t lumi, - SaveReferenceTag ref, - int minStatus, - std::string const& fileupdate) { - assert(!"NIY"); - } - template - void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { + IGetter::~IGetter() {} + + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) {} + DQMStore::~DQMStore() {} + + void DQMStore::save(std::string const& filename, + std::string const& path, + std::string const& pattern, + std::string const& rewrite, + uint32_t run, + uint32_t lumi, + SaveReferenceTag ref, + int minStatus, + std::string const& fileupdate) { assert(!"NIY"); } - template - bool DQMStore::open(std::string const& filename, - bool overwrite, - std::string const& path, - std::string const& prepend, - OpenRunDirs stripdirs, - bool fileMustExist) { + void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { assert(!"NIY"); } - template - bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { + bool DQMStore::open(std::string const& filename, + bool overwrite, + std::string const& path, + std::string const& prepend, + OpenRunDirs stripdirs, + bool fileMustExist) { assert(!"NIY"); } + bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { assert(!"NIY"); } - template - void DQMStore::showDirStructure() const { - assert(!"NIY"); - } + void DQMStore::showDirStructure() const { assert(!"NIY"); } - template - std::vector DQMStore::getMatchingContents(std::string const& pattern) const { - assert(!"NIY"); - } + std::vector DQMStore::getMatchingContents(std::string const& pattern) const { assert(!"NIY"); } } // namespace implementation } // namespace dqm - -template class dqm::implementation::DQMStore; - -template class dqm::implementation::IGetter>; - -template class dqm::implementation::IBooker>; diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index bd5fbe07d82d7..917359f078284 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -43,7 +43,6 @@ namespace dqm::impl { MonitorElement::~MonitorElement() { // TODO: this is only as long as we use the edm::Service DQMStore. delete mutable_; - delete refvalue_; } //utility function to check the consistency of the axis labels @@ -369,53 +368,58 @@ namespace dqm::impl { return result; } - const QReport *MonitorElement::getQReport(const std::string &qtname) const { - QReport *qr; + const MonitorElementData::QReport *MonitorElement::getQReport(const std::string &qtname) const { + MonitorElementData::MonitorElementData::QReport *qr; DQMNet::QValue *qv; const_cast(this)->getQReport(false, qtname, qr, qv); return qr; } - std::vector MonitorElement::getQReports() const { - std::vector result; - result.reserve(qreports_.size()); - for (size_t i = 0, e = qreports_.size(); i != e; ++i) { - const_cast(this)->qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&qreports_[i])); + // TODO: what was this logic ever supposed to do? + std::vector MonitorElement::getQReports() const { + auto access = this->access(); + std::vector result; + result.reserve(access.value.qreports_.size()); + for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) { + // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); + result.push_back(const_cast(&access.value.qreports_[i])); } return result; } - std::vector MonitorElement::getQWarnings() const { - std::vector result; - result.reserve(qreports_.size()); - for (size_t i = 0, e = qreports_.size(); i != e; ++i) + std::vector MonitorElement::getQWarnings() const { + auto access = this->access(); + std::vector result; + result.reserve(access.value.qreports_.size()); + for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) if (data_.qreports[i].code == dqm::qstatus::WARNING) { - const_cast(this)->qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&qreports_[i])); + // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); + result.push_back(const_cast(&access.value.qreports_[i])); } return result; } - std::vector MonitorElement::getQErrors() const { - std::vector result; - result.reserve(qreports_.size()); - for (size_t i = 0, e = qreports_.size(); i != e; ++i) + std::vector MonitorElement::getQErrors() const { + auto access = this->access(); + std::vector result; + result.reserve(access.value.qreports_.size()); + for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) if (data_.qreports[i].code == dqm::qstatus::ERROR) { - const_cast(this)->qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&qreports_[i])); + // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); + result.push_back(const_cast(&access.value.qreports_[i])); } return result; } - std::vector MonitorElement::getQOthers() const { - std::vector result; - result.reserve(qreports_.size()); - for (size_t i = 0, e = qreports_.size(); i != e; ++i) + std::vector MonitorElement::getQOthers() const { + auto access = this->access(); + std::vector result; + result.reserve(access.value.qreports_.size()); + for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) if (data_.qreports[i].code != dqm::qstatus::STATUS_OK && data_.qreports[i].code != dqm::qstatus::WARNING && data_.qreports[i].code != dqm::qstatus::ERROR) { - const_cast(this)->qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&qreports_[i])); + // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); + result.push_back(const_cast(&access.value.qreports_[i])); } return result; } @@ -902,14 +906,17 @@ namespace dqm::impl { copyFunctions(from, orig); } - // --- Operations on MEs that are normally reset at end of monitoring cycle --- - void MonitorElement::getQReport(bool create, const std::string &qtname, QReport *&qr, DQMNet::QValue *&qv) { - assert(qreports_.size() == data_.qreports.size()); + void MonitorElement::getQReport(bool create, + const std::string &qtname, + MonitorElementData::QReport *&qr, + DQMNet::QValue *&qv) { + auto access = this->accessMut(); + assert(access.value.qreports_.size() == data_.qreports.size()); qr = nullptr; qv = nullptr; - size_t pos = 0, end = qreports_.size(); + size_t pos = 0, end = access.value.qreports_.size(); while (pos < end && data_.qreports[pos].qtname != qtname) ++pos; @@ -917,7 +924,6 @@ namespace dqm::impl { return; else if (pos == end) { data_.qreports.emplace_back(); - qreports_.push_back(QReport(nullptr)); DQMNet::QValue &q = data_.qreports.back(); q.code = dqm::qstatus::DID_NOT_RUN; @@ -925,10 +931,10 @@ namespace dqm::impl { q.qtname = qtname; q.message = "NO_MESSAGE_ASSIGNED"; q.algorithm = "UNKNOWN_ALGORITHM"; - qreports_[pos].qvalue_ = &q; + access.value.qreports_.push_back(MonitorElementData::QReport(&q)); } - qr = &qreports_[pos]; + qr = &access.value.qreports_[pos]; qv = &data_.qreports[pos]; } diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index 6c6f5615405c5..10075e2f474fd 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -111,10 +111,12 @@ struct MonitorElementData { /// (not relevant for all quality tests!) const std::vector& getBadChannels() const { return badChannels_; } - QReport(DQMNet::QValue* value) : qvalue_(value) {} + void setBadChannels(std::vector badChannels) { badChannels_ = badChannels; } + + QReport(QValue* value) : qvalue_(value) {} private: - DQMNet::QValue* qvalue_; //< Pointer to the actual data. + QValue* qvalue_; //< Pointer to the actual data. std::vector badChannels_; //< Bad channels from QCriterion. }; From 7cc564cfb99c663f32c2a83b5d13c41196f3c83b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 19 Nov 2019 15:23:21 +0100 Subject: [PATCH 010/112] Some formatting change. --- DQMServices/Core/src/DQMStore.cc | 720 +++++++++++++++---------------- 1 file changed, 357 insertions(+), 363 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 92947b921f108..42b4d918daa5a 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -9,388 +9,382 @@ #include #include -namespace dqm { - - namespace implementation { - std::string const& NavigatorBase::pwd() { return cwd_; } - void NavigatorBase::cd() { setCurrentFolder(""); } - void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(cwd_ + dir); } - void NavigatorBase::goUp() { cd(".."); } - void NavigatorBase::setCurrentFolder(std::string const& fullpath) { - MonitorElementData::Path path; - path.set(fullpath, MonitorElementData::Path::Type::DIR); - assert(this); - cwd_ = path.getDirname(); - } - - IBooker::IBooker(DQMStore* store) { - store_ = store; - scope_ = MonitorElementData::Scope::RUN; - } - - IBooker::~IBooker() {} - - MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) { - auto oldscope = scope_; - scope_ = newscope; - return oldscope; - } - - MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { - MonitorElementData* data = new MonitorElementData(); - MonitorElementData::Key key; - key.kind_ = kind; - std::string fullpath = pwd() + std::string(name.View()); - key.path_.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); - key.scope_ = scope_; - data->key_ = key; - { - //MonitorElementData::Value::Access value(data->value_); - //value.object = std::unique_ptr(object); - } - - std::unique_ptr me = - std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); - assert(me); - MonitorElement* me_ptr = store_->putME(std::move(me)); - assert(me_ptr); - return me_ptr; - } +namespace dqm::implementation { + + std::string const& NavigatorBase::pwd() { return cwd_; } + void NavigatorBase::cd() { setCurrentFolder(""); } + void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(cwd_ + dir); } + void NavigatorBase::goUp() { cd(".."); } + void NavigatorBase::setCurrentFolder(std::string const& fullpath) { + MonitorElementData::Path path; + path.set(fullpath, MonitorElementData::Path::Type::DIR); + assert(this); + cwd_ = path.getDirname(); + } + + IBooker::IBooker(DQMStore* store) { + store_ = store; + scope_ = MonitorElementData::Scope::RUN; + } + + IBooker::~IBooker() {} + + MonitorElementData::Scope IBooker::setScope(MonitorElementData::Scope newscope) { + auto oldscope = scope_; + scope_ = newscope; + return oldscope; + } + + MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { + MonitorElementData* data = new MonitorElementData(); + MonitorElementData::Key key; + key.kind_ = kind; + std::string fullpath = pwd() + std::string(name.View()); + key.path_.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = scope_; + data->key_ = key; + { + //MonitorElementData::Value::Access value(data->value_); + //value.object = std::unique_ptr(object); + } + + std::unique_ptr me = + std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); + assert(me); + MonitorElement* me_ptr = store_->putME(std::move(me)); + assert(me_ptr); + return me_ptr; + } + + MonitorElement* DQMStore::putME(std::unique_ptr&& me) { + //TODO + return nullptr; + } + + void DQMStore::printTrace(std::string const& message) { + edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { + std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; + std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; + + void* array[10]; + size_t size; + char** strings; + int demangle_status = 0; + std::vector clean_trace; + + // glibc/libgcc backtrace functionality, declared in execinfo.h. + size = backtrace(array, 10); + strings = backtrace_symbols(array, size); + + size_t level = 1; + char* demangled = nullptr; + for (; level < size; ++level) { + std::cmatch match; + bool ok = std::regex_match(strings[level], match, s_rxtrace); + + if (!ok) { + edm::LogWarning("DQMStoreBacktrace") << "failed match" << level << strings[level]; + continue; + } - MonitorElement* DQMStore::putME(std::unique_ptr&& me) { - //TODO - return nullptr; - } + if (match[2].length() == 0) { + // no symbol, ignore. + continue; + } - void DQMStore::printTrace(std::string const& message) { - edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { - std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; - std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; - - void* array[10]; - size_t size; - char** strings; - int demangle_status = 0; - std::vector clean_trace; - - // glibc/libgcc backtrace functionality, declared in execinfo.h. - size = backtrace(array, 10); - strings = backtrace_symbols(array, size); - - size_t level = 1; - char* demangled = nullptr; - for (; level < size; ++level) { - std::cmatch match; - bool ok = std::regex_match(strings[level], match, s_rxtrace); - - if (!ok) { - edm::LogWarning("DQMStoreBacktrace") << "failed match" << level << strings[level]; - continue; - } - - if (match[2].length() == 0) { - // no symbol, ignore. - continue; - } - - // demangle name to human readable form - demangled = abi::__cxa_demangle(std::string(match[2]).c_str(), nullptr, nullptr, &demangle_status); - if (!demangled || demangle_status != 0) { - edm::LogWarning("DQMStoreBacktrace") << "failed demangle! status " << demangle_status << " on " << match[2]; - continue; - } - - if (std::regex_match(demangled, s_rxself)) { - // ignore framework/internal methods - free(demangled); - demangled = nullptr; - continue; - } else { - // keep the demangled name and the address. - // The address can be resolved to a line number in gdb attached to - // the process, using `list *0x`, but it can only be done in - // the running process and we can"t easily do it in this code. - clean_trace.push_back(std::string(demangled) + std::string(match[3])); - free(demangled); - demangled = nullptr; - } + // demangle name to human readable form + demangled = abi::__cxa_demangle(std::string(match[2]).c_str(), nullptr, nullptr, &demangle_status); + if (!demangled || demangle_status != 0) { + edm::LogWarning("DQMStoreBacktrace") << "failed demangle! status " << demangle_status << " on " << match[2]; + continue; } - if (clean_trace.size() > 0) { - logger << message << " at "; - for (auto const& s : clean_trace) { - logger << s << "; "; - } + if (std::regex_match(demangled, s_rxself)) { + // ignore framework/internal methods + free(demangled); + demangled = nullptr; + continue; } else { - logger << message << " : failed to collect stack trace."; + // keep the demangled name and the address. + // The address can be resolved to a line number in gdb attached to + // the process, using `list *0x`, but it can only be done in + // the running process and we can"t easily do it in this code. + clean_trace.push_back(std::string(demangled) + std::string(match[3])); + free(demangled); + demangled = nullptr; } + } - free(strings); - }); - } + if (clean_trace.size() > 0) { + logger << message << " at "; + for (auto const& s : clean_trace) { + logger << s << "; "; + } + } else { + logger << message << " : failed to collect stack trace."; + } - MonitorElement* IBooker::bookInt(TString const& name) { - return bookME(name, MonitorElementData::Kind::INT, nullptr); - } - MonitorElement* IBooker::bookFloat(TString const& name) { - return bookME(name, MonitorElementData::Kind::INT, nullptr); - } - MonitorElement* IBooker::bookString(TString const& name, TString const& value) { - return bookME(name, MonitorElementData::Kind::INT, nullptr); - } - MonitorElement* IBooker::book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { - auto th1 = new TH1F(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { - auto th1 = new TH1F(name, title, nchX, xbinsize); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1D(TString const& name, TH1F* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { - auto th1 = new TH1S(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1S, th1); - } - MonitorElement* IBooker::book1S(TString const& name, TH1S* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1S, th1); - } - MonitorElement* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { - auto th1 = new TH1D(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1D, th1); - } - MonitorElement* IBooker::book1DD(TString const& name, TH1D* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1D, th1); - } - MonitorElement* IBooker::book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2D(TString const& name, TH2F* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2S(TString const& name, TH2S* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2D, th2); - } - MonitorElement* IBooker::book2DD(TString const& name, TH2D* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2D, th2); - } - MonitorElement* IBooker::book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ) { - auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); - return bookME(name, MonitorElementData::Kind::TH3F, th3); - } - MonitorElement* IBooker::book3D(TString const& name, TH3F* object) { - auto th3 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH3F, th3); - } - MonitorElement* IBooker::bookProfile(TString const& name, + free(strings); + }); + } + + MonitorElement* IBooker::bookInt(TString const& name) { return bookME(name, MonitorElementData::Kind::INT, nullptr); } + MonitorElement* IBooker::bookFloat(TString const& name) { + return bookME(name, MonitorElementData::Kind::INT, nullptr); + } + MonitorElement* IBooker::bookString(TString const& name, TString const& value) { + return bookME(name, MonitorElementData::Kind::INT, nullptr); + } + MonitorElement* IBooker::book1D( + TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { + auto th1 = new TH1F(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + MonitorElement* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { + auto th1 = new TH1F(name, title, nchX, xbinsize); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + MonitorElement* IBooker::book1D(TString const& name, TH1F* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1F, th1); + } + MonitorElement* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { + auto th1 = new TH1S(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1S, th1); + } + MonitorElement* IBooker::book1S(TString const& name, TH1S* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1S, th1); + } + MonitorElement* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { + auto th1 = new TH1D(name, title, nchX, lowX, highX); + return bookME(name, MonitorElementData::Kind::TH1D, th1); + } + MonitorElement* IBooker::book1DD(TString const& name, TH1D* object) { + auto th1 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH1D, th1); + } + MonitorElement* IBooker::book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + MonitorElement* IBooker::book2D( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { + auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + MonitorElement* IBooker::book2D(TString const& name, TH2F* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2F, th2); + } + MonitorElement* IBooker::book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + MonitorElement* IBooker::book2S( + TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { + auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + MonitorElement* IBooker::book2S(TString const& name, TH2S* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2S, th2); + } + MonitorElement* IBooker::book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY) { + auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); + return bookME(name, MonitorElementData::Kind::TH2D, th2); + } + MonitorElement* IBooker::book2DD(TString const& name, TH2D* object) { + auto th2 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH2D, th2); + } + MonitorElement* IBooker::book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ) { + auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); + return bookME(name, MonitorElementData::Kind::TH3F, th3); + } + MonitorElement* IBooker::book3D(TString const& name, TH3F* object) { + auto th3 = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TH3F, th3); + } + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int /* nchY */, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + int /* nchY */, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + MonitorElement* IBooker::bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + double lowY, + double highY, + char const* option) { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + MonitorElement* IBooker::bookProfile(TString const& name, TProfile* object) { + auto tprofile = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); + } + MonitorElement* IBooker::bookProfile2D(TString const& name, TString const& title, int nchX, double lowX, double highX, - int /* nchY */, + int nchY, double lowY, double highY, + double lowZ, + double highZ, char const* option) { - auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); + } + MonitorElement* IBooker::bookProfile2D(TString const& name, TString const& title, int nchX, double lowX, double highX, + int nchY, double lowY, double highY, + int /* nchZ */, + double lowZ, + double highZ, char const* option) { - auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int /* nchY */, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, TProfile* object) { - auto tprofile = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option) { - auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int /* nchZ */, - double lowZ, - double highZ, - char const* option) { - auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { - auto tprofile = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - - void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { - //TODO - } - - MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { - //TODO - return nullptr; - } - - void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { - //TODO - } - - std::vector IGetter::getContents(std::string const& path) const { - assert(!"NIY"); - } - void IGetter::getContents(std::vector& into, bool showContents) const { assert(!"NIY"); } - - std::vector IGetter::getAllContents(std::string const& path) const { - assert(!"NIY"); - } - std::vector IGetter::getAllContents(std::string const& path, - uint32_t runNumber, - uint32_t lumi) const { - assert(!"NIY"); - } - - MonitorElement* IGetter::get(std::string const& fullpath) const { assert(!"NIY"); } - - MonitorElement* IGetter::getElement(std::string const& path) const { assert(!"NIY"); } - - std::vector IGetter::getSubdirs() const { assert(!"NIY"); } - std::vector IGetter::getMEs() const { assert(!"NIY"); } - bool IGetter::dirExists(std::string const& path) const { assert(!"NIY"); } - - IGetter::IGetter(DQMStore* store) { store_ = store; } - - IGetter::~IGetter() {} - - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) {} - DQMStore::~DQMStore() {} - - void DQMStore::save(std::string const& filename, - std::string const& path, - std::string const& pattern, - std::string const& rewrite, - uint32_t run, - uint32_t lumi, - SaveReferenceTag ref, - int minStatus, - std::string const& fileupdate) { - assert(!"NIY"); - } - void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { - assert(!"NIY"); - } - bool DQMStore::open(std::string const& filename, - bool overwrite, - std::string const& path, - std::string const& prepend, - OpenRunDirs stripdirs, - bool fileMustExist) { - assert(!"NIY"); - } - bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { assert(!"NIY"); } - - void DQMStore::showDirStructure() const { assert(!"NIY"); } - - std::vector DQMStore::getMatchingContents(std::string const& pattern) const { assert(!"NIY"); } - - } // namespace implementation -} // namespace dqm + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); + } + MonitorElement* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { + auto tprofile = static_cast(object->Clone(name)); + return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); + } + + void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { + //TODO + } + + MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { + //TODO + return nullptr; + } + + void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { + //TODO + } + + std::vector IGetter::getContents(std::string const& path) const { assert(!"NIY"); } + void IGetter::getContents(std::vector& into, bool showContents) const { assert(!"NIY"); } + + std::vector IGetter::getAllContents(std::string const& path) const { + assert(!"NIY"); + } + std::vector IGetter::getAllContents(std::string const& path, + uint32_t runNumber, + uint32_t lumi) const { + assert(!"NIY"); + } + + MonitorElement* IGetter::get(std::string const& fullpath) const { assert(!"NIY"); } + + MonitorElement* IGetter::getElement(std::string const& path) const { assert(!"NIY"); } + + std::vector IGetter::getSubdirs() const { assert(!"NIY"); } + std::vector IGetter::getMEs() const { assert(!"NIY"); } + bool IGetter::dirExists(std::string const& path) const { assert(!"NIY"); } + + IGetter::IGetter(DQMStore* store) { store_ = store; } + + IGetter::~IGetter() {} + + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) {} + DQMStore::~DQMStore() {} + + void DQMStore::save(std::string const& filename, + std::string const& path, + std::string const& pattern, + std::string const& rewrite, + uint32_t run, + uint32_t lumi, + SaveReferenceTag ref, + int minStatus, + std::string const& fileupdate) { + assert(!"NIY"); + } + void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { + assert(!"NIY"); + } + bool DQMStore::open(std::string const& filename, + bool overwrite, + std::string const& path, + std::string const& prepend, + OpenRunDirs stripdirs, + bool fileMustExist) { + assert(!"NIY"); + } + bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { assert(!"NIY"); } + + void DQMStore::showDirStructure() const { assert(!"NIY"); } + + std::vector DQMStore::getMatchingContents(std::string const& pattern) const { assert(!"NIY"); } + +} // namespace dqm::implementation From 7d0603e2eafaa40a8333aab3efc6c30198eff146 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 20 Nov 2019 12:06:39 +0100 Subject: [PATCH 011/112] Allow passing a lambda to book*() calls. This should only be called when a new object is needed, and can freely modify the ROOT object. --- DQMServices/Core/interface/DQMStore.h | 460 +++++++++++++++++++------- DQMServices/Core/src/DQMStore.cc | 195 +---------- 2 files changed, 351 insertions(+), 304 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 13474bf4bb357..270c1b0da9786 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -6,6 +6,8 @@ #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" +#include + // TODO: Remove at some point: #define TRACE(msg) \ std::cout << "TRACE: " << __FILE__ << ":" << __LINE__ << "(" << __FUNCTION__ << ") " << msg << std::endl; @@ -40,117 +42,351 @@ namespace dqm { class IBooker : public dqm::implementation::NavigatorBase { public: - virtual MonitorElement* bookInt(TString const& name); - virtual MonitorElement* bookFloat(TString const& name); - virtual MonitorElement* bookString(TString const& name, TString const& value); - virtual MonitorElement* book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX); - virtual MonitorElement* book1D(TString const& name, TString const& title, int nchX, float const* xbinsize); - virtual MonitorElement* book1D(TString const& name, TH1F* object); - virtual MonitorElement* book1S(TString const& name, TString const& title, int nchX, double lowX, double highX); - virtual MonitorElement* book1S(TString const& name, TH1S* object); - virtual MonitorElement* book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX); - virtual MonitorElement* book1DD(TString const& name, TH1D* object); - virtual MonitorElement* book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual MonitorElement* book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - virtual MonitorElement* book2D(TString const& name, TH2F* object); - virtual MonitorElement* book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual MonitorElement* book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize); - virtual MonitorElement* book2S(TString const& name, TH2S* object); - virtual MonitorElement* book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY); - virtual MonitorElement* book2DD(TString const& name, TH2D* object); - virtual MonitorElement* book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ); - virtual MonitorElement* book3D(TString const& name, TH3F* object); - virtual MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - char const* option = "s"); - virtual MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option = "s"); - virtual MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int nchY, - double lowY, - double highY, - char const* option = "s"); - virtual MonitorElement* bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option = "s"); - virtual MonitorElement* bookProfile(TString const& name, TProfile* object); - virtual MonitorElement* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option = "s"); - virtual MonitorElement* bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ, - char const* option = "s"); - virtual MonitorElement* bookProfile2D(TString const& name, TProfile2D* object); + struct NOOP { + // functor to be passed as a default argument that does not do anything. + void operator()(TH1*) const {}; + void operator()() const {}; + }; + + template + MonitorElement* bookInt(TString const& name, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::INT, [=]() { + onbooking(); + return nullptr; + }); + } + template + MonitorElement* bookFloat(TString const& name, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind ::REAL, [=]() { + onbooking(); + return nullptr; + }); + } + template + MonitorElement* bookString(TString const& name, TString const& value, FUNC onbooking = NOOP()) { + // TODO: value unused! + return bookME(name, MonitorElementData::Kind::STRING, [=]() { + onbooking(); + return nullptr; + }); + } + + template + MonitorElement* book1D(TString const& name, + TString const& title, + int const nchX, + double const lowX, + double const highX, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1F, [=]() { + auto th1 = new TH1F(name, title, nchX, lowX, highX); + onbooking(th1); + return th1; + }); + } + template + MonitorElement* book1D( + TString const& name, TString const& title, int nchX, float const* xbinsize, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1F, [=]() { + auto th1 = new TH1F(name, title, nchX, xbinsize); + onbooking(th1); + return th1; + }); + } + template + MonitorElement* book1D(TString const& name, TH1F* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1F, [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }); + } + + template + MonitorElement* book1S( + TString const& name, TString const& title, int nchX, double lowX, double highX, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1S, [=]() { + auto th1 = new TH1S(name, title, nchX, lowX, highX); + onbooking(th1); + return th1; + }); + } + template + MonitorElement* book1S(TString const& name, TH1S* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1S, [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }); + } + + template + MonitorElement* book1DD( + TString const& name, TString const& title, int nchX, double lowX, double highX, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1D, [=]() { + auto th1 = new TH1D(name, title, nchX, lowX, highX); + onbooking(th1); + return th1; + }); + } + template + MonitorElement* book1DD(TString const& name, TH1D* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH1D, [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }); + } + + template + MonitorElement* book2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2F, [=]() { + auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2D(TString const& name, + TString const& title, + int nchX, + float const* xbinsize, + int nchY, + float const* ybinsize, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2F, [=]() { + auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2D(TString const& name, TH2F* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2F, [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2S(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2S, [=]() { + auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2S(TString const& name, + TString const& title, + int nchX, + float const* xbinsize, + int nchY, + float const* ybinsize, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2S, [=]() { + auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2S(TString const& name, TH2S* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2S, [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2DD(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2D, [=]() { + auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); + onbooking(th2); + return th2; + }); + } + template + MonitorElement* book2DD(TString const& name, TH2D* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH2D, [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }); + } + + template + MonitorElement* book3D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int nchZ, + double lowZ, + double highZ, + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH3F, [=]() { + auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); + onbooking(th3); + return th3; + }); + } + template + MonitorElement* book3D(TString const& name, TH3F* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TH3F, [=]() { + auto th3 = static_cast(object->Clone(name)); + onbooking(th3); + return th3; + }); + } + + template + MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int /* nchY */, + double lowY, + double highY, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + double lowY, + double highY, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { + auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + int /* nchY */, + double lowY, + double highY, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile(TString const& name, + TString const& title, + int nchX, + double const* xbinsize, + double lowY, + double highY, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { + auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile(TString const& name, TProfile* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { + auto tprofile = static_cast(object->Clone(name)); + onbooking(tprofile); + return tprofile; + }); + } + + template + MonitorElement* bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + double lowZ, + double highZ, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE2D, [=]() { + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile2D(TString const& name, + TString const& title, + int nchX, + double lowX, + double highX, + int nchY, + double lowY, + double highY, + int /* nchZ */, + double lowZ, + double highZ, + char const* option = "s", + FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE2D, [=]() { + auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); + onbooking(tprofile); + return tprofile; + }); + } + template + MonitorElement* bookProfile2D(TString const& name, TProfile2D* object, FUNC onbooking = NOOP()) { + return bookME(name, MonitorElementData::Kind::TPROFILE2D, [=]() { + auto tprofile = static_cast(object->Clone(name)); + onbooking(tprofile); + return tprofile; + }); + } virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); @@ -158,7 +394,7 @@ namespace dqm { protected: IBooker(DQMStore* store); - MonitorElement* bookME(TString const& name, MonitorElementData::Kind kind, TH1* object); + MonitorElement* bookME(TString const& name, MonitorElementData::Kind kind, std::function makeobject); DQMStore* store_; MonitorElementData::Scope scope_; diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 42b4d918daa5a..24ee36a7a85fd 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -35,7 +35,9 @@ namespace dqm::implementation { return oldscope; } - MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, TH1* object) { + MonitorElement* IBooker::bookME(TString const& name, + MonitorElementData::Kind kind, + std::function makeobject) { MonitorElementData* data = new MonitorElementData(); MonitorElementData::Key key; key.kind_ = kind; @@ -128,197 +130,6 @@ namespace dqm::implementation { }); } - MonitorElement* IBooker::bookInt(TString const& name) { return bookME(name, MonitorElementData::Kind::INT, nullptr); } - MonitorElement* IBooker::bookFloat(TString const& name) { - return bookME(name, MonitorElementData::Kind::INT, nullptr); - } - MonitorElement* IBooker::bookString(TString const& name, TString const& value) { - return bookME(name, MonitorElementData::Kind::INT, nullptr); - } - MonitorElement* IBooker::book1D( - TString const& name, TString const& title, int const nchX, double const lowX, double const highX) { - auto th1 = new TH1F(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1D(TString const& name, TString const& title, int nchX, float const* xbinsize) { - auto th1 = new TH1F(name, title, nchX, xbinsize); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1D(TString const& name, TH1F* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1F, th1); - } - MonitorElement* IBooker::book1S(TString const& name, TString const& title, int nchX, double lowX, double highX) { - auto th1 = new TH1S(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1S, th1); - } - MonitorElement* IBooker::book1S(TString const& name, TH1S* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1S, th1); - } - MonitorElement* IBooker::book1DD(TString const& name, TString const& title, int nchX, double lowX, double highX) { - auto th1 = new TH1D(name, title, nchX, lowX, highX); - return bookME(name, MonitorElementData::Kind::TH1D, th1); - } - MonitorElement* IBooker::book1DD(TString const& name, TH1D* object) { - auto th1 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH1D, th1); - } - MonitorElement* IBooker::book2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2F(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2D( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - auto th2 = new TH2F(name, title, nchX, xbinsize, nchY, ybinsize); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2D(TString const& name, TH2F* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2F, th2); - } - MonitorElement* IBooker::book2S(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2S(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2S( - TString const& name, TString const& title, int nchX, float const* xbinsize, int nchY, float const* ybinsize) { - auto th2 = new TH2S(name, title, nchX, xbinsize, nchY, ybinsize); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2S(TString const& name, TH2S* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2S, th2); - } - MonitorElement* IBooker::book2DD(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY) { - auto th2 = new TH2D(name, title, nchX, lowX, highX, nchY, lowY, highY); - return bookME(name, MonitorElementData::Kind::TH2D, th2); - } - MonitorElement* IBooker::book2DD(TString const& name, TH2D* object) { - auto th2 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH2D, th2); - } - MonitorElement* IBooker::book3D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int nchZ, - double lowZ, - double highZ) { - auto th3 = new TH3F(name, title, nchX, lowX, highX, nchY, lowY, highY, nchZ, lowZ, highZ); - return bookME(name, MonitorElementData::Kind::TH3F, th3); - } - MonitorElement* IBooker::book3D(TString const& name, TH3F* object) { - auto th3 = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TH3F, th3); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int /* nchY */, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, lowX, highX, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - int /* nchY */, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, - TString const& title, - int nchX, - double const* xbinsize, - double lowY, - double highY, - char const* option) { - auto tprofile = new TProfile(name, title, nchX, xbinsize, lowY, highY, option); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile(TString const& name, TProfile* object) { - auto tprofile = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TPROFILE, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - double lowZ, - double highZ, - char const* option) { - auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, - TString const& title, - int nchX, - double lowX, - double highX, - int nchY, - double lowY, - double highY, - int /* nchZ */, - double lowZ, - double highZ, - char const* option) { - auto tprofile = new TProfile2D(name, title, nchX, lowX, highX, nchY, lowY, highY, lowZ, highZ, option); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - MonitorElement* IBooker::bookProfile2D(TString const& name, TProfile2D* object) { - auto tprofile = static_cast(object->Clone(name)); - return bookME(name, MonitorElementData::Kind::TPROFILE2D, tprofile); - } - void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { //TODO } From 7804282d0d7c70e69dfab0074174e9375d1c51a4 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 20 Nov 2019 14:36:27 +0100 Subject: [PATCH 012/112] Implement more booking --- DQMServices/Core/interface/DQMStore.h | 38 ++++++-- DQMServices/Core/interface/MonitorElement.h | 16 ++-- DQMServices/Core/src/DQMStore.cc | 88 +++++++++++++++---- .../interface/MonitorElementCollection.h | 1 + 4 files changed, 107 insertions(+), 36 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 270c1b0da9786..7ca3fd3fefe22 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -42,12 +42,18 @@ namespace dqm { class IBooker : public dqm::implementation::NavigatorBase { public: + // functor to be passed as a default argument that does not do anything. struct NOOP { - // functor to be passed as a default argument that does not do anything. void operator()(TH1*) const {}; void operator()() const {}; }; + // + // Booking Methods, templated to allow passing in lambdas. + // The variations taking ROOT object pointers do NOT take ownership of + // the object; it will be clone'd. + // + template MonitorElement* bookInt(TString const& name, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::INT, [=]() { @@ -388,16 +394,25 @@ namespace dqm { }); } - virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); + // + // all non-template interfaces are virtual. + // + virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); virtual ~IBooker(); protected: IBooker(DQMStore* store); - MonitorElement* bookME(TString const& name, MonitorElementData::Kind kind, std::function makeobject); + virtual uint64_t setModuleID(uint64_t moduleID); + virtual edm::LuminosityBlockID setRunLumi(edm::LuminosityBlockID runlumi); + virtual MonitorElement* bookME(TString const& name, + MonitorElementData::Kind kind, + std::function makeobject); DQMStore* store_; MonitorElementData::Scope scope_; + uint64_t moduleID_; + edm::LuminosityBlockID runlumi_; }; class IGetter : public dqm::implementation::NavigatorBase { @@ -512,9 +527,12 @@ namespace dqm { template void meBookerGetter(iFunc f){}; - // Will take ownership of the ROOT object in `me`, deleting it if not - // needed. + // Add ME to DQMStore datastructures. The object will be deleted if a + // similar object is already present. + // For global ME MonitorElement* putME(std::unique_ptr&& me); + // For local ME + MonitorElement* putME(std::unique_ptr&& me, uint64_t moduleID); // Log a backtrace on booking. void printTrace(std::string const& message); void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); @@ -542,10 +560,16 @@ namespace dqm { // required. Prototype MEs are reset *before* inserting, so fill calls // can go into prototype MEs and not be lost. // Key is (run, lumi), potentially one or both 0 for SCOPE::RUN or SCOPE::JOB - std::map, std::set> - globalMEs_; + // NEVER modify the key_ of a ME in these datastructures. Since we use + // pointers, this may be possible (not everything is const), but it could + // still corrupt the datastructure. + std::map> globalMEs_; // Key is (moduleID [, run]), run is only needed for edm::global. + // Legacy MEs have moduleID 0. std::map> localMEs_; + + // universal verbose flag. + int verbose_; }; } // namespace implementation diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index f079a5b48a763..ccf2fe45c46fc 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -196,21 +196,15 @@ namespace dqm::impl { uint32_t flags() const { return data_.flags; } /// get name of ME - const std::string &getName() const { return data_.objname; } + const std::string &getName() const { return access().key.path_.getObjectname(); } /// get pathname of parent folder - const std::string &getPathname() const { return *data_.dirname; } + const std::string &getPathname() const { return access().key.path_.getDirname(); } /// get full name of ME including Pathname - const std::string getFullname() const { - std::string path; - path.reserve(data_.dirname->size() + data_.objname.size() + 2); - path += *data_.dirname; - if (!data_.dirname->empty()) - path += '/'; - path += data_.objname; - return path; - } + const std::string getFullname() const { return access().key.path_.getFullname(); } + + const edm::LuminosityBlockID getRunLumi() { return acces().key_.id_; } /// true if ME was updated in last monitoring cycle bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 24ee36a7a85fd..a6bca2656d916 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -34,36 +34,85 @@ namespace dqm::implementation { scope_ = newscope; return oldscope; } + uint64_t MonitorElementData::setModuleID(uint64_t moduleID) { + auto oldid = moduleID_; + moduleID_ = moduleID; + return oldid; + } + + edm::LuminosityBlockID setRunLumi(edm::LuminosityBlockID runlumi) { + auto oldrunlumi = runlumi_; + runlumi_ = runlumi; + return oldrunlumi; + } MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, std::function makeobject) { - MonitorElementData* data = new MonitorElementData(); - MonitorElementData::Key key; - key.kind_ = kind; + MonitorElementData::Path path; std::string fullpath = pwd() + std::string(name.View()); - key.path_.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); - key.scope_ = scope_; - data->key_ = key; - { - //MonitorElementData::Value::Access value(data->value_); - //value.object = std::unique_ptr(object); + path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); + MonitorElement* me store_->findME(path); + printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)")); + if (me == nullptr) { + // no existing global ME found. We need to instantiate one, and put it + // into the DQMStore. This will typically be a prototype, unless run and + // lumi are set and we proces a legacy booking call. + TH1* th1 = makeobject(); + MonitorElementData medata; + medata.key_.path_ = path; + medata.key_.kind_ = kind; + medata.key_.scope_ = this->scope_; + // will be 0 ( = prototype) in the common case. + medata.key_.id_ = edm::LuminosityBlockID(this->run_, this_->lumi_); + medata.value_.object_ = std::unique_ptr(th1); + MonitorElement* me_ptr = new MonitorElement(data, /* is_owned */ true, /* is_readonly */ false); + me = store_->putME(me_ptr); } + // me now points to a global ME owned by the DQMStore. + assert(me); - std::unique_ptr me = - std::make_unique(data, /* is_owned */ true, /* is_readonly */ false); + // each booking call returns a unique "local" ME, which the DQMStore keeps + // in a container associated with the module (and potentially run, for + // DQMGlobalEDAnalyzer). This will later be update to point to different + // MEData (kept in a global ME) as needed. + MonitorElement* local_me = new MonitorElement(me); + me = store_->putME(local_me); + // me now points to a local ME owned by the DQMStore. assert(me); - MonitorElement* me_ptr = store_->putME(std::move(me)); - assert(me_ptr); - return me_ptr; + return me; } - MonitorElement* DQMStore::putME(std::unique_ptr&& me) { - //TODO - return nullptr; + MonitorElement* DQMStore::putME(MonitorElement* me) { + assert(me); + auto existing_new = globalMEs_[me->getRunLumi()].insert(me); + if (existing_new.second = true) { + // successfully inserted, return new object + return me; + } else { + // already present, return old object + delete me; + assert(!"Currently, this should never happen."); + return *(existing_new.first); + } + } + + MonitorElement* DQMStore::putME(MonitorElement* me, uint64_t moduleID) { + assert(me); + auto existing_new = localMEs_[moduleID].insert(me); + if (existing_new.second = true) { + // successfully inserted, return new object + return me; + } else { + // already present, return old object + delete me; + assert(!"Currently, this should never happen."); + return *(existing_new.first); + } } void DQMStore::printTrace(std::string const& message) { + if (verbose_ < 3) return; edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; @@ -167,7 +216,10 @@ namespace dqm::implementation { IGetter::~IGetter() {} - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) {} + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) { + verbose_ = pset.getUntrackedParameter("verbose", 0); + } + DQMStore::~DQMStore() {} void DQMStore::save(std::string const& filename, diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index 10075e2f474fd..2ec8186d11ba0 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -166,6 +166,7 @@ struct MonitorElementData { std::string const& getDirname() const { return dirname_; } std::string const& getObjectname() const { return objname_; } + std::string const& getFullname() const { return dirname_ + objname_; } // Clean up the path and normalize it to preserve certain invariants. // Instead of reasoning about whatever properties of paths, we just parse From 5e1010c5e95232bbfd5eced809c4fd804634ba9d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 20 Nov 2019 15:12:29 +0100 Subject: [PATCH 013/112] Adjust APIs to fit each other. --- DQMServices/Core/interface/DQMStore.h | 6 +++-- DQMServices/Core/interface/MonitorElement.h | 10 ++++++-- DQMServices/Core/src/DQMStore.cc | 25 ++++++++++++------- DQMServices/Core/src/MonitorElement.cc | 18 +++++++++++-- .../interface/MonitorElementCollection.h | 2 +- 5 files changed, 45 insertions(+), 16 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 7ca3fd3fefe22..d12a8c1d2d9f2 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -530,9 +530,11 @@ namespace dqm { // Add ME to DQMStore datastructures. The object will be deleted if a // similar object is already present. // For global ME - MonitorElement* putME(std::unique_ptr&& me); + MonitorElement* putME(MonitorElement* me); // For local ME - MonitorElement* putME(std::unique_ptr&& me, uint64_t moduleID); + MonitorElement* putME(MonitorElement* me, uint64_t moduleID); + // Find a global ME of matching name, in any state. + MonitorElement* findME(MonitorElementData::Path const& path); // Log a backtrace on booking. void printTrace(std::string const& message); void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index ccf2fe45c46fc..85302eefe6b0c 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -103,6 +103,7 @@ namespace dqm::impl { std::atomic frozen_; // only set if this ME is in a product already std::atomic mutable_; // only set if there is a mutable copy of this ME + bool is_owned_; // true if we are responsible for deleting the mutable object. /** * To do anything to the MEs data, one needs to obtain an access object. * This object will contain the lock guard if one is needed. We differentiate @@ -169,6 +170,7 @@ namespace dqm::impl { return existing->accessMut(); } else { // we won the race, and our clone is the real one now. + this->is_owned_ = true; return clone->accessMut(); } // in either case, if somebody destroyed the mutable object between us @@ -178,7 +180,11 @@ namespace dqm::impl { } public: - MonitorElement(MonitorElementData *data, bool is_owned, bool is_readonly){}; + // Create ME using this data. A ROOT object pointer may be moved into the + // new ME. The new ME will own this data. + MonitorElement(MonitorElementData&& data); + // Create a new ME sharing data with this existing ME. + MonitorElement(MonitorElement* me); MonitorElement &operator=(const MonitorElement &) = delete; MonitorElement &operator=(MonitorElement &&) = delete; virtual ~MonitorElement(); @@ -204,7 +210,7 @@ namespace dqm::impl { /// get full name of ME including Pathname const std::string getFullname() const { return access().key.path_.getFullname(); } - const edm::LuminosityBlockID getRunLumi() { return acces().key_.id_; } + const edm::LuminosityBlockID getRunLumi() { return access().key.id_; } /// true if ME was updated in last monitoring cycle bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index a6bca2656d916..7d8e20b704cdd 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -34,13 +34,13 @@ namespace dqm::implementation { scope_ = newscope; return oldscope; } - uint64_t MonitorElementData::setModuleID(uint64_t moduleID) { + uint64_t IBooker::setModuleID(uint64_t moduleID) { auto oldid = moduleID_; moduleID_ = moduleID; return oldid; } - edm::LuminosityBlockID setRunLumi(edm::LuminosityBlockID runlumi) { + edm::LuminosityBlockID IBooker::setRunLumi(edm::LuminosityBlockID runlumi) { auto oldrunlumi = runlumi_; runlumi_ = runlumi; return oldrunlumi; @@ -52,8 +52,8 @@ namespace dqm::implementation { MonitorElementData::Path path; std::string fullpath = pwd() + std::string(name.View()); path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); - MonitorElement* me store_->findME(path); - printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)")); + MonitorElement* me = store_->findME(path); + store_->printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)")); if (me == nullptr) { // no existing global ME found. We need to instantiate one, and put it // into the DQMStore. This will typically be a prototype, unless run and @@ -64,9 +64,9 @@ namespace dqm::implementation { medata.key_.kind_ = kind; medata.key_.scope_ = this->scope_; // will be 0 ( = prototype) in the common case. - medata.key_.id_ = edm::LuminosityBlockID(this->run_, this_->lumi_); + medata.key_.id_ = this->runlumi_; medata.value_.object_ = std::unique_ptr(th1); - MonitorElement* me_ptr = new MonitorElement(data, /* is_owned */ true, /* is_readonly */ false); + MonitorElement* me_ptr = new MonitorElement(std::move(medata)); me = store_->putME(me_ptr); } // me now points to a global ME owned by the DQMStore. @@ -86,7 +86,7 @@ namespace dqm::implementation { MonitorElement* DQMStore::putME(MonitorElement* me) { assert(me); auto existing_new = globalMEs_[me->getRunLumi()].insert(me); - if (existing_new.second = true) { + if (existing_new.second == true) { // successfully inserted, return new object return me; } else { @@ -100,7 +100,7 @@ namespace dqm::implementation { MonitorElement* DQMStore::putME(MonitorElement* me, uint64_t moduleID) { assert(me); auto existing_new = localMEs_[moduleID].insert(me); - if (existing_new.second = true) { + if (existing_new.second == true) { // successfully inserted, return new object return me; } else { @@ -111,8 +111,15 @@ namespace dqm::implementation { } } + MonitorElement* DQMStore::findME(MonitorElementData::Path const& path) { + //TODO + + + } + void DQMStore::printTrace(std::string const& message) { - if (verbose_ < 3) return; + if (verbose_ < 3) + return; edm::LogWarning("DQMStoreBooking").log([&](auto& logger) { std::regex s_rxtrace{"(.*)\\((.*)\\+0x.*\\).*(\\[.*\\])"}; std::regex s_rxself{"^[^()]*dqm::implementation::.*|^[^()]*edm::.*|.*edm::convertException::wrap.*"}; diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 917359f078284..3796065a847c5 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -40,9 +40,23 @@ namespace dqm::impl { return h; } + MonitorElement::MonitorElement(MonitorElementData&& data) { + this->mutable_ = new MutableMonitorElementData(); + this->frozen_ = nullptr; + this->mutable_.load()->data_ = std::move(data); + this->is_owned_ = true; + // TODO: update DQMNet::CoreObject. + } + MonitorElement::MonitorElement(MonitorElement* me) { + this->mutable_ = me->mutable_.load(); + this->frozen_ = me->frozen_.load(); + this->is_owned_ = false; + // TODO: update DQMNet::CoreObject. + } + MonitorElement::~MonitorElement() { - // TODO: this is only as long as we use the edm::Service DQMStore. - delete mutable_; + if (is_owned_) + delete mutable_; } //utility function to check the consistency of the axis labels diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index 2ec8186d11ba0..cd92584f0cd66 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -166,7 +166,7 @@ struct MonitorElementData { std::string const& getDirname() const { return dirname_; } std::string const& getObjectname() const { return objname_; } - std::string const& getFullname() const { return dirname_ + objname_; } + std::string getFullname() const { return dirname_ + objname_; } // Clean up the path and normalize it to preserve certain invariants. // Instead of reasoning about whatever properties of paths, we just parse From 26b29f6feebc25d618c730a5ab9801b4f242de31 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 20 Nov 2019 16:51:34 +0100 Subject: [PATCH 014/112] Setup more booking logic. Including the heterogeneous queries into the DQMStore sets. --- DQMServices/Core/interface/DQMEDAnalyzer.h | 2 +- .../Core/interface/DQMGlobalEDAnalyzer.h | 4 +- DQMServices/Core/interface/DQMOneEDAnalyzer.h | 2 +- DQMServices/Core/interface/DQMStore.h | 54 +++++++++++++++---- DQMServices/Core/interface/MonitorElement.h | 44 +++++++++++++-- DQMServices/Core/src/DQMStore.cc | 15 ++++-- DQMServices/Core/src/MonitorElement.cc | 4 +- .../interface/MonitorElementCollection.h | 2 +- 8 files changed, 102 insertions(+), 25 deletions(-) diff --git a/DQMServices/Core/interface/DQMEDAnalyzer.h b/DQMServices/Core/interface/DQMEDAnalyzer.h index d8df411160f6a..c988eefafe968 100644 --- a/DQMServices/Core/interface/DQMEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMEDAnalyzer.h @@ -37,7 +37,7 @@ class DQMEDAnalyzer : public edm::one::EDProducerbookHistograms(booker, run, setup); }, - run.run(), + /* run */ 0, // MEs booked here apply for all runs this->moduleDescription().id(), this->getCanSaveByLumi()); edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); diff --git a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h index db8f63afd36fe..0c6226ac651f0 100644 --- a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h @@ -42,10 +42,10 @@ class DQMGlobalEDAnalyzer bookHistograms(b, run, setup, *h); }, run.run(), - /* moduleID */ 0, + this->moduleDescription().id(), /* canSaveByLumi */ false); - // Populate run numbers, in case booking only books prototypes. // We will not call enterLumi per-lumi, since this is strictly run-based. + // The MEs are booked for a fixed run, given in the transaction. return h; } diff --git a/DQMServices/Core/interface/DQMOneEDAnalyzer.h b/DQMServices/Core/interface/DQMOneEDAnalyzer.h index e7c3dcc0d578f..ca8a56c689d47 100644 --- a/DQMServices/Core/interface/DQMOneEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMOneEDAnalyzer.h @@ -38,7 +38,7 @@ class DQMOneEDAnalyzer booker.cd(); this->bookHistograms(booker, run, setup); }, - run.run(), + /* run */ 0, // run is assigned later, in enterLumi this->moduleDescription().id(), this->getCanSaveByLumi()); edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index d12a8c1d2d9f2..d4d9b2d5df65d 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -7,6 +7,7 @@ #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" #include +#include // TODO: Remove at some point: #define TRACE(msg) \ @@ -18,13 +19,6 @@ namespace dqm { using MonitorElement = dqm::legacy::MonitorElement; class DQMStore; - struct MEComparison { - using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. - bool operator()(MonitorElement* left, MonitorElement* right) const { return false; } - bool operator()(MonitorElement* left, MonitorElementData::Path const& right) const { return false; } - bool operator()(MonitorElementData::Path const& left, MonitorElement* right) const { return false; } - }; - // The common implementation to change folders class NavigatorBase { public: @@ -523,9 +517,40 @@ namespace dqm { public: // internal -- figure out better protection. template - void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi){}; + void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi) { + auto lock = std::scoped_lock(this->booking_mutex_); + IBooker& booker = *this; + // TODO: this may need to become more elaborate. + auto oldscope = + booker.setScope(canSaveByLumi ? MonitorElementData::Scope::LUMI : MonitorElementData::Scope::RUN); + assert(moduleId != 0 || !"moduleID must be set for normal booking transaction"); + // run should be 0 unless this is DQMGlobalEDAnalyzer. + // The run number goes into the moduleID, since for the DQMGlobalEDAnalyzer, + // we need *different* localMEs within the same module. + // Access via this-> to allow access to protected member + auto oldmoduleid = this->setModuleID((((uint64_t)run) << 32) + moduleId); + assert(oldmoduleid == 0 || !"Nested booking transaction?"); + // in a proper transaction we book prototypes, except DQMGlobalEDAnalyzer + // (run is set, MEs are booked for a fixed run). + auto oldrunlumi = this->setRunLumi(edm::LuminosityBlockID(run, 0)); + + f(booker); + + booker.setScope(oldscope); + this->setModuleID(oldmoduleid); + this->setRunLumi(oldrunlumi); + }; + template - void meBookerGetter(iFunc f){}; + void meBookerGetter(iFunc f) { + auto lock = std::scoped_lock(this->booking_mutex_); + // here, we make much less assumptions compared to bookTransaction. + // This is essentially legacy semantics except we actually take the lock. + f(*this, *this); + // TODO: we should maybe make sure that Scope changes are reset here, + // but also it makes sense to inherit the Scope from the environement + // (e.g. when meBookerGetter is called *inside* a booking transaction). + }; // Add ME to DQMStore datastructures. The object will be deleted if a // similar object is already present. @@ -565,10 +590,17 @@ namespace dqm { // NEVER modify the key_ of a ME in these datastructures. Since we use // pointers, this may be possible (not everything is const), but it could // still corrupt the datastructure. - std::map> globalMEs_; + std::map> globalMEs_; // Key is (moduleID [, run]), run is only needed for edm::global. // Legacy MEs have moduleID 0. - std::map> localMEs_; + std::map> localMEs_; + // Whenever modifying these sets, take tihs mutex. It's recursive, so we + // can be liberal -- lock on any access, but also lock on the full booking + // transaction. The former is required since also the MEComparison is not + // really thread safe, the latter since booking still uses a single, + // shared IBooker instance (`this`!), and the transaction needs to be + // atomic. + std::recursive_mutex booking_mutex_; // universal verbose flag. int verbose_; diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 85302eefe6b0c..dccc162815064 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -96,6 +96,42 @@ namespace dqm::impl { using Scalar = MonitorElementData::Scalar; using Kind = MonitorElementData::Kind; + // Comparison helper used in DQMStore to insert into sets. This needs deep + // private access to the MEData, that is why it lives here. + struct MEComparison { + using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. + + // no locking here. We assume this is called from the DQMStore set + // operations, which need to be protected by a lock anyways. + bool operator()(MonitorElement *left, MonitorElement *right) const { + MonitorElementData const *l_frozen = left->frozen_.load(); + MutableMonitorElementData const *l_mutable = left->mutable_.load(); + MonitorElementData const *r_frozen = right->frozen_.load(); + MutableMonitorElementData const *r_mutable = right->mutable_.load(); + + MonitorElementData::Path const &l = l_mutable ? l_mutable->data_.key_.path_ : l_frozen->key_.path_; + MonitorElementData::Path const &r = r_mutable ? r_mutable->data_.key_.path_ : r_frozen->key_.path_; + + return (*this)(l, r); // call implementation below + } + bool operator()(MonitorElement *left, MonitorElementData::Path const &right) const { + MonitorElementData const *l_frozen = left->frozen_.load(); + MutableMonitorElementData const *l_mutable = left->mutable_.load(); + MonitorElementData::Path const &l = l_mutable ? l_mutable->data_.key_.path_ : l_frozen->key_.path_; + return (*this)(l, right); // call implementation below + } + bool operator()(MonitorElementData::Path const &left, MonitorElement *right) const { + MonitorElementData const *r_frozen = right->frozen_.load(); + MutableMonitorElementData const *r_mutable = right->mutable_.load(); + MonitorElementData::Path const &r = r_mutable ? r_mutable->data_.key_.path_ : r_frozen->key_.path_; + return (*this)(left, r); // call implementation below + } + bool operator()(MonitorElementData::Path const &left, MonitorElementData::Path const &right) const { + return std::make_tuple(left.getDirname(), left.getObjectname()) < + std::make_tuple(right.getDirname(), right.getObjectname()); + } + }; + protected: DQMNet::CoreObject data_; //< Core object information. // TODO: we only use the ::Value part so far. @@ -103,7 +139,7 @@ namespace dqm::impl { std::atomic frozen_; // only set if this ME is in a product already std::atomic mutable_; // only set if there is a mutable copy of this ME - bool is_owned_; // true if we are responsible for deleting the mutable object. + bool is_owned_; // true if we are responsible for deleting the mutable object. /** * To do anything to the MEs data, one needs to obtain an access object. * This object will contain the lock guard if one is needed. We differentiate @@ -180,11 +216,11 @@ namespace dqm::impl { } public: - // Create ME using this data. A ROOT object pointer may be moved into the + // Create ME using this data. A ROOT object pointer may be moved into the // new ME. The new ME will own this data. - MonitorElement(MonitorElementData&& data); + MonitorElement(MonitorElementData &&data); // Create a new ME sharing data with this existing ME. - MonitorElement(MonitorElement* me); + MonitorElement(MonitorElement *me); MonitorElement &operator=(const MonitorElement &) = delete; MonitorElement &operator=(MonitorElement &&) = delete; virtual ~MonitorElement(); diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 7d8e20b704cdd..085ab2986754d 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -84,6 +84,7 @@ namespace dqm::implementation { } MonitorElement* DQMStore::putME(MonitorElement* me) { + auto lock = std::scoped_lock(this->booking_mutex_); assert(me); auto existing_new = globalMEs_[me->getRunLumi()].insert(me); if (existing_new.second == true) { @@ -98,6 +99,7 @@ namespace dqm::implementation { } MonitorElement* DQMStore::putME(MonitorElement* me, uint64_t moduleID) { + auto lock = std::scoped_lock(this->booking_mutex_); assert(me); auto existing_new = localMEs_[moduleID].insert(me); if (existing_new.second == true) { @@ -112,9 +114,16 @@ namespace dqm::implementation { } MonitorElement* DQMStore::findME(MonitorElementData::Path const& path) { - //TODO - - + auto lock = std::scoped_lock(this->booking_mutex_); + for (auto& [runlumi, meset] : this->globalMEs_) { + auto it = meset.find(path); + if (it != meset.end()) { + // no guarantee on which ME we return here -- only that clone'ing this + // would give a valid ME for that path. + return *it; + } + } + return nullptr; } void DQMStore::printTrace(std::string const& message) { diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 3796065a847c5..022578f319c96 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -40,14 +40,14 @@ namespace dqm::impl { return h; } - MonitorElement::MonitorElement(MonitorElementData&& data) { + MonitorElement::MonitorElement(MonitorElementData &&data) { this->mutable_ = new MutableMonitorElementData(); this->frozen_ = nullptr; this->mutable_.load()->data_ = std::move(data); this->is_owned_ = true; // TODO: update DQMNet::CoreObject. } - MonitorElement::MonitorElement(MonitorElement* me) { + MonitorElement::MonitorElement(MonitorElement *me) { this->mutable_ = me->mutable_.load(); this->frozen_ = me->frozen_.load(); this->is_owned_ = false; diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index cd92584f0cd66..9ffa5cce46e67 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -166,7 +166,7 @@ struct MonitorElementData { std::string const& getDirname() const { return dirname_; } std::string const& getObjectname() const { return objname_; } - std::string getFullname() const { return dirname_ + objname_; } + std::string getFullname() const { return dirname_ + objname_; } // Clean up the path and normalize it to preserve certain invariants. // Instead of reasoning about whatever properties of paths, we just parse From 079f18d745d422db11304784618150580c086b61 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 20 Nov 2019 18:43:30 +0100 Subject: [PATCH 015/112] Implement enterLumi/leaveLumi. This is the hard part -- at least half of it. This logic creates clones if and only if they are required. The other half is the cleanup code to run after saving, that needs to delete all clones but a prototype after saving. Not entirely sure yet what to do with harvesting -- we will have global and local MEs as well, but I am not sure yet if we should separate them by module ID. get*() will probably just return global MEs. --- DQMServices/Core/interface/DQMEDAnalyzer.h | 5 +- .../Core/interface/DQMGlobalEDAnalyzer.h | 9 +- DQMServices/Core/interface/DQMOneEDAnalyzer.h | 5 +- DQMServices/Core/interface/DQMStore.h | 31 +++--- DQMServices/Core/interface/MonitorElement.h | 16 ++- DQMServices/Core/src/DQMStore.cc | 105 ++++++++++++++++-- DQMServices/Core/src/MonitorElement.cc | 35 ++++++ 7 files changed, 171 insertions(+), 35 deletions(-) diff --git a/DQMServices/Core/interface/DQMEDAnalyzer.h b/DQMServices/Core/interface/DQMEDAnalyzer.h index c988eefafe968..9617f4bc6a9bc 100644 --- a/DQMServices/Core/interface/DQMEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMEDAnalyzer.h @@ -37,10 +37,9 @@ class DQMEDAnalyzer : public edm::one::EDProducerbookHistograms(booker, run, setup); }, - /* run */ 0, // MEs booked here apply for all runs this->moduleDescription().id(), this->getCanSaveByLumi()); - edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); + edm::Service()->enterLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); } void beginLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const& setup) final {} @@ -57,7 +56,7 @@ class DQMEDAnalyzer : public edm::one::EDProducer()->leaveLumi(run.run(), 0, this->moduleDescription().id()); + edm::Service()->leaveLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); run.emplace(runToken_); } diff --git a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h index 0c6226ac651f0..6cb1209093d11 100644 --- a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h @@ -41,11 +41,11 @@ class DQMGlobalEDAnalyzer b.cd(); bookHistograms(b, run, setup, *h); }, - run.run(), - this->moduleDescription().id(), + // The run number is part of the module ID here, since we want distinct + // local MEs for each run cache. + (((uint64_t)run.run()) << 32) + this->moduleDescription().id(), /* canSaveByLumi */ false); - // We will not call enterLumi per-lumi, since this is strictly run-based. - // The MEs are booked for a fixed run, given in the transaction. + dqmstore_->enterLumi(run.run(), /* lumi */ 0, (((uint64_t)run.run()) << 32) + this->moduleDescription().id()); return h; } @@ -57,6 +57,7 @@ class DQMGlobalEDAnalyzer void globalEndRunProduce(edm::Run& run, edm::EventSetup const& setup) const final { auto const& h = *this->runCache(run.index()); dqmEndRun(run, setup, h); + dqmstore_->leaveLumi(run.run(), /* lumi */ 0, (((uint64_t)run.run()) << 32) + this->moduleDescription().id()); run.emplace(runToken_); } diff --git a/DQMServices/Core/interface/DQMOneEDAnalyzer.h b/DQMServices/Core/interface/DQMOneEDAnalyzer.h index ca8a56c689d47..c9317275caf90 100644 --- a/DQMServices/Core/interface/DQMOneEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMOneEDAnalyzer.h @@ -38,17 +38,16 @@ class DQMOneEDAnalyzer booker.cd(); this->bookHistograms(booker, run, setup); }, - /* run */ 0, // run is assigned later, in enterLumi this->moduleDescription().id(), this->getCanSaveByLumi()); - edm::Service()->enterLumi(run.run(), 0, this->moduleDescription().id()); + edm::Service()->enterLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); } void accumulate(edm::Event const& event, edm::EventSetup const& setup) final { analyze(event, setup); } void endRunProduce(edm::Run& run, edm::EventSetup const& setup) final { dqmEndRun(run, setup); - edm::Service()->leaveLumi(run.run(), 0, this->moduleDescription().id()); + edm::Service()->leaveLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); run.emplace(runToken_); } diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index d4d9b2d5df65d..39544463b3453 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -517,22 +517,18 @@ namespace dqm { public: // internal -- figure out better protection. template - void bookTransaction(iFunc f, uint32_t run, uint32_t moduleId, bool canSaveByLumi) { + void bookTransaction(iFunc f, uint32_t moduleId, bool canSaveByLumi) { auto lock = std::scoped_lock(this->booking_mutex_); IBooker& booker = *this; // TODO: this may need to become more elaborate. auto oldscope = booker.setScope(canSaveByLumi ? MonitorElementData::Scope::LUMI : MonitorElementData::Scope::RUN); assert(moduleId != 0 || !"moduleID must be set for normal booking transaction"); - // run should be 0 unless this is DQMGlobalEDAnalyzer. - // The run number goes into the moduleID, since for the DQMGlobalEDAnalyzer, - // we need *different* localMEs within the same module. // Access via this-> to allow access to protected member - auto oldmoduleid = this->setModuleID((((uint64_t)run) << 32) + moduleId); + auto oldmoduleid = this->setModuleID(moduleId); assert(oldmoduleid == 0 || !"Nested booking transaction?"); - // in a proper transaction we book prototypes, except DQMGlobalEDAnalyzer - // (run is set, MEs are booked for a fixed run). - auto oldrunlumi = this->setRunLumi(edm::LuminosityBlockID(run, 0)); + // always book prototypes (except for Scope::JOB, where we can use these directly). + auto oldrunlumi = this->setRunLumi(edm::LuminosityBlockID()); f(booker); @@ -552,6 +548,12 @@ namespace dqm { // (e.g. when meBookerGetter is called *inside* a booking transaction). }; + // modules are expected to call these callbacks when they change run/lumi. + // The DQMStore then updates the module's MEs, potentially cloning them + // if there are concurrent runs/lumis. + void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); + void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); + // Add ME to DQMStore datastructures. The object will be deleted if a // similar object is already present. // For global ME @@ -559,18 +561,15 @@ namespace dqm { // For local ME MonitorElement* putME(MonitorElement* me, uint64_t moduleID); // Find a global ME of matching name, in any state. - MonitorElement* findME(MonitorElementData::Path const& path); + // MELIKE can be a MonitorElementData::Path or MonitorElement*. + template + MonitorElement* findME(MELIKE const& path); // Log a backtrace on booking. void printTrace(std::string const& message); - void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); - void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); - - // Clone data including the underlying ROOT object (calls ->Clone()). - static MonitorElementData* cloneMonitorElementData(MonitorElementData const* input); private: - // TODO: MEComparison need to deref pointers, and allow comparison to - // bare strings, and provide `is_transparent` for heterogeneous lookup. + // MEComparison is a name-only comparison on MEs and Paths, allowing + // heterogeneous lookup. // The ME objects here are lightweight, all the important stuff is in the // MEData. However we never handle MEData directly, but always keep it // wrapped in MEs (created as needed). MEs can share MEData. diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index dccc162815064..d83d73049fed1 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -219,10 +219,19 @@ namespace dqm::impl { // Create ME using this data. A ROOT object pointer may be moved into the // new ME. The new ME will own this data. MonitorElement(MonitorElementData &&data); + // Create new ME and take ownership of this data. + MonitorElement(MutableMonitorElementData *data); // Create a new ME sharing data with this existing ME. MonitorElement(MonitorElement *me); MonitorElement &operator=(const MonitorElement &) = delete; MonitorElement &operator=(MonitorElement &&) = delete; + // return a new clone of the data of this ME. Calls ->Clone(), new object + // is owned by the returned value. + MonitorElementData cloneMEData(); + // Remove access and ownership to the data. The flag is used for a sanity check. + MutableMonitorElementData *release(bool expectOwned); + // re-initialize this ME as a shared copy of the other. + void switchData(MonitorElement *other); virtual ~MonitorElement(); /// Compare monitor elements, for ordering in sets. @@ -248,6 +257,8 @@ namespace dqm::impl { const edm::LuminosityBlockID getRunLumi() { return access().key.id_; } + const MonitorElementData::Scope getScope() { return access().key.scope_; } + /// true if ME was updated in last monitoring cycle bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } @@ -259,10 +270,11 @@ namespace dqm::impl { void setResetMe(bool /* flag */) { data_.flags |= DQMNet::DQM_PROP_RESET; } /// true if ME is meant to be stored for each luminosity section - bool getLumiFlag() const { return data_.flags & DQMNet::DQM_PROP_LUMI; } + bool getLumiFlag() const { return access().key.scope_ == MonitorElementData::Scope::LUMI; } /// this ME is meant to be stored for each luminosity section - void setLumiFlag() { data_.flags |= DQMNet::DQM_PROP_LUMI; } + // we can't support this any more, but the ME might be safed by lumi anyways! + void setLumiFlag() { assert(getLumiFlag()); } /// this ME is meant to be an efficiency plot that must not be /// normalized when drawn in the DQM GUI. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 085ab2986754d..9ed4c392ada9e 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -113,7 +113,8 @@ namespace dqm::implementation { } } - MonitorElement* DQMStore::findME(MonitorElementData::Path const& path) { + template + MonitorElement* DQMStore::findME(MELIKE const& path) { auto lock = std::scoped_lock(this->booking_mutex_); for (auto& [runlumi, meset] : this->globalMEs_) { auto it = meset.find(path); @@ -196,16 +197,106 @@ namespace dqm::implementation { } void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { - //TODO - } + // Make sure global MEs for the run/lumi exist (depending on scope), and + // point the local MEs for this module to these global MEs. - MonitorElementData* DQMStore::cloneMonitorElementData(MonitorElementData const* input) { - //TODO - return nullptr; + auto lock = std::scoped_lock(this->booking_mutex_); + + // these are the MEs we need to update. + auto& localset = this->localMEs_[moduleID]; + // this is where they need to point to. + auto& targetset = this->globalMEs_[edm::LuminosityBlockID(run, lumi)]; + // this is where we can get MEs to reuse. + auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()]; + + auto checkScope = [run, lumi](MonitorElementData::Scope scope) { + if (scope == MonitorElementData::Scope::JOB) { + return (run == 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::RUN) { + return (run != 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::LUMI) { + return (lumi != 0); + } + assert(!"Impossible Scope."); + return false; + }; + + for (MonitorElement* me : localset) { + auto target = targetset.find(me); // lookup by path, thanks to MEComparison + if (target != targetset.end()) { + // we already have a ME, just use it! + } else { + // look for a prototype to reuse. + auto proto = prototypes.find(me); + if (proto != prototypes.end()) { + // first, check if this ME needs updating at all. We can only check + // the scope once we have an actual global ME instance, the local ME + // might not have any data attached! + if (checkScope((*proto)->getScope()) == false) { + continue; + } // else + // reuse that. + MonitorElement* oldme = *proto; + prototypes.erase(proto); + auto medata = oldme->release(/* expectOwned */ true); // destroy the ME, get its data. + delete oldme; + // in this situation, nobody should be filling the ME concurrently. + medata->data_.key_.id_ = edm::LuminosityBlockID(run, lumi); + auto newme = new MonitorElement(medata); + auto result = targetset.insert(newme); + assert(result.second); // was new insertion + target = result.first; // iterator to new ME + } else { + // no prototype available. That means we have concurrent Lumis/Runs, + // and need to make a clone now. + auto anyme = this->findME(me); + assert(anyme || !"local ME without any global ME!"); + if (checkScope(anyme->getScope()) == false) { + continue; + } // else + MonitorElementData newdata = anyme->cloneMEData(); + auto newme = new MonitorElement(std::move(newdata)); + newme->Reset(); // we cloned a ME in use, not an empty prototype + auto result = targetset.insert(newme); + assert(result.second); // was new insertion + target = result.first; // iterator to new ME + } + } + // now we have the proper global ME in the right place, point the local there. + me->switchData(*target); + } } void DQMStore::leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { - //TODO + // here, we remove the pointers in the local MEs. No deletion or recycling + // yet -- this has to happen after the output module had a chance to do its + // work. We just leave the global MEs where they are. This is purely an + // accounting step, the cleanup code has to check that nobody is using the + // ME any more, and here we make sure that is the case. + + auto lock = std::scoped_lock(this->booking_mutex_); + + // these are the MEs we need to update. + auto& localset = this->localMEs_[moduleID]; + + auto checkScope = [run, lumi](MonitorElementData::Scope scope) { + if (scope == MonitorElementData::Scope::JOB) { + return (run == 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::RUN) { + return (run != 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::LUMI) { + return (lumi != 0); + } + assert(!"Impossible Scope."); + return false; + }; + + for (MonitorElement* me : localset) { + if (checkScope(me->getScope()) == true) { + // if we left the scope, simply release the data. + me->release(/* expectOwned */ false); + } + } } std::vector IGetter::getContents(std::string const& path) const { assert(!"NIY"); } diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 022578f319c96..b80d3cd3ec2aa 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -47,6 +47,12 @@ namespace dqm::impl { this->is_owned_ = true; // TODO: update DQMNet::CoreObject. } + MonitorElement::MonitorElement(MutableMonitorElementData *data) { + this->mutable_ = data; + this->frozen_ = nullptr; + this->is_owned_ = true; + // TODO: update DQMNet::CoreObject. + } MonitorElement::MonitorElement(MonitorElement *me) { this->mutable_ = me->mutable_.load(); this->frozen_ = me->frozen_.load(); @@ -54,6 +60,35 @@ namespace dqm::impl { // TODO: update DQMNet::CoreObject. } + MonitorElementData MonitorElement::cloneMEData() { + MonitorElementData out; + auto access = this->access(); + out.key_ = access.key; + out.value_.scalar_ = access.value.scalar_; + if (access.value.object_) { + out.value_.object_ = std::unique_ptr(static_cast(access.value.object_->Clone())); + } + return out; + } + + MutableMonitorElementData *MonitorElement::release(bool expectOwned) { + assert(this->is_owned_ == expectOwned); + MutableMonitorElementData *data = this->mutable_.load(); + this->mutable_ = nullptr; + this->frozen_ = nullptr; + this->is_owned_ = false; + assert(!expectOwned || data); + return data; + } + + void MonitorElement::switchData(MonitorElement *other) { + assert(other); + this->mutable_ = other->mutable_.load(); + this->frozen_ = other->frozen_.load(); + this->is_owned_ = false; + // TODO: update DQMNet::CoreObject. + } + MonitorElement::~MonitorElement() { if (is_owned_) delete mutable_; From ba2289b125ae5e9568b8ae0520a08daa0588b188 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 21 Nov 2019 12:59:13 +0100 Subject: [PATCH 016/112] Changes to make everything compile. This includes adding a restriction to the tempalte callback argument for the booking calls to prevent ambiguities. --- DQM/L1TMonitor/src/L1TMP7ZeroSupp.cc | 6 +- DQMServices/Core/interface/DQMStore.h | 69 ++++++++++++-------- HLTrigger/Timer/plugins/FastTimerService.cc | 5 +- HLTrigger/Timer/plugins/ThroughputService.cc | 5 +- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/DQM/L1TMonitor/src/L1TMP7ZeroSupp.cc b/DQM/L1TMonitor/src/L1TMP7ZeroSupp.cc index 52f745974607c..1a613ced55bbf 100644 --- a/DQM/L1TMonitor/src/L1TMP7ZeroSupp.cc +++ b/DQM/L1TMonitor/src/L1TMP7ZeroSupp.cc @@ -95,7 +95,7 @@ void L1TMP7ZeroSupp::bookCapIdHistograms(DQMStore::IBooker& ibooker, const unsig sizeTitleText = "cumulated caption id " + std::to_string(id) + " block "; } - zeroSuppValMap_[id] = ibooker.book1D("zeroSuppVal", summaryTitleText, NBINLABELS, 0, NBINLABELS); + zeroSuppValMap_[id] = ibooker.book1D("zeroSuppVal", summaryTitleText, (int)NBINLABELS, 0, (int)NBINLABELS); zeroSuppValMap_[id]->setAxisTitle("ZS status", 1); zeroSuppValMap_[id]->setBinLabel(EVTS + 1, "events", 1); zeroSuppValMap_[id]->setBinLabel(EVTSGOOD + 1, "good events", 1); @@ -111,7 +111,7 @@ void L1TMP7ZeroSupp::bookCapIdHistograms(DQMStore::IBooker& ibooker, const unsig zeroSuppValMap_[id]->setBinLabel(ZSBXBLKSBADFALSEPOS + 1, "BX false pos.", 1); zeroSuppValMap_[id]->setBinLabel(ZSBXBLKSBADFALSENEG + 1, "BX false neg.", 1); - errorSummaryNumMap_[id] = ibooker.book1D("errorSummaryNum", summaryTitleText, RNBINLABELS, 0, RNBINLABELS); + errorSummaryNumMap_[id] = ibooker.book1D("errorSummaryNum", summaryTitleText, (int)RNBINLABELS, 0, (int)RNBINLABELS); errorSummaryNumMap_[id]->setBinLabel(REVTS + 1, "bad events", 1); errorSummaryNumMap_[id]->setBinLabel(RBLKS + 1, "bad blocks", 1); errorSummaryNumMap_[id]->setBinLabel(RBLKSFALSEPOS + 1, "false pos.", 1); @@ -120,7 +120,7 @@ void L1TMP7ZeroSupp::bookCapIdHistograms(DQMStore::IBooker& ibooker, const unsig errorSummaryNumMap_[id]->setBinLabel(RBXBLKSFALSEPOS + 1, "BX false pos.", 1); errorSummaryNumMap_[id]->setBinLabel(RBXBLKSFALSENEG + 1, "BX false neg.", 1); - errorSummaryDenMap_[id] = ibooker.book1D("errorSummaryDen", "denominators", RNBINLABELS, 0, RNBINLABELS); + errorSummaryDenMap_[id] = ibooker.book1D("errorSummaryDen", "denominators", (int)RNBINLABELS, 0, (int)RNBINLABELS); errorSummaryDenMap_[id]->setBinLabel(REVTS + 1, "# events", 1); errorSummaryDenMap_[id]->setBinLabel(RBLKS + 1, "# blocks", 1); errorSummaryDenMap_[id]->setBinLabel(RBLKSFALSEPOS + 1, "# blocks", 1); diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 39544463b3453..318563e468d36 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -6,6 +6,7 @@ #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" +#include #include #include @@ -48,21 +49,33 @@ namespace dqm { // the object; it will be clone'd. // - template + // The function argument as an optional template parameter adds a lot of + // ambiguity to the overload resolution, since it accepts *anything* by + // default (and it does not help that we rely on implicit conversions for + // almost all of the arguments in many cases, converting string literal + // to TString and ints to floats, and 0 also prefers to convert to float* + // and so on ...). + // So, we use SFINAE to restrict the template parameter type, but that is + // also not that easy: there is no way to check for sth. callable in + // type_traits (`is_function` is not the right thing!), so instead we + // check for not-numeric things, which works most of the time (though e.g. + // enum constants somehow still pass as not arithmetic and need an + // explicit cast to resolve the ambiguity). + template ::value, int> = 0> MonitorElement* bookInt(TString const& name, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::INT, [=]() { onbooking(); return nullptr; }); } - template + template ::value, int> = 0> MonitorElement* bookFloat(TString const& name, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind ::REAL, [=]() { onbooking(); return nullptr; }); } - template + template ::value, int> = 0> MonitorElement* bookString(TString const& name, TString const& value, FUNC onbooking = NOOP()) { // TODO: value unused! return bookME(name, MonitorElementData::Kind::STRING, [=]() { @@ -71,7 +84,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* book1D(TString const& name, TString const& title, int const nchX, @@ -84,7 +97,7 @@ namespace dqm { return th1; }); } - template + template ::value, int> = 0> MonitorElement* book1D( TString const& name, TString const& title, int nchX, float const* xbinsize, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1F, [=]() { @@ -93,7 +106,7 @@ namespace dqm { return th1; }); } - template + template ::value, int> = 0> MonitorElement* book1D(TString const& name, TH1F* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1F, [=]() { auto th1 = static_cast(object->Clone(name)); @@ -102,7 +115,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* book1S( TString const& name, TString const& title, int nchX, double lowX, double highX, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1S, [=]() { @@ -111,7 +124,7 @@ namespace dqm { return th1; }); } - template + template ::value, int> = 0> MonitorElement* book1S(TString const& name, TH1S* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1S, [=]() { auto th1 = static_cast(object->Clone(name)); @@ -120,7 +133,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* book1DD( TString const& name, TString const& title, int nchX, double lowX, double highX, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1D, [=]() { @@ -129,7 +142,7 @@ namespace dqm { return th1; }); } - template + template ::value, int> = 0> MonitorElement* book1DD(TString const& name, TH1D* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH1D, [=]() { auto th1 = static_cast(object->Clone(name)); @@ -138,7 +151,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* book2D(TString const& name, TString const& title, int nchX, @@ -154,7 +167,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2D(TString const& name, TString const& title, int nchX, @@ -168,7 +181,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2D(TString const& name, TH2F* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH2F, [=]() { auto th2 = static_cast(object->Clone(name)); @@ -176,7 +189,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2S(TString const& name, TString const& title, int nchX, @@ -192,7 +205,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2S(TString const& name, TString const& title, int nchX, @@ -206,7 +219,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2S(TString const& name, TH2S* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH2S, [=]() { auto th2 = static_cast(object->Clone(name)); @@ -214,7 +227,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2DD(TString const& name, TString const& title, int nchX, @@ -230,7 +243,7 @@ namespace dqm { return th2; }); } - template + template ::value, int> = 0> MonitorElement* book2DD(TString const& name, TH2D* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH2D, [=]() { auto th2 = static_cast(object->Clone(name)); @@ -239,7 +252,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* book3D(TString const& name, TString const& title, int nchX, @@ -258,7 +271,7 @@ namespace dqm { return th3; }); } - template + template ::value, int> = 0> MonitorElement* book3D(TString const& name, TH3F* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TH3F, [=]() { auto th3 = static_cast(object->Clone(name)); @@ -267,7 +280,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -284,7 +297,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -300,7 +313,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -316,7 +329,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TString const& title, int nchX, @@ -331,7 +344,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TProfile* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { auto tprofile = static_cast(object->Clone(name)); @@ -340,7 +353,7 @@ namespace dqm { }); } - template + template ::value, int> = 0> MonitorElement* bookProfile2D(TString const& name, TString const& title, int nchX, @@ -359,7 +372,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile2D(TString const& name, TString const& title, int nchX, @@ -379,7 +392,7 @@ namespace dqm { return tprofile; }); } - template + template ::value, int> = 0> MonitorElement* bookProfile2D(TString const& name, TProfile2D* object, FUNC onbooking = NOOP()) { return bookME(name, MonitorElementData::Kind::TPROFILE2D, [=]() { auto tprofile = static_cast(object->Clone(name)); diff --git a/HLTrigger/Timer/plugins/FastTimerService.cc b/HLTrigger/Timer/plugins/FastTimerService.cc index dca9eef18c81f..129ae435e33fd 100644 --- a/HLTrigger/Timer/plugins/FastTimerService.cc +++ b/HLTrigger/Timer/plugins/FastTimerService.cc @@ -856,7 +856,7 @@ void FastTimerService::preGlobalBeginRun(edm::GlobalContext const& gc) { // book the DQM plots if (enable_dqm_) { // define a callback to book the MonitorElements - auto bookTransactionCallback = [&, this](dqm::reco::DQMStore::IBooker& booker) { + auto bookTransactionCallback = [&, this](dqm::reco::DQMStore::IBooker& booker, dqm::reco::DQMStore::IGetter&) { booker.setCurrentFolder(dqm_path_); plots_->book(booker, callgraph_, @@ -872,8 +872,7 @@ void FastTimerService::preGlobalBeginRun(edm::GlobalContext const& gc) { }; // book MonitorElements for this stream - edm::Service()->bookTransaction( - bookTransactionCallback, gc.luminosityBlockID().run(), /* moduleID */ 0, /* canSaveByLumi*/ false); + edm::Service()->meBookerGetter(bookTransactionCallback); } } } diff --git a/HLTrigger/Timer/plugins/ThroughputService.cc b/HLTrigger/Timer/plugins/ThroughputService.cc index 568a1ed0741e6..7814bdcb95118 100644 --- a/HLTrigger/Timer/plugins/ThroughputService.cc +++ b/HLTrigger/Timer/plugins/ThroughputService.cc @@ -55,7 +55,7 @@ void ThroughputService::preGlobalBeginRun(edm::GlobalContext const& gc) { double range = bins * m_time_resolution; // define a callback that can book the histograms - auto bookTransactionCallback = [&, this](DQMStore::IBooker& booker) { + auto bookTransactionCallback = [&, this](DQMStore::IBooker& booker, DQMStore::IGetter&) { booker.setCurrentFolder(m_dqm_path); m_sourced_events = booker.book1D("throughput_sourced", "Throughput (sourced events)", bins, 0., range); m_sourced_events->setXTitle("time [s]"); @@ -66,8 +66,7 @@ void ThroughputService::preGlobalBeginRun(edm::GlobalContext const& gc) { }; // book MonitorElement's for this run - edm::Service()->bookTransaction( - bookTransactionCallback, gc.luminosityBlockID().run(), /* moduleID */ 0, /* canSaveByLumi */ false); + edm::Service()->meBookerGetter(bookTransactionCallback); } else { std::cerr << "No DQMStore service, aborting." << std::endl; abort(); From b443f5312af8e8e58546ef81acdd5f417c9f00f4 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 21 Nov 2019 15:25:26 +0100 Subject: [PATCH 017/112] Updates and fixes around DQMNet::CoreObject. plus some bugfixes in other places. syncCoreObject is pretty inefficient, and in general I replaced many complicated, efficient things with slow, safe, value-based code. The hope is that the compiler will optimize away some of the values and that this code does not run much anyways. Later improvements are possible. --- .../Components/plugins/QualityTester.cc | 2 +- DQMServices/Core/interface/DQMStore.h | 8 +- DQMServices/Core/interface/MonitorElement.h | 14 +- DQMServices/Core/src/DQMStore.cc | 4 +- DQMServices/Core/src/MonitorElement.cc | 152 +++++++++++------- DQMServices/Core/test/DQMTestMultiThread.cc | 3 +- .../Demo/plugins/DemoNormalDQMEDAnalyzer.cc | 2 +- .../interface/MonitorElementCollection.h | 16 +- 8 files changed, 117 insertions(+), 84 deletions(-) diff --git a/DQMServices/Components/plugins/QualityTester.cc b/DQMServices/Components/plugins/QualityTester.cc index d64cb595f0423..2099d929c4dce 100644 --- a/DQMServices/Components/plugins/QualityTester.cc +++ b/DQMServices/Components/plugins/QualityTester.cc @@ -162,7 +162,7 @@ void QualityTester::performTests(DQMStore::IGetter& igetter) { assert(qv); qtest->runTest(me, *qr, *qv); // this propagates the result into the DQMNet object flags - me->updateQReportStats(); + me->syncCoreObject(); } } diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 318563e468d36..c506d5ec18894 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -416,10 +416,10 @@ namespace dqm { MonitorElementData::Kind kind, std::function makeobject); - DQMStore* store_; - MonitorElementData::Scope scope_; - uint64_t moduleID_; - edm::LuminosityBlockID runlumi_; + DQMStore* store_ = nullptr; + MonitorElementData::Scope scope_ = MonitorElementData::Scope::JOB; + uint64_t moduleID_ = 0; + edm::LuminosityBlockID runlumi_ = edm::LuminosityBlockID(); }; class IGetter : public dqm::implementation::NavigatorBase { diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index d83d73049fed1..5567ac3bbe6ab 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -232,6 +232,9 @@ namespace dqm::impl { MutableMonitorElementData *release(bool expectOwned); // re-initialize this ME as a shared copy of the other. void switchData(MonitorElement *other); + // copy applicable fileds into the DQMNet core object for compatibility. + // In a few places these flags are also still used by the ME. + void syncCoreObject(); virtual ~MonitorElement(); /// Compare monitor elements, for ordering in sets. @@ -365,14 +368,16 @@ namespace dqm::impl { /// will not be normalized when rendered within the DQM GUI. bool isEfficiency() const { return data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT; } + // used to implement getQErrors et. al. + template + std::vector filterQReports(FILTER filter) const; + /// get QReport corresponding to (null pointer if QReport does not exist) const MonitorElementData::QReport *getQReport(const std::string &qtname) const; /// get map of QReports std::vector getQReports() const; /// access QReport, potentially adding it. void getQReport(bool create, const std::string &qtname, MonitorElementData::QReport *&qr, DQMNet::QValue *&qv); - /// propagate QReport status bits after change - void updateQReportStats(); /// get warnings from last set of quality tests std::vector getQWarnings() const; @@ -483,11 +488,6 @@ namespace dqm::impl { void addProfiles(TProfile2D *h1, TProfile2D *h2, TProfile2D *sum, float c1, float c2); void copyFunctions(TH1 *from, TH1 *to); void copyFrom(TH1 *from); - - public: - const uint32_t run() const { return data_.run; } - const uint32_t lumi() const { return data_.lumi; } - const uint32_t moduleId() const { return data_.moduleId; } }; } // namespace dqm::impl diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 9ed4c392ada9e..0a3e441ac6ffa 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -77,7 +77,7 @@ namespace dqm::implementation { // DQMGlobalEDAnalyzer). This will later be update to point to different // MEData (kept in a global ME) as needed. MonitorElement* local_me = new MonitorElement(me); - me = store_->putME(local_me); + me = store_->putME(local_me, this->moduleID_); // me now points to a local ME owned by the DQMStore. assert(me); return me; @@ -263,6 +263,8 @@ namespace dqm::implementation { } } // now we have the proper global ME in the right place, point the local there. + // This is only safe if the name is exactly the same -- else it might corrupt + // the tree structure of the set! me->switchData(*target); } } diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index b80d3cd3ec2aa..470ecdca3b329 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -45,19 +45,19 @@ namespace dqm::impl { this->frozen_ = nullptr; this->mutable_.load()->data_ = std::move(data); this->is_owned_ = true; - // TODO: update DQMNet::CoreObject. + syncCoreObject(); } MonitorElement::MonitorElement(MutableMonitorElementData *data) { this->mutable_ = data; this->frozen_ = nullptr; this->is_owned_ = true; - // TODO: update DQMNet::CoreObject. + syncCoreObject(); } MonitorElement::MonitorElement(MonitorElement *me) { this->mutable_ = me->mutable_.load(); this->frozen_ = me->frozen_.load(); this->is_owned_ = false; - // TODO: update DQMNet::CoreObject. + syncCoreObject(); } MonitorElementData MonitorElement::cloneMEData() { @@ -86,7 +86,70 @@ namespace dqm::impl { this->mutable_ = other->mutable_.load(); this->frozen_ = other->frozen_.load(); this->is_owned_ = false; - // TODO: update DQMNet::CoreObject. + syncCoreObject(); + } + + void MonitorElement::syncCoreObject() { + auto access = this->access(); + data_.flags &= ~DQMNet::DQM_PROP_TYPE_MASK; + data_.flags |= (int)access.key.kind_; + + // mark as updated. + data_.flags |= DQMNet::DQM_PROP_NEW; + + // lumi flag is approximately equivalent to Scope::LUMI. + data_.flags &= ~DQMNet::DQM_PROP_LUMI; + if (access.key.scope_ == MonitorElementData::Scope::LUMI) { + data_.flags |= DQMNet::DQM_PROP_LUMI; + } + + // these are unsupported and always off. + data_.flags &= ~DQMNet::DQM_PROP_HAS_REFERENCE; + data_.flags &= ~DQMNet::DQM_PROP_TAGGED; + data_.flags &= ~DQMNet::DQM_PROP_RESET; + data_.flags &= ~DQMNet::DQM_PROP_ACCUMULATE; + + // we use ROOT's internal efficiency flag as the truth + data_.flags &= ~DQMNet::DQM_PROP_EFFICIENCY_PLOT; + if (access.value.object_ && access.value.object_->TestBit(TH1::kIsAverage)) { + data_.flags |= DQMNet::DQM_PROP_EFFICIENCY_PLOT; + } + + data_.tag = 0; + + // don't touch version (a timestamp). + + // we could set proper values here, but nobody should use them. + data_.run = 0; + data_.lumi = 0; + + // these are relics from the threaded migration and should not be used anywhere. + data_.streamId = 0; + data_.moduleId = 0; + + // leaking a pointer here, but that should be fine. + data_.dirname = &access.key.path_.getDirname(); + + data_.objname = access.key.path_.getObjectname(); + + data_.flags &= ~DQMNet::DQM_PROP_REPORT_ALARM; + data_.qreports.clear(); + for (QReport const &qr : access.value.qreports_) { + data_.qreports.push_back(qr.getValue()); + switch (qr.getStatus()) { + case dqm::qstatus::STATUS_OK: + break; + case dqm::qstatus::WARNING: + data_.flags |= DQMNet::DQM_PROP_REPORT_WARN; + break; + case dqm::qstatus::ERROR: + data_.flags |= DQMNet::DQM_PROP_REPORT_ERROR; + break; + default: + data_.flags |= DQMNet::DQM_PROP_REPORT_OTHER; + break; + } + } } MonitorElement::~MonitorElement() { @@ -424,53 +487,38 @@ namespace dqm::impl { return qr; } - // TODO: what was this logic ever supposed to do? - std::vector MonitorElement::getQReports() const { + template + std::vector MonitorElement::filterQReports(FILTER filter) const { auto access = this->access(); std::vector result; - result.reserve(access.value.qreports_.size()); - for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) { - // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&access.value.qreports_[i])); + for (MonitorElementData::QReport const &qr : access.value.qreports_) { + if (filter(qr)) { + // const_cast here because this API always violated cons'ness. Should + // make the result type const and fix all usages. + result.push_back(const_cast(&qr)); + } } return result; } + std::vector MonitorElement::getQReports() const { + return filterQReports([](MonitorElementData::QReport const &qr) { return true; }); + } + std::vector MonitorElement::getQWarnings() const { - auto access = this->access(); - std::vector result; - result.reserve(access.value.qreports_.size()); - for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) - if (data_.qreports[i].code == dqm::qstatus::WARNING) { - // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&access.value.qreports_[i])); - } - return result; + return filterQReports( + [](MonitorElementData::QReport const &qr) { return qr.getStatus() == dqm::qstatus::WARNING; }); } std::vector MonitorElement::getQErrors() const { - auto access = this->access(); - std::vector result; - result.reserve(access.value.qreports_.size()); - for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) - if (data_.qreports[i].code == dqm::qstatus::ERROR) { - // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&access.value.qreports_[i])); - } - return result; + return filterQReports([](MonitorElementData::QReport const &qr) { return qr.getStatus() == dqm::qstatus::ERROR; }); } std::vector MonitorElement::getQOthers() const { - auto access = this->access(); - std::vector result; - result.reserve(access.value.qreports_.size()); - for (size_t i = 0, e = access.value.qreports_.size(); i != e; ++i) - if (data_.qreports[i].code != dqm::qstatus::STATUS_OK && data_.qreports[i].code != dqm::qstatus::WARNING && - data_.qreports[i].code != dqm::qstatus::ERROR) { - // TODO, WTF?: access.value.qreports_[i].qvalue_ = const_cast(&data_.qreports[i]); - result.push_back(const_cast(&access.value.qreports_[i])); - } - return result; + return filterQReports([](MonitorElementData::QReport const &qr) { + return qr.getStatus() != dqm::qstatus::STATUS_OK && qr.getStatus() != dqm::qstatus::WARNING && + qr.getStatus() != dqm::qstatus::ERROR; + }); } void MonitorElement::incompatible(const char *func) const { @@ -972,38 +1020,18 @@ namespace dqm::impl { if (pos == end && !create) return; else if (pos == end) { - data_.qreports.emplace_back(); - - DQMNet::QValue &q = data_.qreports.back(); + DQMNet::QValue q; q.code = dqm::qstatus::DID_NOT_RUN; q.qtresult = 0; q.qtname = qtname; q.message = "NO_MESSAGE_ASSIGNED"; q.algorithm = "UNKNOWN_ALGORITHM"; - access.value.qreports_.push_back(MonitorElementData::QReport(&q)); + access.value.qreports_.push_back(MonitorElementData::QReport(q)); + syncCoreObject(); } qr = &access.value.qreports_[pos]; - qv = &data_.qreports[pos]; - } - - /// Refresh QReport stats, usually after MEs were read in from a file. - void MonitorElement::updateQReportStats() { - data_.flags &= ~DQMNet::DQM_PROP_REPORT_ALARM; - for (auto &qreport : data_.qreports) - switch (qreport.code) { - case dqm::qstatus::STATUS_OK: - break; - case dqm::qstatus::WARNING: - data_.flags |= DQMNet::DQM_PROP_REPORT_WARN; - break; - case dqm::qstatus::ERROR: - data_.flags |= DQMNet::DQM_PROP_REPORT_ERROR; - break; - default: - data_.flags |= DQMNet::DQM_PROP_REPORT_OTHER; - break; - } + qv = &(qr->getValue()); } // ------------------------------------------------------------------- diff --git a/DQMServices/Core/test/DQMTestMultiThread.cc b/DQMServices/Core/test/DQMTestMultiThread.cc index 57112839ba20e..c5c78ce1080ed 100644 --- a/DQMServices/Core/test/DQMTestMultiThread.cc +++ b/DQMServices/Core/test/DQMTestMultiThread.cc @@ -48,8 +48,7 @@ void DQMTestMultiThread::bookHistograms(DQMStore::IBooker &b, void DQMTestMultiThread::analyze(const edm::Event &iEvent, const edm::EventSetup &) { myHisto->Fill(fill_value_); } void DQMTestMultiThread::dumpMe(MonitorElement const &me, bool printStat /* = false */) { - std::cout << "Run: " << me.run() << " Lumi: " << me.lumi() << " LumiFlag: " << me.getLumiFlag() - << " moduleId: " << me.moduleId() << " fullpathname: " << me.getPathname(); + std::cout << " LumiFlag: " << me.getLumiFlag() << " fullpathname: " << me.getPathname(); if (printStat) std::cout << " Mean: " << me.getMean() << " RMS: " << me.getRMS() << " Entries: " << std::setprecision(9) << me.getEntries(); diff --git a/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc b/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc index b8616d5f32eb4..a095d636002f0 100644 --- a/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc +++ b/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc @@ -70,7 +70,7 @@ void DemoNormalDQMEDAnalyzer::bookHistograms(DQMStore::IBooker& ibook, edm::EventSetup const& iSetup) { ibook.setCurrentFolder(folder_); - example_ = ibook.book1D("EXAMPLE", "Example 1D", 20, 0., 10.); + example_ = ibook.book1D("EXAMPLE", "Example 1D", 20, 0., 10., [](TH1*) { std::cout << "booked!\n"; }); example2D_ = ibook.book2D("EXAMPLE_2D", "Example 2D", 20, 0, 20, 15, 0, 15); example3D_ = ibook.book3D("EXAMPLE_3D", "Example 3D", 20, 0, 20, 15, 0, 15, 25, 0, 25); exampleTProfile_ = ibook.bookProfile("EXAMPLE_TPROFILE", "Example TProfile", 20, 0, 20, 15, 0, 15); diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index 9ffa5cce46e67..eae7a443871ae 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -92,17 +92,21 @@ struct MonitorElementData { } }; + /// access underlying value + QValue& getValue() { return qvalue_; }; + QValue const& getValue() const { return qvalue_; }; + /// get test status - int getStatus() const { return qvalue_->code; } + int getStatus() const { return qvalue_.code; } /// get test result i.e. prob value - float getQTresult() const { return qvalue_->qtresult; } + float getQTresult() const { return qvalue_.qtresult; } /// get message attached to test - const std::string& getMessage() const { return qvalue_->message; } + const std::string& getMessage() const { return qvalue_.message; } /// get name of quality test - const std::string& getQRName() const { return qvalue_->qtname; } + const std::string& getQRName() const { return qvalue_.qtname; } /// get quality test algorithm const std::string& getAlgorithm() const { return qvalue_->algorithm; } @@ -113,10 +117,10 @@ struct MonitorElementData { void setBadChannels(std::vector badChannels) { badChannels_ = badChannels; } - QReport(QValue* value) : qvalue_(value) {} + QReport(QValue value) : qvalue_(value) {} private: - QValue* qvalue_; //< Pointer to the actual data. + QValue qvalue_; //< Pointer to the actual data. std::vector badChannels_; //< Bad channels from QCriterion. }; From 08c05be449208b24bcdc7d4e58b4a9547c3ccb4c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 21 Nov 2019 16:54:14 +0100 Subject: [PATCH 018/112] Implement some booking logic. Not sure if it works at all, for now. --- DQMServices/Core/interface/DQMStore.h | 12 ++++ DQMServices/Core/interface/MonitorElement.h | 2 + DQMServices/Core/src/DQMStore.cc | 74 ++++++++++++++++++--- DQMServices/Core/src/MonitorElement.cc | 21 +++--- 4 files changed, 87 insertions(+), 22 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index c506d5ec18894..f48d7ebd2e5bb 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -424,6 +424,16 @@ namespace dqm { class IGetter : public dqm::implementation::NavigatorBase { public: + // The IGetter interface is not really suitable for concurrent lumis/runs, + // so we don't even try much to get it right. Harvesting needs to run + // sequentially for now. Therefore, the methods just return the next-best + // global MEs that they find, ignoring Scope and run/lumi. + // Since these are global MEs, they may be deleted at some point; don't + // store the pointers! + // They are also shared with other modules. That is save when running + // multi-threaded as long as getTH1() etc. are not used, but of course + // all dependencies need to be properly declared to get correct results. + // TODO: review and possibly rename the all methods below: // get MEs that are direct children of full path `path` virtual std::vector getContents(std::string const& path) const; @@ -464,6 +474,8 @@ namespace dqm { class DQMStore : public IGetter, public IBooker { public: + // IGetter uses the globalMEs_ directly. + friend IGetter; // TODO: There are no references any more. we should gt rid of these. enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; enum OpenRunDirs { KeepRunDirs, StripRunDirs }; diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 5567ac3bbe6ab..44e14c9ab979b 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -232,6 +232,8 @@ namespace dqm::impl { MutableMonitorElementData *release(bool expectOwned); // re-initialize this ME as a shared copy of the other. void switchData(MonitorElement *other); + // re-initialize taking ownership of this data. + void switchData(MutableMonitorElementData *data); // copy applicable fileds into the DQMNet core object for compatibility. // In a few places these flags are also still used by the ME. void syncCoreObject(); diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 0a3e441ac6ffa..8a4b9f033e8f3 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -80,6 +80,7 @@ namespace dqm::implementation { me = store_->putME(local_me, this->moduleID_); // me now points to a local ME owned by the DQMStore. assert(me); + // TODO: maybe return global ME for legacy/harvesting bookings. return me; } @@ -239,11 +240,13 @@ namespace dqm::implementation { MonitorElement* oldme = *proto; prototypes.erase(proto); auto medata = oldme->release(/* expectOwned */ true); // destroy the ME, get its data. - delete oldme; // in this situation, nobody should be filling the ME concurrently. medata->data_.key_.id_ = edm::LuminosityBlockID(run, lumi); - auto newme = new MonitorElement(medata); - auto result = targetset.insert(newme); + // We reuse the ME object here, even if we don't have to. This ensures + // that when running single-threaded without concurrent lumis/runs, + // the global MEs will also live forever and allow legacy usages. + oldme->switchData(medata); + auto result = targetset.insert(oldme); assert(result.second); // was new insertion target = result.first; // iterator to new ME } else { @@ -301,21 +304,72 @@ namespace dqm::implementation { } } - std::vector IGetter::getContents(std::string const& path) const { assert(!"NIY"); } + std::vector IGetter::getContents(std::string const& pathname) const { + std::vector out; + MonitorElementData::Path path; + path.set(pathname, MonitorElementData::Path::Type::DIR); + for (auto& [runlumi, meset] : store_->globalMEs_) { + auto it = meset.lower_bound(path); + // rfind can be used as a prefix match. + while (it != meset.end() && (*it)->getPathname() == path.getDirname()) { + out.push_back(*it); + ++it; + } + } + return out; + } + void IGetter::getContents(std::vector& into, bool showContents) const { assert(!"NIY"); } - std::vector IGetter::getAllContents(std::string const& path) const { - assert(!"NIY"); + std::vector IGetter::getAllContents(std::string const& pathname) const { + std::vector out; + MonitorElementData::Path path; + path.set(pathname, MonitorElementData::Path::Type::DIR); + // make sure this is normalized by getting it from Path object. + auto path_str = path.getFullname(); + for (auto& [runlumi, meset] : store_->globalMEs_) { + auto it = meset.lower_bound(path); + // rfind can be used as a prefix match. + while (it != meset.end() && (*it)->getPathname().rfind(path_str, 0) == 0) { + out.push_back(*it); + ++it; + } + } + return out; } - std::vector IGetter::getAllContents(std::string const& path, + std::vector IGetter::getAllContents(std::string const& pathname, uint32_t runNumber, uint32_t lumi) const { - assert(!"NIY"); + std::vector out; + MonitorElementData::Path path; + path.set(pathname, MonitorElementData::Path::Type::DIR); + // make sure this is normalized by getting it from Path object. + auto path_str = path.getFullname(); + auto meset = store_->globalMEs_[edm::LuminosityBlockID(runNumber, lumi)]; + auto it = meset.lower_bound(path); + // rfind can be used as a prefix match. + while (it != meset.end() && (*it)->getFullname().rfind(path_str, 0) == 0) { + out.push_back(*it); + ++it; + } + return out; } - MonitorElement* IGetter::get(std::string const& fullpath) const { assert(!"NIY"); } + MonitorElement* IGetter::get(std::string const& fullpath) const { + MonitorElementData::Path path; + path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); + // this only really makes sense if there is only one instance of this ME, + // but the signature of this mthod also only makes sense in that case. + return store_->findME(path); + } - MonitorElement* IGetter::getElement(std::string const& path) const { assert(!"NIY"); } + MonitorElement* IGetter::getElement(std::string const& path) const { + auto result = this->get(path); + if (result == nullptr) { + throw cms::Exception("iGetter Error") << "ME " << path << " was requested but not found."; + } + return result; + } std::vector IGetter::getSubdirs() const { assert(!"NIY"); } std::vector IGetter::getMEs() const { assert(!"NIY"); } diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 470ecdca3b329..af6d7eb79a909 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -47,18 +47,8 @@ namespace dqm::impl { this->is_owned_ = true; syncCoreObject(); } - MonitorElement::MonitorElement(MutableMonitorElementData *data) { - this->mutable_ = data; - this->frozen_ = nullptr; - this->is_owned_ = true; - syncCoreObject(); - } - MonitorElement::MonitorElement(MonitorElement *me) { - this->mutable_ = me->mutable_.load(); - this->frozen_ = me->frozen_.load(); - this->is_owned_ = false; - syncCoreObject(); - } + MonitorElement::MonitorElement(MutableMonitorElementData *data) { switchData(data); } + MonitorElement::MonitorElement(MonitorElement *me) { switchData(me); } MonitorElementData MonitorElement::cloneMEData() { MonitorElementData out; @@ -89,6 +79,13 @@ namespace dqm::impl { syncCoreObject(); } + void MonitorElement::switchData(MutableMonitorElementData *data) { + this->mutable_ = data; + this->frozen_ = nullptr; + this->is_owned_ = true; + syncCoreObject(); + } + void MonitorElement::syncCoreObject() { auto access = this->access(); data_.flags &= ~DQMNet::DQM_PROP_TYPE_MASK; From 7f6fe761adbd1681b679cbff4e4833dac2eea4d4 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 21 Nov 2019 17:32:28 +0100 Subject: [PATCH 019/112] Remove the dirname* optimization from DQMNet. Since we can have MEs whithout data attached, we need to keep a copy of the name somewhere: Either by using a map or inside the ME. Since DQMNet::CoreObject already has the name, we also use it to store the path. Doing that with this optimization would be possible, but complicated, so I just removed it with the minimum amount of changes. --- DQMServices/Core/interface/DQMNet.h | 14 ++++----- DQMServices/Core/interface/MonitorElement.h | 35 +++++++-------------- DQMServices/Core/src/DQMNet.cc | 18 +++++------ DQMServices/Core/src/MonitorElement.cc | 2 +- 4 files changed, 29 insertions(+), 40 deletions(-) diff --git a/DQMServices/Core/interface/DQMNet.h b/DQMServices/Core/interface/DQMNet.h index 87a85865e4344..7bbcff9e8be2b 100644 --- a/DQMServices/Core/interface/DQMNet.h +++ b/DQMServices/Core/interface/DQMNet.h @@ -93,7 +93,7 @@ class DQMNet { uint32_t lumi; uint32_t streamId; uint32_t moduleId; - const std::string *dirname; + std::string dirname; std::string objname; QReports qreports; }; @@ -167,10 +167,10 @@ class DQMNet { if (a.lumi == b.lumi) { if (a.streamId == b.streamId) { if (a.moduleId == b.moduleId) { - if (*a.dirname == *b.dirname) { + if (a.dirname == b.dirname) { return a.objname < b.objname; } - return *a.dirname < *b.dirname; + return a.dirname < b.dirname; } return a.moduleId < b.moduleId; } @@ -187,7 +187,7 @@ class DQMNet { struct HashEqual { bool operator()(const Object &a, const Object &b) const { - return a.hash == b.hash && *a.dirname == *b.dirname && a.objname == b.objname; + return a.hash == b.hash && a.dirname == b.dirname && a.objname == b.objname; } }; @@ -401,7 +401,7 @@ class DQMImplNet : public DQMNet { std::string path(name, 0, dirpos); ObjType proto; proto.hash = dqmhash(name.c_str(), name.size()); - proto.dirname = &path; + proto.dirname = path; proto.objname.append(name, namepos, std::string::npos); typename ObjectMap::iterator pos; @@ -441,7 +441,7 @@ class DQMImplNet : public DQMNet { o.tag = 0; o.version = 0; o.lastreq = 0; - o.dirname = &*ip->dirs.insert(name.substr(0, dirpos)).first; + o.dirname = *ip->dirs.insert(name.substr(0, dirpos)).first; o.objname.append(name, namepos, std::string::npos); o.hash = dqmhash(name.c_str(), name.size()); return const_cast(&*ip->objs.insert(o).first); @@ -523,7 +523,7 @@ class DQMImplNet : public DQMNet { for (pi = peers_.begin(), pe = peers_.end(); pi != pe; ++pi) for (oi = pi->second.objs.begin(), oe = pi->second.objs.end(); oi != oe; ++oi, ++numobjs) if (all || (oi->flags & DQM_PROP_NEW)) - size += 9 * sizeof(uint32_t) + oi->dirname->size() + oi->objname.size() + 1 + oi->scalar.size() + + size += 9 * sizeof(uint32_t) + oi->dirname.size() + oi->objname.size() + 1 + oi->scalar.size() + oi->qdata.size() + (oi->lastreq > 0 ? oi->rawdata.size() : 0); msg->data.reserve(msg->data.size() + size + 8 * sizeof(uint32_t)); diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 44e14c9ab979b..187e4a7818768 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -101,34 +101,23 @@ namespace dqm::impl { struct MEComparison { using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. - // no locking here. We assume this is called from the DQMStore set - // operations, which need to be protected by a lock anyways. + auto make_tuple(MonitorElement* me) const { + return std::make_tuple(me->getPathname(), me->getName()); + } + auto make_tuple(MonitorElementData::Path const& path) const { + return std::make_tuple(path.getDirname(), path.getObjectname()); + } bool operator()(MonitorElement *left, MonitorElement *right) const { - MonitorElementData const *l_frozen = left->frozen_.load(); - MutableMonitorElementData const *l_mutable = left->mutable_.load(); - MonitorElementData const *r_frozen = right->frozen_.load(); - MutableMonitorElementData const *r_mutable = right->mutable_.load(); - - MonitorElementData::Path const &l = l_mutable ? l_mutable->data_.key_.path_ : l_frozen->key_.path_; - MonitorElementData::Path const &r = r_mutable ? r_mutable->data_.key_.path_ : r_frozen->key_.path_; - - return (*this)(l, r); // call implementation below + return make_tuple(left) < make_tuple(right); } bool operator()(MonitorElement *left, MonitorElementData::Path const &right) const { - MonitorElementData const *l_frozen = left->frozen_.load(); - MutableMonitorElementData const *l_mutable = left->mutable_.load(); - MonitorElementData::Path const &l = l_mutable ? l_mutable->data_.key_.path_ : l_frozen->key_.path_; - return (*this)(l, right); // call implementation below + return make_tuple(left) < make_tuple(right); } bool operator()(MonitorElementData::Path const &left, MonitorElement *right) const { - MonitorElementData const *r_frozen = right->frozen_.load(); - MutableMonitorElementData const *r_mutable = right->mutable_.load(); - MonitorElementData::Path const &r = r_mutable ? r_mutable->data_.key_.path_ : r_frozen->key_.path_; - return (*this)(left, r); // call implementation below + return make_tuple(left) < make_tuple(right); } bool operator()(MonitorElementData::Path const &left, MonitorElementData::Path const &right) const { - return std::make_tuple(left.getDirname(), left.getObjectname()) < - std::make_tuple(right.getDirname(), right.getObjectname()); + return make_tuple(left) < make_tuple(right); } }; @@ -252,10 +241,10 @@ namespace dqm::impl { uint32_t flags() const { return data_.flags; } /// get name of ME - const std::string &getName() const { return access().key.path_.getObjectname(); } + const std::string &getName() const { return this->data_.objname; } /// get pathname of parent folder - const std::string &getPathname() const { return access().key.path_.getDirname(); } + const std::string &getPathname() const { return this->data_.dirname; } /// get full name of ME including Pathname const std::string getFullname() const { return access().key.path_.getFullname(); } diff --git a/DQMServices/Core/src/DQMNet.cc b/DQMServices/Core/src/DQMNet.cc index 7fab0fac44a5e..ab9e3ce8eb262 100644 --- a/DQMServices/Core/src/DQMNet.cc +++ b/DQMServices/Core/src/DQMNet.cc @@ -286,7 +286,7 @@ DQMNet::reinstateObject(DQMStore *store, Object &o) // Reconstruct the main object MonitorElement *obj = 0; - store->setCurrentFolder(*o.dirname); + store->setCurrentFolder(o.dirname); switch (o.flags & DQM_PROP_TYPE_MASK) { case DQM_PROP_TYPE_INT: @@ -351,7 +351,7 @@ DQMNet::reinstateObject(DQMStore *store, Object &o) logme() << "ERROR: unexpected monitor element of type " << (o.flags & DQM_PROP_TYPE_MASK) << " called '" - << *o.dirname << '/' << o.objname << "'\n"; + << o.dirname << '/' << o.objname << "'\n"; return false; } @@ -401,11 +401,11 @@ void DQMNet::sendObjectToPeer(Bucket *msg, Object &o, bool data) { objdata.insert(objdata.end(), &o.rawdata[0], &o.rawdata[0] + o.rawdata.size()); uint32_t words[9]; - uint32_t namelen = o.dirname->size() + o.objname.size() + 1; + uint32_t namelen = o.dirname.size() + o.objname.size() + 1; uint32_t datalen = objdata.size(); uint32_t qlen = o.qdata.size(); - if (o.dirname->empty()) + if (o.dirname.empty()) --namelen; words[0] = 9 * sizeof(uint32_t) + namelen + datalen + qlen; @@ -421,8 +421,8 @@ void DQMNet::sendObjectToPeer(Bucket *msg, Object &o, bool data) { msg->data.reserve(msg->data.size() + words[0]); copydata(msg, &words[0], 9 * sizeof(uint32_t)); if (namelen) { - copydata(msg, &(*o.dirname)[0], o.dirname->size()); - if (!o.dirname->empty()) + copydata(msg, &(o.dirname)[0], o.dirname.size()); + if (!o.dirname.empty()) copydata(msg, "/", 1); copydata(msg, &o.objname[0], o.objname.size()); } @@ -1219,7 +1219,7 @@ void DQMBasicNet::reserveLocalSpace(uint32_t size) { local_->objs.resize(size); /// Update the network cache for an object. The caller must call /// sendLocalChanges() later to push out the changes. void DQMBasicNet::updateLocalObject(Object &o) { - o.dirname = &*local_->dirs.insert(*o.dirname).first; + o.dirname = *local_->dirs.insert(o.dirname).first; std::pair info(local_->objs.insert(o)); if (!info.second) { // Somewhat hackish. Sets are supposedly immutable, but we @@ -1245,8 +1245,8 @@ bool DQMBasicNet::removeLocalExcept(const std::set &known) { ObjectMap::iterator i, e; for (i = local_->objs.begin(), e = local_->objs.end(); i != e;) { path.clear(); - path.reserve(i->dirname->size() + i->objname.size() + 2); - path += *i->dirname; + path.reserve(i->dirname.size() + i->objname.size() + 2); + path += i->dirname; if (!path.empty()) path += '/'; path += i->objname; diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index af6d7eb79a909..b326fcff72967 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -125,7 +125,7 @@ namespace dqm::impl { data_.moduleId = 0; // leaking a pointer here, but that should be fine. - data_.dirname = &access.key.path_.getDirname(); + data_.dirname = access.key.path_.getDirname(); data_.objname = access.key.path_.getObjectname(); From 89f3874aef170f5a94e68afc9bb35086afea8cc3 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 21 Nov 2019 18:44:53 +0100 Subject: [PATCH 020/112] Fix some more logic bugs. Incluing adding an API to MonitorElement to check if there actually if there actually is data inside it. --- DQMServices/Core/interface/MonitorElement.h | 10 ++++++---- DQMServices/Core/src/DQMStore.cc | 21 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 187e4a7818768..cca7500893b14 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -101,10 +101,8 @@ namespace dqm::impl { struct MEComparison { using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. - auto make_tuple(MonitorElement* me) const { - return std::make_tuple(me->getPathname(), me->getName()); - } - auto make_tuple(MonitorElementData::Path const& path) const { + auto make_tuple(MonitorElement *me) const { return std::make_tuple(me->getPathname(), me->getName()); } + auto make_tuple(MonitorElementData::Path const &path) const { return std::make_tuple(path.getDirname(), path.getObjectname()); } bool operator()(MonitorElement *left, MonitorElement *right) const { @@ -228,6 +226,10 @@ namespace dqm::impl { void syncCoreObject(); virtual ~MonitorElement(); + // check if the ME is currently backed by MEData; if false (almost) any + // access will throw. + bool isValid() const { return mutable_.load() || frozen_.load(); } + /// Compare monitor elements, for ordering in sets. bool operator<(const MonitorElement &x) const { return DQMNet::setOrder(data_, x.data_); } diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 8a4b9f033e8f3..1912ce13a442c 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -52,8 +52,14 @@ namespace dqm::implementation { MonitorElementData::Path path; std::string fullpath = pwd() + std::string(name.View()); path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); + + // We should check if there is a local ME for this module and name already. + // However, there is nothing wrong with creating a new local ME and then + // having putME() drop it, so we do that (it's easier). + MonitorElement* me = store_->findME(path); store_->printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)")); + if (me == nullptr) { // no existing global ME found. We need to instantiate one, and put it // into the DQMStore. This will typically be a prototype, unless run and @@ -72,6 +78,15 @@ namespace dqm::implementation { // me now points to a global ME owned by the DQMStore. assert(me); + if (this->moduleID_ == 0) { + // this is a legacy/global/harvesting booking. In this case, we do not + // create a local ME and return the global directly. It is not advisable + // to hold this pointer, as we may delete the global ME later, but we + // promise to keep it valid for the entire job if there are no concurrent + // runs/lumis. + return me; + } + // each booking call returns a unique "local" ME, which the DQMStore keeps // in a container associated with the module (and potentially run, for // DQMGlobalEDAnalyzer). This will later be update to point to different @@ -80,7 +95,6 @@ namespace dqm::implementation { me = store_->putME(local_me, this->moduleID_); // me now points to a local ME owned by the DQMStore. assert(me); - // TODO: maybe return global ME for legacy/harvesting bookings. return me; } @@ -108,8 +122,8 @@ namespace dqm::implementation { return me; } else { // already present, return old object + edm::LogInfo("DQMStore") << "ME " << me->getFullname() << " booked twice in the same module."; delete me; - assert(!"Currently, this should never happen."); return *(existing_new.first); } } @@ -297,7 +311,8 @@ namespace dqm::implementation { }; for (MonitorElement* me : localset) { - if (checkScope(me->getScope()) == true) { + // we have to be very careful with the ME here, it might not be backed by data at all. + if (me->isValid() && checkScope(me->getScope()) == true) { // if we left the scope, simply release the data. me->release(/* expectOwned */ false); } From 661699e2c790b4224c4722984a78038c1e131cf3 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 22 Nov 2019 15:22:39 +0100 Subject: [PATCH 021/112] Implement transition watching and cleanup. This completes the ME life cycle. --- DQMServices/Core/interface/DQMStore.h | 4 ++ DQMServices/Core/src/DQMStore.cc | 96 ++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index f48d7ebd2e5bb..8ad48cd38f3e8 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -578,6 +578,10 @@ namespace dqm { // if there are concurrent runs/lumis. void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); + // this is triggered by a framework hook to remove/recycle MEs after a + // run/lumi is saved. We do this via the edm::Service interface to make + // sure it runs after *all* output modules, even if there are multiple. + void cleanupLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi); // Add ME to DQMStore datastructures. The object will be deleted if a // similar object is already present. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 1912ce13a442c..8437a1a791216 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -2,6 +2,7 @@ #define DQM_DEPRECATED #include "DQMServices/Core/interface/DQMStore.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ServiceRegistry/interface/GlobalContext.h" #include #include #include @@ -69,8 +70,19 @@ namespace dqm::implementation { medata.key_.path_ = path; medata.key_.kind_ = kind; medata.key_.scope_ = this->scope_; - // will be 0 ( = prototype) in the common case. - medata.key_.id_ = this->runlumi_; + + // will be (0,0) ( = prototype) in the common case. + // This branching is for harvesting, where we have run/lumi in the booker. + if (this->scope_ == MonitorElementData::Scope::JOB) { + medata.key_.id_ = edm::LuminosityBlockID(); + } else if (this->scope_ == MonitorElementData::Scope::RUN) { + medata.key_.id_ = edm::LuminosityBlockID(this->runlumi_.run(), 0); + } else if (this->scope_ == MonitorElementData::Scope::LUMI) { + medata.key_.id_ = this->runlumi_; + } else { + assert(!"Illegal scope"); + } + medata.value_.object_ = std::unique_ptr(th1); MonitorElement* me_ptr = new MonitorElement(std::move(medata)); me = store_->putME(me_ptr); @@ -319,6 +331,70 @@ namespace dqm::implementation { } } + void DQMStore::cleanupLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) { + // now, we are done with the lumi, no modules have any work to do on these + // MEs, and the output modules have saved this lumi/run. Remove/recycle + // the MEs here. + + auto lock = std::scoped_lock(this->booking_mutex_); + + // in case of end-job cleanup we need different logic because of the + // prototype set. + assert(run != 0 || lumi != 0); + auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()]; + + // these are the MEs we need to get rid of... + auto meset = std::set(); + // ... we take them out first. + meset.swap(this->globalMEs_[edm::LuminosityBlockID(run, lumi)]); + + // temporary buffer for the MEs to recycle, we must not change the key + // while they are in a set. + auto torecycle = std::vector(); + + // here, this is only a sanity check and not functionally needed. + auto checkScope = [run, lumi](MonitorElementData::Scope scope) { + if (scope == MonitorElementData::Scope::JOB) { + assert(run == 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::RUN) { + assert(run != 0 && lumi == 0); + } else if (scope == MonitorElementData::Scope::LUMI) { + assert(lumi != 0); + } else { + assert(!"Impossible Scope."); + } + }; + + for (MonitorElement* me : meset) { + assert(me->isValid()); // global MEs should always be valid. + checkScope(me->getScope()); // we should only see MEs of one scope here. + auto other = this->findME(me); + if (other) { + // we still have a global one, so we can just remove this. + delete me; + } else { + // we will modify the ME, so it needs to be out of the set. + // use a temporary vector to be save. + torecycle.push_back(me); + } + } + + meset.clear(); + + for (MonitorElement* me : torecycle) { + auto medata = me->release(/* expectOwned */ true); // destroy the ME, get its data. + medata->data_.key_.id_ = edm::LuminosityBlockID(); // prototype + // We reuse the ME object here, even if we don't have to. This ensures + // that when running single-threaded without concurrent lumis/runs, + // the global MEs will also live forever and allow legacy usages. + me->switchData(medata); + // reset here (not later) to still catch random legacy fill calls. + me->Reset(); + auto result = prototypes.insert(me); + assert(result.second); // was new insertion, else findME should succeed + } + } + std::vector IGetter::getContents(std::string const& pathname) const { std::vector out; MonitorElementData::Path path; @@ -394,8 +470,22 @@ namespace dqm::implementation { IGetter::~IGetter() {} - DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&) : IGetter(this), IBooker(this) { + DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry& ar) : IGetter(this), IBooker(this) { verbose_ = pset.getUntrackedParameter("verbose", 0); + + // Set lumi and run for legacy booking. + // This is no more than a guess with concurrent runs/lumis, but should be + // correct for purely sequential legacy stuff. + // TODO: detect concurrent runs/lumis and disable legacy API in case? + ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); + ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); + + // Delete/recycle MEs after output modules have run. + ar.watchPostGlobalEndRun( + [this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), 0); }); + ar.watchPostGlobalEndLumi([this](edm::GlobalContext const& gc) { + this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().value()); + }); } DQMStore::~DQMStore() {} From 48dce796284f440feec32e14591f3d7678b27cf2 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 15:08:32 +0100 Subject: [PATCH 022/112] Import DQMRootSource from @andrius-k. Using commit 70fc23c56872f2ee43330ac2ab6cb6b72ddb1682 on his "new DQMstore" branch. This might revert changes from #28064, needs to be fixed later. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 1286 ++++++++------------ 1 file changed, 478 insertions(+), 808 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 792f64c459f2f..868b1952a2edc 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -14,25 +14,16 @@ #include #include #include -#include -#include -#include #include "TFile.h" #include "TTree.h" #include "TString.h" -#include "TH1.h" -#include "TH2.h" -#include "TProfile.h" // user include files #include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/Sources/interface/PuttableSourceBase.h" #include "FWCore/Catalog/interface/InputFileCatalog.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" #include "DQMServices/Core/interface/DQMStore.h" -#include "FWCore/ServiceRegistry/interface/Service.h" -#include "FWCore/MessageLogger/interface/JobReport.h" -//#include "FWCore/Utilities/interface/GlobalIdentifier.h" -#include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/ExceptionPropagate.h" #include "FWCore/Framework/interface/RunPrincipal.h" @@ -48,198 +39,171 @@ #include "FWCore/Framework/interface/InputSourceMacros.h" #include "FWCore/Framework/interface/FileBlock.h" -#include "DataFormats/Provenance/interface/EventRange.h" -#include "DataFormats/Provenance/interface/EventID.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/Utilities/interface/TimeOfDay.h" -#include "DataFormats/Provenance/interface/ProcessHistory.h" -#include "DataFormats/Provenance/interface/ProcessHistoryID.h" -#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" -#include "FWCore/ParameterSet/interface/Registry.h" - -#include "FWCore/Utilities/interface/Digest.h" -#include "FWCore/Utilities/interface/InputType.h" - #include "format.h" namespace { - typedef dqm::legacy::MonitorElement MonitorElement; - typedef dqm::legacy::DQMStore DQMStore; - - //adapter functions - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1F* iHist) { - //std::cout <<"create: hist size "<GetEffectiveEntries()<CanExtendAllAxes() && iToAdd->CanExtendAllAxes()) { - TList list; - list.Add(iToAdd); - if (-1 == iOriginal->Merge(&list)) { - edm::LogError("MergeFailure") << "Failed to merge DQM element " << iOriginal->GetName(); + typedef dqm::harvesting::MonitorElement MonitorElement; + + // TODO: this should probably be moved somewhere else + class DQMMergeHelper { + public: + // Utility function to check the consistency of the axis labels + // Taken from TH1::CheckBinLabels which is not public + static bool CheckBinLabels(const TAxis* a1, const TAxis * a2) { + // Check that axis have same labels + THashList *l1 = (const_cast(a1))->GetLabels(); + THashList *l2 = (const_cast(a2))->GetLabels(); + + if (!l1 && !l2 ) + return true; + if (!l1 || !l2 ) { + return false; } - } else { - if (iOriginal->GetNbinsX() == iToAdd->GetNbinsX() && - iOriginal->GetXaxis()->GetXmin() == iToAdd->GetXaxis()->GetXmin() && - iOriginal->GetXaxis()->GetXmax() == iToAdd->GetXaxis()->GetXmax() && - iOriginal->GetNbinsY() == iToAdd->GetNbinsY() && - iOriginal->GetYaxis()->GetXmin() == iToAdd->GetYaxis()->GetXmin() && - iOriginal->GetYaxis()->GetXmax() == iToAdd->GetYaxis()->GetXmax() && - iOriginal->GetNbinsZ() == iToAdd->GetNbinsZ() && - iOriginal->GetZaxis()->GetXmin() == iToAdd->GetZaxis()->GetXmin() && - iOriginal->GetZaxis()->GetXmax() == iToAdd->GetZaxis()->GetXmax() && - MonitorElement::CheckBinLabels(iOriginal->GetXaxis(), iToAdd->GetXaxis()) && - MonitorElement::CheckBinLabels(iOriginal->GetYaxis(), iToAdd->GetYaxis()) && - MonitorElement::CheckBinLabels(iOriginal->GetZaxis(), iToAdd->GetZaxis())) { - iOriginal->Add(iToAdd); - } else { - edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels'" - << iOriginal->GetName() << "' not merged."; + + // Check now labels sizes are the same + if (l1->GetSize() != l2->GetSize() ) { + return false; } - } - } - void mergeWithElement(MonitorElement* iElement, TH1F* iHist) { - //std::cout <<"merge: hist size "<getName() <<" "<GetEffectiveEntries()<getTH1F(), iHist); - } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1S* iHist) { - return iStore.book1S(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH1S* iHist) { mergeTogether(iElement->getTH1S(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH1D* iHist) { - return iStore.book1DD(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH1D* iHist) { mergeTogether(iElement->getTH1D(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2F* iHist) { - return iStore.book2D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2F* iHist) { mergeTogether(iElement->getTH2F(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2S* iHist) { - return iStore.book2S(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2S* iHist) { mergeTogether(iElement->getTH2S(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH2D* iHist) { - return iStore.book2DD(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH2D* iHist) { mergeTogether(iElement->getTH2D(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TH3F* iHist) { - return iStore.book3D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TH3F* iHist) { mergeTogether(iElement->getTH3F(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile* iHist) { - return iStore.bookProfile(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TProfile* iHist) { mergeTogether(iElement->getTProfile(), iHist); } - MonitorElement* createElement(DQMStore& iStore, const char* iName, TProfile2D* iHist) { - return iStore.bookProfile2D(iName, iHist); - } - void mergeWithElement(MonitorElement* iElement, TProfile2D* iHist) { - mergeTogether(iElement->getTProfile2D(), iHist); - } + for (int i = 1; i <= a1->GetNbins(); ++i) { + TString label1 = a1->GetBinLabel(i); + TString label2 = a2->GetBinLabel(i); + if (label1 != label2) { + return false; + } + } - MonitorElement* createElement(DQMStore& iStore, const char* iName, Long64_t& iValue) { - MonitorElement* e = iStore.bookInt(iName); - e->Fill(iValue); - return e; - } + return true; + } - //NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h - void mergeWithElement(MonitorElement* iElement, Long64_t& iValue) { - const std::string& name = iElement->getFullname(); - if (name.find("EventInfo/processedEvents") != std::string::npos) { - iElement->Fill(iValue + iElement->getIntValue()); - } else if (name.find("EventInfo/iEvent") != std::string::npos || - name.find("EventInfo/iLumiSection") != std::string::npos) { - if (iValue > iElement->getIntValue()) { - iElement->Fill(iValue); + // NOTE: the merge logic comes from DataFormats/Histograms/interface/MEtoEDMFormat.h + static void mergeTogether(TH1* original, TH1* toAdd) { + if (original->CanExtendAllAxes() && toAdd->CanExtendAllAxes()) { + TList list; + list.Add(toAdd); + if (original->Merge(&list) == -1) { + edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName(); + } + } + else { + // TODO: Redo. What's wrong with this implementation? + if (original->GetNbinsX() == toAdd->GetNbinsX() && + original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() && + original->GetXaxis()->GetXmax() == toAdd->GetXaxis()->GetXmax() && + original->GetNbinsY() == toAdd->GetNbinsY() && + original->GetYaxis()->GetXmin() == toAdd->GetYaxis()->GetXmin() && + original->GetYaxis()->GetXmax() == toAdd->GetYaxis()->GetXmax() && + original->GetNbinsZ() == toAdd->GetNbinsZ() && + original->GetZaxis()->GetXmin() == toAdd->GetZaxis()->GetXmin() && + original->GetZaxis()->GetXmax() == toAdd->GetZaxis()->GetXmax() && + CheckBinLabels(original->GetXaxis(), toAdd->GetXaxis()) && + CheckBinLabels(original->GetYaxis(), toAdd->GetYaxis()) && + CheckBinLabels(original->GetZaxis(), toAdd->GetZaxis())) { + original->Add(toAdd); + } + else { + edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels '" + << original->GetName() << "' not merged."; + } } - } else { - iElement->Fill(iValue); } - } - - MonitorElement* createElement(DQMStore& iStore, const char* iName, double& iValue) { - MonitorElement* e = iStore.bookFloat(iName); - e->Fill(iValue); - return e; - } - void mergeWithElement(MonitorElement* iElement, double& iValue) { - //no merging, take the last one - iElement->Fill(iValue); - } - MonitorElement* createElement(DQMStore& iStore, const char* iName, std::string* iValue) { - return iStore.bookString(iName, *iValue); - } - void mergeWithElement(MonitorElement* iElement, std::string* iValue) { - //no merging, take the last one - iElement->Fill(*iValue); - } + }; - void splitName(const std::string& iFullName, std::string& oPath, const char*& oName) { - oPath = iFullName; - size_t index = oPath.find_last_of('/'); - if (index == std::string::npos) { - oPath = std::string(); - oName = iFullName.c_str(); - } else { - oPath.resize(index); - oName = iFullName.c_str() + index + 1; - } - } + using MonitorElementsFromFile = std::map, std::vector>; - struct RunLumiToRange { - unsigned int m_run, m_lumi, m_historyIDIndex; + // This struct allows to find all MEs belonging to a run-lumi pair + // All files will be open at once so m_file property indicates the file where data is saved. + struct FileMetadata { + unsigned int m_run; + unsigned int m_lumi; ULong64_t m_beginTime; ULong64_t m_endTime; - ULong64_t m_firstIndex, m_lastIndex; //last is inclusive - unsigned int m_type; //A value in TypeIndex + ULong64_t m_firstIndex; + ULong64_t m_lastIndex; // Last is inclusive + unsigned int m_type; + TFile* m_file; + + // This will be used when sorting a vector + bool operator < (const FileMetadata& obj) const { + if(m_run == obj.m_run) + return m_lumi < obj.m_lumi; + else + return m_run < obj.m_run; + } + + void describe() { + std::cout << "read r:" << m_run + << " l:" << m_lumi + << " bt:" << m_beginTime + << " et:" << m_endTime + << " fi:" << m_firstIndex + << " li:" << m_lastIndex + << " type:" << m_type + << " file: " << m_file << std::endl; + } }; class TreeReaderBase { public: - TreeReaderBase() {} + TreeReaderBase(MonitorElementData::Kind kind) : m_kind(kind) {} virtual ~TreeReaderBase() {} - MonitorElement* read(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) { return doRead(iIndex, iStore, iIsLumi); } + virtual void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) = 0; virtual void setTree(TTree* iTree) = 0; protected: + MonitorElementData::Kind m_kind; TTree* m_tree; - - private: - virtual MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) = 0; }; template class TreeObjectReader : public TreeReaderBase { public: - TreeObjectReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) {} - MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override { + TreeObjectReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) { + assert(m_kind != MonitorElementData::Kind::INT); + assert(m_kind != MonitorElementData::Kind::REAL); + assert(m_kind != MonitorElementData::Kind::STRING); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElement* element = iStore.get(*m_fullName); - try { - if (nullptr == element) { - std::string path; - const char* name; - splitName(*m_fullName, path, name); - iStore.setCurrentFolder(path); - element = createElement(iStore, name, m_buffer); - if (iIsLumi) { - element->setLumiFlag(); - } - } else { - mergeWithElement(element, m_buffer); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + // TODO: What should the range be for per run MEs? Now lumi will be 0 and not the max lumi for that run. + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Merge with already existing ME! + MonitorElementData::Value::Access value(meData->value_); + DQMMergeHelper::mergeTogether(value.object.get(), m_buffer); + merged = true; + break; } - } catch (cms::Exception& e) { - e.addContext(std::string("While reading element ") + *m_fullName); - e.raise(); } - return element; + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + value.object = std::unique_ptr((T*)(m_buffer->Clone())); + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(meData); + } } + void setTree(TTree* iTree) override { m_tree = iTree; m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); @@ -254,27 +218,107 @@ namespace { uint32_t m_tag; }; + class TreeStringReader : public TreeReaderBase { + public: + TreeStringReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) { + assert(m_kind == MonitorElementData::Kind::STRING); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method + m_tree->GetEntry(iIndex); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Keep the latest one + MonitorElementData::Value::Access value(meData->value_); + value.scalar.str = *m_value; + merged = true; + break; + } + } + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + value.scalar.str = *m_value; + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + } + } + + void setTree(TTree* iTree) override { + m_tree = iTree; + m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); + m_tree->SetBranchAddress(kFlagBranch, &m_tag); + m_tree->SetBranchAddress(kValueBranch, &m_value); + } + + private: + TTree* m_tree; + std::string* m_fullName; + std::string* m_value; + uint32_t m_tag; + }; + template class TreeSimpleReader : public TreeReaderBase { public: - TreeSimpleReader() : m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) {} - MonitorElement* doRead(ULong64_t iIndex, DQMStore& iStore, bool iIsLumi) override { + TreeSimpleReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) { + assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL); + } + + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElement* element = iStore.get(*m_fullName); - if (nullptr == element) { - std::string path; - const char* name; - splitName(*m_fullName, path, name); - iStore.setCurrentFolder(path); - element = createElement(iStore, name, m_buffer); - if (iIsLumi) { - element->setLumiFlag(); + + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); + + std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; + bool merged = false; + for (MonitorElementData* meData : runLumiMEs) { + if(meData->key_ == key) { + // Keep the latest one + MonitorElementData::Value::Access value(meData->value_); + if(m_kind == MonitorElementData::Kind::INT) + value.scalar.num = m_buffer; + else if(m_kind == MonitorElementData::Kind::REAL) + value.scalar.real = m_buffer; + merged = true; + break; } - } else { - mergeWithElement(element, m_buffer); } - return element; + + if(!merged) { + MonitorElementData* meData = new MonitorElementData(); + meData->key_ = key; + { + MonitorElementData::Value::Access value(meData->value_); + if(m_kind == MonitorElementData::Kind::INT) + value.scalar.num = m_buffer; + else if(m_kind == MonitorElementData::Kind::REAL) + value.scalar.real = m_buffer; + } + + mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + } } + void setTree(TTree* iTree) override { m_tree = iTree; m_tree->SetBranchAddress(kFullNameBranch, &m_fullName); @@ -289,9 +333,9 @@ namespace { uint32_t m_tag; }; -} // namespace +} // namespace -class DQMRootSource : public edm::InputSource { +class DQMRootSource : public edm::PuttableSourceBase { public: DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&); ~DQMRootSource() override; @@ -304,102 +348,60 @@ class DQMRootSource : public edm::InputSource { static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: - DQMRootSource(const DQMRootSource&) = delete; // stop default - - class RunPHIDKey { - public: - RunPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run) : processHistoryID_(phid), run_(run) {} - edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; } - unsigned int run() const { return run_; } - bool operator<(RunPHIDKey const& right) const { - if (processHistoryID_ == right.processHistoryID()) { - return run_ < right.run(); - } - return processHistoryID_ < right.processHistoryID(); - } - - private: - edm::ProcessHistoryID processHistoryID_; - unsigned int run_; - }; - - class RunLumiPHIDKey { - public: - RunLumiPHIDKey(edm::ProcessHistoryID const& phid, unsigned int run, unsigned int lumi) - : processHistoryID_(phid), run_(run), lumi_(lumi) {} - edm::ProcessHistoryID const& processHistoryID() const { return processHistoryID_; } - unsigned int run() const { return run_; } - unsigned int lumi() const { return lumi_; } - bool operator<(RunLumiPHIDKey const& right) const { - if (processHistoryID_ == right.processHistoryID()) { - if (run_ == right.run()) { - return lumi_ < right.lumi(); - } - return run_ < right.run(); - } - return processHistoryID_ < right.processHistoryID(); - } - - private: - edm::ProcessHistoryID processHistoryID_; - unsigned int run_; - unsigned int lumi_; - }; + DQMRootSource(const DQMRootSource&) = delete; edm::InputSource::ItemType getNextItemType() override; - //NOTE: the following is really read next run auxiliary + + std::unique_ptr readFile_() override; std::shared_ptr readRunAuxiliary_() override; std::shared_ptr readLuminosityBlockAuxiliary_() override; void readRun_(edm::RunPrincipal& rpCache) override; void readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) override; void readEvent_(edm::EventPrincipal&) override; - std::unique_ptr readFile_() override; - void closeFile_() override; + // Read MEs from m_fileMetadatas to m_MEsFromFile till run or lumi transition + void readElements(); + // True if m_currentIndex points to an element that has a different + // run or lumi than the previous element (a transition needs to happen). + // False otherwise. + bool isRunOrLumiTransition() const; + void readNextItemType(); + // These methods will be called by the framework. + // MEs in m_MEsFromFile will be put to products. + void beginRun(edm::Run& run); + void beginLuminosityBlock(edm::LuminosityBlock& lumi); + + // If the run matches the filterOnRun configuration parameter, the run + // (and all its lumis) will be kept. + // Otherwise, check if a run and a lumi are in the range that needs to be processed. + // Range is retrieved from lumisToProcess configuration parameter. + // If at least one lumi of a run needs to be kept, per run MEs of that run will also be kept. + bool keepIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const; void logFileAction(char const* msg, char const* fileName) const; - void readNextItemType(); - bool setupFile(unsigned int iIndex); - void readElements(); - bool skipIt(edm::RunNumber_t, edm::LuminosityBlockNumber_t) const; - const DQMRootSource& operator=(const DQMRootSource&) = delete; // stop default // ---------- member data -------------------------------- - edm::InputFileCatalog m_catalog; - edm::RunAuxiliary m_runAux; - edm::LuminosityBlockAuxiliary m_lumiAux; - edm::InputSource::ItemType m_nextItemType; - size_t m_fileIndex; - std::string m_presentlyOpenFileName; - std::list::iterator m_nextIndexItr; - std::list::iterator m_presentIndexItr; - std::vector m_runlumiToRange; - std::unique_ptr m_file; - std::vector m_trees; - std::vector > m_treeReaders; - - std::list m_orderedIndices; - edm::ProcessHistoryID m_lastSeenReducedPHID; - unsigned int m_lastSeenRun; - edm::ProcessHistoryID m_lastSeenReducedPHID2; - unsigned int m_lastSeenRun2; - unsigned int m_lastSeenLumi2; - unsigned int m_filterOnRun; + // Properties from python config bool m_skipBadFiles; + unsigned int m_filterOnRun; + edm::InputFileCatalog m_catalog; std::vector m_lumisToProcess; - std::vector m_runsToProcess; - - bool m_justOpenedFileSoNeedToGenerateRunTransition; - bool m_shouldReadMEs; - std::set m_lumiElements; - std::set m_runElements; - std::vector m_historyIDs; - std::vector m_reducedHistoryIDs; - edm::JobReport::Token m_jrToken; + edm::InputSource::ItemType m_nextItemType; + // Each ME type gets its own reader + std::vector> m_treeReaders; + + // Index of currenlty processed row in m_fileMetadatas + unsigned int m_currentIndex; + // All open DQMIO files + std::vector m_openFiles; + // MEs read from files and merged if needed. Ready to be put into products + MonitorElementsFromFile m_MEsFromFile; + // An item here is a row read from DQMIO indices (metadata) table + std::vector m_fileMetadatas; }; // @@ -423,633 +425,301 @@ void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& description descriptions.addDefault(desc); } + // // constructors and destructor // + DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSourceDescription& iDesc) - : edm::InputSource(iPSet, iDesc), + : edm::PuttableSourceBase(iPSet, iDesc), + m_skipBadFiles(iPSet.getUntrackedParameter("skipBadFiles", false)), + m_filterOnRun(iPSet.getUntrackedParameter("filterOnRun", 0)), m_catalog(iPSet.getUntrackedParameter >("fileNames"), iPSet.getUntrackedParameter("overrideCatalog")), - m_nextItemType(edm::InputSource::IsFile), - m_fileIndex(0), - m_trees(kNIndicies, static_cast(nullptr)), - m_treeReaders(kNIndicies, std::shared_ptr()), - m_lastSeenReducedPHID(), - m_lastSeenRun(0), - m_lastSeenReducedPHID2(), - m_lastSeenRun2(0), - m_lastSeenLumi2(0), - m_filterOnRun(iPSet.getUntrackedParameter("filterOnRun", 0)), - m_skipBadFiles(iPSet.getUntrackedParameter("skipBadFiles", false)), m_lumisToProcess(iPSet.getUntrackedParameter >( "lumisToProcess", std::vector())), - m_justOpenedFileSoNeedToGenerateRunTransition(false), - m_shouldReadMEs(true) { + m_nextItemType(edm::InputSource::IsFile), + m_treeReaders(kNIndicies, std::shared_ptr()), + m_currentIndex(0), + m_openFiles (std::vector()), + m_MEsFromFile(MonitorElementsFromFile()), + m_fileMetadatas(std::vector()) { edm::sortAndRemoveOverlaps(m_lumisToProcess); - for (std::vector::const_iterator itr = m_lumisToProcess.begin(); - itr != m_lumisToProcess.end(); - ++itr) - m_runsToProcess.push_back(itr->startRun()); - if (m_fileIndex == m_catalog.fileNames().size()) { + if (m_catalog.fileNames().size() == 0) { m_nextItemType = edm::InputSource::IsStop; - } else { - m_treeReaders[kIntIndex].reset(new TreeSimpleReader()); - m_treeReaders[kFloatIndex].reset(new TreeSimpleReader()); - m_treeReaders[kStringIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1SIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH1DIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2SIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH2DIndex].reset(new TreeObjectReader()); - m_treeReaders[kTH3FIndex].reset(new TreeObjectReader()); - m_treeReaders[kTProfileIndex].reset(new TreeObjectReader()); - m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader()); } + else { + m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT)); + m_treeReaders[kFloatIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::REAL)); + m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING)); + m_treeReaders[kTH1FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1F)); + m_treeReaders[kTH1SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1S)); + m_treeReaders[kTH1DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1D)); + m_treeReaders[kTH2FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2F)); + m_treeReaders[kTH2SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2S)); + m_treeReaders[kTH2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2D)); + m_treeReaders[kTH3FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH3F)); + m_treeReaders[kTProfileIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE)); + m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE2D)); + } + + produces("DQMGenerationRecoRun"); + produces("DQMGenerationRecoLumi"); } -// DQMRootSource::DQMRootSource(const DQMRootSource& rhs) -// { -// // do actual copying here; -// } - DQMRootSource::~DQMRootSource() { - if (m_file.get() != nullptr && m_file->IsOpen()) { - m_file->Close(); - logFileAction(" Closed file ", m_presentlyOpenFileName.c_str()); + for(auto &file : m_openFiles) { + if (file != nullptr && file->IsOpen()) { + file->Close(); + logFileAction("Closed file", ""); + } } } -// -// assignment operators -// -// const DQMRootSource& DQMRootSource::operator=(const DQMRootSource& rhs) -// { -// //An exception safe implementation is -// DQMRootSource temp(rhs); -// swap(rhs); -// -// return *this; -// } - // // member functions // -void DQMRootSource::readEvent_(edm::EventPrincipal&) { - //std::cout << "readEvent_" << std::endl; -} edm::InputSource::ItemType DQMRootSource::getNextItemType() { - //std::cout <<"getNextItemType "< DQMRootSource::readRunAuxiliary_() { - //std::cout <<"readRunAuxiliary_"< runLumiRange.m_historyIDIndex); - //std::cout <<"readRunAuxiliary_ "<(m_runAux); -} +// We will read the metadata of all files and fill m_fileMetadatas vector +std::unique_ptr DQMRootSource::readFile_() { + const int numFiles = m_catalog.fileNames().size(); + m_openFiles.reserve(numFiles); -std::shared_ptr DQMRootSource::readLuminosityBlockAuxiliary_() { - //std::cout <<"readLuminosityBlockAuxiliary_"< runLumiRange.m_historyIDIndex); - //std::cout <<"lumi "<(m_lumiAux); -} + // TODO: add support to fallback files: https://github.com/cms-sw/cmssw/pull/28064/files + for (auto &filename : m_catalog.fileNames()) { + TFile* file; -void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { - assert(m_presentIndexItr != m_orderedIndices.end()); - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; + // Try to open a file + try { + file = TFile::Open(filename.c_str()); - m_justOpenedFileSoNeedToGenerateRunTransition = false; - unsigned int runID = rpCache.id().run(); - assert(runID == runLumiRange.m_run); + // Exception will be trapped so we pull it out ourselves + std::exception_ptr e = edm::threadLocalException::getException(); + if (e != std::exception_ptr()) { + edm::threadLocalException::setException(std::exception_ptr()); + std::rethrow_exception(e); + } + + m_openFiles.insert(m_openFiles.begin(), file); + } catch (cms::Exception const& e) { + if (!m_skipBadFiles) { + edm::Exception ex(edm::errors::FileOpenError, "", e); + ex.addContext("Opening DQM Root file"); + ex << "\nInput file " << filename + << " was not found, could not be opened, or is corrupted.\n"; + throw ex; + } + } - m_shouldReadMEs = (m_filterOnRun == 0 || (m_filterOnRun != 0 && m_filterOnRun == runID)); - if (m_lastSeenRun != runID || m_lastSeenReducedPHID != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) { - if (m_shouldReadMEs) { - // TODO: Worry about resetting depending on Collate option here. + // Check if a file is usable + if (!file->IsZombie()) { + logFileAction("Successfully opened file ", filename.c_str()); + } else { + if (!m_skipBadFiles) { + edm::Exception ex(edm::errors::FileOpenError); + ex << "Input file " << filename.c_str() << " could not be opened.\n"; + ex.addContext("Opening DQM Root file"); + throw ex; + } } - m_lastSeenReducedPHID = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex); - m_lastSeenRun = runID; - } - readNextItemType(); + // Check file format version, which is encoded in the Title of the TFile + if (strcmp(file->GetTitle(), "1") != 0) { + edm::Exception ex(edm::errors::FileReadError); + ex << "Input file " << filename.c_str() << " does not appear to be a DQM Root file.\n"; + } - //NOTE: it is possible to have a Run when all we have stored is lumis - if (runLumiRange.m_lumi == 0) { - readElements(); + // Read metadata from the file + TTree* indicesTree = dynamic_cast(file->Get(kIndicesTree)); + assert(indicesTree != nullptr); + + FileMetadata temp; + // Each line of metadata will be read into the coresponding fields of temp. + indicesTree->SetBranchAddress(kRunBranch, &temp.m_run); + indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi); + indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime); + indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime); + indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type); + indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex); + indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex); + + for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) { + indicesTree->GetEntry(index); + temp.m_file = file; + + if(keepIt(temp.m_run, temp.m_lumi)) { + m_fileMetadatas.push_back(temp); + } + } } - edm::Service jr; - jr->reportInputRunNumber(rpCache.id().run()); - - rpCache.fillRunPrincipal(processHistoryRegistryForUpdate()); -} + // Sort to make sure runs and lumis appear in sequential order + std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); -void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) { - assert(m_presentIndexItr != m_orderedIndices.end()); - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; - assert(runLumiRange.m_run == lbCache.id().run()); - assert(runLumiRange.m_lumi == lbCache.id().luminosityBlock()); - - //NOTE: need to reset all lumi block elements at this point - if ((m_lastSeenLumi2 != runLumiRange.m_lumi || m_lastSeenRun2 != runLumiRange.m_run || - m_lastSeenReducedPHID2 != m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex)) && - m_shouldReadMEs) { - edm::Service store; - std::vector allMEs = (*store).getAllContents(""); - //for(auto const& ME : allMEs) { - // // We do not want to reset Run Products here! - // if (ME->getLumiFlag()) { - // ME->Reset(); - // } - //} - m_lastSeenReducedPHID2 = m_reducedHistoryIDs.at(runLumiRange.m_historyIDIndex); - m_lastSeenRun2 = runLumiRange.m_run; - m_lastSeenLumi2 = runLumiRange.m_lumi; - } + for(auto &metadata : m_fileMetadatas) + metadata.describe(); - readNextItemType(); - readElements(); + // Stop if there's nothing to process. Otherwise start the run. + if(m_fileMetadatas.size() == 0) + m_nextItemType = edm::InputSource::IsStop; + else + m_nextItemType = edm::InputSource::IsRun; - edm::Service jr; - jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock()); + // We have to return something but not sure why + return std::unique_ptr(new edm::FileBlock); +} - lbCache.fillLuminosityBlockPrincipal(processHistoryRegistry().getMapped(lbCache.aux().processHistoryID())); +std::shared_ptr DQMRootSource::readRunAuxiliary_() { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + auto runAux = edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime)); + return std::make_shared(runAux); } -std::unique_ptr DQMRootSource::readFile_() { - auto const numFiles = m_catalog.fileNames().size(); - while (m_fileIndex < numFiles && not setupFile(m_fileIndex++)) { - } +std::shared_ptr DQMRootSource::readLuminosityBlockAuxiliary_() { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + auto lumiAux = edm::LuminosityBlockAuxiliary(edm::LuminosityBlockID(metadata.m_run, metadata.m_lumi), + edm::Timestamp(metadata.m_beginTime), + edm::Timestamp(metadata.m_endTime)); + return std::make_shared(lumiAux); +} - if (m_file.get() == nullptr) { - //last file in list was bad - m_nextItemType = edm::InputSource::IsStop; - return std::unique_ptr(new edm::FileBlock); - } +void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { + // Read elements of a current run. + do { + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; + if (metadata.m_lumi == 0) { + readElements(); + } + m_currentIndex++; + } while(!isRunOrLumiTransition()); readNextItemType(); - while (m_presentIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi)) - ++m_presentIndexItr; - - edm::Service jr; - std::string guid{m_file->GetUUID().AsString()}; - std::transform(guid.begin(), guid.end(), guid.begin(), (int (*)(int))std::toupper); - m_jrToken = jr->inputFileOpened(m_presentlyOpenFileName, - m_catalog.logicalFileNames()[m_fileIndex - 1], - std::string(), - std::string(), - "DQMRootSource", - "source", - std::move(guid), - std::vector()); - - return std::unique_ptr(new edm::FileBlock); } -void DQMRootSource::closeFile_() { - if (m_file.get() == nullptr) { - return; - } - edm::Service jr; - jr->inputFileClosed(edm::InputType::Primary, m_jrToken); -} - -void DQMRootSource::readElements() { - edm::Service store; - RunLumiToRange runLumiRange = m_runlumiToRange[*m_presentIndexItr]; - bool shouldContinue = false; +void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) { + // Read elements of a current lumi. do { - shouldContinue = false; - ++m_presentIndexItr; - while (m_presentIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_presentIndexItr].m_run, m_runlumiToRange[*m_presentIndexItr].m_lumi)) - ++m_presentIndexItr; - - if (runLumiRange.m_type != kNoTypesStored) { - std::shared_ptr reader = m_treeReaders[runLumiRange.m_type]; - ULong64_t index = runLumiRange.m_firstIndex; - ULong64_t endIndex = runLumiRange.m_lastIndex + 1; - for (; index != endIndex; ++index) { - bool isLumi = runLumiRange.m_lumi != 0; - if (m_shouldReadMEs) - reader->read(index, *store, isLumi); - - //std::cout << runLumiRange.m_run << " " << runLumiRange.m_lumi <<" "< reader = m_treeReaders[metadata.m_type]; + TTree* tree = dynamic_cast(metadata.m_file->Get(kTypeNames[metadata.m_type])); + reader->setTree(tree); - bool shouldContinue = false; - do { - shouldContinue = false; - while (m_nextIndexItr != m_orderedIndices.end() && - skipIt(m_runlumiToRange[*m_nextIndexItr].m_run, m_runlumiToRange[*m_nextIndexItr].m_lumi)) - ++m_nextIndexItr; - - if (m_nextIndexItr == m_orderedIndices.end()) { - //go to next file - m_nextItemType = edm::InputSource::IsFile; - //std::cout <<"going to next file"<read(index, m_MEsFromFile, metadata.m_run, metadata.m_lumi); } } } -bool DQMRootSource::setupFile(unsigned int iIndex) { - if (m_file.get() != nullptr && iIndex > 0) { - m_file->Close(); - logFileAction(" Closed file ", m_presentlyOpenFileName.c_str()); +bool DQMRootSource::isRunOrLumiTransition() const { + if(m_currentIndex == 0) { + return false; } - logFileAction(" Initiating request to open file ", m_catalog.fileNames()[iIndex].c_str()); - - auto fallbackFileName = m_catalog.fallbackFileNames()[iIndex]; - bool hasFallback = !fallbackFileName.empty() && fallbackFileName != m_catalog.fileNames()[iIndex]; - - m_presentlyOpenFileName.clear(); - m_file.reset(); - - std::unique_ptr newFile; - std::list originalInfo; - try { - // ROOT's context management implicitly assumes that a file is opened and - // closed on the same thread. To avoid the problem, we declare a local - // TContext object; when it goes out of scope, its destructor unregisters - // the context, guaranteeing the context is unregistered in the same thread - // it was registered in. - { - TDirectory::TContext contextEraser; - newFile = std::unique_ptr(TFile::Open(m_catalog.fileNames()[iIndex].c_str())); - } - //Since ROOT6, we can not propagate an exception through ROOT's plugin - // system so we trap them and then pull from this function - std::exception_ptr e = edm::threadLocalException::getException(); - if (e != std::exception_ptr()) { - edm::threadLocalException::setException(std::exception_ptr()); - std::rethrow_exception(e); - } - } catch (cms::Exception const& e) { - if (!hasFallback) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError, "", e); - ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << m_catalog.fileNames()[iIndex] - << " was not found, could not be opened, or is corrupted.\n"; - throw ex; - } - } - originalInfo = e.additionalInfo(); // save in case of fallback error - newFile.reset(); - } - if (newFile && not newFile->IsZombie()) { - m_presentlyOpenFileName = m_catalog.fileNames()[iIndex]; - logFileAction(" Successfully opened file ", m_presentlyOpenFileName.c_str()); - } else { - if (!hasFallback) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError); - ex << "Input file " << m_catalog.fileNames()[iIndex].c_str() << " could not be opened.\n"; - ex.addContext("Opening DQM Root file"); - throw ex; - } - } - newFile.reset(); + if(m_currentIndex > m_fileMetadatas.size() - 1) { + // We reached the end + return true; } - if (!newFile && hasFallback) { - logFileAction(" Initiating request to open fallback file ", fallbackFileName.c_str()); - try { - { - TDirectory::TContext contextEraser; - newFile = std::unique_ptr(TFile::Open(fallbackFileName.c_str())); - } - std::exception_ptr e = edm::threadLocalException::getException(); - if (e != std::exception_ptr()) { - edm::threadLocalException::setException(std::exception_ptr()); - std::rethrow_exception(e); - } - } catch (cms::Exception const& e) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError, "", e); - ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName - << " were not found, could not be opened, or are corrupted.\n"; - for (auto const& s : originalInfo) { - ex.addAdditionalInfo(s); - } - throw ex; - } - } - if (not newFile->IsZombie()) { - m_presentlyOpenFileName = fallbackFileName; - logFileAction(" Successfully opened fallback file ", m_presentlyOpenFileName.c_str()); - } else { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileOpenError); - ex << "Input file " << m_catalog.fileNames()[iIndex] << " and fallback input file " << fallbackFileName - << " could not be opened.\n"; - ex.addContext("Opening DQM Root file"); - for (auto const& s : originalInfo) { - ex.addAdditionalInfo(s); - } - throw ex; - } - } - } + FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1]; + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - //Check file format version, which is encoded in the Title of the TFile - if (0 != strcmp(newFile->GetTitle(), "1")) { - if (m_skipBadFiles) { - return false; - } else { - edm::Exception ex(edm::errors::FileReadError); - ex << "Input file " << m_presentlyOpenFileName << " does not appear to be a DQM Root file.\n"; - throw ex; - } - } + return previousMetadata.m_run != metadata.m_run || previousMetadata.m_lumi != metadata.m_lumi; +} - //Get meta Data - TDirectory* metaDir = newFile->GetDirectory(kMetaDataDirectoryAbsolute); - if (nullptr == metaDir) { - if (!m_skipBadFiles) { - edm::Exception ex(edm::errors::FileReadError); - ex << "Input file " << m_presentlyOpenFileName - << " appears to be corrupted since it does not have the proper internal structure.\n" - " Check to see if the file was closed properly.\n"; - ex.addContext("Opening DQM Root file"); - throw ex; - } else { - return false; - } +void DQMRootSource::readNextItemType() { + if(m_currentIndex == 0) { + m_nextItemType = edm::InputSource::IsRun; } - m_file = std::move(newFile); //passed all tests so now we want to use this file - TTree* parameterSetTree = dynamic_cast(metaDir->Get(kParameterSetTree)); - assert(nullptr != parameterSetTree); - - edm::pset::Registry* psr = edm::pset::Registry::instance(); - assert(nullptr != psr); - { - std::string blob; - std::string* pBlob = &blob; - parameterSetTree->SetBranchAddress(kParameterSetBranch, &pBlob); - for (unsigned int index = 0; index != parameterSetTree->GetEntries(); ++index) { - parameterSetTree->GetEntry(index); - edm::ParameterSet::registerFromString(blob); - } + else if(m_currentIndex > m_fileMetadatas.size() - 1) { + // We reached the end + m_nextItemType = edm::InputSource::IsStop; } + else { + FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1]; + FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - { - TTree* processHistoryTree = dynamic_cast(metaDir->Get(kProcessHistoryTree)); - assert(nullptr != processHistoryTree); - unsigned int phIndex = 0; - processHistoryTree->SetBranchAddress(kPHIndexBranch, &phIndex); - std::string processName; - std::string* pProcessName = &processName; - processHistoryTree->SetBranchAddress(kProcessConfigurationProcessNameBranch, &pProcessName); - std::string parameterSetIDBlob; - std::string* pParameterSetIDBlob = ¶meterSetIDBlob; - processHistoryTree->SetBranchAddress(kProcessConfigurationParameterSetIDBranch, &pParameterSetIDBlob); - std::string releaseVersion; - std::string* pReleaseVersion = &releaseVersion; - processHistoryTree->SetBranchAddress(kProcessConfigurationReleaseVersion, &pReleaseVersion); - std::string passID; - std::string* pPassID = &passID; - processHistoryTree->SetBranchAddress(kProcessConfigurationPassID, &pPassID); - - edm::ProcessHistoryRegistry& phr = processHistoryRegistryForUpdate(); - std::vector configs; - configs.reserve(5); - m_historyIDs.clear(); - m_reducedHistoryIDs.clear(); - for (unsigned int i = 0; i != processHistoryTree->GetEntries(); ++i) { - processHistoryTree->GetEntry(i); - if (phIndex == 0) { - if (not configs.empty()) { - edm::ProcessHistory ph(configs); - m_historyIDs.push_back(ph.id()); - phr.registerProcessHistory(ph); - m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id())); - } - configs.clear(); - } - edm::ParameterSetID psetID(parameterSetIDBlob); - edm::ProcessConfiguration pc(processName, psetID, releaseVersion, passID); - configs.push_back(pc); + if(previousMetadata.m_run != metadata.m_run) { + m_nextItemType = edm::InputSource::IsRun; } - if (not configs.empty()) { - edm::ProcessHistory ph(configs); - m_historyIDs.push_back(ph.id()); - phr.registerProcessHistory(ph); - m_reducedHistoryIDs.push_back(phr.reducedProcessHistoryID(ph.id())); - //std::cout <<"inserted "<(m_file->Get(kIndicesTree)); - assert(nullptr != indicesTree); - - m_runlumiToRange.clear(); - m_runlumiToRange.reserve(indicesTree->GetEntries()); - m_orderedIndices.clear(); - - RunLumiToRange temp; - indicesTree->SetBranchAddress(kRunBranch, &temp.m_run); - indicesTree->SetBranchAddress(kLumiBranch, &temp.m_lumi); - indicesTree->SetBranchAddress(kBeginTimeBranch, &temp.m_beginTime); - indicesTree->SetBranchAddress(kEndTimeBranch, &temp.m_endTime); - indicesTree->SetBranchAddress(kProcessHistoryIndexBranch, &temp.m_historyIDIndex); - indicesTree->SetBranchAddress(kTypeBranch, &temp.m_type); - indicesTree->SetBranchAddress(kFirstIndex, &temp.m_firstIndex); - indicesTree->SetBranchAddress(kLastIndex, &temp.m_lastIndex); - - //Need to reorder items since if there was a merge done the same Run - //and/or Lumi can appear multiple times but we want to process them - //all at once - - //We use a std::list for m_orderedIndices since inserting into the - //middle of a std::list does not disrupt the iterators to already - //existing entries - - //The Map is used to see if a Run/Lumi pair has appeared before - typedef std::map::iterator> RunLumiToLastEntryMap; - RunLumiToLastEntryMap runLumiToLastEntryMap; - - //Need to group all lumis for the same run together and move the run - //entry to the beginning - typedef std::map::iterator, std::list::iterator> > - RunToFirstLastEntryMap; - RunToFirstLastEntryMap runToFirstLastEntryMap; - - for (Long64_t index = 0; index != indicesTree->GetEntries(); ++index) { - indicesTree->GetEntry(index); - // std::cout <<"read r:"<::iterator itLastOfRun = m_orderedIndices.end(); - - RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey); - bool needNewEntryInRunFirstLastEntryMap = true; - if (itRunFirstLastEntryFind != runToFirstLastEntryMap.end()) { - needNewEntryInRunFirstLastEntryMap = false; - if (temp.m_lumi != 0) { - //lumis go to the end - itLastOfRun = itRunFirstLastEntryFind->second.second; - //we want to insert after this one so must advance the iterator - ++itLastOfRun; - } else { - //runs go at the beginning - itLastOfRun = itRunFirstLastEntryFind->second.first; - } - } - std::list::iterator iter = m_orderedIndices.insert(itLastOfRun, index); - runLumiToLastEntryMap[runLumi] = iter; - if (needNewEntryInRunFirstLastEntryMap) - runToFirstLastEntryMap[runKey] = std::make_pair(iter, iter); - else { - if (temp.m_lumi != 0) { - //lumis go at end - runToFirstLastEntryMap[runKey].second = iter; - } else { - //since we haven't yet seen this run/lumi combination it means we haven't yet seen - // a run so we can put this first - runToFirstLastEntryMap[runKey].first = iter; - } - } - } else { - //We need to do a merge since the run/lumi already appeared. Put it after the existing entry - //std::cout <<" found a second instance of "<::iterator itNext = itFind->second; - ++itNext; - std::list::iterator iter = m_orderedIndices.insert(itNext, index); - RunToFirstLastEntryMap::iterator itRunFirstLastEntryFind = runToFirstLastEntryMap.find(runKey); - if (itRunFirstLastEntryFind->second.second == itFind->second) { - //if the previous one was the last in the run, we need to update to make this one the last - itRunFirstLastEntryFind->second.second = iter; - } - itFind->second = iter; - } - } - m_nextIndexItr = m_orderedIndices.begin(); - m_presentIndexItr = m_orderedIndices.begin(); - - if (m_nextIndexItr != m_orderedIndices.end()) { - for (size_t index = 0; index < kNIndicies; ++index) { - m_trees[index] = dynamic_cast(m_file->Get(kTypeNames[index])); - assert(nullptr != m_trees[index]); - m_treeReaders[index]->setTree(m_trees[index]); - } +void DQMRootSource::beginRun(edm::Run& run) { + TRACE("Begin run: " + std::to_string(run.run())) + + std::unique_ptr product = std::make_unique(); + + auto mes = m_MEsFromFile[std::make_tuple(run.run(), 0)]; + + TRACE("Found MEs: " + std::to_string(mes.size())) + + for(MonitorElementData* meData_ptr : mes) { + product->push_back(meData_ptr); } - //After a file open, the framework expects to see a new 'IsRun' - m_justOpenedFileSoNeedToGenerateRunTransition = true; - return true; + run.put(std::move(product), "DQMGenerationRecoRun"); + + // Remove already processed MEs + m_MEsFromFile[std::make_tuple(run.run(), 0)] = std::vector(); } -bool DQMRootSource::skipIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { - if (!m_runsToProcess.empty() && edm::search_all(m_runsToProcess, run) && lumi == 0) { - return false; +void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { + TRACE("Begin lumi: " + std::to_string(lumi.luminosityBlock())) + + std::unique_ptr product = std::make_unique(); + + auto mes = m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())]; + + TRACE("Found MEs: " + std::to_string(mes.size())) + + for(MonitorElementData* meData_ptr : mes) { + assert(meData_ptr != nullptr); + product->push_back(meData_ptr); } + + lumi.put(std::move(product), "DQMGenerationRecoLumi"); - edm::LuminosityBlockID lumiID = edm::LuminosityBlockID(run, lumi); - edm::LuminosityBlockRange lumiRange = edm::LuminosityBlockRange(lumiID, lumiID); - bool (*lt)(edm::LuminosityBlockRange const&, edm::LuminosityBlockRange const&) = &edm::lessThan; - if (!m_lumisToProcess.empty() && !binary_search_all(m_lumisToProcess, lumiRange, lt)) { + // Remove already processed MEs + m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())] = std::vector(); +} + +bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { + if(run == m_filterOnRun) return true; + + for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) { + if(run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) { + if(lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) { + return true; + } + else if (lumi == 0) { + return true; + } + } } return false; } From f3ecb650752306c957032463da1aa5fdcbad054c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 15:15:02 +0100 Subject: [PATCH 023/112] Code-format. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 156 ++++++++++----------- 1 file changed, 72 insertions(+), 84 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 868b1952a2edc..efffaf9acc3b5 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -50,22 +50,22 @@ namespace { // TODO: this should probably be moved somewhere else class DQMMergeHelper { - public: + public: // Utility function to check the consistency of the axis labels // Taken from TH1::CheckBinLabels which is not public - static bool CheckBinLabels(const TAxis* a1, const TAxis * a2) { + static bool CheckBinLabels(const TAxis* a1, const TAxis* a2) { // Check that axis have same labels - THashList *l1 = (const_cast(a1))->GetLabels(); - THashList *l2 = (const_cast(a2))->GetLabels(); - - if (!l1 && !l2 ) + THashList* l1 = (const_cast(a1))->GetLabels(); + THashList* l2 = (const_cast(a2))->GetLabels(); + + if (!l1 && !l2) return true; - if (!l1 || !l2 ) { + if (!l1 || !l2) { return false; } // Check now labels sizes are the same - if (l1->GetSize() != l2->GetSize() ) { + if (l1->GetSize() != l2->GetSize()) { return false; } @@ -88,8 +88,7 @@ namespace { if (original->Merge(&list) == -1) { edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName(); } - } - else { + } else { // TODO: Redo. What's wrong with this implementation? if (original->GetNbinsX() == toAdd->GetNbinsX() && original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() && @@ -104,8 +103,7 @@ namespace { CheckBinLabels(original->GetYaxis(), toAdd->GetYaxis()) && CheckBinLabels(original->GetZaxis(), toAdd->GetZaxis())) { original->Add(toAdd); - } - else { + } else { edm::LogError("MergeFailure") << "Found histograms with different axis limits or different labels '" << original->GetName() << "' not merged."; } @@ -123,27 +121,22 @@ namespace { ULong64_t m_beginTime; ULong64_t m_endTime; ULong64_t m_firstIndex; - ULong64_t m_lastIndex; // Last is inclusive + ULong64_t m_lastIndex; // Last is inclusive unsigned int m_type; TFile* m_file; // This will be used when sorting a vector - bool operator < (const FileMetadata& obj) const { - if(m_run == obj.m_run) + bool operator<(const FileMetadata& obj) const { + if (m_run == obj.m_run) return m_lumi < obj.m_lumi; else return m_run < obj.m_run; } void describe() { - std::cout << "read r:" << m_run - << " l:" << m_lumi - << " bt:" << m_beginTime - << " et:" << m_endTime - << " fi:" << m_firstIndex - << " li:" << m_lastIndex - << " type:" << m_type - << " file: " << m_file << std::endl; + std::cout << "read r:" << m_run << " l:" << m_lumi << " bt:" << m_beginTime << " et:" << m_endTime + << " fi:" << m_firstIndex << " li:" << m_lastIndex << " type:" << m_type << " file: " << m_file + << std::endl; } }; @@ -163,12 +156,13 @@ namespace { template class TreeObjectReader : public TreeReaderBase { public: - TreeObjectReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) { + TreeObjectReader(MonitorElementData::Kind kind) + : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) { assert(m_kind != MonitorElementData::Kind::INT); assert(m_kind != MonitorElementData::Kind::REAL); assert(m_kind != MonitorElementData::Kind::STRING); } - + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -183,7 +177,7 @@ namespace { std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; bool merged = false; for (MonitorElementData* meData : runLumiMEs) { - if(meData->key_ == key) { + if (meData->key_ == key) { // Merge with already existing ME! MonitorElementData::Value::Access value(meData->value_); DQMMergeHelper::mergeTogether(value.object.get(), m_buffer); @@ -192,7 +186,7 @@ namespace { } } - if(!merged) { + if (!merged) { MonitorElementData* meData = new MonitorElementData(); meData->key_ = key; { @@ -220,10 +214,11 @@ namespace { class TreeStringReader : public TreeReaderBase { public: - TreeStringReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) { + TreeStringReader(MonitorElementData::Kind kind) + : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) { assert(m_kind == MonitorElementData::Kind::STRING); } - + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -237,7 +232,7 @@ namespace { std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; bool merged = false; for (MonitorElementData* meData : runLumiMEs) { - if(meData->key_ == key) { + if (meData->key_ == key) { // Keep the latest one MonitorElementData::Value::Access value(meData->value_); value.scalar.str = *m_value; @@ -246,7 +241,7 @@ namespace { } } - if(!merged) { + if (!merged) { MonitorElementData* meData = new MonitorElementData(); meData->key_ = key; { @@ -275,10 +270,11 @@ namespace { template class TreeSimpleReader : public TreeReaderBase { public: - TreeSimpleReader(MonitorElementData::Kind kind) : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) { + TreeSimpleReader(MonitorElementData::Kind kind) + : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) { assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL); } - + void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -292,26 +288,26 @@ namespace { std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; bool merged = false; for (MonitorElementData* meData : runLumiMEs) { - if(meData->key_ == key) { + if (meData->key_ == key) { // Keep the latest one MonitorElementData::Value::Access value(meData->value_); - if(m_kind == MonitorElementData::Kind::INT) + if (m_kind == MonitorElementData::Kind::INT) value.scalar.num = m_buffer; - else if(m_kind == MonitorElementData::Kind::REAL) + else if (m_kind == MonitorElementData::Kind::REAL) value.scalar.real = m_buffer; merged = true; break; } } - if(!merged) { + if (!merged) { MonitorElementData* meData = new MonitorElementData(); meData->key_ = key; { MonitorElementData::Value::Access value(meData->value_); - if(m_kind == MonitorElementData::Kind::INT) + if (m_kind == MonitorElementData::Kind::INT) value.scalar.num = m_buffer; - else if(m_kind == MonitorElementData::Kind::REAL) + else if (m_kind == MonitorElementData::Kind::REAL) value.scalar.real = m_buffer; } @@ -333,7 +329,7 @@ namespace { uint32_t m_tag; }; -} // namespace +} // namespace class DQMRootSource : public edm::PuttableSourceBase { public: @@ -361,7 +357,7 @@ class DQMRootSource : public edm::PuttableSourceBase { // Read MEs from m_fileMetadatas to m_MEsFromFile till run or lumi transition void readElements(); - // True if m_currentIndex points to an element that has a different + // True if m_currentIndex points to an element that has a different // run or lumi than the previous element (a transition needs to happen). // False otherwise. bool isRunOrLumiTransition() const; @@ -372,7 +368,7 @@ class DQMRootSource : public edm::PuttableSourceBase { void beginRun(edm::Run& run); void beginLuminosityBlock(edm::LuminosityBlock& lumi); - // If the run matches the filterOnRun configuration parameter, the run + // If the run matches the filterOnRun configuration parameter, the run // (and all its lumis) will be kept. // Otherwise, check if a run and a lumi are in the range that needs to be processed. // Range is retrieved from lumisToProcess configuration parameter. @@ -414,13 +410,13 @@ class DQMRootSource : public edm::PuttableSourceBase { void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; - desc.addUntracked >("fileNames")->setComment("Names of files to be processed."); + desc.addUntracked>("fileNames")->setComment("Names of files to be processed."); desc.addUntracked("filterOnRun", 0)->setComment("Just limit the process to the selected run."); desc.addUntracked("skipBadFiles", false)->setComment("Skip the file if it is not valid"); desc.addUntracked("overrideCatalog", std::string()) ->setComment("An alternate file catalog to use instead of the standard site one."); std::vector defaultLumis; - desc.addUntracked >("lumisToProcess", defaultLumis) + desc.addUntracked>("lumisToProcess", defaultLumis) ->setComment("Skip any lumi inside the specified run:lumi range."); descriptions.addDefault(desc); @@ -434,22 +430,21 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou : edm::PuttableSourceBase(iPSet, iDesc), m_skipBadFiles(iPSet.getUntrackedParameter("skipBadFiles", false)), m_filterOnRun(iPSet.getUntrackedParameter("filterOnRun", 0)), - m_catalog(iPSet.getUntrackedParameter >("fileNames"), + m_catalog(iPSet.getUntrackedParameter>("fileNames"), iPSet.getUntrackedParameter("overrideCatalog")), - m_lumisToProcess(iPSet.getUntrackedParameter >( + m_lumisToProcess(iPSet.getUntrackedParameter>( "lumisToProcess", std::vector())), m_nextItemType(edm::InputSource::IsFile), m_treeReaders(kNIndicies, std::shared_ptr()), m_currentIndex(0), - m_openFiles (std::vector()), + m_openFiles(std::vector()), m_MEsFromFile(MonitorElementsFromFile()), m_fileMetadatas(std::vector()) { edm::sortAndRemoveOverlaps(m_lumisToProcess); if (m_catalog.fileNames().size() == 0) { m_nextItemType = edm::InputSource::IsStop; - } - else { + } else { m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT)); m_treeReaders[kFloatIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::REAL)); m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING)); @@ -469,7 +464,7 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou } DQMRootSource::~DQMRootSource() { - for(auto &file : m_openFiles) { + for (auto& file : m_openFiles) { if (file != nullptr && file->IsOpen()) { file->Close(); logFileAction("Closed file", ""); @@ -481,9 +476,7 @@ DQMRootSource::~DQMRootSource() { // member functions // -edm::InputSource::ItemType DQMRootSource::getNextItemType() { - return m_nextItemType; -} +edm::InputSource::ItemType DQMRootSource::getNextItemType() { return m_nextItemType; } // We will read the metadata of all files and fill m_fileMetadatas vector std::unique_ptr DQMRootSource::readFile_() { @@ -491,7 +484,7 @@ std::unique_ptr DQMRootSource::readFile_() { m_openFiles.reserve(numFiles); // TODO: add support to fallback files: https://github.com/cms-sw/cmssw/pull/28064/files - for (auto &filename : m_catalog.fileNames()) { + for (auto& filename : m_catalog.fileNames()) { TFile* file; // Try to open a file @@ -510,8 +503,7 @@ std::unique_ptr DQMRootSource::readFile_() { if (!m_skipBadFiles) { edm::Exception ex(edm::errors::FileOpenError, "", e); ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << filename - << " was not found, could not be opened, or is corrupted.\n"; + ex << "\nInput file " << filename << " was not found, could not be opened, or is corrupted.\n"; throw ex; } } @@ -552,7 +544,7 @@ std::unique_ptr DQMRootSource::readFile_() { indicesTree->GetEntry(index); temp.m_file = file; - if(keepIt(temp.m_run, temp.m_lumi)) { + if (keepIt(temp.m_run, temp.m_lumi)) { m_fileMetadatas.push_back(temp); } } @@ -561,11 +553,11 @@ std::unique_ptr DQMRootSource::readFile_() { // Sort to make sure runs and lumis appear in sequential order std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); - for(auto &metadata : m_fileMetadatas) + for (auto& metadata : m_fileMetadatas) metadata.describe(); // Stop if there's nothing to process. Otherwise start the run. - if(m_fileMetadatas.size() == 0) + if (m_fileMetadatas.size() == 0) m_nextItemType = edm::InputSource::IsStop; else m_nextItemType = edm::InputSource::IsRun; @@ -576,7 +568,8 @@ std::unique_ptr DQMRootSource::readFile_() { std::shared_ptr DQMRootSource::readRunAuxiliary_() { FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - auto runAux = edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime)); + auto runAux = + edm::RunAuxiliary(metadata.m_run, edm::Timestamp(metadata.m_beginTime), edm::Timestamp(metadata.m_endTime)); return std::make_shared(runAux); } @@ -596,7 +589,7 @@ void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { readElements(); } m_currentIndex++; - } while(!isRunOrLumiTransition()); + } while (!isRunOrLumiTransition()); readNextItemType(); } @@ -606,13 +599,12 @@ void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) do { readElements(); m_currentIndex++; - } while(!isRunOrLumiTransition()); + } while (!isRunOrLumiTransition()); readNextItemType(); } -void DQMRootSource::readEvent_(edm::EventPrincipal&) { -} +void DQMRootSource::readEvent_(edm::EventPrincipal&) {} void DQMRootSource::readElements() { FileMetadata metadata = m_fileMetadatas[m_currentIndex]; @@ -632,11 +624,11 @@ void DQMRootSource::readElements() { } bool DQMRootSource::isRunOrLumiTransition() const { - if(m_currentIndex == 0) { + if (m_currentIndex == 0) { return false; } - if(m_currentIndex > m_fileMetadatas.size() - 1) { + if (m_currentIndex > m_fileMetadatas.size() - 1) { // We reached the end return true; } @@ -648,21 +640,18 @@ bool DQMRootSource::isRunOrLumiTransition() const { } void DQMRootSource::readNextItemType() { - if(m_currentIndex == 0) { + if (m_currentIndex == 0) { m_nextItemType = edm::InputSource::IsRun; - } - else if(m_currentIndex > m_fileMetadatas.size() - 1) { + } else if (m_currentIndex > m_fileMetadatas.size() - 1) { // We reached the end m_nextItemType = edm::InputSource::IsStop; - } - else { + } else { FileMetadata previousMetadata = m_fileMetadatas[m_currentIndex - 1]; FileMetadata metadata = m_fileMetadatas[m_currentIndex]; - if(previousMetadata.m_run != metadata.m_run) { + if (previousMetadata.m_run != metadata.m_run) { m_nextItemType = edm::InputSource::IsRun; - } - else if(previousMetadata.m_lumi != metadata.m_lumi) { + } else if (previousMetadata.m_lumi != metadata.m_lumi) { m_nextItemType = edm::InputSource::IsLumi; } } @@ -677,12 +666,12 @@ void DQMRootSource::beginRun(edm::Run& run) { TRACE("Found MEs: " + std::to_string(mes.size())) - for(MonitorElementData* meData_ptr : mes) { + for (MonitorElementData* meData_ptr : mes) { product->push_back(meData_ptr); } run.put(std::move(product), "DQMGenerationRecoRun"); - + // Remove already processed MEs m_MEsFromFile[std::make_tuple(run.run(), 0)] = std::vector(); } @@ -696,11 +685,11 @@ void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { TRACE("Found MEs: " + std::to_string(mes.size())) - for(MonitorElementData* meData_ptr : mes) { + for (MonitorElementData* meData_ptr : mes) { assert(meData_ptr != nullptr); product->push_back(meData_ptr); } - + lumi.put(std::move(product), "DQMGenerationRecoLumi"); // Remove already processed MEs @@ -708,15 +697,14 @@ void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { } bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { - if(run == m_filterOnRun) + if (run == m_filterOnRun) return true; - + for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) { - if(run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) { - if(lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) { + if (run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) { + if (lumi >= lumiToProcess.startLumi() && lumi <= lumiToProcess.endLumi()) { return true; - } - else if (lumi == 0) { + } else if (lumi == 0) { return true; } } From 155fc2aebf03c7b004a03b5c0cce8fc70d3988d3 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 16:18:34 +0100 Subject: [PATCH 024/112] Re-add DQMStore dependence. --- DQMServices/Core/interface/DQMStore.h | 4 + DQMServices/Core/src/DQMStore.cc | 12 +- DQMServices/FwkIO/plugins/DQMRootSource.cc | 172 +++++++-------------- 3 files changed, 68 insertions(+), 120 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 8ad48cd38f3e8..47dc95771e18f 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -452,6 +452,10 @@ namespace dqm { // return ME identified by full path `path`, or nullptr virtual MonitorElement* get(std::string const& fullpath) const; + // This is the most specific way to get a ME, specifying also run and + // lumi in the key. Primarily for internal use. + virtual MonitorElement* get(MonitorElementData::Key const& key) const; + // same as get, throws an exception if histogram not found // Deprecated simply because it is barely used. DQM_DEPRECATED diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 8437a1a791216..572440ade5651 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -450,10 +450,20 @@ namespace dqm::implementation { MonitorElementData::Path path; path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); // this only really makes sense if there is only one instance of this ME, - // but the signature of this mthod also only makes sense in that case. + // but the signature of this method also only makes sense in that case. return store_->findME(path); } + MonitorElement* IGetter::get(MonitorElementData::Key const& key) const { + auto meset = store_->globalMEs_[key.id_]; + auto it = meset.find(key.path_); + if (it != meset.end()) { + assert((*it)->getScope() == key.scope_); + return *it; + } + return nullptr; + } + MonitorElement* IGetter::getElement(std::string const& path) const { auto result = this->get(path); if (result == nullptr) { diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index efffaf9acc3b5..bde1dc77b49e3 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -18,12 +18,14 @@ #include "TTree.h" #include "TString.h" -// user include files +#include "DQMServices/Core/interface/DQMStore.h" +#include "DataFormats/Histograms/interface/DQMToken.h" +#include "FWCore/ServiceRegistry/interface/Service.h" + #include "FWCore/Framework/interface/InputSource.h" #include "FWCore/Sources/interface/PuttableSourceBase.h" #include "FWCore/Catalog/interface/InputFileCatalog.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "DQMServices/Core/interface/DQMStore.h" #include "FWCore/Utilities/interface/ExceptionPropagate.h" #include "FWCore/Framework/interface/RunPrincipal.h" @@ -47,6 +49,7 @@ namespace { typedef dqm::harvesting::MonitorElement MonitorElement; + typedef dqm::harvesting::DQMStore DQMStore; // TODO: this should probably be moved somewhere else class DQMMergeHelper { @@ -89,7 +92,8 @@ namespace { edm::LogError("MergeFailure") << "Failed to merge DQM element " << original->GetName(); } } else { - // TODO: Redo. What's wrong with this implementation? + // TODO: Redo. This is both more strict than what ROOT checks for yet + // allows cases where ROOT fails with merging. if (original->GetNbinsX() == toAdd->GetNbinsX() && original->GetXaxis()->GetXmin() == toAdd->GetXaxis()->GetXmin() && original->GetXaxis()->GetXmax() == toAdd->GetXaxis()->GetXmax() && @@ -111,8 +115,6 @@ namespace { } }; - using MonitorElementsFromFile = std::map, std::vector>; - // This struct allows to find all MEs belonging to a run-lumi pair // All files will be open at once so m_file property indicates the file where data is saved. struct FileMetadata { @@ -145,7 +147,7 @@ namespace { TreeReaderBase(MonitorElementData::Kind kind) : m_kind(kind) {} virtual ~TreeReaderBase() {} - virtual void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) = 0; + virtual void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) = 0; virtual void setTree(TTree* iTree) = 0; protected: @@ -163,7 +165,7 @@ namespace { assert(m_kind != MonitorElementData::Kind::STRING); } - void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -171,30 +173,19 @@ namespace { key.kind_ = m_kind; key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - // TODO: What should the range be for per run MEs? Now lumi will be 0 and not the max lumi for that run. - key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); - - std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; - bool merged = false; - for (MonitorElementData* meData : runLumiMEs) { - if (meData->key_ == key) { - // Merge with already existing ME! - MonitorElementData::Value::Access value(meData->value_); - DQMMergeHelper::mergeTogether(value.object.get(), m_buffer); - merged = true; - break; - } - } - - if (!merged) { - MonitorElementData* meData = new MonitorElementData(); - meData->key_ = key; - { - MonitorElementData::Value::Access value(meData->value_); - value.object = std::unique_ptr((T*)(m_buffer->Clone())); - } + key.id_ = edm::LuminosityBlockID(run, lumi); - mesFromFile[std::make_tuple(run, lumi)].push_back(meData); + auto existing = dqmstore->get(key); + if (existing) { + // TODO: make sure there is sufficient locking here. + DQMMergeHelper::mergeTogether(existing->getTH1(), m_buffer); + } else { + // We make our own MEs here, to avoid a round-trip through the booking API. + MonitorElementData meData; + meData.key_ = key; + meData.value_.object_ = std::unique_ptr((T*)(m_buffer->Clone())); + auto me = new MonitorElement(std::move(meData)); + dqmstore->putME(me); } } @@ -219,7 +210,7 @@ namespace { assert(m_kind == MonitorElementData::Kind::STRING); } - void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -227,29 +218,18 @@ namespace { key.kind_ = m_kind; key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); - - std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; - bool merged = false; - for (MonitorElementData* meData : runLumiMEs) { - if (meData->key_ == key) { - // Keep the latest one - MonitorElementData::Value::Access value(meData->value_); - value.scalar.str = *m_value; - merged = true; - break; - } - } + key.id_ = edm::LuminosityBlockID(run, lumi); - if (!merged) { - MonitorElementData* meData = new MonitorElementData(); - meData->key_ = key; - { - MonitorElementData::Value::Access value(meData->value_); - value.scalar.str = *m_value; - } - - mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + auto existing = dqmstore->get(key); + if (existing) { + existing->Fill(*m_value); + } else { + // We make our own MEs here, to avoid a round-trip through the booking API. + MonitorElementData meData; + meData.key_ = key; + meData.value_.scalar_.str = *m_value; + auto me = new MonitorElement(std::move(meData)); + dqmstore->putME(me); } } @@ -275,7 +255,7 @@ namespace { assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL); } - void read(ULong64_t iIndex, MonitorElementsFromFile& mesFromFile, int run, int lumi) override { + void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) override { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); @@ -283,35 +263,21 @@ namespace { key.kind_ = m_kind; key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - key.coveredrange_ = edm::LuminosityBlockRange(run, lumi, run, lumi); - - std::vector runLumiMEs = mesFromFile[std::make_tuple(run, lumi)]; - bool merged = false; - for (MonitorElementData* meData : runLumiMEs) { - if (meData->key_ == key) { - // Keep the latest one - MonitorElementData::Value::Access value(meData->value_); - if (m_kind == MonitorElementData::Kind::INT) - value.scalar.num = m_buffer; - else if (m_kind == MonitorElementData::Kind::REAL) - value.scalar.real = m_buffer; - merged = true; - break; - } - } + key.id_ = edm::LuminosityBlockID(run, lumi); - if (!merged) { - MonitorElementData* meData = new MonitorElementData(); - meData->key_ = key; - { - MonitorElementData::Value::Access value(meData->value_); + auto existing = dqmstore->get(key); + if (existing) { + existing->Fill(m_buffer); + } else { + // We make our own MEs here, to avoid a round-trip through the booking API. + MonitorElementData meData; + meData.key_ = key; if (m_kind == MonitorElementData::Kind::INT) - value.scalar.num = m_buffer; + meData.value_.scalar_.num = m_buffer; else if (m_kind == MonitorElementData::Kind::REAL) - value.scalar.real = m_buffer; - } - - mesFromFile[std::make_tuple(run, lumi)].push_back(std::move(meData)); + meData.value_.scalar_.real = m_buffer; + auto me = new MonitorElement(std::move(meData)); + dqmstore->putME(me); } } @@ -355,7 +321,7 @@ class DQMRootSource : public edm::PuttableSourceBase { void readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) override; void readEvent_(edm::EventPrincipal&) override; - // Read MEs from m_fileMetadatas to m_MEsFromFile till run or lumi transition + // Read MEs from m_fileMetadatas to DQMStore till run or lumi transition void readElements(); // True if m_currentIndex points to an element that has a different // run or lumi than the previous element (a transition needs to happen). @@ -364,7 +330,7 @@ class DQMRootSource : public edm::PuttableSourceBase { void readNextItemType(); // These methods will be called by the framework. - // MEs in m_MEsFromFile will be put to products. + // MEs in DQMStore will be put to products. void beginRun(edm::Run& run); void beginLuminosityBlock(edm::LuminosityBlock& lumi); @@ -394,8 +360,6 @@ class DQMRootSource : public edm::PuttableSourceBase { unsigned int m_currentIndex; // All open DQMIO files std::vector m_openFiles; - // MEs read from files and merged if needed. Ready to be put into products - MonitorElementsFromFile m_MEsFromFile; // An item here is a row read from DQMIO indices (metadata) table std::vector m_fileMetadatas; }; @@ -438,7 +402,6 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou m_treeReaders(kNIndicies, std::shared_ptr()), m_currentIndex(0), m_openFiles(std::vector()), - m_MEsFromFile(MonitorElementsFromFile()), m_fileMetadatas(std::vector()) { edm::sortAndRemoveOverlaps(m_lumisToProcess); @@ -459,8 +422,8 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE2D)); } - produces("DQMGenerationRecoRun"); - produces("DQMGenerationRecoLumi"); + produces("DQMGenerationRecoRun"); + produces("DQMGenerationRecoLumi"); } DQMRootSource::~DQMRootSource() { @@ -618,7 +581,7 @@ void DQMRootSource::readElements() { ULong64_t endIndex = metadata.m_lastIndex + 1; for (; index != endIndex; ++index) { - reader->read(index, m_MEsFromFile, metadata.m_run, metadata.m_lumi); + reader->read(index, edm::Service().operator->(), metadata.m_run, metadata.m_lumi); } } } @@ -658,42 +621,13 @@ void DQMRootSource::readNextItemType() { } void DQMRootSource::beginRun(edm::Run& run) { - TRACE("Begin run: " + std::to_string(run.run())) - - std::unique_ptr product = std::make_unique(); - - auto mes = m_MEsFromFile[std::make_tuple(run.run(), 0)]; - - TRACE("Found MEs: " + std::to_string(mes.size())) - - for (MonitorElementData* meData_ptr : mes) { - product->push_back(meData_ptr); - } - + std::unique_ptr product = std::make_unique(); run.put(std::move(product), "DQMGenerationRecoRun"); - - // Remove already processed MEs - m_MEsFromFile[std::make_tuple(run.run(), 0)] = std::vector(); } void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { - TRACE("Begin lumi: " + std::to_string(lumi.luminosityBlock())) - - std::unique_ptr product = std::make_unique(); - - auto mes = m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())]; - - TRACE("Found MEs: " + std::to_string(mes.size())) - - for (MonitorElementData* meData_ptr : mes) { - assert(meData_ptr != nullptr); - product->push_back(meData_ptr); - } - + std::unique_ptr product = std::make_unique(); lumi.put(std::move(product), "DQMGenerationRecoLumi"); - - // Remove already processed MEs - m_MEsFromFile[std::make_tuple(lumi.run(), lumi.luminosityBlock())] = std::vector(); } bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { From bebd00cd234a4b87ea3f6ddfb62022ffefcdb9ed Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 16:25:40 +0100 Subject: [PATCH 025/112] Import matching BuildFile.xml --- DQMServices/FwkIO/plugins/BuildFile.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/DQMServices/FwkIO/plugins/BuildFile.xml b/DQMServices/FwkIO/plugins/BuildFile.xml index 241f02c440603..f18877c32b8bd 100644 --- a/DQMServices/FwkIO/plugins/BuildFile.xml +++ b/DQMServices/FwkIO/plugins/BuildFile.xml @@ -1,5 +1,6 @@ + From 3fc7e5cc07985dba6c1cf33d4e73c1547cbe7885 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 17:04:13 +0100 Subject: [PATCH 026/112] Add changes from @dan131riley as in #28064. Not tested yet. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 94 +++++++++++++++++----- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index bde1dc77b49e3..a7443b1127bbd 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -272,10 +272,10 @@ namespace { // We make our own MEs here, to avoid a round-trip through the booking API. MonitorElementData meData; meData.key_ = key; - if (m_kind == MonitorElementData::Kind::INT) - meData.value_.scalar_.num = m_buffer; - else if (m_kind == MonitorElementData::Kind::REAL) - meData.value_.scalar_.real = m_buffer; + if (m_kind == MonitorElementData::Kind::INT) + meData.value_.scalar_.num = m_buffer; + else if (m_kind == MonitorElementData::Kind::REAL) + meData.value_.scalar_.real = m_buffer; auto me = new MonitorElement(std::move(meData)); dqmstore->putME(me); } @@ -446,9 +446,12 @@ std::unique_ptr DQMRootSource::readFile_() { const int numFiles = m_catalog.fileNames().size(); m_openFiles.reserve(numFiles); - // TODO: add support to fallback files: https://github.com/cms-sw/cmssw/pull/28064/files - for (auto& filename : m_catalog.fileNames()) { + for (auto& fileitem : m_catalog.fileCatalogItems()) { + auto filename = fileitem.fileName(); + auto fallbackname = fileitem.fallbackFileName(); + bool hasFallback = !fallbackname.empty() && fallbackname != filename; TFile* file; + std::list originalInfo; // Try to open a file try { @@ -461,28 +464,83 @@ std::unique_ptr DQMRootSource::readFile_() { std::rethrow_exception(e); } - m_openFiles.insert(m_openFiles.begin(), file); } catch (cms::Exception const& e) { - if (!m_skipBadFiles) { - edm::Exception ex(edm::errors::FileOpenError, "", e); - ex.addContext("Opening DQM Root file"); - ex << "\nInput file " << filename << " was not found, could not be opened, or is corrupted.\n"; - throw ex; + if (!hasFallback) { + if (m_skipBadFiles) { + continue; + } else { + edm::Exception ex(edm::errors::FileOpenError, "", e); + ex.addContext("Opening DQM Root file"); + ex << "\nInput file " << filename << " was not found, could not be opened, or is corrupted.\n"; + throw ex; + } } + originalInfo = e.additionalInfo(); // save in case of fallback error } // Check if a file is usable - if (!file->IsZombie()) { + if (file && !file->IsZombie()) { logFileAction("Successfully opened file ", filename.c_str()); } else { - if (!m_skipBadFiles) { - edm::Exception ex(edm::errors::FileOpenError); - ex << "Input file " << filename.c_str() << " could not be opened.\n"; - ex.addContext("Opening DQM Root file"); - throw ex; + if (!hasFallback) { + if (!m_skipBadFiles) { + edm::Exception ex(edm::errors::FileOpenError); + ex << "Input file " << filename.c_str() << " could not be opened.\n"; + ex.addContext("Opening DQM Root file"); + throw ex; + } + } + if (file) { + delete file; + file = nullptr; + } + } + + if (!file && hasFallback) { + logFileAction(" Initiating request to open fallback file ", fallbackname.c_str()); + try { + { + TDirectory::TContext contextEraser; + file = TFile::Open(fallbackname.c_str()); + } + std::exception_ptr e = edm::threadLocalException::getException(); + if (e != std::exception_ptr()) { + edm::threadLocalException::setException(std::exception_ptr()); + std::rethrow_exception(e); + } + } catch (cms::Exception const& e) { + if (m_skipBadFiles) { + continue; + } else { + edm::Exception ex(edm::errors::FileOpenError, "", e); + ex.addContext("Opening DQM Root file"); + ex << "\nInput file " << filename << " and fallback input file " << fallbackname + << " were not found, could not be opened, or are corrupted.\n"; + for (auto const& s : originalInfo) { + ex.addAdditionalInfo(s); + } + throw ex; + } + } + if (not file->IsZombie()) { + logFileAction(" Successfully opened fallback file ", fallbackname.c_str()); + } else { + if (m_skipBadFiles) { + continue; + } else { + edm::Exception ex(edm::errors::FileOpenError); + ex << "Input file " << filename << " and fallback input file " << fallbackname << " could not be opened.\n"; + ex.addContext("Opening DQM Root file"); + for (auto const& s : originalInfo) { + ex.addAdditionalInfo(s); + } + throw ex; + } } } + m_openFiles.insert(m_openFiles.begin(), file); + // Check file format version, which is encoded in the Title of the TFile if (strcmp(file->GetTitle(), "1") != 0) { edm::Exception ex(edm::errors::FileReadError); From 01bd412599bee3500fef73506377ed47af498125 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 27 Nov 2019 18:35:12 +0100 Subject: [PATCH 027/112] Ooops... some perf fixes ... to make things acceptably fast. The make_tuple in MEComparison is another hotspot, not fixed here. --- DQMServices/Core/src/DQMStore.cc | 2 +- DataFormats/Histograms/interface/MonitorElementCollection.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 572440ade5651..bd221dd31b695 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -455,7 +455,7 @@ namespace dqm::implementation { } MonitorElement* IGetter::get(MonitorElementData::Key const& key) const { - auto meset = store_->globalMEs_[key.id_]; + auto const& meset = store_->globalMEs_[key.id_]; auto it = meset.find(key.path_); if (it != meset.end()) { assert((*it)->getScope() == key.scope_); diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index eae7a443871ae..0c9d5a3ceb5d3 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -181,7 +181,7 @@ struct MonitorElementData { void set(std::string path, Path::Type type) { std::string in(path); std::vector buf; - std::regex dir("^/*([^/]+)"); + static std::regex const dir("^/*([^/]+)"); std::smatch m; while (std::regex_search(in, m, dir)) { From d7d8bbaf89d53117bc72e8d16bf756e6db514438 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 28 Nov 2019 11:27:21 +0100 Subject: [PATCH 028/112] WIP: harvesting test harness --- DQMServices/Demo/test/getfiles | 3 ++ DQMServices/Demo/test/harvesting.py | 56 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 DQMServices/Demo/test/getfiles create mode 100644 DQMServices/Demo/test/harvesting.py diff --git a/DQMServices/Demo/test/getfiles b/DQMServices/Demo/test/getfiles new file mode 100644 index 0000000000000..06d821143c57a --- /dev/null +++ b/DQMServices/Demo/test/getfiles @@ -0,0 +1,3 @@ +RUN=325172 +../../Components/test/dqmiofilecopy.sh /DoubleMuon/Run2018D-12Nov2019_UL2018-v2/DQMIO $RUN >> harvesting.py +echo "process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange('$RUN:1-$(($RUN+1)):0')" >> harvesting.py diff --git a/DQMServices/Demo/test/harvesting.py b/DQMServices/Demo/test/harvesting.py new file mode 100644 index 0000000000000..9b97373e9c17a --- /dev/null +++ b/DQMServices/Demo/test/harvesting.py @@ -0,0 +1,56 @@ + +import FWCore.ParameterSet.Config as cms +process = cms.Process('HARVESTING') + +process.Tracer = cms.Service("Tracer") + +process.load('Configuration.StandardSequences.Services_cff') + +process.source = cms.Source("DQMRootSource", + fileNames = cms.untracked.vstring('') +) + +process.options = cms.untracked.PSet( + FailPath = cms.untracked.vstring(), + IgnoreCompletely = cms.untracked.vstring(), + Rethrow = cms.untracked.vstring('ProductNotFound'), + SkipEvent = cms.untracked.vstring(), + allowUnscheduled = cms.obsolete.untracked.bool, + canDeleteEarly = cms.untracked.vstring(), + emptyRunLumiMode = cms.obsolete.untracked.string, + eventSetup = cms.untracked.PSet( + forceNumberOfConcurrentIOVs = cms.untracked.PSet( + + ), + numberOfConcurrentIOVs = cms.untracked.uint32(1) + ), + fileMode = cms.untracked.string('FULLMERGE'), + forceEventSetupCacheClearOnNewRun = cms.untracked.bool(False), + makeTriggerResults = cms.obsolete.untracked.bool, + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfStreams = cms.untracked.uint32(0), + numberOfThreads = cms.untracked.uint32(1), + printDependencies = cms.untracked.bool(False), + sizeOfStackForThreadsInKB = cms.optional.untracked.uint32, + throwIfIllegalParameter = cms.untracked.bool(True), + wantSummary = cms.untracked.bool(False) +) +process.source.fileNames = cms.untracked.vstring( + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/3C5DD0BD-4370-AD40-BF5C-2FDCE02A327A.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/E78D6DAD-556D-014F-84D4-B244EDE72106.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/81CAE175-9616-954E-B0F3-2C6E82BEBCC9.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/046EF9CC-812A-D441-9C11-3AF6F3D345DB.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/649FC63D-1924-724F-BC17-22626DFA636E.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/9604B80E-2AF1-7840-A8C0-1F689ECC6694.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/BC3E3EE5-4DF0-2A4F-BE3A-DDB72741E8D7.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/C76145B9-E43F-6F49-A3F8-0C8E567A27E2.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/8759972F-8155-6443-AA7A-CFC344539084.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/02342C8D-AA50-BB43-A232-AED595EBAC38.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/69D71662-C61D-C746-BF11-AB0FD4A1079A.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/D6BF8E7C-13AF-3D47-99E7-22C2E044E568.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/838E45E8-4A96-184B-A473-B4B3137EC215.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/E4DBF9BB-C9DC-FA44-9BE1-84CA9AD257D2.root', + 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/2334BC68-85EA-7D49-9ED3-E78A84FF1BEF.root', +) +process.source.lumisToProcess = cms.untracked.VLuminosityBlockRange('325172:1-325173:0') From 03f46f4ba72794670d88a6dac170e22d58c95508 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 28 Nov 2019 18:25:30 +0100 Subject: [PATCH 029/112] Implement reScope option. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 117 ++++++++++++--------- 1 file changed, 67 insertions(+), 50 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index a7443b1127bbd..be353d1c2a7cb 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -144,22 +144,43 @@ namespace { class TreeReaderBase { public: - TreeReaderBase(MonitorElementData::Kind kind) : m_kind(kind) {} + TreeReaderBase(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) + : m_kind(kind), m_rescope(rescope) {} virtual ~TreeReaderBase() {} + MonitorElementData::Key makeKey(std::string const& fullname, int run, int lumi) { + MonitorElementData::Key key; + key.kind_ = m_kind; + key.path_.set(fullname, MonitorElementData::Path::Type::DIR_AND_NAME); + if (m_rescope == MonitorElementData::Scope::LUMI) { + // no rescoping + key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; + key.id_ = edm::LuminosityBlockID(run, lumi); + } else if (m_rescope == MonitorElementData::Scope::RUN) { + // everything becomes run, we'll never see Scope::JOB inside DQMIO files. + key.scope_ = MonitorElementData::Scope::RUN; + key.id_ = edm::LuminosityBlockID(run, 0); + } else if (m_rescope == MonitorElementData::Scope::JOB) { + // Everything is aggregated over the entire job. + key.scope_ = MonitorElementData::Scope::JOB; + key.id_ = edm::LuminosityBlockID(0, 0); + } else { + assert(!"Invalid Scope in rescope option."); + } + return key; + } virtual void read(ULong64_t iIndex, DQMStore* dqmstore, int run, int lumi) = 0; virtual void setTree(TTree* iTree) = 0; protected: MonitorElementData::Kind m_kind; - TTree* m_tree; + MonitorElementData::Scope m_rescope; }; template class TreeObjectReader : public TreeReaderBase { public: - TreeObjectReader(MonitorElementData::Kind kind) - : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(nullptr), m_tag(0) { + TreeObjectReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) { assert(m_kind != MonitorElementData::Kind::INT); assert(m_kind != MonitorElementData::Kind::REAL); assert(m_kind != MonitorElementData::Kind::STRING); @@ -169,12 +190,7 @@ namespace { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElementData::Key key; - key.kind_ = m_kind; - key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); - key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - key.id_ = edm::LuminosityBlockID(run, lumi); - + auto key = makeKey(*m_fullName, run, lumi); auto existing = dqmstore->get(key); if (existing) { // TODO: make sure there is sufficient locking here. @@ -197,16 +213,15 @@ namespace { } private: - TTree* m_tree; - std::string* m_fullName; - T* m_buffer; - uint32_t m_tag; + TTree* m_tree = nullptr; + std::string* m_fullName = nullptr; + T* m_buffer = nullptr; + uint32_t m_tag = 0; }; class TreeStringReader : public TreeReaderBase { public: - TreeStringReader(MonitorElementData::Kind kind) - : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_value(nullptr), m_tag(0) { + TreeStringReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) { assert(m_kind == MonitorElementData::Kind::STRING); } @@ -214,13 +229,9 @@ namespace { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElementData::Key key; - key.kind_ = m_kind; - key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); - key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - key.id_ = edm::LuminosityBlockID(run, lumi); - + auto key = makeKey(*m_fullName, run, lumi); auto existing = dqmstore->get(key); + if (existing) { existing->Fill(*m_value); } else { @@ -241,17 +252,16 @@ namespace { } private: - TTree* m_tree; - std::string* m_fullName; - std::string* m_value; - uint32_t m_tag; + TTree* m_tree = nullptr; + std::string* m_fullName = nullptr; + std::string* m_value = nullptr; + uint32_t m_tag = 0; }; template class TreeSimpleReader : public TreeReaderBase { public: - TreeSimpleReader(MonitorElementData::Kind kind) - : TreeReaderBase(kind), m_tree(nullptr), m_fullName(nullptr), m_buffer(0), m_tag(0) { + TreeSimpleReader(MonitorElementData::Kind kind, MonitorElementData::Scope rescope) : TreeReaderBase(kind, rescope) { assert(m_kind == MonitorElementData::Kind::INT || m_kind == MonitorElementData::Kind::REAL); } @@ -259,13 +269,9 @@ namespace { // This will populate the fields as defined in setTree method m_tree->GetEntry(iIndex); - MonitorElementData::Key key; - key.kind_ = m_kind; - key.path_.set(*m_fullName, MonitorElementData::Path::Type::DIR_AND_NAME); - key.scope_ = lumi == 0 ? MonitorElementData::Scope::RUN : MonitorElementData::Scope::LUMI; - key.id_ = edm::LuminosityBlockID(run, lumi); - + auto key = makeKey(*m_fullName, run, lumi); auto existing = dqmstore->get(key); + if (existing) { existing->Fill(m_buffer); } else { @@ -289,10 +295,10 @@ namespace { } private: - TTree* m_tree; - std::string* m_fullName; - T m_buffer; - uint32_t m_tag; + TTree* m_tree = nullptr; + std::string* m_fullName = nullptr; + T m_buffer = 0; + uint32_t m_tag = 0; }; } // namespace @@ -351,6 +357,7 @@ class DQMRootSource : public edm::PuttableSourceBase { unsigned int m_filterOnRun; edm::InputFileCatalog m_catalog; std::vector m_lumisToProcess; + MonitorElementData::Scope m_rescope; edm::InputSource::ItemType m_nextItemType; // Each ME type gets its own reader @@ -376,6 +383,10 @@ void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& description edm::ParameterSetDescription desc; desc.addUntracked>("fileNames")->setComment("Names of files to be processed."); desc.addUntracked("filterOnRun", 0)->setComment("Just limit the process to the selected run."); + desc.addUntracked("reScope", "") + ->setComment( + "Accumulate histograms more coarsely." + " Options: \"RUN\": turn LUMI histograms into RUN histograms, \"JOB\": turn everything into JOB histograms."); desc.addUntracked("skipBadFiles", false)->setComment("Skip the file if it is not valid"); desc.addUntracked("overrideCatalog", std::string()) ->setComment("An alternate file catalog to use instead of the standard site one."); @@ -398,6 +409,11 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou iPSet.getUntrackedParameter("overrideCatalog")), m_lumisToProcess(iPSet.getUntrackedParameter>( "lumisToProcess", std::vector())), + m_rescope(std::map{ + {"", MonitorElementData::Scope::LUMI}, + {"LUMI", MonitorElementData::Scope::LUMI}, + {"RUN", MonitorElementData::Scope::RUN}, + {"JOB", MonitorElementData::Scope::JOB}}[iPSet.getUntrackedParameter("reScope", "")]), m_nextItemType(edm::InputSource::IsFile), m_treeReaders(kNIndicies, std::shared_ptr()), m_currentIndex(0), @@ -408,18 +424,19 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou if (m_catalog.fileNames().size() == 0) { m_nextItemType = edm::InputSource::IsStop; } else { - m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT)); - m_treeReaders[kFloatIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::REAL)); - m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING)); - m_treeReaders[kTH1FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1F)); - m_treeReaders[kTH1SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1S)); - m_treeReaders[kTH1DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1D)); - m_treeReaders[kTH2FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2F)); - m_treeReaders[kTH2SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2S)); - m_treeReaders[kTH2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2D)); - m_treeReaders[kTH3FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH3F)); - m_treeReaders[kTProfileIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE)); - m_treeReaders[kTProfile2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE2D)); + m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT, m_rescope)); + m_treeReaders[kFloatIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::REAL, m_rescope)); + m_treeReaders[kStringIndex].reset(new TreeStringReader(MonitorElementData::Kind::STRING, m_rescope)); + m_treeReaders[kTH1FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1F, m_rescope)); + m_treeReaders[kTH1SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1S, m_rescope)); + m_treeReaders[kTH1DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH1D, m_rescope)); + m_treeReaders[kTH2FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2F, m_rescope)); + m_treeReaders[kTH2SIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2S, m_rescope)); + m_treeReaders[kTH2DIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH2D, m_rescope)); + m_treeReaders[kTH3FIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TH3F, m_rescope)); + m_treeReaders[kTProfileIndex].reset(new TreeObjectReader(MonitorElementData::Kind::TPROFILE, m_rescope)); + m_treeReaders[kTProfile2DIndex].reset( + new TreeObjectReader(MonitorElementData::Kind::TPROFILE2D, m_rescope)); } produces("DQMGenerationRecoRun"); From d0231af846655dfe1eec1d88a74b9795388b10a3 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 28 Nov 2019 20:11:41 +0100 Subject: [PATCH 030/112] Fix a few bugs to get DQMIO round trip to work. Including reScope for harvesting! --- DQMServices/Core/src/DQMStore.cc | 25 ++++++++++++------- DQMServices/Demo/test/harvesting.py | 19 +++++++++++++- .../FwkIO/plugins/DQMRootOutputModule.cc | 18 ++++++------- DQMServices/FwkIO/plugins/DQMRootSource.cc | 9 +++++++ 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index bd221dd31b695..19e838613668b 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -436,7 +436,7 @@ namespace dqm::implementation { path.set(pathname, MonitorElementData::Path::Type::DIR); // make sure this is normalized by getting it from Path object. auto path_str = path.getFullname(); - auto meset = store_->globalMEs_[edm::LuminosityBlockID(runNumber, lumi)]; + auto const& meset = store_->globalMEs_[edm::LuminosityBlockID(runNumber, lumi)]; auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getFullname().rfind(path_str, 0) == 0) { @@ -487,15 +487,22 @@ namespace dqm::implementation { // This is no more than a guess with concurrent runs/lumis, but should be // correct for purely sequential legacy stuff. // TODO: detect concurrent runs/lumis and disable legacy API in case? - ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); - ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); - - // Delete/recycle MEs after output modules have run. - ar.watchPostGlobalEndRun( - [this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), 0); }); - ar.watchPostGlobalEndLumi([this](edm::GlobalContext const& gc) { - this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().value()); + // TODO: There is no callback for "after the last output module wrote", so + // instead we infer that from seeing the next run/lumi. Won't work with + // concurrent runs/lumis. + ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { + if (this->runlumi_.run() != 0) { + this->cleanupLumi(this->runlumi_.run(), 0); + } + this->setRunLumi(gc.luminosityBlockID()); + }); + ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { + if (this->runlumi_.luminosityBlock() != 0) { + this->cleanupLumi(this->runlumi_.run(), this->runlumi_.luminosityBlock()); + } + this->setRunLumi(gc.luminosityBlockID()); }); + // no cleanup at end of job, we don't really need it. } DQMStore::~DQMStore() {} diff --git a/DQMServices/Demo/test/harvesting.py b/DQMServices/Demo/test/harvesting.py index 9b97373e9c17a..af1af944bfd15 100644 --- a/DQMServices/Demo/test/harvesting.py +++ b/DQMServices/Demo/test/harvesting.py @@ -7,7 +7,8 @@ process.load('Configuration.StandardSequences.Services_cff') process.source = cms.Source("DQMRootSource", - fileNames = cms.untracked.vstring('') + fileNames = cms.untracked.vstring(''), + reScope = cms.untracked.string("RUN") ) process.options = cms.untracked.PSet( @@ -36,6 +37,22 @@ throwIfIllegalParameter = cms.untracked.bool(True), wantSummary = cms.untracked.bool(False) ) +process.out = cms.OutputModule( + "DQMRootOutputModule", + fileName = cms.untracked.string("harvesting_out.root"), + outputCommands = cms.untracked.vstring( + 'keep *' + ) +) +process.dqmSaver = cms.EDAnalyzer("DQMFileSaver", + convention = cms.untracked.string('Offline'), + fileFormat = cms.untracked.string('ROOT'), + producer = cms.untracked.string('DQM'), + workflow = cms.untracked.string('/A/B/C'), + dirName = cms.untracked.string('.'), +) +process.o = cms.EndPath(process.out + process.dqmSaver) + process.source.fileNames = cms.untracked.vstring( 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/3C5DD0BD-4370-AD40-BF5C-2FDCE02A327A.root', 'file:_DoubleMuon_Run2018D-12Nov2019_UL2018-v2_DQMIO/E78D6DAD-556D-014F-84D4-B244EDE72106.root', diff --git a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc index c1f0cd997335a..ebdaa367bcad6 100644 --- a/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc +++ b/DQMServices/FwkIO/plugins/DQMRootOutputModule.cc @@ -384,11 +384,10 @@ void DQMRootOutputModule::writeLuminosityBlock(edm::LuminosityBlockForOutput con return; std::vector items(dstore->getAllContents("", m_run, m_lumi)); for (std::vector::iterator it = items.begin(), itEnd = items.end(); it != itEnd; ++it) { - if ((*it)->getLumiFlag()) { - std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); - assert(itFound != m_dqmKindToTypeIndex.end()); - m_treeHelpers[itFound->second]->fill(*it); - } + assert((*it)->getScope() == MonitorElementData::Scope::LUMI); + std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); + assert(itFound != m_dqmKindToTypeIndex.end()); + m_treeHelpers[itFound->second]->fill(*it); } const edm::ProcessHistoryID& id = iLumi.processHistoryID(); @@ -441,11 +440,10 @@ void DQMRootOutputModule::writeRun(edm::RunForOutput const& iRun) { std::vector items(dstore->getAllContents("", m_run, 0)); for (std::vector::iterator it = items.begin(), itEnd = items.end(); it != itEnd; ++it) { - if (not(*it)->getLumiFlag()) { - std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); - assert(itFound != m_dqmKindToTypeIndex.end()); - m_treeHelpers[itFound->second]->fill(*it); - } + assert((*it)->getScope() == MonitorElementData::Scope::RUN); + std::map::iterator itFound = m_dqmKindToTypeIndex.find((int)(*it)->kind()); + assert(itFound != m_dqmKindToTypeIndex.end()); + m_treeHelpers[itFound->second]->fill(*it); } const edm::ProcessHistoryID& id = iRun.processHistoryID(); diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index be353d1c2a7cb..aab173f9a2769 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -43,6 +43,7 @@ #include "FWCore/Framework/interface/FileBlock.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/MessageLogger/interface/JobReport.h" #include "FWCore/Utilities/interface/TimeOfDay.h" #include "format.h" @@ -630,6 +631,10 @@ void DQMRootSource::readRun_(edm::RunPrincipal& rpCache) { } while (!isRunOrLumiTransition()); readNextItemType(); + + edm::Service jr; + jr->reportInputRunNumber(rpCache.id().run()); + rpCache.fillRunPrincipal(processHistoryRegistryForUpdate()); } void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) { @@ -640,6 +645,10 @@ void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) } while (!isRunOrLumiTransition()); readNextItemType(); + + edm::Service jr; + jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock()); + lbCache.fillLuminosityBlockPrincipal(processHistoryRegistryForUpdate()); } void DQMRootSource::readEvent_(edm::EventPrincipal&) {} From 1a4063d8a819876a27396fe75ca9635a0203a0bd Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 29 Nov 2019 12:10:51 +0100 Subject: [PATCH 031/112] Some random changes to survive 136.85 step3. The main thing is to make sure that only Scope::RUN histograms are used in dqmEndRun. The changes here are more than required, but still not complete. In the longer term, we may want to restrict dqmEndRun to DQMOneEDAnalyzer and completely ban endJob in DQM*EDAnalyzer. The manual setScope would not be required then, since DQMOneEDAnalyzer would use RUN scope by default. Alternatively, we *could* support endRun in edm::stream, running it only on a single stream, but that is unnecessarily complicated. --- .../LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc | 1 + DQM/CastorMonitor/src/CastorMonitorModule.cc | 3 +++ DQMOffline/JetMET/src/METAnalyzer.cc | 8 +++++++- DQMOffline/L1Trigger/interface/L1TEGammaOffline.h | 1 - DQMOffline/L1Trigger/src/L1TEGammaOffline.cc | 6 +++--- .../EcalDigis/src/EcalSelectiveReadoutValidation.cc | 2 ++ Validation/HcalDigis/src/ZDCDigiStudy.cc | 3 +++ Validation/RecoMuon/src/GlobalMuonMatchAnalyzer.cc | 3 +++ 8 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc b/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc index 4bcb9d81a2176..32e33e6243d5c 100644 --- a/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc +++ b/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc @@ -672,6 +672,7 @@ void CorrPCCProducer::resetBlock() { //-------------------------------------------------------------------------------------------------- void CorrPCCProducer::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& iRun, edm::EventSetup const& context) { ibooker.setCurrentFolder("AlCaReco/LumiPCC/"); + ibooker.setScope(MonitorElementData::Scope::RUN); Type1FracMon = ibooker.book1D("type1Fraction", "Type1Fraction;Lumisection;Type 1 Fraction", maxLS, 0, maxLS); Type1ResMon = ibooker.book1D("type1Residual", "Type1Residual;Lumisection;Type 1 Residual", maxLS, 0, maxLS); Type2ResMon = ibooker.book1D("type2Residual", "Type2Residual;Lumisection;Type 2 Residual", maxLS, 0, maxLS); diff --git a/DQM/CastorMonitor/src/CastorMonitorModule.cc b/DQM/CastorMonitor/src/CastorMonitorModule.cc index 710377b210901..99b6f9c018836 100644 --- a/DQM/CastorMonitor/src/CastorMonitorModule.cc +++ b/DQM/CastorMonitor/src/CastorMonitorModule.cc @@ -62,7 +62,10 @@ void CastorMonitorModule::bookHistograms(DQMStore::IBooker &ibooker, const edm::Run &iRun, const edm::EventSetup &iSetup) { if (DigiMon_) { + // Run histos only since there is endRun processing. + auto scope = ibooker.setScope(MonitorElementData::Scope::RUN); DigiMon_->bookHistograms(ibooker, iRun, iSetup); + ibooker.setScope(scope); } if (RecHitMon_) { RecHitMon_->bookHistograms(ibooker, iRun); diff --git a/DQMOffline/JetMET/src/METAnalyzer.cc b/DQMOffline/JetMET/src/METAnalyzer.cc index 1b644d69c9b8d..f1d47af9e480c 100644 --- a/DQMOffline/JetMET/src/METAnalyzer.cc +++ b/DQMOffline/JetMET/src/METAnalyzer.cc @@ -196,6 +196,9 @@ METAnalyzer::~METAnalyzer() { void METAnalyzer::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& iRun, edm::EventSetup const&) { std::string DirName = FolderName_ + metCollectionLabel_.label(); ibooker.setCurrentFolder(DirName); + // since this module does things in dqmEndRun, we need to make sure to have + // per-run histograms. + ibooker.setScope(MonitorElementData::Scope::RUN); if (!folderNames_.empty()) { folderNames_.clear(); @@ -334,8 +337,11 @@ void METAnalyzer::bookMonitorElement(std::string DirName, hMET = ibooker.book1D("MET", "MET", 200, 0, 1000); hMET_2 = ibooker.book1D("MET_2", "MET Range 2", 200, 0, 2000); hSumET = ibooker.book1D("SumET", "SumET", 400, 0, 4000); + + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); hMETSig = ibooker.book1D("METSig", "METSig", 51, 0, 51); - hMETSig->setLumiFlag(); + ibooker.setScope(scope); + hMETPhi = ibooker.book1D("METPhi", "METPhi", 60, -M_PI, M_PI); hMET_logx = ibooker.book1D("MET_logx", "MET_logx", 40, -1, 9); hSumET_logx = ibooker.book1D("SumET_logx", "SumET_logx", 40, -1, 9); diff --git a/DQMOffline/L1Trigger/interface/L1TEGammaOffline.h b/DQMOffline/L1Trigger/interface/L1TEGammaOffline.h index 3e8ae02033f4a..e8e386c417d06 100644 --- a/DQMOffline/L1Trigger/interface/L1TEGammaOffline.h +++ b/DQMOffline/L1Trigger/interface/L1TEGammaOffline.h @@ -49,7 +49,6 @@ class L1TEGammaOffline : public DQMOneEDAnalyzer<> { void bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) override; void analyze(edm::Event const& e, edm::EventSetup const& eSetup) override; void dqmEndRun(edm::Run const& run, edm::EventSetup const& eSetup) override; - void endJob() override; private: bool passesLooseEleId(reco::GsfElectron const& electron) const; diff --git a/DQMOffline/L1Trigger/src/L1TEGammaOffline.cc b/DQMOffline/L1Trigger/src/L1TEGammaOffline.cc index 0640a7bc24302..b2dc84d1e1744 100644 --- a/DQMOffline/L1Trigger/src/L1TEGammaOffline.cc +++ b/DQMOffline/L1Trigger/src/L1TEGammaOffline.cc @@ -572,7 +572,7 @@ void L1TEGammaOffline::fillPhotons(edm::Event const& e, const unsigned int nVert // -------------------------------------- endRun -------------------------------------------- // void L1TEGammaOffline::dqmEndRun(edm::Run const& run, edm::EventSetup const& eSetup) { - edm::LogInfo("L1TEGammaOffline") << "L1TEGammaOffline::endRun" << std::endl; + normalise2DHistogramsToBinArea(); } // @@ -581,6 +581,8 @@ void L1TEGammaOffline::dqmEndRun(edm::Run const& run, edm::EventSetup const& eSe void L1TEGammaOffline::bookElectronHistos(DQMStore::IBooker& ibooker) { ibooker.cd(); ibooker.setCurrentFolder(histFolder_); + // since there are endRun manipulations. + ibooker.setScope(MonitorElementData::Scope::RUN); dqmoffline::l1t::HistDefinition nVertexDef = histDefinitions_[PlotConfig::nVertex]; h_nVertex_ = ibooker.book1D(nVertexDef.name, nVertexDef.title, nVertexDef.nbinsX, nVertexDef.xmin, nVertexDef.xmax); @@ -941,8 +943,6 @@ void L1TEGammaOffline::bookPhotonHistos(DQMStore::IBooker& ibooker) { ibooker.cd(); } -void L1TEGammaOffline::endJob() { normalise2DHistogramsToBinArea(); } - void L1TEGammaOffline::normalise2DHistogramsToBinArea() { std::vector monElementstoNormalize = {h_L1EGammaETvsElectronET_EB_, h_L1EGammaETvsElectronET_EE_, diff --git a/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc b/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc index ad7e1024c877b..91ad29b44e36c 100644 --- a/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc +++ b/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc @@ -857,7 +857,9 @@ void EcalSelectiveReadoutValidation::bookHistograms(DQMStore::IBooker& ibooker, edm::EventSetup const&) { ibooker.setCurrentFolder("EcalDigisV/SelectiveReadout"); + auto scope = ibooker.setScope(MonitorElementData::Scope::RUN); meL1aRate_ = bookFloat(ibooker, "l1aRate_"); + ibooker.setScope(scope); meDccVol_ = bookProfile(ibooker, "hDccVol", //"EcalDccEventSizeComputed", diff --git a/Validation/HcalDigis/src/ZDCDigiStudy.cc b/Validation/HcalDigis/src/ZDCDigiStudy.cc index e79cf9a276c9c..18415bcffd0fc 100644 --- a/Validation/HcalDigis/src/ZDCDigiStudy.cc +++ b/Validation/HcalDigis/src/ZDCDigiStudy.cc @@ -40,6 +40,9 @@ ZDCDigiStudy::~ZDCDigiStudy() {} void ZDCDigiStudy::bookHistograms(DQMStore::IBooker& ib, edm::Run const& run, edm::EventSetup const& es) { ib.setCurrentFolder("ZDCDigiValidation"); + // run histos only since there is dqmEndRun processing. + ib.setScope(MonitorElementData::Scope::RUN); + //Histograms for Hits ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// //# Below we are filling the histograms made in the .h file. The syntax is as follows: # diff --git a/Validation/RecoMuon/src/GlobalMuonMatchAnalyzer.cc b/Validation/RecoMuon/src/GlobalMuonMatchAnalyzer.cc index c3a78454034f7..0b6a2affa228f 100644 --- a/Validation/RecoMuon/src/GlobalMuonMatchAnalyzer.cc +++ b/Validation/RecoMuon/src/GlobalMuonMatchAnalyzer.cc @@ -225,6 +225,9 @@ void GlobalMuonMatchAnalyzer::bookHistograms(DQMStore::IBooker &ibooker, // Tk Associator ibooker.cd(); + // Run histos only for dqmEndRun handling + ibooker.setScope(MonitorElementData::Scope::RUN); + std::string dirName = "Matcher/"; // ibooker.setCurrentFolder("RecoMuonV/Matcher"); ibooker.setCurrentFolder(dirName); From 11c3be50b555280aeb77f37bdc679a8a94d5845f Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 29 Nov 2019 17:51:30 +0100 Subject: [PATCH 032/112] Make DummyFillDQMStore.cc slightly less legacy. --- DQMServices/FwkIO/test/DummyFillDQMStore.cc | 45 +++++++++---------- .../FwkIO/test/create_empty_file_cfg.py | 2 +- DQMServices/FwkIO/test/create_file1_cfg.py | 2 +- DQMServices/FwkIO/test/create_file2_cfg.py | 2 +- DQMServices/FwkIO/test/create_file3_cfg.py | 2 +- DQMServices/FwkIO/test/create_file4_cfg.py | 2 +- .../FwkIO/test/create_file_multi_types_cfg.py | 2 +- .../FwkIO/test/create_lumi_only_file_cfg.py | 2 +- ...eate_one_run_one_lumi_run_only_file_cfg.py | 2 +- .../FwkIO/test/create_run_lumi_file_cfg.py | 2 +- .../FwkIO/test/create_run_only_file_cfg.py | 2 +- 11 files changed, 31 insertions(+), 34 deletions(-) diff --git a/DQMServices/FwkIO/test/DummyFillDQMStore.cc b/DQMServices/FwkIO/test/DummyFillDQMStore.cc index 0e263576f56ed..0414b892c319e 100644 --- a/DQMServices/FwkIO/test/DummyFillDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyFillDQMStore.cc @@ -20,15 +20,13 @@ #include // user include files -#include "FWCore/Framework/interface/Frameworkfwd.h" -#include "FWCore/Framework/interface/EDAnalyzer.h" +#include "DQMServices/Core/interface/DQMOneEDAnalyzer.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "DQMServices/Core/interface/DQMStore.h" #include "FWCore/ServiceRegistry/interface/Service.h" // @@ -116,24 +114,25 @@ namespace { } // namespace -class DummyFillDQMStore : public edm::EDAnalyzer { +class DummyFillDQMStore : public DQMOneLumiEDAnalyzer<> { public: explicit DummyFillDQMStore(const edm::ParameterSet&); ~DummyFillDQMStore(); static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); -private: - virtual void beginJob(); - virtual void analyze(const edm::Event&, const edm::EventSetup&); - virtual void endJob(); + virtual void bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) override; + virtual void analyze(edm::Event const&, edm::EventSetup const&) override; + virtual void dqmEndRun(edm::Run const&, edm::EventSetup const&) override; + virtual void dqmBeginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) override; + virtual void dqmEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) override; + - virtual void beginRun(edm::Run const&, edm::EventSetup const&); - virtual void endRun(edm::Run const&, edm::EventSetup const&); - virtual void beginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&); - virtual void endLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&); + +private: // ----------member data --------------------------- + edm::ParameterSet iConfig; std::vector > m_runFillers; std::vector > m_lumiFillers; bool m_fillRuns; @@ -154,11 +153,17 @@ class DummyFillDQMStore : public edm::EDAnalyzer { DummyFillDQMStore::DummyFillDQMStore(const edm::ParameterSet& iConfig) : m_fillRuns(iConfig.getUntrackedParameter("fillRuns")), m_fillLumis(iConfig.getUntrackedParameter("fillLumis")) { + this->iConfig = iConfig; +} + +void DummyFillDQMStore::bookHistograms(DQMStore::IBooker&, edm::Run const&, edm::EventSetup const&) { + // this is fine as long as we don't do concurrent booking. edm::Service dstore; typedef std::vector PSets; const PSets& elements = iConfig.getUntrackedParameter >("elements"); if (m_fillRuns) { + dstore->setScope(MonitorElementData::Scope::RUN); m_runFillers.reserve(elements.size()); for (PSets::const_iterator it = elements.begin(), itEnd = elements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { @@ -173,6 +178,7 @@ DummyFillDQMStore::DummyFillDQMStore(const edm::ParameterSet& iConfig) } if (m_fillLumis) { + dstore->setScope(MonitorElementData::Scope::LUMI); m_lumiFillers.reserve(elements.size()); for (PSets::const_iterator it = elements.begin(), itEnd = elements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { @@ -217,17 +223,8 @@ void DummyFillDQMStore::analyze(edm::Event const& iEvent, edm::EventSetup const& */ } -// ------------ method called once each job just before starting event loop ------------ -void DummyFillDQMStore::beginJob() {} - -// ------------ method called once each job just after ending the event loop ------------ -void DummyFillDQMStore::endJob() {} - -// ------------ method called when starting to processes a run ------------ -void DummyFillDQMStore::beginRun(edm::Run const&, edm::EventSetup const&) {} - // ------------ method called when ending the processing of a run ------------ -void DummyFillDQMStore::endRun(edm::Run const&, edm::EventSetup const&) { +void DummyFillDQMStore::dqmEndRun(edm::Run const&, edm::EventSetup const&) { for (std::vector >::iterator it = m_runFillers.begin(), itEnd = m_runFillers.end(); it != itEnd; ++it) { @@ -236,7 +233,7 @@ void DummyFillDQMStore::endRun(edm::Run const&, edm::EventSetup const&) { } // ------------ method called when starting to processes a luminosity block ------------ -void DummyFillDQMStore::beginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) { +void DummyFillDQMStore::dqmBeginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) { for (std::vector >::iterator it = m_lumiFillers.begin(), itEnd = m_lumiFillers.end(); it != itEnd; ++it) { @@ -245,7 +242,7 @@ void DummyFillDQMStore::beginLuminosityBlock(edm::LuminosityBlock const&, edm::E } // ------------ method called when ending the processing of a luminosity block ------------ -void DummyFillDQMStore::endLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) { +void DummyFillDQMStore::dqmEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) { for (std::vector >::iterator it = m_lumiFillers.begin(), itEnd = m_lumiFillers.end(); it != itEnd; ++it) { diff --git a/DQMServices/FwkIO/test/create_empty_file_cfg.py b/DQMServices/FwkIO/test/create_empty_file_cfg.py index af5d5582c3e52..fa41bf131ba53 100644 --- a/DQMServices/FwkIO/test/create_empty_file_cfg.py +++ b/DQMServices/FwkIO/test/create_empty_file_cfg.py @@ -16,7 +16,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_file1_cfg.py b/DQMServices/FwkIO/test/create_file1_cfg.py index dca0d853c14b2..f4b8bd4c3583f 100644 --- a/DQMServices/FwkIO/test/create_file1_cfg.py +++ b/DQMServices/FwkIO/test/create_file1_cfg.py @@ -16,7 +16,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_file2_cfg.py b/DQMServices/FwkIO/test/create_file2_cfg.py index 582253f77dda4..340cfdd136ed2 100644 --- a/DQMServices/FwkIO/test/create_file2_cfg.py +++ b/DQMServices/FwkIO/test/create_file2_cfg.py @@ -16,7 +16,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_file3_cfg.py b/DQMServices/FwkIO/test/create_file3_cfg.py index d063b0db214a2..486784bca40f5 100644 --- a/DQMServices/FwkIO/test/create_file3_cfg.py +++ b/DQMServices/FwkIO/test/create_file3_cfg.py @@ -17,7 +17,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i+1))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_file4_cfg.py b/DQMServices/FwkIO/test/create_file4_cfg.py index cc11bddc4e5ae..f87d7456b920f 100644 --- a/DQMServices/FwkIO/test/create_file4_cfg.py +++ b/DQMServices/FwkIO/test/create_file4_cfg.py @@ -18,7 +18,7 @@ # A dummy tracked parameter is added to force the ProcessHistoryID to be different # It serves no other purpose. -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True), diff --git a/DQMServices/FwkIO/test/create_file_multi_types_cfg.py b/DQMServices/FwkIO/test/create_file_multi_types_cfg.py index 4297ed4a01c3d..6eaaa0918e213 100644 --- a/DQMServices/FwkIO/test/create_file_multi_types_cfg.py +++ b/DQMServices/FwkIO/test/create_file_multi_types_cfg.py @@ -22,7 +22,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_lumi_only_file_cfg.py b/DQMServices/FwkIO/test/create_lumi_only_file_cfg.py index 548011decaddf..6a770d4ea9729 100644 --- a/DQMServices/FwkIO/test/create_lumi_only_file_cfg.py +++ b/DQMServices/FwkIO/test/create_lumi_only_file_cfg.py @@ -13,7 +13,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(False), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_one_run_one_lumi_run_only_file_cfg.py b/DQMServices/FwkIO/test/create_one_run_one_lumi_run_only_file_cfg.py index 197b34f9dd12b..9263c27a3cf80 100644 --- a/DQMServices/FwkIO/test/create_one_run_one_lumi_run_only_file_cfg.py +++ b/DQMServices/FwkIO/test/create_one_run_one_lumi_run_only_file_cfg.py @@ -13,7 +13,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(False)) diff --git a/DQMServices/FwkIO/test/create_run_lumi_file_cfg.py b/DQMServices/FwkIO/test/create_run_lumi_file_cfg.py index ee333cd05b7a5..33cc14c10446e 100644 --- a/DQMServices/FwkIO/test/create_run_lumi_file_cfg.py +++ b/DQMServices/FwkIO/test/create_run_lumi_file_cfg.py @@ -13,7 +13,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(True)) diff --git a/DQMServices/FwkIO/test/create_run_only_file_cfg.py b/DQMServices/FwkIO/test/create_run_only_file_cfg.py index 01eb03d3deceb..1e06e191c8d42 100644 --- a/DQMServices/FwkIO/test/create_run_only_file_cfg.py +++ b/DQMServices/FwkIO/test/create_run_only_file_cfg.py @@ -13,7 +13,7 @@ title=cms.untracked.string("Foo"+str(i)), value=cms.untracked.double(i))) -process.filler = cms.EDAnalyzer("DummyFillDQMStore", +process.filler = cms.EDProducer("DummyFillDQMStore", elements=cms.untracked.VPSet(*elements), fillRuns = cms.untracked.bool(True), fillLumis = cms.untracked.bool(False)) From 1ed3afda085b657f4b94d140880a0d28c9e1a523 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 29 Nov 2019 17:52:01 +0100 Subject: [PATCH 033/112] Call enterLumi also before booking. This is needed if there are multiple runs, and we book multiple times, to make sure the old histograms are already prepared in booking. --- DQMServices/Core/interface/DQMOneEDAnalyzer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/DQMServices/Core/interface/DQMOneEDAnalyzer.h b/DQMServices/Core/interface/DQMOneEDAnalyzer.h index c9317275caf90..6a67d757d2ac7 100644 --- a/DQMServices/Core/interface/DQMOneEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMOneEDAnalyzer.h @@ -32,6 +32,7 @@ class DQMOneEDAnalyzer } void beginRun(edm::Run const& run, edm::EventSetup const& setup) final { + edm::Service()->enterLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); dqmBeginRun(run, setup); edm::Service()->bookTransaction( [this, &run, &setup](DQMStore::IBooker& booker) { From 906e8c1fdf2201781b4fc3fefaf773bc49e28a19 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 2 Dec 2019 14:52:44 +0100 Subject: [PATCH 034/112] Some fixes to make more DQMIO tests pass. Involves bugfixes in DQMStore as well as the DQMRootSource. The current cleanup logic is pretty much dommed to fail, and it is not really possible to get DummyReadDQMStore (relying on legacy emulation behavior) to work now. We could fix DummyReadDQMStore to not rely on the legacy-keep-all-pointers-avlive behavior, but actually we should keep supporting that for whatever else needs it. --- DQMServices/Core/src/DQMStore.cc | 34 +++++++++++++++------ DQMServices/FwkIO/plugins/DQMRootSource.cc | 28 ++++++++++++++++- DQMServices/FwkIO/test/DummyFillDQMStore.cc | 5 ++- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 19e838613668b..8020fc956be6b 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -55,8 +55,7 @@ namespace dqm::implementation { path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); // We should check if there is a local ME for this module and name already. - // However, there is nothing wrong with creating a new local ME and then - // having putME() drop it, so we do that (it's easier). + // However, it is easier to do that in putME(). MonitorElement* me = store_->findME(path); store_->printTrace("Booking " + std::string(name) + (me ? " (existing)" : " (new)")); @@ -103,8 +102,8 @@ namespace dqm::implementation { // in a container associated with the module (and potentially run, for // DQMGlobalEDAnalyzer). This will later be update to point to different // MEData (kept in a global ME) as needed. - MonitorElement* local_me = new MonitorElement(me); - me = store_->putME(local_me, this->moduleID_); + // putME creates the local ME object as needed. + me = store_->putME(me, this->moduleID_); // me now points to a local ME owned by the DQMStore. assert(me); return me; @@ -128,15 +127,29 @@ namespace dqm::implementation { MonitorElement* DQMStore::putME(MonitorElement* me, uint64_t moduleID) { auto lock = std::scoped_lock(this->booking_mutex_); assert(me); - auto existing_new = localMEs_[moduleID].insert(me); - if (existing_new.second == true) { + auto& localmes = localMEs_[moduleID]; + auto existing = localmes.find(me); + if (existing == localmes.end()) { + // insert new local ME + MonitorElement* local_me = new MonitorElement(me); + auto existing_new = localmes.insert(local_me); // successfully inserted, return new object - return me; + assert(existing_new.second == true); // insert successful + return local_me; } else { // already present, return old object + auto local_me = *existing; edm::LogInfo("DQMStore") << "ME " << me->getFullname() << " booked twice in the same module."; - delete me; - return *(existing_new.first); + // the existing local ME might not have data attached (e.g. in 2nd run) + // in that case, we attach the global ME provided by booking above. + // This may be a prototype or of a random run/lumi, but it ensures that + // even LUMI histos are always valid after booking (as we promise for + // legacy modules -- for sequential runs/lumis, there is only ever one + // global ME, and the local one points to it). + if (!local_me->isValid()) { + local_me->switchData(me); + } + return local_me; } } @@ -494,6 +507,9 @@ namespace dqm::implementation { if (this->runlumi_.run() != 0) { this->cleanupLumi(this->runlumi_.run(), 0); } + if (this->runlumi_.luminosityBlock() != 0) { + this->cleanupLumi(this->runlumi_.run(), this->runlumi_.luminosityBlock()); + } this->setRunLumi(gc.luminosityBlockID()); }); ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index aab173f9a2769..3d06275b86d0a 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -592,6 +592,27 @@ std::unique_ptr DQMRootSource::readFile_() { // Sort to make sure runs and lumis appear in sequential order std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); + // If we have lumisections without matching runs, insert dummy runs here. + unsigned int run = 0; + auto toadd = std::vector(); + for (auto& metadata : m_fileMetadatas) { + if (run < metadata.m_run && metadata.m_lumi != 0) { + // run transition and lumi transition at the same time! + FileMetadata dummy{0}; // zero initialize + dummy.m_run = metadata.m_run; + dummy.m_lumi = 0; + dummy.m_type = kNoTypesStored; + toadd.push_back(dummy); + } + run = metadata.m_run; + } + + if (!toadd.empty()) { + // rather than trying to insert at the right places, just append and sort again. + m_fileMetadatas.insert(m_fileMetadatas.end(), toadd.begin(), toadd.end()); + std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); + } + for (auto& metadata : m_fileMetadatas) metadata.describe(); @@ -715,8 +736,13 @@ void DQMRootSource::beginLuminosityBlock(edm::LuminosityBlock& lumi) { } bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi) const { - if (run == m_filterOnRun) + if (m_filterOnRun != 0 && run != m_filterOnRun) { + return false; + } + + if (m_lumisToProcess.size() == 0) { return true; + } for (edm::LuminosityBlockRange const& lumiToProcess : m_lumisToProcess) { if (run >= lumiToProcess.startRun() && run <= lumiToProcess.endRun()) { diff --git a/DQMServices/FwkIO/test/DummyFillDQMStore.cc b/DQMServices/FwkIO/test/DummyFillDQMStore.cc index 0414b892c319e..21f060bc211a7 100644 --- a/DQMServices/FwkIO/test/DummyFillDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyFillDQMStore.cc @@ -127,10 +127,7 @@ class DummyFillDQMStore : public DQMOneLumiEDAnalyzer<> { virtual void dqmBeginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) override; virtual void dqmEndLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) override; - - private: - // ----------member data --------------------------- edm::ParameterSet iConfig; std::vector > m_runFillers; @@ -164,6 +161,7 @@ void DummyFillDQMStore::bookHistograms(DQMStore::IBooker&, edm::Run const&, edm: const PSets& elements = iConfig.getUntrackedParameter >("elements"); if (m_fillRuns) { dstore->setScope(MonitorElementData::Scope::RUN); + m_runFillers.clear(); m_runFillers.reserve(elements.size()); for (PSets::const_iterator it = elements.begin(), itEnd = elements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { @@ -179,6 +177,7 @@ void DummyFillDQMStore::bookHistograms(DQMStore::IBooker&, edm::Run const&, edm: if (m_fillLumis) { dstore->setScope(MonitorElementData::Scope::LUMI); + m_lumiFillers.clear(); m_lumiFillers.reserve(elements.size()); for (PSets::const_iterator it = elements.begin(), itEnd = elements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { From 2be8eefab76e53efcf3c95967842cb1f30784f6d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 2 Dec 2019 19:15:02 +0100 Subject: [PATCH 035/112] Make DummyReadDQMStore re-get its MEs. This is the recommended way to do it, and works even if we allocate to many copies. --- DQMServices/FwkIO/test/DummyReadDQMStore.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/DQMServices/FwkIO/test/DummyReadDQMStore.cc b/DQMServices/FwkIO/test/DummyReadDQMStore.cc index fcc1256235dbc..e0228f79ae3a6 100644 --- a/DQMServices/FwkIO/test/DummyReadDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyReadDQMStore.cc @@ -63,11 +63,9 @@ namespace { virtual ~TH1FReader(){}; void read() { + m_element = m_store->get(m_name); if (0 == m_element) { - m_element = m_store->get(m_name); - if (0 == m_element) { - throw cms::Exception("MissingElement") << "The element: " << m_name << " was not found"; - } + throw cms::Exception("MissingElement") << "The element: " << m_name << " was not found"; } TH1F* hist = m_element->getTH1F(); assert(m_indexToCheck < m_means.size()); From f9ea7f37a5a948904e2bbb7d8876cacff2ae91b7 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 2 Dec 2019 19:16:26 +0100 Subject: [PATCH 036/112] Fix DQMRootSource to properly recycle MEs. Since the MEs read in the input source don't have local MEs with them, the enterLumi logic for recycling does not apply. But we need to recycle them, to hold up the guarantees for legacy modules. --- DQMServices/Core/interface/DQMStore.h | 2 ++ DQMServices/Core/src/DQMStore.cc | 41 ++++++++++++++++++++++ DQMServices/FwkIO/plugins/DQMRootSource.cc | 6 ++-- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 47dc95771e18f..705352b0694e1 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -577,6 +577,8 @@ namespace dqm { // (e.g. when meBookerGetter is called *inside* a booking transaction). }; + // For input modules: trigger recycling without local ME/enterLumi/moduleID. + MonitorElement* findOrRecycle(MonitorElementData::Key const&); // modules are expected to call these callbacks when they change run/lumi. // The DQMStore then updates the module's MEs, potentially cloning them // if there are concurrent runs/lumis. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 8020fc956be6b..81398d60c4a8c 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -236,6 +236,47 @@ namespace dqm::implementation { }); } + MonitorElement* DQMStore::findOrRecycle(MonitorElementData::Key const& key) { + // This is specifically for DQMRootSource, or other input modules. These + // are special in that they use the legacy interface (no moduleID, no local + // MEs) but need to be able to handle concurrent lumisections correctly. + // The logic is very similar to that in enterLumi; this is enterLumi for + // Input Modules. + auto lock = std::scoped_lock(this->booking_mutex_); + auto existing = this->get(key); + if (existing) { + // exactly matching ME found, needs merging with the new data. + return existing; + } // else + + // this is where we'd expect the ME. + auto& targetset = this->globalMEs_[key.id_]; + // this is where we can get MEs to reuse. + auto& prototypes = this->globalMEs_[edm::LuminosityBlockID()]; + + auto proto = prototypes.find(key.path_); + if (proto != prototypes.end()) { + MonitorElement* oldme = *proto; + assert(oldme->getScope() == key.scope_); + prototypes.erase(proto); + auto medata = oldme->release(/* expectOwned */ true); // destroy the ME, get its data. + // in this situation, nobody should be filling the ME concurrently. + medata->data_.key_.id_ = key.id_; + // We reuse the ME object here, even if we don't have to. This ensures + // that when running single-threaded without concurrent lumis/runs, + // the global MEs will also live forever and allow legacy usages. + oldme->switchData(medata); + auto result = targetset.insert(oldme); + assert(result.second); // was new insertion + auto newme = *result.first; // iterator to new ME + assert(oldme == newme); // recycling! + // newme is reset and ready to accept data. + return newme; + } // else + + return nullptr; + } + void DQMStore::enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID) { // Make sure global MEs for the run/lumi exist (depending on scope), and // point the local MEs for this module to these global MEs. diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 3d06275b86d0a..c8523bf7a4e01 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -192,7 +192,7 @@ namespace { m_tree->GetEntry(iIndex); auto key = makeKey(*m_fullName, run, lumi); - auto existing = dqmstore->get(key); + auto existing = dqmstore->findOrRecycle(key); if (existing) { // TODO: make sure there is sufficient locking here. DQMMergeHelper::mergeTogether(existing->getTH1(), m_buffer); @@ -231,7 +231,7 @@ namespace { m_tree->GetEntry(iIndex); auto key = makeKey(*m_fullName, run, lumi); - auto existing = dqmstore->get(key); + auto existing = dqmstore->findOrRecycle(key); if (existing) { existing->Fill(*m_value); @@ -271,7 +271,7 @@ namespace { m_tree->GetEntry(iIndex); auto key = makeKey(*m_fullName, run, lumi); - auto existing = dqmstore->get(key); + auto existing = dqmstore->findOrRecycle(key); if (existing) { existing->Fill(m_buffer); From 214a997fa871c2364af86ed4aab24956a48be338 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 3 Dec 2019 12:47:20 +0100 Subject: [PATCH 037/112] Update test to expect merging jobs in any case. I don't see how the old behavior was ever useful. Also filterOnRun is unused apart from tests, it should be removed. The tests should rather test lumisToProcess, which is used in production. --- ...ck_merged_file1_file3_file2_filterOnRun1_cfg.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/DQMServices/FwkIO/test/check_merged_file1_file3_file2_filterOnRun1_cfg.py b/DQMServices/FwkIO/test/check_merged_file1_file3_file2_filterOnRun1_cfg.py index 283b47172e1c0..0ec5ca6a77804 100644 --- a/DQMServices/FwkIO/test/check_merged_file1_file3_file2_filterOnRun1_cfg.py +++ b/DQMServices/FwkIO/test/check_merged_file1_file3_file2_filterOnRun1_cfg.py @@ -14,16 +14,11 @@ nRuns = 1 nHists = 10 nLumiPerRun = 20 +nJobsPerRun = 2 startIndex = 0 lastIndex =-1 for i in range(0,nRuns): for l in range(0,nLumiPerRun): - if l == 10: - for j in range(0,nHists): - lastIndex +=1 - values.append(("Foo"+str(j), 0, 1.0)) - expectedIndices.append( (i+1,0,3,startIndex,lastIndex) ) - startIndex = lastIndex+1 for j in range(0,nHists): lastIndex +=1 values.append(("Foo"+str(j)+"_lumi", 0, 1.0)) @@ -31,17 +26,16 @@ startIndex = lastIndex+1 for j in range(0,nHists): lastIndex +=1 - values.append(("Foo"+str(j), 0, 1.0)) + values.append(("Foo"+str(j), 0, 1.0*nJobsPerRun)) expectedIndices.append( (i+1,0,3,startIndex,lastIndex) ) startIndex = lastIndex+1 - -expected = 2*nRuns*nHists+nRuns*nLumiPerRun*nHists +expected = nRuns*nHists+nRuns*nLumiPerRun*nHists if expected != th1fs.GetEntries(): print("wrong number of entries in TH1Fs",th1fs.GetEntries(),"expected",expected) sys.exit(1) -if 2*nRuns+nRuns*nLumiPerRun != indices.GetEntries(): +if nRuns+nRuns*nLumiPerRun != indices.GetEntries(): print("wrong number of entries in Indices", indices.GetEntries()) sys.exit(1) From e49ba5151599abec5b700b49757f04821ec2e66d Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 3 Dec 2019 15:49:52 +0100 Subject: [PATCH 038/112] Fix DQMIO unit tests. The main change is to make DummyReadDQMStore less stupid so it does not rely on the sequential ordering of things. There is a semantic change in read_merged_file1_file3_file4_cfg.py, which expected run to be not merged, while the new code merges whatever makes sense. --- DQMServices/FwkIO/test/DummyReadDQMStore.cc | 113 ++++++------------ .../FwkIO/test/create_empty_file_cfg.py | 4 + DQMServices/FwkIO/test/create_file1_cfg.py | 4 + DQMServices/FwkIO/test/create_file2_cfg.py | 4 + DQMServices/FwkIO/test/create_file3_cfg.py | 4 + DQMServices/FwkIO/test/create_file4_cfg.py | 4 + .../FwkIO/test/read_file1_file2_cfg.py | 4 + .../FwkIO/test/read_file1_file3_cfg.py | 10 +- .../FwkIO/test/read_merged_file1_file2_cfg.py | 4 + .../test/read_merged_file1_file3_file2_cfg.py | 10 +- .../test/read_merged_file1_file3_file4_cfg.py | 21 ++-- 11 files changed, 87 insertions(+), 95 deletions(-) diff --git a/DQMServices/FwkIO/test/DummyReadDQMStore.cc b/DQMServices/FwkIO/test/DummyReadDQMStore.cc index e0228f79ae3a6..92f8556d98347 100644 --- a/DQMServices/FwkIO/test/DummyReadDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyReadDQMStore.cc @@ -41,17 +41,19 @@ namespace { class ReaderBase { public: virtual ~ReaderBase() = default; - virtual void read() = 0; + virtual void read(int run, int lumi) = 0; }; - class TH1FReader : public ReaderBase { + class TH1Reader : public ReaderBase { public: - TH1FReader(const edm::ParameterSet& iPSet, DQMStore& iStore, bool iSetLumiFlag) + TH1Reader(const edm::ParameterSet& iPSet, DQMStore& iStore, bool iSetLumiFlag) : m_store(&iStore), m_element(0), + m_runs(iPSet.getUntrackedParameter >("runs")), + m_lumis(iPSet.getUntrackedParameter >("lumis")), m_means(iPSet.getUntrackedParameter >("means")), - m_entries(iPSet.getUntrackedParameter >("entries")), - m_indexToCheck(0) { + m_entries(iPSet.getUntrackedParameter >("entries")) + { assert(m_means.size() == m_entries.size()); std::string extension; if (iSetLumiFlag) { @@ -60,90 +62,45 @@ namespace { m_name = iPSet.getUntrackedParameter("name") + extension; } - virtual ~TH1FReader(){}; + virtual ~TH1Reader(){}; + + void read(int run, int lumi) { + double expected_mean = -1, expected_entries = -1; + for (unsigned int i = 0; i < m_runs.size(); i++) { + if (m_runs[i] == run && m_lumis[i] == lumi) { + expected_mean = m_means[i]; + expected_entries = m_entries[i]; + } + } + assert(expected_entries != -1 || !"Unexpected run/lumi!"); - void read() { m_element = m_store->get(m_name); if (0 == m_element) { throw cms::Exception("MissingElement") << "The element: " << m_name << " was not found"; } - TH1F* hist = m_element->getTH1F(); - assert(m_indexToCheck < m_means.size()); + TH1* hist = m_element->getTH1(); - if (hist->GetEntries() != m_entries[m_indexToCheck]) { + if (hist->GetEntries() != expected_entries) { throw cms::Exception("WrongEntries") - << "The element: " << m_name << " for index " << m_indexToCheck << " was expected to have " - << m_entries[m_indexToCheck] << " entries but instead has " << hist->GetEntries(); - } - - if (hist->GetMean() != m_means[m_indexToCheck]) { - throw cms::Exception("WrongEntries") - << "The element: " << m_name << " for index " << m_indexToCheck << " was expected to have " - << m_means[m_indexToCheck] << " mean but instead has " << hist->GetMean(); - } - - ++m_indexToCheck; - } - - private: - std::string m_name; - DQMStore* m_store; - MonitorElement* m_element; - std::vector m_means; - std::vector m_entries; - unsigned int m_indexToCheck; - }; - - class TH2FReader : public ReaderBase { - public: - TH2FReader(const edm::ParameterSet& iPSet, DQMStore& iStore, bool iSetLumiFlag) - : m_store(&iStore), - m_element(0), - m_means(iPSet.getUntrackedParameter >("means")), - m_entries(iPSet.getUntrackedParameter >("entries")), - m_indexToCheck(0) { - assert(m_means.size() == m_entries.size()); - std::string extension; - if (iSetLumiFlag) { - extension = "_lumi"; - } - m_name = iPSet.getUntrackedParameter("name") + extension; - } - - virtual ~TH2FReader(){}; - - void read() { - if (0 == m_element) { - m_element = m_store->get(m_name); - if (0 == m_element) { - throw cms::Exception("MissingElement") << "The element: " << m_name << " was not found"; - } + << "The element: " << m_name << " for run " << run << " lumi " << lumi << " was expected to have " + << expected_entries << " entries but instead has " << hist->GetEntries(); } - TH2F* hist = m_element->getTH2F(); - assert(m_indexToCheck < m_means.size()); - if (hist->GetEntries() != m_entries[m_indexToCheck]) { + if (hist->GetMean() != expected_mean) { throw cms::Exception("WrongEntries") - << "The element: " << m_name << " for index " << m_indexToCheck << " was expected to have " - << m_entries[m_indexToCheck] << " entries but instead has " << hist->GetEntries(); - } - - if (hist->GetMean() != m_means[m_indexToCheck]) { - throw cms::Exception("WrongMeans") - << "The element: " << m_name << " for index " << m_indexToCheck << " was expected to have " - << m_means[m_indexToCheck] << " mean but instead has " << hist->GetMean(); + << "The element: " << m_name << " for run " << run << " lumi " << lumi << " was expected to have " + << expected_mean << " mean but instead has " << hist->GetMean(); } - - ++m_indexToCheck; } private: std::string m_name; DQMStore* m_store; MonitorElement* m_element; + std::vector m_runs; + std::vector m_lumis; std::vector m_means; std::vector m_entries; - unsigned int m_indexToCheck; }; } // namespace @@ -190,10 +147,10 @@ DummyReadDQMStore::DummyReadDQMStore(const edm::ParameterSet& iConfig) { for (PSets::const_iterator it = runElements.begin(), itEnd = runElements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { case 1: - m_runReaders.push_back(std::shared_ptr(new TH1FReader(*it, *dstore, false))); + m_runReaders.push_back(std::shared_ptr(new TH1Reader(*it, *dstore, false))); break; case 2: - m_runReaders.push_back(std::shared_ptr(new TH2FReader(*it, *dstore, false))); + m_runReaders.push_back(std::shared_ptr(new TH1Reader(*it, *dstore, false))); break; } } @@ -203,10 +160,10 @@ DummyReadDQMStore::DummyReadDQMStore(const edm::ParameterSet& iConfig) { for (PSets::const_iterator it = lumiElements.begin(), itEnd = lumiElements.end(); it != itEnd; ++it) { switch (it->getUntrackedParameter("type", 1)) { case 1: - m_lumiReaders.push_back(std::shared_ptr(new TH1FReader(*it, *dstore, true))); + m_lumiReaders.push_back(std::shared_ptr(new TH1Reader(*it, *dstore, true))); break; case 2: - m_lumiReaders.push_back(std::shared_ptr(new TH2FReader(*it, *dstore, true))); + m_lumiReaders.push_back(std::shared_ptr(new TH1Reader(*it, *dstore, true))); break; } } @@ -252,11 +209,11 @@ void DummyReadDQMStore::endJob() {} void DummyReadDQMStore::beginRun(edm::Run const&, edm::EventSetup const&) {} // ------------ method called when ending the processing of a run ------------ -void DummyReadDQMStore::endRun(edm::Run const&, edm::EventSetup const&) { +void DummyReadDQMStore::endRun(edm::Run const& run, edm::EventSetup const&) { for (std::vector >::iterator it = m_runReaders.begin(), itEnd = m_runReaders.end(); it != itEnd; ++it) { - (*it)->read(); + (*it)->read(run.run(), 0); } } @@ -264,11 +221,11 @@ void DummyReadDQMStore::endRun(edm::Run const&, edm::EventSetup const&) { void DummyReadDQMStore::beginLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) {} // ------------ method called when ending the processing of a luminosity block ------------ -void DummyReadDQMStore::endLuminosityBlock(edm::LuminosityBlock const&, edm::EventSetup const&) { +void DummyReadDQMStore::endLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const&) { for (std::vector >::iterator it = m_lumiReaders.begin(), itEnd = m_lumiReaders.end(); it != itEnd; ++it) { - (*it)->read(); + (*it)->read(lumi.run(), lumi.luminosityBlock()); } } diff --git a/DQMServices/FwkIO/test/create_empty_file_cfg.py b/DQMServices/FwkIO/test/create_empty_file_cfg.py index fa41bf131ba53..bae7da35c3026 100644 --- a/DQMServices/FwkIO/test/create_empty_file_cfg.py +++ b/DQMServices/FwkIO/test/create_empty_file_cfg.py @@ -27,6 +27,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(1) )) @@ -34,6 +36,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,10)]), means = cms.untracked.vdouble([i for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,10)]) )) diff --git a/DQMServices/FwkIO/test/create_file1_cfg.py b/DQMServices/FwkIO/test/create_file1_cfg.py index f4b8bd4c3583f..545ff846378e1 100644 --- a/DQMServices/FwkIO/test/create_file1_cfg.py +++ b/DQMServices/FwkIO/test/create_file1_cfg.py @@ -27,6 +27,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(1) )) @@ -34,6 +36,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,10)]), means = cms.untracked.vdouble([i for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,10)]) )) diff --git a/DQMServices/FwkIO/test/create_file2_cfg.py b/DQMServices/FwkIO/test/create_file2_cfg.py index 340cfdd136ed2..ade5a50293d52 100644 --- a/DQMServices/FwkIO/test/create_file2_cfg.py +++ b/DQMServices/FwkIO/test/create_file2_cfg.py @@ -27,6 +27,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(1) )) @@ -34,6 +36,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+11 for x in range(0,10)]), means = cms.untracked.vdouble([i for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,10)]) )) diff --git a/DQMServices/FwkIO/test/create_file3_cfg.py b/DQMServices/FwkIO/test/create_file3_cfg.py index 486784bca40f5..47128e9e3c9b5 100644 --- a/DQMServices/FwkIO/test/create_file3_cfg.py +++ b/DQMServices/FwkIO/test/create_file3_cfg.py @@ -28,6 +28,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(2), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i+1), entries=cms.untracked.vdouble(1) )) @@ -35,6 +37,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([2 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,10)]), means = cms.untracked.vdouble([i+1 for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,10)]) )) diff --git a/DQMServices/FwkIO/test/create_file4_cfg.py b/DQMServices/FwkIO/test/create_file4_cfg.py index f87d7456b920f..d5df533cc0dd5 100644 --- a/DQMServices/FwkIO/test/create_file4_cfg.py +++ b/DQMServices/FwkIO/test/create_file4_cfg.py @@ -30,6 +30,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(1) )) @@ -37,6 +39,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+100 for x in range(0,10)]), means = cms.untracked.vdouble([i for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,10)]) )) diff --git a/DQMServices/FwkIO/test/read_file1_file2_cfg.py b/DQMServices/FwkIO/test/read_file1_file2_cfg.py index 0ebfd8e49e508..53985972542be 100644 --- a/DQMServices/FwkIO/test/read_file1_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_file1_file2_cfg.py @@ -24,6 +24,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(2) )) @@ -31,6 +33,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,20)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,20)]), means = cms.untracked.vdouble([i for x in range(0,20)]), entries=cms.untracked.vdouble([1 for x in range(0,20)]) )) diff --git a/DQMServices/FwkIO/test/read_file1_file3_cfg.py b/DQMServices/FwkIO/test/read_file1_file3_cfg.py index 5709bd459e140..21b63acbd0834 100644 --- a/DQMServices/FwkIO/test/read_file1_file3_cfg.py +++ b/DQMServices/FwkIO/test/read_file1_file3_cfg.py @@ -24,15 +24,19 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), - means = cms.untracked.vdouble([i+x for x in (0,1)]), - entries=cms.untracked.vdouble([1 for x in (0,1)]) + runs = cms.untracked.vint32([1, 2]), + lumis = cms.untracked.vint32([0, 0]), + means = cms.untracked.vdouble([i, i+1]), + entries=cms.untracked.vdouble([1, 1]) )) readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)] + [2 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,10)] + [x+1 for x in range(0,10)]), #file3 has means shifted by 1 - means = cms.untracked.vdouble([i+x//10 for x in range(0,20)]), + means = cms.untracked.vdouble([i for x in range(0,10)] + [i+1 for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,20)]) )) diff --git a/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py index 1068978033f63..fa95038afec4b 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py @@ -25,6 +25,8 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32(1), + lumis = cms.untracked.vint32(0), means = cms.untracked.vdouble(i), entries=cms.untracked.vdouble(2) )) @@ -32,6 +34,8 @@ readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,20)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,20)]), means = cms.untracked.vdouble([i for x in range(0,20)]), entries=cms.untracked.vdouble([1 for x in range(0,20)]) )) diff --git a/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py index 4d55cab41a97e..88f8765208f8c 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py @@ -25,15 +25,19 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), - means = cms.untracked.vdouble([i+x for x in (0,1)]), - entries=cms.untracked.vdouble([x for x in (2,1)]) + runs = cms.untracked.vint32([1, 2]), + lumis = cms.untracked.vint32([0, 0]), + means = cms.untracked.vdouble([i, i+1]), + entries=cms.untracked.vdouble([2, 1]) )) readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,20)] + [2 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,20)] + [x+1 for x in range(0,10)]), #file3, which is run 2 has means shifted by 1 - means = cms.untracked.vdouble([i+x//20 for x in range(0,30)]), + means = cms.untracked.vdouble([i for x in range(0,20)] + [i+1 for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,30)]) )) diff --git a/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py index 8e301f9384e63..99dcc7fc1032b 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py @@ -16,22 +16,17 @@ seq.append(cms.EventID(r,l,0)) #end lumi seq.append(cms.EventID(r,l,0)) -#end run -seq.append(cms.EventID(r,0,0)) -r = 2 -#begin run -seq.append(cms.EventID(r,0,0)) -for l in range(1,11): +for l in range(100,110): #begin lumi seq.append(cms.EventID(r,l,0)) #end lumi seq.append(cms.EventID(r,l,0)) #end run seq.append(cms.EventID(r,0,0)) -r = 1 +r = 2 #begin run seq.append(cms.EventID(r,0,0)) -for l in range(100,110): +for l in range(1,11): #begin lumi seq.append(cms.EventID(r,l,0)) #end lumi @@ -45,15 +40,19 @@ readRunElements = list() for i in range(0,10): readRunElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), - means = cms.untracked.vdouble([i+x for x in (0,1,0)]), - entries=cms.untracked.vdouble([x for x in (1,1,1)]) + runs = cms.untracked.vint32([1,2]), + lumis = cms.untracked.vint32([0,0]), + means = cms.untracked.vdouble([i, i+1]), + entries=cms.untracked.vdouble([2, 1]) )) readLumiElements=list() for i in range(0,10): readLumiElements.append(cms.untracked.PSet(name=cms.untracked.string("Foo"+str(i)), + runs = cms.untracked.vint32([1 for x in range(0,10)] + [1 for x in range(0,10)] + [2 for x in range(0,10)]), + lumis = cms.untracked.vint32([x+1 for x in range(0,10)] + [x+100 for x in range(0,10)] + [x+1 for x in range(0,10)]), #file3, which is run 2 has means shifted by 1 - means = cms.untracked.vdouble([(i+x//10-x//20-x//20) for x in range(0,30)]), + means = cms.untracked.vdouble([i for x in range(0,10)] + [i for x in range(0,10)] + [i+1 for x in range(0,10)]), entries=cms.untracked.vdouble([1 for x in range(0,30)]) )) From 9efa80622a782930bbeb5f944ad47780f25d6b67 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 4 Dec 2019 11:41:26 +0100 Subject: [PATCH 039/112] Remove dqmEndRun from DQMEDAnalyzer. This includes lots of automated changes to remove "empty" dqmBeginRun and dqmEndRun. dqmBeginRun remains allowed in DQMEDAnalyzer; it is called before booking and ME cannot (and must not) be accessed there. Any local member intialization should be fine to do there; it will be called for each of the edm::stream instancs once we switch back to stream. Technically all the code could simply be moved into bookHistograms, but having a separate beginRun makes more sense logically. Many dqmEndRun that only contain log messages and printouts where removed, even if they were more than basic debugging output. The remaining modules with non-trivial dqmEndRuns are converted to DQMOneEDAnalyzer. Commands like sed -i '/dqmBeginRun/,+3d' $(ag \ 'dqm(Begin|End)Run[^\n]*\n[^\n]*LogInfo[^\n]*\n\s*}' -l) were used to delete "useless" bodies. --- DQMServices/Core/interface/DQMEDAnalyzer.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DQMServices/Core/interface/DQMEDAnalyzer.h b/DQMServices/Core/interface/DQMEDAnalyzer.h index 9617f4bc6a9bc..756d15e1b04e2 100644 --- a/DQMServices/Core/interface/DQMEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMEDAnalyzer.h @@ -42,7 +42,9 @@ class DQMEDAnalyzer : public edm::one::EDProducer()->enterLumi(run.run(), /* lumi */ 0, this->moduleDescription().id()); } - void beginLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const& setup) final {} + void beginLuminosityBlock(edm::LuminosityBlock const& lumi, edm::EventSetup const& setup) final { + edm::Service()->enterLumi(lumi.run(), lumi.luminosityBlock(), this->moduleDescription().id()); + } void accumulate(edm::Event const& event, edm::EventSetup const& setup) final { analyze(event, setup); } From 46954bcb0aa05c35fb1ab10d6ccfc4f198707382 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 5 Dec 2019 18:05:01 +0100 Subject: [PATCH 040/112] Migrate some modules away from setLumiFlag. We shuold eventually remove it entirely, but there are a few weird corner cases to think about. --- DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc | 7 +++---- DQM/EcalCommon/src/MESetEcal.cc | 8 +++++--- DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc | 5 ++--- DQMServices/Components/plugins/DQMDcsInfo.cc | 2 +- DQMServices/FwkIO/test/DummyFillDQMStore.cc | 6 ------ DQMServices/FwkIO/test/DummyReadDQMStore.cc | 3 +-- 6 files changed, 12 insertions(+), 19 deletions(-) diff --git a/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc b/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc index 8cd0b5a232755..9a174ce18260d 100644 --- a/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc +++ b/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc @@ -34,22 +34,21 @@ void DTDCSByLumiSummary::dqmEndLuminosityBlock(DQMStore::IBooker& ibooker, if (!bookingdone) { ibooker.setCurrentFolder("DT/EventInfo/DCSContents"); - totalDCSFraction = ibooker.bookFloat("DTDCSSummary"); - totalDCSFraction->setLumiFlag(); // set LumiFlag to DCS content value (save it by lumi) - globalHVSummary = ibooker.book2D("HVGlbSummary", "HV Status Summary", 1, 1, 13, 5, -2, 3); globalHVSummary->setAxisTitle("Sectors", 1); globalHVSummary->setAxisTitle("Wheel", 2); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + totalDCSFraction = ibooker.bookFloat("DTDCSSummary"); for (int wh = -2; wh <= 2; wh++) { stringstream wheel_str; wheel_str << wh; MonitorElement* FractionWh = ibooker.bookFloat("DT_Wheel" + wheel_str.str()); - FractionWh->setLumiFlag(); // set LumiFlag to DCS content value (save it by lumi) totalDCSFractionWh.push_back(FractionWh); } + ibooker.setScope(scope); globalHVSummary->Reset(); diff --git a/DQM/EcalCommon/src/MESetEcal.cc b/DQM/EcalCommon/src/MESetEcal.cc index 4e4b01bdefa47..88345af3ad7ba 100644 --- a/DQM/EcalCommon/src/MESetEcal.cc +++ b/DQM/EcalCommon/src/MESetEcal.cc @@ -70,6 +70,11 @@ namespace ecaldqm { void MESetEcal::book(DQMStore::IBooker &_ibooker) { using namespace std; + _ibooker.setScope(MonitorElementData::Scope::RUN); + if (lumiFlag_) { + _ibooker.setScope(MonitorElementData::Scope::LUMI); + } + clear(); vector mePaths(generatePaths()); @@ -264,9 +269,6 @@ namespace ecaldqm { me->getTH1()->SetUniqueID(uint32_t(2 * (actualObject + 1) + (isMap ? 1 : 0))); } - if (lumiFlag_) - me->setLumiFlag(); - mes_.push_back(me); } diff --git a/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc b/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc index ca063fec82f57..fafdda66dc2b7 100644 --- a/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc +++ b/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc @@ -129,10 +129,9 @@ void SiStripDcsInfo::bookStatus(DQMStore& dqm_store) { else dqm_store.setCurrentFolder("SiStrip/EventInfo"); + auto scope = dqm_store.setScope(MonitorElementData::Scope::LUMI); DcsFraction_ = dqm_store.bookFloat("DCSSummary"); - DcsFraction_->setLumiFlag(); - dqm_store.cd(); if (!strip_dir.empty()) dqm_store.setCurrentFolder(strip_dir + "/EventInfo/DCSContents"); @@ -141,9 +140,9 @@ void SiStripDcsInfo::bookStatus(DQMStore& dqm_store) { for (auto& [suffix, subDetME] : SubDetMEsMap) { std::string const me_name{"SiStrip_" + suffix}; subDetME.DcsFractionME = dqm_store.bookFloat(me_name); - subDetME.DcsFractionME->setLumiFlag(); } bookedStatus_ = true; + dqm_store.setScope(scope); dqm_store.cd(); } diff --git a/DQMServices/Components/plugins/DQMDcsInfo.cc b/DQMServices/Components/plugins/DQMDcsInfo.cc index ad926426e20b9..289ef6e2ebc87 100644 --- a/DQMServices/Components/plugins/DQMDcsInfo.cc +++ b/DQMServices/Components/plugins/DQMDcsInfo.cc @@ -45,8 +45,8 @@ void DQMDcsInfo::bookHistograms(DQMStore::IBooker& ibooker, ibooker.cd(); ibooker.setCurrentFolder(subsystemname_ + "/" + dcsinfofolder_); + ibooker.setScope(MonitorElementData::Scope::LUMI); DCSbyLS_ = ibooker.book1D("DCSbyLS", "DCS", 25, 0., 25.); - DCSbyLS_->setLumiFlag(); // initialize for (bool& dc : dcs) diff --git a/DQMServices/FwkIO/test/DummyFillDQMStore.cc b/DQMServices/FwkIO/test/DummyFillDQMStore.cc index 21f060bc211a7..032e820660e85 100644 --- a/DQMServices/FwkIO/test/DummyFillDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyFillDQMStore.cc @@ -56,9 +56,6 @@ namespace { m_steps, m_min, iPSet.getUntrackedParameter("highX")); - if (iSetLumiFlag) { - m_element->setLumiFlag(); - } m_hist = m_element->getTH1F(); m_valueToFill = iPSet.getUntrackedParameter("value"); } @@ -92,9 +89,6 @@ namespace { iPSet.getUntrackedParameter("nchY"), iPSet.getUntrackedParameter("lowY"), iPSet.getUntrackedParameter("highY")); - if (iSetLumiFlag) { - m_element->setLumiFlag(); - } m_hist = m_element->getTH2F(); m_valueToFill = iPSet.getUntrackedParameter("value"); } diff --git a/DQMServices/FwkIO/test/DummyReadDQMStore.cc b/DQMServices/FwkIO/test/DummyReadDQMStore.cc index 92f8556d98347..d8e258990b073 100644 --- a/DQMServices/FwkIO/test/DummyReadDQMStore.cc +++ b/DQMServices/FwkIO/test/DummyReadDQMStore.cc @@ -52,8 +52,7 @@ namespace { m_runs(iPSet.getUntrackedParameter >("runs")), m_lumis(iPSet.getUntrackedParameter >("lumis")), m_means(iPSet.getUntrackedParameter >("means")), - m_entries(iPSet.getUntrackedParameter >("entries")) - { + m_entries(iPSet.getUntrackedParameter >("entries")) { assert(m_means.size() == m_entries.size()); std::string extension; if (iSetLumiFlag) { From 9b649f4af294199d3be2bce49fbd6281c6958733 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 5 Dec 2019 18:06:03 +0100 Subject: [PATCH 041/112] Implement some more harvesting APIs. Very inefficiently, but these things should be deprecated. --- DQMServices/Core/src/DQMStore.cc | 42 ++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 81398d60c4a8c..d5640ea837944 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -526,9 +526,32 @@ namespace dqm::implementation { return result; } - std::vector IGetter::getSubdirs() const { assert(!"NIY"); } + std::vector IGetter::getSubdirs() const { + // This is terribly inefficient, esp. if this method is then used to + // recursively enumerate whatever getAllContents would return anyways. + // But that is fine, any such code should just use getAllContents instead. + std::set subdirs; + for (auto me : this->getAllContents(this->cwd_)) { + auto name = me->getPathname(); + auto subdirname = name.substr(this->cwd_.length(), std::string::npos); + auto dirname = subdirname.substr(0, subdirname.find("/")); + subdirs.insert(dirname); + } + std::vector out; + for (auto dir : subdirs) { + if (dir.length() == 0) + continue; + out.push_back(dir); + } + return out; + } + std::vector IGetter::getMEs() const { assert(!"NIY"); } - bool IGetter::dirExists(std::string const& path) const { assert(!"NIY"); } + + bool IGetter::dirExists(std::string const& path) const { + // we don't claim this is fast. + return this->getAllContents(path).size() > 0; + } IGetter::IGetter(DQMStore* store) { store_ = store; } @@ -573,7 +596,9 @@ namespace dqm::implementation { SaveReferenceTag ref, int minStatus, std::string const& fileupdate) { - assert(!"NIY"); + TRACE("filename " << filename << " path " << path << " pattern " << pattern << " rewrite " << rewrite << " run " + << run << " lumi " << lumi << " minStatus " << minStatus << " fileupdate " << fileupdate); + //assert(!"NIY"); } void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { assert(!"NIY"); @@ -584,9 +609,16 @@ namespace dqm::implementation { std::string const& prepend, OpenRunDirs stripdirs, bool fileMustExist) { - assert(!"NIY"); + TRACE("filename " << filename << " overwrite " << overwrite << " path " << path << " prepend " << prepend + << " fileMustExist " << fileMustExist); + //assert(!"NIY"); + return false; + } + bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { + TRACE("filename " << filename << " fileMustExist " << fileMustExist); + //assert(!"NIY"); + return false; } - bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { assert(!"NIY"); } void DQMStore::showDirStructure() const { assert(!"NIY"); } From 21d37fbdb9cb532afe5dff94a880590df091106b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 6 Dec 2019 11:50:26 +0100 Subject: [PATCH 042/112] Implement getMEs. --- DQMServices/Core/src/DQMStore.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index d5640ea837944..9e510ce0e1258 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -546,7 +546,14 @@ namespace dqm::implementation { return out; } - std::vector IGetter::getMEs() const { assert(!"NIY"); } + std::vector IGetter::getMEs() const { + auto mes = this->getContents(this->cwd_); + std::vector out; + for (auto me : mes) { + out.push_back(me->getName()); + } + return out; + } bool IGetter::dirExists(std::string const& path) const { // we don't claim this is fast. From 08bcb21f616017bdd38d40e5da5a6d288fdf7f0c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 6 Dec 2019 11:51:53 +0100 Subject: [PATCH 043/112] Fix deadlock in syncCoreObject() --- DQMServices/Core/interface/MonitorElement.h | 1 + DQMServices/Core/src/MonitorElement.cc | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index cca7500893b14..2884f13b31c6c 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -224,6 +224,7 @@ namespace dqm::impl { // copy applicable fileds into the DQMNet core object for compatibility. // In a few places these flags are also still used by the ME. void syncCoreObject(); + void syncCoreObject(AccessMut &access); virtual ~MonitorElement(); // check if the ME is currently backed by MEData; if false (almost) any diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index b326fcff72967..bd1e221d91728 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -87,7 +87,11 @@ namespace dqm::impl { } void MonitorElement::syncCoreObject() { - auto access = this->access(); + auto access = this->accessMut(); + syncCoreObject(access); + } + + void MonitorElement::syncCoreObject(AccessMut &access) { data_.flags &= ~DQMNet::DQM_PROP_TYPE_MASK; data_.flags |= (int)access.key.kind_; @@ -1024,7 +1028,7 @@ namespace dqm::impl { q.message = "NO_MESSAGE_ASSIGNED"; q.algorithm = "UNKNOWN_ALGORITHM"; access.value.qreports_.push_back(MonitorElementData::QReport(q)); - syncCoreObject(); + syncCoreObject(access); } qr = &access.value.qreports_[pos]; From 394615711ec3c2f0e58f0eb8afe3676e73cc3a0a Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 6 Dec 2019 18:25:15 +0100 Subject: [PATCH 044/112] Fix DQMRootSource after rebase. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index c8523bf7a4e01..c381dea8f8416 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -35,6 +35,7 @@ #include "FWCore/Framework/interface/LuminosityBlock.h" #include "DataFormats/Provenance/interface/LuminosityBlockID.h" #include "DataFormats/Provenance/interface/LuminosityBlockRange.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" @@ -669,7 +670,7 @@ void DQMRootSource::readLuminosityBlock_(edm::LuminosityBlockPrincipal& lbCache) edm::Service jr; jr->reportInputLumiSection(lbCache.id().run(), lbCache.id().luminosityBlock()); - lbCache.fillLuminosityBlockPrincipal(processHistoryRegistryForUpdate()); + lbCache.fillLuminosityBlockPrincipal(processHistoryRegistry().getMapped(lbCache.aux().processHistoryID())); } void DQMRootSource::readEvent_(edm::EventPrincipal&) {} From a5bc2382bfaea5b52eaf0c7bdd158773164bdbdd Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 11:59:28 +0100 Subject: [PATCH 045/112] Use new framework callbacks. --- DQMServices/Core/src/DQMStore.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 9e510ce0e1258..0766a94f3620e 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -570,25 +570,18 @@ namespace dqm::implementation { // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be // correct for purely sequential legacy stuff. - // TODO: detect concurrent runs/lumis and disable legacy API in case? - // TODO: There is no callback for "after the last output module wrote", so - // instead we infer that from seeing the next run/lumi. Won't work with - // concurrent runs/lumis. ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { - if (this->runlumi_.run() != 0) { - this->cleanupLumi(this->runlumi_.run(), 0); - } - if (this->runlumi_.luminosityBlock() != 0) { - this->cleanupLumi(this->runlumi_.run(), this->runlumi_.luminosityBlock()); - } this->setRunLumi(gc.luminosityBlockID()); }); ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { - if (this->runlumi_.luminosityBlock() != 0) { - this->cleanupLumi(this->runlumi_.run(), this->runlumi_.luminosityBlock()); - } this->setRunLumi(gc.luminosityBlockID()); }); + ar.watchPostGlobalWriteLumi([this](edm::GlobalContext const& gc) { + this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock()); + }); + ar.watchPostGlobalWriteRun([this](edm::GlobalContext const& gc) { + this->cleanupLumi(gc.luminosityBlockID().run(), 0); + }); // no cleanup at end of job, we don't really need it. } From 50e631b1bda43f176f36386d5299feb54eff944e Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 12:17:30 +0100 Subject: [PATCH 046/112] Add assertion to catch unexpected clone's --- DQMServices/Core/interface/DQMStore.h | 3 +++ DQMServices/Core/src/DQMStore.cc | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 705352b0694e1..761abef8ddb32 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -638,6 +638,9 @@ namespace dqm { // universal verbose flag. int verbose_; + // If set to true, error out whenever things happen that are not safe for + // legacy modules. + bool assertLegacySafe_; }; } // namespace implementation diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 0766a94f3620e..a94e17c5a9ea8 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -337,6 +337,11 @@ namespace dqm::implementation { if (checkScope(anyme->getScope()) == false) { continue; } // else + + // whenever we clone global MEs, it is no longer safe to hold + // pointers to them. + assert(!assertLegacySafe_); + MonitorElementData newdata = anyme->cloneMEData(); auto newme = new MonitorElement(std::move(newdata)); newme->Reset(); // we cloned a ME in use, not an empty prototype @@ -566,6 +571,7 @@ namespace dqm::implementation { DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry& ar) : IGetter(this), IBooker(this) { verbose_ = pset.getUntrackedParameter("verbose", 0); + assertLegacySafe_ = pset.getUntrackedParameter("assertLegacySafe", true); // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be From c3703d01e03b5d6bd1bd61a0c579f302081e9b4c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 12:22:10 +0100 Subject: [PATCH 047/112] Correctly use virtual on NavigatorBase. Else the cwd in IGetter and IBooker can get out of sync, since the DQMStore overrides are not always used. --- DQMServices/Core/interface/DQMStore.h | 28 ++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 761abef8ddb32..7a6a7ebe67686 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -23,12 +23,14 @@ namespace dqm { // The common implementation to change folders class NavigatorBase { public: - void cd(); - void cd(std::string const& dir); + virtual void cd(); + virtual void cd(std::string const& dir); // This is the only method that is allowed to change cwd_ value - void setCurrentFolder(std::string const& fullpath); - void goUp(); - std::string const& pwd(); + virtual void setCurrentFolder(std::string const& fullpath); + virtual void goUp(); + virtual std::string const& pwd(); + + virtual ~NavigatorBase() {} protected: NavigatorBase(){}; @@ -406,7 +408,7 @@ namespace dqm { // virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); - virtual ~IBooker(); + ~IBooker() override; protected: IBooker(DQMStore* store); @@ -468,7 +470,7 @@ namespace dqm { // returns whether there are objects at full path `path` virtual bool dirExists(std::string const& path) const; - virtual ~IGetter(); + ~IGetter() override; protected: IGetter(DQMStore* store); @@ -485,7 +487,7 @@ namespace dqm { enum OpenRunDirs { KeepRunDirs, StripRunDirs }; DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&); - ~DQMStore(); + ~DQMStore() override; // ------------------------------------------------------------------------ // ---------------------- public I/O -------------------------------------- @@ -525,23 +527,23 @@ namespace dqm { // ------------------------------------------------------------------------ // ------------ IBooker/IGetter overrides to prevent ambiguity ------------ - virtual void cd() { + void cd() override { this->IBooker::cd(); this->IGetter::cd(); } - virtual void cd(std::string const& dir) { + void cd(std::string const& dir) override { this->IBooker::cd(dir); this->IGetter::cd(dir); } - virtual void setCurrentFolder(std::string const& fullpath) { + void setCurrentFolder(std::string const& fullpath) override { this->IBooker::setCurrentFolder(fullpath); this->IGetter::setCurrentFolder(fullpath); } - virtual void goUp() { + void goUp() override { this->IBooker::goUp(); this->IGetter::goUp(); } - std::string const& pwd() { return this->IBooker::pwd(); } + std::string const& pwd() override { return this->IBooker::pwd(); } public: // internal -- figure out better protection. From ec3d8a7bef4bc83281185caf7aaff2988858e159 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 12:35:11 +0100 Subject: [PATCH 048/112] Code-checks && code-format. --- DQMServices/Core/src/DQMStore.cc | 21 ++++++++------------- DQMServices/FwkIO/plugins/DQMRootSource.cc | 14 +++++++------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index a94e17c5a9ea8..1c8e703516487 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -223,7 +223,7 @@ namespace dqm::implementation { } } - if (clean_trace.size() > 0) { + if (!clean_trace.empty()) { logger << message << " at "; for (auto const& s : clean_trace) { logger << s << "; "; @@ -537,7 +537,7 @@ namespace dqm::implementation { // But that is fine, any such code should just use getAllContents instead. std::set subdirs; for (auto me : this->getAllContents(this->cwd_)) { - auto name = me->getPathname(); + const auto& name = me->getPathname(); auto subdirname = name.substr(this->cwd_.length(), std::string::npos); auto dirname = subdirname.substr(0, subdirname.find("/")); subdirs.insert(dirname); @@ -562,7 +562,7 @@ namespace dqm::implementation { bool IGetter::dirExists(std::string const& path) const { // we don't claim this is fast. - return this->getAllContents(path).size() > 0; + return !this->getAllContents(path).empty(); } IGetter::IGetter(DQMStore* store) { store_ = store; } @@ -576,18 +576,13 @@ namespace dqm::implementation { // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be // correct for purely sequential legacy stuff. - ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { - this->setRunLumi(gc.luminosityBlockID()); - }); - ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { - this->setRunLumi(gc.luminosityBlockID()); - }); + ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); + ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); ar.watchPostGlobalWriteLumi([this](edm::GlobalContext const& gc) { - this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock()); - }); - ar.watchPostGlobalWriteRun([this](edm::GlobalContext const& gc) { - this->cleanupLumi(gc.luminosityBlockID().run(), 0); + this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock()); }); + ar.watchPostGlobalWriteRun( + [this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), 0); }); // no cleanup at end of job, we don't really need it. } diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index c381dea8f8416..762eaafdb654f 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -339,8 +339,8 @@ class DQMRootSource : public edm::PuttableSourceBase { // These methods will be called by the framework. // MEs in DQMStore will be put to products. - void beginRun(edm::Run& run); - void beginLuminosityBlock(edm::LuminosityBlock& lumi); + void beginRun(edm::Run& run) override; + void beginLuminosityBlock(edm::LuminosityBlock& lumi) override; // If the run matches the filterOnRun configuration parameter, the run // (and all its lumis) will be kept. @@ -423,7 +423,7 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou m_fileMetadatas(std::vector()) { edm::sortAndRemoveOverlaps(m_lumisToProcess); - if (m_catalog.fileNames().size() == 0) { + if (m_catalog.fileNames().empty()) { m_nextItemType = edm::InputSource::IsStop; } else { m_treeReaders[kIntIndex].reset(new TreeSimpleReader(MonitorElementData::Kind::INT, m_rescope)); @@ -466,8 +466,8 @@ std::unique_ptr DQMRootSource::readFile_() { m_openFiles.reserve(numFiles); for (auto& fileitem : m_catalog.fileCatalogItems()) { - auto filename = fileitem.fileName(); - auto fallbackname = fileitem.fallbackFileName(); + const auto& filename = fileitem.fileName(); + const auto& fallbackname = fileitem.fallbackFileName(); bool hasFallback = !fallbackname.empty() && fallbackname != filename; TFile* file; std::list originalInfo; @@ -618,7 +618,7 @@ std::unique_ptr DQMRootSource::readFile_() { metadata.describe(); // Stop if there's nothing to process. Otherwise start the run. - if (m_fileMetadatas.size() == 0) + if (m_fileMetadatas.empty()) m_nextItemType = edm::InputSource::IsStop; else m_nextItemType = edm::InputSource::IsRun; @@ -741,7 +741,7 @@ bool DQMRootSource::keepIt(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lu return false; } - if (m_lumisToProcess.size() == 0) { + if (m_lumisToProcess.empty()) { return true; } From de25b355e5b3254527bfaac31331bb68f6b36d7c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 9 Dec 2019 17:47:13 +0100 Subject: [PATCH 049/112] Also create local MEs for legacy booked histograms. --- DQMServices/Core/src/DQMStore.cc | 60 +++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 1c8e703516487..207bb1adb984c 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -77,7 +77,14 @@ namespace dqm::implementation { } else if (this->scope_ == MonitorElementData::Scope::RUN) { medata.key_.id_ = edm::LuminosityBlockID(this->runlumi_.run(), 0); } else if (this->scope_ == MonitorElementData::Scope::LUMI) { - medata.key_.id_ = this->runlumi_; + // In the messy case of legacy-booking a LUMI ME in beginRun (or + // similar), where we don't have a valid lumi number yet, make sure to + // book a prototype instead. + if (this->runlumi_.run() != 0 && this->runlumi_.luminosityBlock() != 0) { + medata.key_.id_ = this->runlumi_; + } else { + medata.key_.id_ = edm::LuminosityBlockID(); + } } else { assert(!"Illegal scope"); } @@ -89,24 +96,28 @@ namespace dqm::implementation { // me now points to a global ME owned by the DQMStore. assert(me); - if (this->moduleID_ == 0) { - // this is a legacy/global/harvesting booking. In this case, we do not - // create a local ME and return the global directly. It is not advisable - // to hold this pointer, as we may delete the global ME later, but we - // promise to keep it valid for the entire job if there are no concurrent - // runs/lumis. - return me; - } - // each booking call returns a unique "local" ME, which the DQMStore keeps // in a container associated with the module (and potentially run, for // DQMGlobalEDAnalyzer). This will later be update to point to different // MEData (kept in a global ME) as needed. // putME creates the local ME object as needed. - me = store_->putME(me, this->moduleID_); + auto localme = store_->putME(me, this->moduleID_); // me now points to a local ME owned by the DQMStore. - assert(me); - return me; + assert(localme); + + if (this->moduleID_ == 0) { + // this is a legacy/global/harvesting booking. In this case, we return + // the global directly. It is not advisable to hold this pointer, as we + // may delete the global ME later, but we promise to keep it valid for + // the entire job if there are no concurrent runs/lumis. (see + // assertLegacySafe option). + // We still created a local ME, so we can drive the lumi-changing for + // legacy modules in watchPreGlobalBeginLumi. + return me; + } else { + // the normal case. + return localme; + } } MonitorElement* DQMStore::putME(MonitorElement* me) { @@ -546,7 +557,7 @@ namespace dqm::implementation { for (auto dir : subdirs) { if (dir.length() == 0) continue; - out.push_back(dir); + out.push_back(this->cwd_ + dir); } return out; } @@ -576,13 +587,30 @@ namespace dqm::implementation { // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be // correct for purely sequential legacy stuff. - ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); - ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); }); + // These transitions should only affect non-DQM*EDAnalyzer based code. + ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { + this->setRunLumi(gc.luminosityBlockID()); + this->enterLumi(gc.luminosityBlockID().run(), /* lumi */ 0, /* moduleID */ 0); + }); + ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { + this->setRunLumi(gc.luminosityBlockID()); + this->enterLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock(), /* moduleID */ 0); + }); + ar.watchPostGlobalEndRun([this](edm::GlobalContext const& gc) { + this->leaveLumi(gc.luminosityBlockID().run(), /* lumi */ 0, /* moduleID */ 0); + }); + ar.watchPostGlobalEndLumi([this](edm::GlobalContext const& gc) { + this->leaveLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock(), /* moduleID */ 0); + }); + + // Trigger cleanup after writing. This is needed for all modules; we can + // only run the cleanup after all output modules have run. ar.watchPostGlobalWriteLumi([this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), gc.luminosityBlockID().luminosityBlock()); }); ar.watchPostGlobalWriteRun( [this](edm::GlobalContext const& gc) { this->cleanupLumi(gc.luminosityBlockID().run(), 0); }); + // no cleanup at end of job, we don't really need it. } From 4db3aebe46ac5a043be770eb376cd05808f7fdfb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 18:46:50 +0100 Subject: [PATCH 050/112] Fix after rebase. --- DataFormats/Histograms/interface/MonitorElementCollection.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DataFormats/Histograms/interface/MonitorElementCollection.h b/DataFormats/Histograms/interface/MonitorElementCollection.h index 0c9d5a3ceb5d3..2d65ddb35ef15 100644 --- a/DataFormats/Histograms/interface/MonitorElementCollection.h +++ b/DataFormats/Histograms/interface/MonitorElementCollection.h @@ -109,7 +109,7 @@ struct MonitorElementData { const std::string& getQRName() const { return qvalue_.qtname; } /// get quality test algorithm - const std::string& getAlgorithm() const { return qvalue_->algorithm; } + const std::string& getAlgorithm() const { return qvalue_.algorithm; } /// get vector of channels that failed test /// (not relevant for all quality tests!) From 4dfdbd1686915da42df03f546a661ca525de6206 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 19:34:17 +0100 Subject: [PATCH 051/112] Clarify DQMGlobalEDAnalyzer. --- DQMServices/Core/interface/DQMGlobalEDAnalyzer.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h index 6cb1209093d11..2ea0937676119 100644 --- a/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h +++ b/DQMServices/Core/interface/DQMGlobalEDAnalyzer.h @@ -43,9 +43,9 @@ class DQMGlobalEDAnalyzer }, // The run number is part of the module ID here, since we want distinct // local MEs for each run cache. - (((uint64_t)run.run()) << 32) + this->moduleDescription().id(), + meId(run), /* canSaveByLumi */ false); - dqmstore_->enterLumi(run.run(), /* lumi */ 0, (((uint64_t)run.run()) << 32) + this->moduleDescription().id()); + dqmstore_->enterLumi(run.run(), /* lumi */ 0, meId(run)); return h; } @@ -57,7 +57,7 @@ class DQMGlobalEDAnalyzer void globalEndRunProduce(edm::Run& run, edm::EventSetup const& setup) const final { auto const& h = *this->runCache(run.index()); dqmEndRun(run, setup, h); - dqmstore_->leaveLumi(run.run(), /* lumi */ 0, (((uint64_t)run.run()) << 32) + this->moduleDescription().id()); + dqmstore_->leaveLumi(run.run(), /* lumi */ 0, meId(run)); run.emplace(runToken_); } @@ -75,6 +75,7 @@ class DQMGlobalEDAnalyzer private: DQMStore* dqmstore_; edm::EDPutTokenT runToken_; + uint64_t meId(edm::Run const& run) const { return (((uint64_t)run.run()) << 32) + this->moduleDescription().id(); } }; #endif // DQMServices_Core_DQMGlobalEDAnalyzer_h From 90209dac8a26885f870d715df756dc3325d614f8 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 19:51:48 +0100 Subject: [PATCH 052/112] Fix type in bookTransaction. --- DQMServices/Core/interface/DQMStore.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 7a6a7ebe67686..04b34adf524c5 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -548,7 +548,7 @@ namespace dqm { public: // internal -- figure out better protection. template - void bookTransaction(iFunc f, uint32_t moduleId, bool canSaveByLumi) { + void bookTransaction(iFunc f, uint64_t moduleId, bool canSaveByLumi) { auto lock = std::scoped_lock(this->booking_mutex_); IBooker& booker = *this; // TODO: this may need to become more elaborate. From fbf012d82a6e059ae3d4fc714914671de32685eb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 19:53:01 +0100 Subject: [PATCH 053/112] Enable concurrent lumis in the DQMStore if needed. --- DQMServices/Demo/test/run_analyzers_cfg.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/DQMServices/Demo/test/run_analyzers_cfg.py b/DQMServices/Demo/test/run_analyzers_cfg.py index e7b09940fb1e6..9a22a50443017 100644 --- a/DQMServices/Demo/test/run_analyzers_cfg.py +++ b/DQMServices/Demo/test/run_analyzers_cfg.py @@ -62,6 +62,9 @@ numberOfConcurrentRuns = cms.untracked.uint32(1) ) +if args.nConcurrent > 1: + process.DQMStore.assertLegacySafe = cms.untracked.bool(False) + for mod in [process.test, process.testglobal, process.testone, process.testonefillrun, process.testonelumi, process.testonelumifilllumi, process.testlegacy, process.testlegacyfillrun, process.testlegacyfilllumi]: mod.howmany = args.howmany From f7f7e682bdadcaacc10d821387bb12f90bd7931b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 19:58:41 +0100 Subject: [PATCH 054/112] Use new per-lumi API in TestDQMEDAnalyzer. --- DQMServices/Demo/test/TestDQMEDAnalyzer.cc | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc index 71ec06dd9ca34..fc4b8353e9da1 100644 --- a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc +++ b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc @@ -3,6 +3,7 @@ #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/MakerMacros.h" #include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "DataFormats/Histograms/interface/MonitorElementCollection.h" template class BookerFiller { @@ -37,34 +38,27 @@ class BookerFiller { "thprofile2d" + num, "2D Profile Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); if (DOLUMI) { + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); ibooker.setCurrentFolder(folder + "/lumi"); + mes_1D.push_back(ibooker.bookFloat("float" + num)); - mes_1D.back()->setLumiFlag(); mes_1D.push_back(ibooker.bookInt("int" + num)); - mes_1D.back()->setLumiFlag(); mes_1D.push_back(ibooker.book1D("th1f" + num, "1D Float Histogram " + num, 101, -0.5, 100.5)); - mes_1D.back()->setLumiFlag(); mes_1D.push_back(ibooker.book1S("th1s" + num, "1D Short Histogram " + num, 101, -0.5, 100.5)); - mes_1D.back()->setLumiFlag(); mes_1D.push_back(ibooker.book1DD("th1d" + num, "1D Double Histogram " + num, 101, -0.5, 100.5)); - mes_1D.back()->setLumiFlag(); mes_2D.push_back(ibooker.book2D("th2f" + num, "2D Float Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5)); - mes_2D.back()->setLumiFlag(); mes_2D.push_back(ibooker.book2S("th2s" + num, "2D Short Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5)); - mes_2D.back()->setLumiFlag(); mes_2D.push_back(ibooker.book2DD("th2d" + num, "2D Double Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5)); - mes_2D.back()->setLumiFlag(); mes_2D.push_back( ibooker.bookProfile("tprofile" + num, "1D Profile Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5)); - mes_2D.back()->setLumiFlag(); mes_3D.push_back( ibooker.book3D("th3f" + num, "3D Float Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); - mes_3D.back()->setLumiFlag(); mes_3D.push_back(ibooker.bookProfile2D( "thprofile2d" + num, "2D Profile Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); - mes_3D.back()->setLumiFlag(); + + ibooker.setScope(scope); } } } From d48242a568cac96c40fce96a550d494908eaa8e2 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 20:15:37 +0100 Subject: [PATCH 055/112] Use initial value for bookString() --- DQMServices/Core/interface/DQMStore.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 04b34adf524c5..be25932175fcc 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -79,11 +79,13 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* bookString(TString const& name, TString const& value, FUNC onbooking = NOOP()) { - // TODO: value unused! - return bookME(name, MonitorElementData::Kind::STRING, [=]() { + std::string initial_value(value); + auto me = bookME(name, MonitorElementData::Kind::STRING, [=]() { onbooking(); return nullptr; }); + me->Fill(initial_value); + return me; } template ::value, int> = 0> From 93db601b2646f776b86f741e699885b012177c45 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 20:16:05 +0100 Subject: [PATCH 056/112] Kill mtEnabled. --- DQMServices/Core/interface/DQMStore.h | 3 --- DQMServices/FileIO/plugins/DQMFileSaverPB.cc | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index be25932175fcc..17af15c79301a 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -514,9 +514,6 @@ namespace dqm { bool fileMustExist = true); bool load(std::string const& filename, OpenRunDirs stripdirs = StripRunDirs, bool fileMustExist = true); - DQM_DEPRECATED - bool mtEnabled() { assert(!"NIY"); } - DQM_DEPRECATED void showDirStructure() const; diff --git a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc index 62b08e8b305da..3b17dd819dd92 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc @@ -83,7 +83,7 @@ void DQMFileSaverPB::saveLumi(const FileParameters& fp) const { if (fms ? fms->getEventsProcessedForLumi(fp.lumi_) : true) { // Save the file in the open directory. - this->savePB(&*store, openHistoFilePathName, store->mtEnabled() ? fp.run_ : 0, fp.lumi_); + this->savePB(&*store, openHistoFilePathName, fp.run_, fp.lumi_); // Now move the the data and json files into the output directory. ::rename(openHistoFilePathName.c_str(), histoFilePathName.c_str()); From a25a4e2c930f1584a302e9eec55dc52b27af0087 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 20:30:19 +0100 Subject: [PATCH 057/112] Properly handle saveByLumi. --- DQMServices/Core/interface/DQMStore.h | 11 ++++++++--- DQMServices/Core/src/DQMStore.cc | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 17af15c79301a..fa36a0ac3a63c 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -550,9 +550,11 @@ namespace dqm { void bookTransaction(iFunc f, uint64_t moduleId, bool canSaveByLumi) { auto lock = std::scoped_lock(this->booking_mutex_); IBooker& booker = *this; - // TODO: this may need to become more elaborate. - auto oldscope = - booker.setScope(canSaveByLumi ? MonitorElementData::Scope::LUMI : MonitorElementData::Scope::RUN); + auto newscope = MonitorElementData::Scope::RUN; + if (canSaveByLumi && this->doSaveByLumi_) { + newscope = MonitorElementData::Scope::LUMI; + } + auto oldscope = booker.setScope(newscope); assert(moduleId != 0 || !"moduleID must be set for normal booking transaction"); // Access via this-> to allow access to protected member auto oldmoduleid = this->setModuleID(moduleId); @@ -642,6 +644,9 @@ namespace dqm { // If set to true, error out whenever things happen that are not safe for // legacy modules. bool assertLegacySafe_; + + // Book MEs by lumi by default whenever possible. + bool doSaveByLumi_; }; } // namespace implementation diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 207bb1adb984c..4e5becede2503 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -583,6 +583,7 @@ namespace dqm::implementation { DQMStore::DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry& ar) : IGetter(this), IBooker(this) { verbose_ = pset.getUntrackedParameter("verbose", 0); assertLegacySafe_ = pset.getUntrackedParameter("assertLegacySafe", true); + doSaveByLumi_ = pset.getUntrackedParameter("saveByLumi", false); // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be From b509c052a6f59d2e5839d99803e1ebdd4e5427a1 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 20:44:28 +0100 Subject: [PATCH 058/112] Fix paths in DQMFileSaverPB. --- DQMServices/FileIO/plugins/DQMFileSaverPB.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc index 3b17dd819dd92..77a51a471d0e6 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverPB.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverPB.cc @@ -219,8 +219,7 @@ void DQMFileSaverPB::savePB(DQMStore* store, std::string const& filename, int ru buffer.WriteObject(me->getRootObject()); } dqmstorepb::ROOTFilePB::Histo& histo = *dqmstore_message.add_histo(); - // TODO: getPathname returns a name with trailing slash? - histo.set_full_pathname(me->getPathname() + "/" + me->getName()); + histo.set_full_pathname(me->getFullname()); uint32_t flags = 0; flags |= (uint32_t)me->kind(); if (me->getLumiFlag()) From 8f3bf3014a9bf957d1788cd4ae972f848186bac4 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 12 Dec 2019 20:48:25 +0100 Subject: [PATCH 059/112] Remove debug output in DQMRootSource. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 762eaafdb654f..c1cd03f4cd85f 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -614,8 +614,8 @@ std::unique_ptr DQMRootSource::readFile_() { std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); } - for (auto& metadata : m_fileMetadatas) - metadata.describe(); + //for (auto& metadata : m_fileMetadatas) + // metadata.describe(); // Stop if there's nothing to process. Otherwise start the run. if (m_fileMetadatas.empty()) From 2b0c322ac6ee158036163fafc2d28a9eb9e85e47 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 13 Dec 2019 11:07:28 +0100 Subject: [PATCH 060/112] Remove setLumiFlag. And migrate all the usages. --- DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc | 3 +- DQM/DTMonitorModule/src/DTDCSByLumiTask.cc | 4 ++- DQM/EcalCommon/src/MESetEcal.cc | 10 ++++--- DQM/EcalCommon/src/MESetNonObject.cc | 5 +++- .../src/ESIntegrityTask.cc | 3 +- DQM/HcalCommon/interface/Container1D.h | 3 -- DQM/HcalCommon/src/Container1D.cc | 6 ---- DQM/HcalTasks/plugins/DigiPhase1Task.cc | 9 +++--- DQM/HcalTasks/plugins/DigiTask.cc | 14 ++++++---- DQM/HcalTasks/plugins/RawTask.cc | 28 +++++++++++-------- DQM/HcalTasks/plugins/RecHitTask.cc | 3 +- DQM/HcalTasks/plugins/TPTask.cc | 3 +- DQM/RPCMonitorDigi/src/RPCDcsInfo.cc | 3 +- .../src/SiPixelDigiSource.cc | 3 +- .../src/SiPixelRawDataErrorSource.cc | 3 +- .../src/FEDHistograms.cc | 3 +- DQM/TrackingMonitor/interface/TrackAnalyzer.h | 1 - DQM/TrackingMonitor/src/TrackAnalyzer.cc | 10 ------- DQM/TrackingMonitor/src/TrackingMonitor.cc | 23 ++++++++------- .../Components/plugins/EDMtoMEConverter.cc | 12 ++++---- DQMServices/Core/interface/MonitorElement.h | 4 --- 21 files changed, 77 insertions(+), 76 deletions(-) diff --git a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc index 066c61dcb4e62..8d8ebe4bd6704 100644 --- a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc +++ b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc @@ -173,6 +173,7 @@ void AlcaBeamMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& } } ibooker.setCurrentFolder(monitorName_ + "Service"); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); theValuesContainer_ = ibooker.bookProfile("hHistoLumiValues", "Histo Lumi Values", 3 * numberOfValuesToSave_, @@ -182,7 +183,7 @@ void AlcaBeamMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& -100., 9000., " "); - theValuesContainer_->setLumiFlag(); + ibooker.setScope(scope); // create and cd into new folder ibooker.setCurrentFolder(monitorName_ + "Validation"); diff --git a/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc b/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc index 2a62e5471a113..e0b6a5808e7a6 100644 --- a/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc +++ b/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc @@ -73,9 +73,11 @@ void DTDCSByLumiTask::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& stringstream wheel_str; wheel_str << wheel; + // Set Lumi scope in order to save histo every LS + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); MonitorElement* ME = ibooker.book1D("hActiveUnits" + wheel_str.str(), "Active Untis x LS Wh" + wheel_str.str(), 2, 0.5, 2.5); - ME->setLumiFlag(); // Set LumiFlag in order to save histo every LS + ibooker.setScope(scope); hActiveUnits.push_back(ME); } diff --git a/DQM/EcalCommon/src/MESetEcal.cc b/DQM/EcalCommon/src/MESetEcal.cc index 88345af3ad7ba..850898186c10a 100644 --- a/DQM/EcalCommon/src/MESetEcal.cc +++ b/DQM/EcalCommon/src/MESetEcal.cc @@ -70,10 +70,9 @@ namespace ecaldqm { void MESetEcal::book(DQMStore::IBooker &_ibooker) { using namespace std; - _ibooker.setScope(MonitorElementData::Scope::RUN); - if (lumiFlag_) { - _ibooker.setScope(MonitorElementData::Scope::LUMI); - } + auto oldscope = MonitorElementData::Scope::RUN; + if (lumiFlag_) + oldscope = _ibooker.setScope(MonitorElementData::Scope::LUMI); clear(); @@ -272,6 +271,9 @@ namespace ecaldqm { mes_.push_back(me); } + if (lumiFlag_) + _ibooker.setScope(oldscope); + active_ = true; } diff --git a/DQM/EcalCommon/src/MESetNonObject.cc b/DQM/EcalCommon/src/MESetNonObject.cc index 74ad76e09c0b5..bcd6b251d8961 100644 --- a/DQM/EcalCommon/src/MESetNonObject.cc +++ b/DQM/EcalCommon/src/MESetNonObject.cc @@ -65,6 +65,9 @@ namespace ecaldqm { size_t slashPos(path_.find_last_of('/')); string name(path_.substr(slashPos + 1)); _ibooker.setCurrentFolder(path_.substr(0, slashPos)); + auto oldscope = MonitorElementData::Scope::RUN; + if (lumiFlag_) + oldscope = _ibooker.setScope(MonitorElementData::Scope::LUMI); MonitorElement *me(nullptr); @@ -172,7 +175,7 @@ namespace ecaldqm { } if (lumiFlag_) - me->setLumiFlag(); + _ibooker.setScope(oldscope); mes_.push_back(me); diff --git a/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc b/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc index 50731164c6154..4fd2a3f180909 100644 --- a/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc +++ b/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc @@ -178,8 +178,9 @@ void ESIntegrityTask::bookHistograms(DQMStore::IBooker& iBooker, edm::Run const& if (doLumiAnalysis_) { sprintf(histo, "ES Good Channel Fraction"); + auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); meDIFraction_ = iBooker.book2D(histo, histo, 3, 1.0, 3.0, 3, 1.0, 3.0); - meDIFraction_->setLumiFlag(); + iBooker.setScope(scope); } } diff --git a/DQM/HcalCommon/interface/Container1D.h b/DQM/HcalCommon/interface/Container1D.h index 78135e90e0e43..1c9e76a117e35 100644 --- a/DQM/HcalCommon/interface/Container1D.h +++ b/DQM/HcalCommon/interface/Container1D.h @@ -222,9 +222,6 @@ namespace hcaldqm { // TO BE USED IN THE FUTURE! virtual void extendAxisRange(int); - // set lumi flags for all mes - virtual void setLumiFlag(); - virtual void showOverflowX(bool showOverflow); virtual void showOverflowY(bool showOverflow); diff --git a/DQM/HcalCommon/src/Container1D.cc b/DQM/HcalCommon/src/Container1D.cc index bbcc7c6d52642..ef53e7b5b3910 100644 --- a/DQM/HcalCommon/src/Container1D.cc +++ b/DQM/HcalCommon/src/Container1D.cc @@ -766,12 +766,6 @@ namespace hcaldqm { } } - /* virtual */ void Container1D::setLumiFlag() { - for (auto const &pair : _mes) { - pair.second->setLumiFlag(); - } - } - void Container1D::showOverflowX(bool showOverflow) { _qx->showOverflow(showOverflow); } void Container1D::showOverflowY(bool showOverflow) { _qy->showOverflow(showOverflow); } diff --git a/DQM/HcalTasks/plugins/DigiPhase1Task.cc b/DQM/HcalTasks/plugins/DigiPhase1Task.cc index 37d2dcf2b7dd0..798a797018242 100644 --- a/DQM/HcalTasks/plugins/DigiPhase1Task.cc +++ b/DQM/HcalTasks/plugins/DigiPhase1Task.cc @@ -260,8 +260,9 @@ DigiPhase1Task::DigiPhase1Task(edm::ParameterSet const& ps) : DQTask(ps) { _cTimingCut_depth.book(ib, _emap, _subsystem); _cOccupancyvsLS_Subdet.book(ib, _emap, _subsystem); - _cOccupancy_depth.book(ib, _emap, _subsystem); _cOccupancyCut_depth.book(ib, _emap, _subsystem); + if (_ptype != fOffline) + _cOccupancy_depth.book(ib, _emap, _subsystem); if (_ptype != fOffline) { // hidefed2crate _cShapeCut_FED.book(ib, _emap, _subsystem); @@ -286,21 +287,21 @@ DigiPhase1Task::DigiPhase1Task(edm::ParameterSet const& ps) : DQTask(ps) { _dhashmap.initialize(_emap, electronicsmap::fE2DHashMap); // MARK THESE HISTOGRAMS AS LUMI BASED FOR OFFLINE PROCESSING + auto scope = ib.setScope(MonitorElementData::Scope::LUMI); if (_ptype == fOffline) { //_cDigiSize_FED.setLumiFlag(); // hidefed2crate : FED stuff not available offline anymore, so this histogram doesn't make sense? - _cOccupancy_depth.setLumiFlag(); + _cOccupancy_depth.book(ib, _emap, _subsystem); } // book Number of Events vs LS histogram ib.setCurrentFolder(_subsystem + "/RunInfo"); meNumEvents1LS = ib.book1D("NumberOfEvents", "NumberOfEvents", 1, 0, 1); - meNumEvents1LS->setLumiFlag(); // book the flag for unknown ids and the online guy as well ib.setCurrentFolder(_subsystem + "/" + _name); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); _unknownIdsPresent = false; - meUnknownIds1LS->setLumiFlag(); + ib.setScope(scope); } /* virtual */ void DigiPhase1Task::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/HcalTasks/plugins/DigiTask.cc b/DQM/HcalTasks/plugins/DigiTask.cc index 50351e4df6be6..874b63c4bedb0 100644 --- a/DQM/HcalTasks/plugins/DigiTask.cc +++ b/DQM/HcalTasks/plugins/DigiTask.cc @@ -656,7 +656,6 @@ DigiTask::DigiTask(edm::ParameterSet const& ps) : DQTask(ps) { _cSumQ_depth.book(ib, _emap, _subsystem); _cSumQvsLS_SubdetPM.book(ib, _emap, _filter_QIE8, _subsystem); _cSumQvsLS_SubdetPM_QIE1011.book(ib, _emap, _filter_QIE1011, _subsystem); - _cDigiSize_Crate.book(ib, _emap, _subsystem); _cADCvsTS_SubdetPM.book(ib, _emap, _filter_QIE8, _subsystem); _cADCvsTS_SubdetPM_QIE1011.book(ib, _emap, _filter_QIE1011, _subsystem); @@ -677,13 +676,16 @@ DigiTask::DigiTask(edm::ParameterSet const& ps) : DQTask(ps) { _cOccupancyCut_ElectronicsuTCA.book(ib, _emap, _filter_VME, _subsystem); _cDigiSize_FED.book(ib, _emap, _subsystem); } + if (_ptype != fOffline) { // else book per-lumi later. + _cDigiSize_Crate.book(ib, _emap, _subsystem); + _cOccupancy_depth.book(ib, _emap, _subsystem); + } _cTimingCut_SubdetPM.book(ib, _emap, _subsystem); _cTimingCut_depth.book(ib, _emap, _subsystem); _cTimingCutvsLS_SubdetPM.book(ib, _emap, _subsystem); _cOccupancyvsLS_Subdet.book(ib, _emap, _subsystem); - _cOccupancy_depth.book(ib, _emap, _subsystem); _cOccupancyCut_depth.book(ib, _emap, _subsystem); _cLETDCTimevsADC_SubdetPM.book(ib, _emap, _subsystem); @@ -775,22 +777,22 @@ DigiTask::DigiTask(edm::ParameterSet const& ps) : DQTask(ps) { } // MARK THESE HISTOGRAMS AS LUMI BASED FOR OFFLINE PROCESSING + auto scope = ib.setScope(MonitorElementData::Scope::LUMI); if (_ptype == fOffline) { - _cDigiSize_Crate.setLumiFlag(); //_cDigiSize_FED.setLumiFlag(); - _cOccupancy_depth.setLumiFlag(); + _cDigiSize_Crate.book(ib, _emap, _subsystem); + _cOccupancy_depth.book(ib, _emap, _subsystem); } // book Number of Events vs LS histogram ib.setCurrentFolder(_subsystem + "/RunInfo"); meNumEvents1LS = ib.book1D("NumberOfEvents", "NumberOfEvents", 1, 0, 1); - meNumEvents1LS->setLumiFlag(); // book the flag for unknown ids and the online guy as well ib.setCurrentFolder(_subsystem + "/" + _name); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); _unknownIdsPresent = false; - meUnknownIds1LS->setLumiFlag(); + ib.setScope(scope); } /* virtual */ void DigiTask::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/HcalTasks/plugins/RawTask.cc b/DQM/HcalTasks/plugins/RawTask.cc index 62caf22233ad2..edec13dd19ccf 100644 --- a/DQM/HcalTasks/plugins/RawTask.cc +++ b/DQM/HcalTasks/plugins/RawTask.cc @@ -181,7 +181,13 @@ RawTask::RawTask(edm::ParameterSet const& ps) : DQTask(ps) { _cBadQuality_FEDVME.book(ib, _emap, _filter_uTCA, _subsystem); _cBadQuality_FEDuTCA.book(ib, _emap, _filter_VME, _subsystem); } - _cBadQuality_depth.book(ib, _emap, _subsystem); + if (_ptype == fOffline) { + auto scope = ib.setScope(MonitorElementData::Scope::LUMI); + _cBadQuality_depth.book(ib, _emap, _subsystem); + ib.setScope(scope); + } else { + _cBadQuality_depth.book(ib, _emap, _subsystem); + } _cBadQualityvsLS.book(ib, _subsystem); _cBadQualityvsBX.book(ib, _subsystem); @@ -197,16 +203,16 @@ RawTask::RawTask(edm::ParameterSet const& ps) : DQTask(ps) { } // FOR OFFLINE PROCESSING MARK THESE HISTOGRAMS AS LUMI BASED - if (_ptype == fOffline) { - if (_ptype != fOffline) { // hidefed2crate - // Note that this is deliberately contradictory for the fed2crate fix, so it can be reversed if fed2crate is ever fixed properly, - _cEvnMsm_ElectronicsVME.setLumiFlag(); - _cBcnMsm_ElectronicsVME.setLumiFlag(); - _cEvnMsm_ElectronicsuTCA.setLumiFlag(); - _cBcnMsm_ElectronicsuTCA.setLumiFlag(); - } - _cBadQuality_depth.setLumiFlag(); - } + //if (_ptype == fOffline) { + //if (_ptype != fOffline) { // hidefed2crate + // Note that this is deliberately contradictory for the fed2crate fix, so it can be reversed if fed2crate is ever fixed properly, + // TODO: set LUMI scope while booking. + // _cEvnMsm_ElectronicsVME.setLumiFlag(); + // _cBcnMsm_ElectronicsVME.setLumiFlag(); + // _cEvnMsm_ElectronicsuTCA.setLumiFlag(); + // _cBcnMsm_ElectronicsuTCA.setLumiFlag(); + //} + //} // initialize hash map _ehashmap.initialize(_emap, hcaldqm::electronicsmap::fD2EHashMap); diff --git a/DQM/HcalTasks/plugins/RecHitTask.cc b/DQM/HcalTasks/plugins/RecHitTask.cc index c78aafc9ba4d9..2d73e8c3ac264 100644 --- a/DQM/HcalTasks/plugins/RecHitTask.cc +++ b/DQM/HcalTasks/plugins/RecHitTask.cc @@ -454,9 +454,10 @@ RecHitTask::RecHitTask(edm::ParameterSet const& ps) : DQTask(ps) { // book some mes... ib.setCurrentFolder(_subsystem + "/" + _name); + auto scope = ib.setScope(MonitorElementData::Scope::LUMI); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); + ib.setScope(scope); _unknownIdsPresent = false; - meUnknownIds1LS->setLumiFlag(); } /* virtual */ void RecHitTask::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/HcalTasks/plugins/TPTask.cc b/DQM/HcalTasks/plugins/TPTask.cc index 33ff752064e72..79f520b4b17f0 100644 --- a/DQM/HcalTasks/plugins/TPTask.cc +++ b/DQM/HcalTasks/plugins/TPTask.cc @@ -725,9 +725,10 @@ TPTask::TPTask(edm::ParameterSet const& ps) : DQTask(ps) { // book the flag for unknown ids and the online guy as well ib.setCurrentFolder(_subsystem + "/" + _name); + auto scope = ib.setScope(MonitorElementData::Scope::LUMI); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); + ib.setScope(scope); _unknownIdsPresent = false; - meUnknownIds1LS->setLumiFlag(); } /* virtual */ void TPTask::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc b/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc index 6940d36b2feda..f37ba07994272 100644 --- a/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc +++ b/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc @@ -18,8 +18,9 @@ void RPCDcsInfo::bookHistograms(DQMStore::IBooker& ibooker, ibooker.cd(); ibooker.setCurrentFolder(subsystemname_ + "/" + dcsinfofolder_); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); DCSbyLS_ = ibooker.book1D("DCSbyLS", "DCS", 1, 0.5, 1.5); - DCSbyLS_->setLumiFlag(); + ibooker.setScope(scope); // initialize dcs = true; diff --git a/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc b/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc index 5e6adb6574525..ee25f5170fc7c 100644 --- a/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc +++ b/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc @@ -1231,8 +1231,9 @@ void SiPixelDigiSource::bookMEs(DQMStore::IBooker& iBooker, const edm::EventSetu char title8[80]; sprintf(title8, "FED Digi Occupancy (NDigis/) vs LumiSections;Lumi Section;FED"); if (modOn) { + auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); averageDigiOccupancy = iBooker.bookProfile("averageDigiOccupancy", title7, 40, -0.5, 39.5, 0., 3.); - averageDigiOccupancy->setLumiFlag(); + iBooker.setScope(scope); avgfedDigiOccvsLumi = iBooker.book2D("avgfedDigiOccvsLumi", title8, 640, 0., 3200., 40, -0.5, 39.5); avgBarrelFedOccvsLumi = iBooker.book1D( "avgBarrelFedOccvsLumi", diff --git a/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc b/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc index 59ca7e68498e0..767a82a6244ed 100644 --- a/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc +++ b/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc @@ -285,8 +285,9 @@ void SiPixelRawDataErrorSource::bookMEs(DQMStore::IBooker &iBooker) { iBooker.setCurrentFolder(topFolderName_ + "/AdditionalPixelErrors"); char title[80]; sprintf(title, "By-LumiSection Error counters"); + auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); byLumiErrors = iBooker.book1D("byLumiErrors", title, 2, 0., 2.); - byLumiErrors->setLumiFlag(); + iBooker.setScope(scope); char title1[80]; sprintf(title1, "Errors per LumiSection;LumiSection;NErrors"); errorRate = iBooker.book1D("errorRate", title1, 5000, 0., 5000.); diff --git a/DQM/SiStripMonitorHardware/src/FEDHistograms.cc b/DQM/SiStripMonitorHardware/src/FEDHistograms.cc index efa040f75086e..8058860fd6abc 100644 --- a/DQM/SiStripMonitorHardware/src/FEDHistograms.cc +++ b/DQM/SiStripMonitorHardware/src/FEDHistograms.cc @@ -895,6 +895,7 @@ void FEDHistograms::bookTopLevelHistograms(DQMStore::IBooker& ibooker, ibooker.setCurrentFolder(lBaseDir + "/PerLumiSection"); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); bookHistogram(ibooker, lumiErrorFraction_, "lumiErrorFraction", @@ -903,10 +904,10 @@ void FEDHistograms::bookTopLevelHistograms(DQMStore::IBooker& ibooker, 0.5, 6.5, "SubDetId"); + ibooker.setScope(scope); //Set special property for lumi ME if (lumiErrorFraction_.enabled && lumiErrorFraction_.monitorEle) { - lumiErrorFraction_.monitorEle->setLumiFlag(); lumiErrorFraction_.monitorEle->setBinLabel(1, "TECB"); lumiErrorFraction_.monitorEle->setBinLabel(2, "TECF"); lumiErrorFraction_.monitorEle->setBinLabel(3, "TIB"); diff --git a/DQM/TrackingMonitor/interface/TrackAnalyzer.h b/DQM/TrackingMonitor/interface/TrackAnalyzer.h index 6b4c1cd3209ad..11ab3c0b0ddc2 100644 --- a/DQM/TrackingMonitor/interface/TrackAnalyzer.h +++ b/DQM/TrackingMonitor/interface/TrackAnalyzer.h @@ -47,7 +47,6 @@ class TrackAnalyzer { void analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup, const reco::Track& track); void doReset(); - void setLumiFlag(); // Compute and locally store the number of Good vertices found // in the event. This information is used as X-axis value in // the hit-efficiency plots derived from the hit patter. This diff --git a/DQM/TrackingMonitor/src/TrackAnalyzer.cc b/DQM/TrackingMonitor/src/TrackAnalyzer.cc index 3c4e0a8d0b6f0..bba6f8ee052a3 100644 --- a/DQM/TrackingMonitor/src/TrackAnalyzer.cc +++ b/DQM/TrackingMonitor/src/TrackAnalyzer.cc @@ -2437,16 +2437,6 @@ void TrackAnalyzer::fillHistosForTrackerSpecific(const reco::Track& track) { } } // -// -- Set Lumi Flag -// -void TrackAnalyzer::setLumiFlag() { - TkParameterMEs tkmes; - if (Chi2oNDF_lumiFlag) - Chi2oNDF_lumiFlag->setLumiFlag(); - if (NumberOfRecHitsPerTrack_lumiFlag) - NumberOfRecHitsPerTrack_lumiFlag->setLumiFlag(); -} -// // -- Apply Reset // void TrackAnalyzer::doReset() { diff --git a/DQM/TrackingMonitor/src/TrackingMonitor.cc b/DQM/TrackingMonitor/src/TrackingMonitor.cc index 64c11e906fe27..8868deba25c12 100644 --- a/DQM/TrackingMonitor/src/TrackingMonitor.cc +++ b/DQM/TrackingMonitor/src/TrackingMonitor.cc @@ -342,11 +342,13 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& if (doLumiAnalysis) { // add by Mia in order to deal with LS transitions ibooker.setCurrentFolder(MEFolderName + "/LSanalysis"); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); histname = "NumberOfTracks_lumiFlag_" + CategoryName; NumberOfTracks_lumiFlag = ibooker.book1D(histname, histname, 3 * TKNoBin, TKNoMin, (TKNoMax + 0.5) * 3. - 0.5); NumberOfTracks_lumiFlag->setAxisTitle("Number of Tracks per Event", 1); NumberOfTracks_lumiFlag->setAxisTitle("Number of Events", 2); + ibooker.setScope(scope); } // book profile plots vs LS : @@ -577,7 +579,13 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& NumberOfTracksVsBXlumi->setAxisTitle("Mean number of Tracks", 2); } - theTrackAnalyzer->initHisto(ibooker, iSetup, *conf); + if (doLumiAnalysis) { + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + theTrackAnalyzer->initHisto(ibooker, iSetup, *conf); + ibooker.setScope(scope); + } else { + theTrackAnalyzer->initHisto(ibooker, iSetup, *conf); + } // book the Seed Property histograms // ---------------------------------------------------------------------------------// @@ -599,10 +607,12 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& if (doSeedLumiAnalysis_) { ibooker.setCurrentFolder(MEFolderName + "/LSanalysis"); + auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); histname = "NumberOfSeeds_lumiFlag_" + seedProducer.label() + "_" + CategoryName; NumberOfSeeds_lumiFlag = ibooker.book1D(histname, histname, TKNoSeedBin, TKNoSeedMin, TKNoSeedMax); NumberOfSeeds_lumiFlag->setAxisTitle("Number of Seeds per Event", 1); NumberOfSeeds_lumiFlag->setAxisTitle("Number of Events", 2); + ibooker.setScope(scope); } } @@ -666,17 +676,6 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& theTrackBuildingAnalyzer->initHisto(ibooker, *conf); - if (doLumiAnalysis) { - if (NumberOfTracks_lumiFlag) - NumberOfTracks_lumiFlag->setLumiFlag(); - theTrackAnalyzer->setLumiFlag(); - } - - if (doAllSeedPlots || doSeedNumberPlot) { - if (doSeedLumiAnalysis_) - NumberOfSeeds_lumiFlag->setLumiFlag(); - } - if (doTrackerSpecific_ || doAllPlots) { ClusterLabels = conf->getParameter >("ClusterLabels"); diff --git a/DQMServices/Components/plugins/EDMtoMEConverter.cc b/DQMServices/Components/plugins/EDMtoMEConverter.cc index dfe52a3e3d30e..ffc3bc7bc850b 100644 --- a/DQMServices/Components/plugins/EDMtoMEConverter.cc +++ b/DQMServices/Components/plugins/EDMtoMEConverter.cc @@ -247,8 +247,11 @@ namespace { } }; - void maybeSetLumiFlag(MonitorElement *me, const edm::Run &) {} - void maybeSetLumiFlag(MonitorElement *me, const edm::LuminosityBlock &) { me->setLumiFlag(); } + // TODO: might need re-scoping to JOB here. + void adjustScope(DQMStore::IBooker &ibooker, const edm::Run &) { ibooker.setScope(MonitorElementData::Scope::RUN); } + void adjustScope(DQMStore::IBooker &ibooker, const edm::LuminosityBlock &) { + ibooker.setScope(MonitorElementData::Scope::LUMI); + } } // namespace @@ -350,9 +353,8 @@ void EDMtoMEConverter::getData(DQMStore::IBooker &iBooker, DQMStore::IGetter &iG } // define new monitor element - MonitorElement *me = - AddMonitorElement::call(iBooker, iGetter, &metoedmobject[i].object, dir, name, iGetFrom); - maybeSetLumiFlag(me, iGetFrom); + adjustScope(iBooker, iGetFrom); + AddMonitorElement::call(iBooker, iGetter, &metoedmobject[i].object, dir, name, iGetFrom); } // end loop thorugh metoedmobject }); diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 2884f13b31c6c..8f9c8b50df884 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -269,10 +269,6 @@ namespace dqm::impl { /// true if ME is meant to be stored for each luminosity section bool getLumiFlag() const { return access().key.scope_ == MonitorElementData::Scope::LUMI; } - /// this ME is meant to be stored for each luminosity section - // we can't support this any more, but the ME might be safed by lumi anyways! - void setLumiFlag() { assert(getLumiFlag()); } - /// this ME is meant to be an efficiency plot that must not be /// normalized when drawn in the DQM GUI. void setEfficiencyFlag() { data_.flags |= DQMNet::DQM_PROP_EFFICIENCY_PLOT; } From 63190563b738d13b8bb294b6980f99ebd6a901c3 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 13 Dec 2019 13:08:44 +0100 Subject: [PATCH 061/112] Fix MEtoEDM. --- .../Components/plugins/MEtoEDMConverter.cc | 16 ++++------------ .../Components/plugins/MEtoEDMConverter.h | 1 - 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/DQMServices/Components/plugins/MEtoEDMConverter.cc b/DQMServices/Components/plugins/MEtoEDMConverter.cc index 0f6b7303628a6..d1427a4730384 100644 --- a/DQMServices/Components/plugins/MEtoEDMConverter.cc +++ b/DQMServices/Components/plugins/MEtoEDMConverter.cc @@ -25,7 +25,6 @@ MEtoEDMConverter::MEtoEDMConverter(const edm::ParameterSet& iPSet) : fName(""), verbosity = iPSet.getUntrackedParameter("Verbosity", 0); frequency = iPSet.getUntrackedParameter("Frequency", 50); path = iPSet.getUntrackedParameter("MEPathToSave"); - enableMultiThread_ = false; // use value of first digit to determine default output level (inclusive) // 0 is none, 1 is basic, 2 is fill output, 3 is gather output verbosity %= 10; @@ -82,11 +81,7 @@ MEtoEDMConverter::MEtoEDMConverter(const edm::ParameterSet& iPSet) : fName(""), MEtoEDMConverter::~MEtoEDMConverter() = default; -void MEtoEDMConverter::beginJob() { - // Determine if we are running multithreading asking to the DQMStore. Not to be moved in the ctor - DQMStore* dbe = edm::Service().operator->(); - enableMultiThread_ = false; //dbe->enableMultiThread_; -} +void MEtoEDMConverter::beginJob() {} void MEtoEDMConverter::endJob() {} @@ -123,8 +118,7 @@ void MEtoEDMConverter::putData(DQMStore::IGetter& iGetter, T& iPutTo, bool iLumi // extract ME information into vectors std::vector::iterator mmi, mme; - std::vector items( - iGetter.getAllContents(path, enableMultiThread_ ? run : 0, enableMultiThread_ ? lumi : 0)); + std::vector items(iGetter.getAllContents(path, run, lumi)); unsigned int n1F = 0; unsigned int n1S = 0; @@ -224,10 +218,8 @@ void MEtoEDMConverter::putData(DQMStore::IGetter& iGetter, T& iPutTo, bool iLumi // store only flagged ME at endLumi transition, and Run-based // histo at endRun transition - if (iLumiOnly && !me->getLumiFlag()) - continue; - if (!iLumiOnly && me->getLumiFlag()) - continue; + // DQMStore should only hand out matching MEs + assert(iLumiOnly == me->getLumiFlag()); // get monitor elements switch (me->kind()) { diff --git a/DQMServices/Components/plugins/MEtoEDMConverter.h b/DQMServices/Components/plugins/MEtoEDMConverter.h index 8ee9cefa2168e..dd838de1c9cfb 100644 --- a/DQMServices/Components/plugins/MEtoEDMConverter.h +++ b/DQMServices/Components/plugins/MEtoEDMConverter.h @@ -87,7 +87,6 @@ class MEtoEDMConverter : public edm::one::EDProducer, std::string fName; int verbosity; int frequency; - bool enableMultiThread_; std::string path; // private statistics information From fae7feb49e6548c10466b5d33d535ca70aecb78c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 13 Dec 2019 19:24:45 +0100 Subject: [PATCH 062/112] Clang says these are unused. They also don't seem very important. --- DQMServices/Core/src/DQMNet.cc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/DQMServices/Core/src/DQMNet.cc b/DQMServices/Core/src/DQMNet.cc index ab9e3ce8eb262..a57b9c3cf02e5 100644 --- a/DQMServices/Core/src/DQMNet.cc +++ b/DQMServices/Core/src/DQMNet.cc @@ -38,15 +38,10 @@ static const Regexp s_rxmeval("<(.*)>(i|f|s|qr)=(.*)"); // TODO: Can't include the header file since that leads to ambiguities. namespace dqm { namespace qstatus { - static const int OTHER = 30; //< Anything but 'ok','warning' or 'error'. - static const int DISABLED = 50; //< Test has been disabled. - static const int INVALID = 60; //< Problem preventing test from running. - static const int INSUF_STAT = 70; //< Insufficient statistics. - static const int DID_NOT_RUN = 90; //< Algorithm did not run. - static const int STATUS_OK = 100; //< Test was succesful. - static const int WARNING = 200; //< Test had some problems. - static const int ERROR = 300; //< Test has failed. - } // namespace qstatus + static const int STATUS_OK = 100; //< Test was succesful. + static const int WARNING = 200; //< Test had some problems. + static const int ERROR = 300; //< Test has failed. + } // namespace qstatus } // namespace dqm ////////////////////////////////////////////////////////////////////// From 9c08d524e2a5b1f75d7ba57f817ef7b2cc6304bc Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 18:05:01 +0100 Subject: [PATCH 063/112] Add a feature to track the life cycle of a ME for debugging. --- DQMServices/Core/interface/DQMStore.h | 6 ++++ DQMServices/Core/src/DQMStore.cc | 45 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index fa36a0ac3a63c..a3af5f8ef4fdd 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -604,6 +604,8 @@ namespace dqm { MonitorElement* findME(MELIKE const& path); // Log a backtrace on booking. void printTrace(std::string const& message); + // print a log message if ME matches trackME_. + void debugTrackME(const char* message, MonitorElement* me) const; private: // MEComparison is a name-only comparison on MEs and Paths, allowing @@ -647,6 +649,10 @@ namespace dqm { // Book MEs by lumi by default whenever possible. bool doSaveByLumi_; + + // if non-empty, debugTrackME calls will log some information whenever a + // ME path contains this string. + std::string trackME_; }; } // namespace implementation diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 4e5becede2503..6c7ff375ff9ef 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -113,9 +113,11 @@ namespace dqm::implementation { // assertLegacySafe option). // We still created a local ME, so we can drive the lumi-changing for // legacy modules in watchPreGlobalBeginLumi. + store_->debugTrackME("bookME (legacy)", me); return me; } else { // the normal case. + store_->debugTrackME("bookME (legacy)", me); return localme; } } @@ -126,6 +128,7 @@ namespace dqm::implementation { auto existing_new = globalMEs_[me->getRunLumi()].insert(me); if (existing_new.second == true) { // successfully inserted, return new object + debugTrackME("putME (global)", me); return me; } else { // already present, return old object @@ -146,6 +149,7 @@ namespace dqm::implementation { auto existing_new = localmes.insert(local_me); // successfully inserted, return new object assert(existing_new.second == true); // insert successful + debugTrackME("putME (local, new)", me); return local_me; } else { // already present, return old object @@ -160,6 +164,7 @@ namespace dqm::implementation { if (!local_me->isValid()) { local_me->switchData(me); } + debugTrackME("putME (local, existing)", me); return local_me; } } @@ -170,6 +175,7 @@ namespace dqm::implementation { for (auto& [runlumi, meset] : this->globalMEs_) { auto it = meset.find(path); if (it != meset.end()) { + debugTrackME("findME (found)", *it); // no guarantee on which ME we return here -- only that clone'ing this // would give a valid ME for that path. return *it; @@ -247,6 +253,32 @@ namespace dqm::implementation { }); } + void DQMStore::debugTrackME(const char* message, MonitorElement* me) const { + const char* scopename[] = {"INVALID", "JOB", "RUN", "LUMI"}; + if (!this->trackME_.empty() && me) { + std::string name = me->getFullname(); + if (name.find(this->trackME_) != std::string::npos) { + edm::LogWarning("DQMStoreTrackME").log([&](auto& logger) { + logger << message << " for " << name; + if (me->isValid()) { + logger << " " << me->getRunLumi() << " scope " << scopename[me->getScope()]; + if (me->kind() >= MonitorElement::Kind::TH1F) { + logger << " entries " << me->getEntries(); + } else if (me->kind() == MonitorElement::Kind::STRING) { + logger << " value " << me->getStringValue(); + } else if (me->kind() == MonitorElement::Kind::REAL) { + logger << " value " << me->getFloatValue(); + } else if (me->kind() == MonitorElement::Kind::INT) { + logger << " value " << me->getIntValue(); + } + } else { + logger << " (invalid)"; + } + }); + } + } + } + MonitorElement* DQMStore::findOrRecycle(MonitorElementData::Key const& key) { // This is specifically for DQMRootSource, or other input modules. These // are special in that they use the legacy interface (no moduleID, no local @@ -257,6 +289,7 @@ namespace dqm::implementation { auto existing = this->get(key); if (existing) { // exactly matching ME found, needs merging with the new data. + debugTrackME("findOrRecycle (found)", existing); return existing; } // else @@ -282,6 +315,7 @@ namespace dqm::implementation { auto newme = *result.first; // iterator to new ME assert(oldme == newme); // recycling! // newme is reset and ready to accept data. + debugTrackME("findOrRecycle (recycled)", newme); return newme; } // else @@ -317,6 +351,7 @@ namespace dqm::implementation { auto target = targetset.find(me); // lookup by path, thanks to MEComparison if (target != targetset.end()) { // we already have a ME, just use it! + debugTrackME("enterLumi (existing)", *target); } else { // look for a prototype to reuse. auto proto = prototypes.find(me); @@ -340,6 +375,7 @@ namespace dqm::implementation { auto result = targetset.insert(oldme); assert(result.second); // was new insertion target = result.first; // iterator to new ME + debugTrackME("enterLumi (reused)", *target); } else { // no prototype available. That means we have concurrent Lumis/Runs, // and need to make a clone now. @@ -359,6 +395,7 @@ namespace dqm::implementation { auto result = targetset.insert(newme); assert(result.second); // was new insertion target = result.first; // iterator to new ME + debugTrackME("enterLumi (allocated)", *target); } } // now we have the proper global ME in the right place, point the local there. @@ -396,6 +433,7 @@ namespace dqm::implementation { // we have to be very careful with the ME here, it might not be backed by data at all. if (me->isValid() && checkScope(me->getScope()) == true) { // if we left the scope, simply release the data. + debugTrackME("leaveLumi (release)", me); me->release(/* expectOwned */ false); } } @@ -441,10 +479,12 @@ namespace dqm::implementation { auto other = this->findME(me); if (other) { // we still have a global one, so we can just remove this. + debugTrackME("cleanupLumi (delete)", me); delete me; } else { // we will modify the ME, so it needs to be out of the set. // use a temporary vector to be save. + debugTrackME("cleanupLumi (recycle)", me); torecycle.push_back(me); } } @@ -473,6 +513,7 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getPathname() == path.getDirname()) { + store_->debugTrackME("getContents (match)", *it); out.push_back(*it); ++it; } @@ -492,6 +533,7 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getPathname().rfind(path_str, 0) == 0) { + store_->debugTrackME("getAllContents (match)", *it); out.push_back(*it); ++it; } @@ -510,6 +552,7 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getFullname().rfind(path_str, 0) == 0) { + store_->debugTrackME("getAllContents (run/lumi match)", *it); out.push_back(*it); ++it; } @@ -529,6 +572,7 @@ namespace dqm::implementation { auto it = meset.find(key.path_); if (it != meset.end()) { assert((*it)->getScope() == key.scope_); + store_->debugTrackME("get (key found)", *it); return *it; } return nullptr; @@ -584,6 +628,7 @@ namespace dqm::implementation { verbose_ = pset.getUntrackedParameter("verbose", 0); assertLegacySafe_ = pset.getUntrackedParameter("assertLegacySafe", true); doSaveByLumi_ = pset.getUntrackedParameter("saveByLumi", false); + trackME_ = pset.getUntrackedParameter("trackME", ""); // Set lumi and run for legacy booking. // This is no more than a guess with concurrent runs/lumis, but should be From f9a3ec39b58430ae3efff7d5e246910561256d48 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 18:05:51 +0100 Subject: [PATCH 064/112] Fix cd() and friends. Wonder how we got that far with this bug... --- DQMServices/Core/interface/DQMStore.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index a3af5f8ef4fdd..2c3cacd06fbd9 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -526,22 +526,14 @@ namespace dqm { // ------------------------------------------------------------------------ // ------------ IBooker/IGetter overrides to prevent ambiguity ------------ - void cd() override { - this->IBooker::cd(); - this->IGetter::cd(); - } - void cd(std::string const& dir) override { - this->IBooker::cd(dir); - this->IGetter::cd(dir); - } + void cd() override { this->IBooker::cd(); } + void cd(std::string const& dir) override { this->IBooker::cd(dir); } void setCurrentFolder(std::string const& fullpath) override { + // only here we keep the two in sync -- all the others call this in the end! this->IBooker::setCurrentFolder(fullpath); this->IGetter::setCurrentFolder(fullpath); } - void goUp() override { - this->IBooker::goUp(); - this->IGetter::goUp(); - } + void goUp() override { this->IBooker::goUp(); } std::string const& pwd() override { return this->IBooker::pwd(); } public: From 640107e18e097ca7d7df474534ab8f47a8b147e8 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 20:17:07 +0100 Subject: [PATCH 065/112] Change default reScope to JOB. This covers all single- and multi run Harvesting. Easier to make it the default than hunt down all the places where the source is configured. Also worrying that no unit tests fail with this change. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index c1cd03f4cd85f..54d18c2a038b3 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -415,7 +415,7 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou {"", MonitorElementData::Scope::LUMI}, {"LUMI", MonitorElementData::Scope::LUMI}, {"RUN", MonitorElementData::Scope::RUN}, - {"JOB", MonitorElementData::Scope::JOB}}[iPSet.getUntrackedParameter("reScope", "")]), + {"JOB", MonitorElementData::Scope::JOB}}[iPSet.getUntrackedParameter("reScope", "JOB")]), m_nextItemType(edm::InputSource::IsFile), m_treeReaders(kNIndicies, std::shared_ptr()), m_currentIndex(0), From e413ab20b429fb3b9dba11d268b71f7910ae2e78 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 20:17:39 +0100 Subject: [PATCH 066/112] Adapt the merge config to not do reScoping. --- Configuration/DataProcessing/python/Merge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Configuration/DataProcessing/python/Merge.py b/Configuration/DataProcessing/python/Merge.py index e3207879d61d8..b34dcb7a197d6 100644 --- a/Configuration/DataProcessing/python/Merge.py +++ b/Configuration/DataProcessing/python/Merge.py @@ -51,8 +51,8 @@ def mergeProcess(*inputFiles, **options): # // input source #// if newDQMIO: - process.source = Source("DQMRootSource") - process.add_(Service("DQMStore", forceResetOnBeginLumi = CfgTypes.untracked.bool(True))) + process.source = Source("DQMRootSource", reScope = CfgTypes.untracked.string("")) + process.add_(Service("DQMStore")) else: process.source = Source("PoolSource") if bypassVersionCheck: From 98fd31a18c7f411ab65465cb7df40a74c718d1bf Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 20:19:49 +0100 Subject: [PATCH 067/112] Use string_view instead of copy. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 54d18c2a038b3..0e8975a80b37f 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -75,8 +75,8 @@ namespace { } for (int i = 1; i <= a1->GetNbins(); ++i) { - TString label1 = a1->GetBinLabel(i); - TString label2 = a2->GetBinLabel(i); + std::string_view label1 = a1->GetBinLabel(i); + std::string_view label2 = a2->GetBinLabel(i); if (label1 != label2) { return false; } From cb390399ffb478deec4881fe50df99f1311e8e0a Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 16 Dec 2019 20:21:42 +0100 Subject: [PATCH 068/112] Remove useless const. --- DQMServices/Core/interface/MonitorElement.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 8f9c8b50df884..8ac20c85b07b7 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -250,11 +250,11 @@ namespace dqm::impl { const std::string &getPathname() const { return this->data_.dirname; } /// get full name of ME including Pathname - const std::string getFullname() const { return access().key.path_.getFullname(); } + std::string getFullname() const { return access().key.path_.getFullname(); } - const edm::LuminosityBlockID getRunLumi() { return access().key.id_; } + edm::LuminosityBlockID getRunLumi() { return access().key.id_; } - const MonitorElementData::Scope getScope() { return access().key.scope_; } + MonitorElementData::Scope getScope() { return access().key.scope_; } /// true if ME was updated in last monitoring cycle bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } From f052a6371f97d611971e98109d8bb4f8c0e0fabb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 17 Dec 2019 17:01:59 +0100 Subject: [PATCH 069/112] Disable normalisation in L1T DQM. It never worked, I suspect. --- DQMOffline/L1Trigger/src/L1TStage2CaloLayer2Offline.cc | 5 ++++- DQMOffline/L1Trigger/src/L1TTauOffline.cc | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/DQMOffline/L1Trigger/src/L1TStage2CaloLayer2Offline.cc b/DQMOffline/L1Trigger/src/L1TStage2CaloLayer2Offline.cc index a847228ee45de..0a8d41d1becdb 100644 --- a/DQMOffline/L1Trigger/src/L1TStage2CaloLayer2Offline.cc +++ b/DQMOffline/L1Trigger/src/L1TStage2CaloLayer2Offline.cc @@ -988,7 +988,10 @@ bool L1TStage2CaloLayer2Offline::doesNotOverlapWithHLTObjects(const l1t::Jet& je return matchedObjects.empty(); } -void L1TStage2CaloLayer2Offline::endJob() { normalise2DHistogramsToBinArea(); } +void L1TStage2CaloLayer2Offline::endJob() { + // TODO: In offline, this runs after histograms are saved! + //normalise2DHistogramsToBinArea(); +} void L1TStage2CaloLayer2Offline::normalise2DHistogramsToBinArea() { std::vector monElementstoNormalize = { diff --git a/DQMOffline/L1Trigger/src/L1TTauOffline.cc b/DQMOffline/L1Trigger/src/L1TTauOffline.cc index 5ef58a0fd0ba5..910a1cfa1fa25 100644 --- a/DQMOffline/L1Trigger/src/L1TTauOffline.cc +++ b/DQMOffline/L1Trigger/src/L1TTauOffline.cc @@ -670,7 +670,10 @@ void L1TTauOffline::getProbeTaus(const edm::Event& iEvent, } } } -void L1TTauOffline::endJob() { normalise2DHistogramsToBinArea(); } +void L1TTauOffline::endJob() { + // TODO: In offline, this runs after histograms are saved! + //normalise2DHistogramsToBinArea(); +} void L1TTauOffline::normalise2DHistogramsToBinArea() { std::vector monElementstoNormalize = {h_L1TauETvsTauET_EB_, From 3703bbe0e5b1875b6ae6465639eadf28a97f89c9 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 17 Dec 2019 17:02:41 +0100 Subject: [PATCH 070/112] Improve diagonstics in DQMStore/ME. --- DQMServices/Core/interface/MonitorElement.h | 4 ++-- DQMServices/Core/src/DQMStore.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 8ac20c85b07b7..d8365eac95e5b 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -156,7 +156,7 @@ namespace dqm::impl { return Access{std::unique_lock(), frozen->key_, frozen->value_}; } // else - throw cms::Exception("LogicError") << "MonitorElement not backed by any data!"; + throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } AccessMut accessMut() { @@ -171,7 +171,7 @@ namespace dqm::impl { } // else auto frozen = frozen_.load(); if (!frozen) { - throw cms::Exception("LogicError") << "MonitorElement not backed by any data!"; + throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } // in case of an immutable object read from edm products, attempt to // make a clone. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 6c7ff375ff9ef..7be4a463733ae 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -117,7 +117,7 @@ namespace dqm::implementation { return me; } else { // the normal case. - store_->debugTrackME("bookME (legacy)", me); + store_->debugTrackME("bookME (normal)", me); return localme; } } From 8d59ad5c0a83739fc1f3bea684909bdd4ed6792c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 12:20:36 +0100 Subject: [PATCH 071/112] Change reScope default in another place. Maybe this fixes the default for harvesting jobs? --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 0e8975a80b37f..381e6373d6634 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -385,10 +385,11 @@ void DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& description edm::ParameterSetDescription desc; desc.addUntracked>("fileNames")->setComment("Names of files to be processed."); desc.addUntracked("filterOnRun", 0)->setComment("Just limit the process to the selected run."); - desc.addUntracked("reScope", "") + desc.addUntracked("reScope", "JOB") ->setComment( "Accumulate histograms more coarsely." - " Options: \"RUN\": turn LUMI histograms into RUN histograms, \"JOB\": turn everything into JOB histograms."); + " Options: \"\": keep unchanged, \"RUN\": turn LUMI histograms into RUN histograms, \"JOB\": turn everything " + "into JOB histograms."); desc.addUntracked("skipBadFiles", false)->setComment("Skip the file if it is not valid"); desc.addUntracked("overrideCatalog", std::string()) ->setComment("An alternate file catalog to use instead of the standard site one."); From 48d11094f2181141022cbc102726d3e701e9c96b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 12:22:03 +0100 Subject: [PATCH 072/112] Fix getAllContents to not return empty prototypes. --- DQMServices/Core/src/DQMStore.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 7be4a463733ae..d4fa05db86d60 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -533,8 +533,12 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getPathname().rfind(path_str, 0) == 0) { - store_->debugTrackME("getAllContents (match)", *it); - out.push_back(*it); + if (runlumi == edm::LuminosityBlockID() && (*it)->getScope() != MonitorElementData::Scope::JOB) { + // skip prototypes + } else { + store_->debugTrackME("getAllContents (match)", *it); + out.push_back(*it); + } ++it; } } From d198f7859d27e4b13d2d74e8fae4b9b2363b5dda Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 12:44:58 +0100 Subject: [PATCH 073/112] Improve legacy save() and remove other legacy APIs --- .../Components/plugins/DQMFileSaver.cc | 2 +- DQMServices/Core/interface/DQMStore.h | 29 ++------------- DQMServices/Core/interface/LegacyIOHelper.h | 14 +++++-- DQMServices/Core/src/DQMStore.cc | 37 ++++--------------- DQMServices/Core/src/LegacyIOHelper.cc | 12 +++++- .../Core/test/DQMSourceExampleConfig.cc | 1 - .../FileIO/plugins/DQMFileSaverOnline.cc | 2 +- 7 files changed, 34 insertions(+), 63 deletions(-) diff --git a/DQMServices/Components/plugins/DQMFileSaver.cc b/DQMServices/Components/plugins/DQMFileSaver.cc index fabed1d57a147..e4d09725344d7 100644 --- a/DQMServices/Components/plugins/DQMFileSaver.cc +++ b/DQMServices/Components/plugins/DQMFileSaver.cc @@ -113,7 +113,7 @@ void DQMFileSaver::saveForOffline(const std::string &workflow, int run, int lumi } LegacyIOHelper h(dbe_); - h.save(filename, run); + h.save(filename, "", run, /* saveall */ true, "RECREATE"); // save the JobReport saveJobReport(filename); diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 2c3cacd06fbd9..5f693144f200d 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -441,9 +441,6 @@ namespace dqm { // TODO: review and possibly rename the all methods below: // get MEs that are direct children of full path `path` virtual std::vector getContents(std::string const& path) const; - // not clear what this is good for. - DQM_DEPRECATED - virtual void getContents(std::vector& into, bool showContents = true) const; // get all elements below full path `path` // we have to discuss semantics here -- are run/lumi ever used? @@ -484,8 +481,6 @@ namespace dqm { public: // IGetter uses the globalMEs_ directly. friend IGetter; - // TODO: There are no references any more. we should gt rid of these. - enum SaveReferenceTag { SaveWithoutReference, SaveWithReference, SaveWithReferenceForQTest }; enum OpenRunDirs { KeepRunDirs, StripRunDirs }; DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&); @@ -493,33 +488,15 @@ namespace dqm { // ------------------------------------------------------------------------ // ---------------------- public I/O -------------------------------------- - // TODO: these need some cleanup, though in general we want to keep the - // functionality. Maybe move it to a different class, and change the (rather - // few) usages. - void save(std::string const& filename, - std::string const& path = "", - std::string const& pattern = "", - std::string const& rewrite = "", - uint32_t run = 0, - uint32_t lumi = 0, - SaveReferenceTag ref = SaveWithReference, - int minStatus = dqm::qstatus::STATUS_OK, - std::string const& fileupdate = "RECREATE"); - void savePB(std::string const& filename, std::string const& path = "", uint32_t run = 0, uint32_t lumi = 0); + DQM_DEPRECATED + void save(std::string const& filename, std::string const& path = ""); + DQM_DEPRECATED bool open(std::string const& filename, bool overwrite = false, std::string const& path = "", std::string const& prepend = "", OpenRunDirs stripdirs = KeepRunDirs, bool fileMustExist = true); - bool load(std::string const& filename, OpenRunDirs stripdirs = StripRunDirs, bool fileMustExist = true); - - DQM_DEPRECATED - void showDirStructure() const; - - // TODO: getting API not part of IGetter. - DQM_DEPRECATED - std::vector getMatchingContents(std::string const& pattern) const; DQMStore(DQMStore const&) = delete; DQMStore& operator=(DQMStore const&) = delete; diff --git a/DQMServices/Core/interface/LegacyIOHelper.h b/DQMServices/Core/interface/LegacyIOHelper.h index 89a037e86c5c7..0a08fc05d2739 100644 --- a/DQMServices/Core/interface/LegacyIOHelper.h +++ b/DQMServices/Core/interface/LegacyIOHelper.h @@ -13,7 +13,8 @@ // interface to use this without adding another dependency. class LegacyIOHelper { public: - typedef dqm::legacy::DQMStore DQMStore; + // use internal type here since we call this from the DQMStore itself. + typedef dqm::implementation::DQMStore DQMStore; typedef dqm::legacy::MonitorElement MonitorElement; LegacyIOHelper(DQMStore* dqmstore) : dbe_(dqmstore){}; @@ -21,8 +22,15 @@ class LegacyIOHelper { // is passed, the paths are rewritten to the "Run Summary" format used by // DQMGUI. The run number does not affect which MEs are saved; this code only // supports non-threaded mode. `fileupdate` is passed to ROOT unchanged. - // TODO: allow to select only RUN, LUMI or JOB histograms. - void save(std::string const& filename, uint32_t const run = 0, std::string const& fileupdate = "RECREATE"); + // The run number passed in is added to the Directory structure inside the + // file ("Run xxxxxx/.../Run Summary/...") if not 0. It is only used to + // select only MEs for that run iff saveall is false, else all MEs (RUN, LUMI + // and JOB) are saved. + void save(std::string const& filename, + std::string const& path = "", + uint32_t const run = 0, + bool saveall = true, + std::string const& fileupdate = "RECREATE"); private: bool createDirectoryIfNeededAndCd(const std::string& path); diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index d4fa05db86d60..87237c93c2430 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -1,6 +1,7 @@ // silence deprecation warnings for the DQMStore itself. #define DQM_DEPRECATED #include "DQMServices/Core/interface/DQMStore.h" +#include "DQMServices/Core/interface/LegacyIOHelper.h" #include "FWCore/MessageLogger/interface/MessageLogger.h" #include "FWCore/ServiceRegistry/interface/GlobalContext.h" #include @@ -521,8 +522,6 @@ namespace dqm::implementation { return out; } - void IGetter::getContents(std::vector& into, bool showContents) const { assert(!"NIY"); } - std::vector IGetter::getAllContents(std::string const& pathname) const { std::vector out; MonitorElementData::Path path; @@ -666,41 +665,19 @@ namespace dqm::implementation { DQMStore::~DQMStore() {} - void DQMStore::save(std::string const& filename, - std::string const& path, - std::string const& pattern, - std::string const& rewrite, - uint32_t run, - uint32_t lumi, - SaveReferenceTag ref, - int minStatus, - std::string const& fileupdate) { - TRACE("filename " << filename << " path " << path << " pattern " << pattern << " rewrite " << rewrite << " run " - << run << " lumi " << lumi << " minStatus " << minStatus << " fileupdate " << fileupdate); - //assert(!"NIY"); - } - void DQMStore::savePB(std::string const& filename, std::string const& path, uint32_t run, uint32_t lumi) { - assert(!"NIY"); + void DQMStore::save(std::string const& filename, std::string const& path) { + LegacyIOHelper h(this); + // no run number passed, will save a flat ROOT file (rather than 'Run xxxxxx/.../Run Summary/...') + h.save(filename, path); } + bool DQMStore::open(std::string const& filename, bool overwrite, std::string const& path, std::string const& prepend, OpenRunDirs stripdirs, bool fileMustExist) { - TRACE("filename " << filename << " overwrite " << overwrite << " path " << path << " prepend " << prepend - << " fileMustExist " << fileMustExist); - //assert(!"NIY"); - return false; - } - bool DQMStore::load(std::string const& filename, OpenRunDirs stripdirs, bool fileMustExist) { - TRACE("filename " << filename << " fileMustExist " << fileMustExist); - //assert(!"NIY"); - return false; + assert(!"NIY"); } - void DQMStore::showDirStructure() const { assert(!"NIY"); } - - std::vector DQMStore::getMatchingContents(std::string const& pattern) const { assert(!"NIY"); } - } // namespace dqm::implementation diff --git a/DQMServices/Core/src/LegacyIOHelper.cc b/DQMServices/Core/src/LegacyIOHelper.cc index 8c7c9d1961fc8..d588e63fd6393 100644 --- a/DQMServices/Core/src/LegacyIOHelper.cc +++ b/DQMServices/Core/src/LegacyIOHelper.cc @@ -11,7 +11,9 @@ #include void LegacyIOHelper::save(std::string const &filename, + std::string const &path /* = "" */, uint32_t const run /* = 0 */, + bool saveall /* = true */, std::string const &fileupdate /* = "RECREATE" */) { // TFile flushes to disk with fsync() on every TDirectory written to // the file. This makes DQM file saving painfully slow, and @@ -31,7 +33,15 @@ void LegacyIOHelper::save(std::string const &filename, TFileNoSync *file = new TFileNoSync(filename.c_str(), fileupdate.c_str()); // open file // Traverse all MEs - auto mes = dbe_->getAllContents(""); // TODO run/lumi and/or job? + std::vector mes; + if (saveall) { + // this is typically used, at endJob there will only be JOB histos here + mes = dbe_->getAllContents(path); + } else { + // at endRun it might make sense to use this, to not save JOB histos yet. + mes = dbe_->getAllContents(path, run, 0); + } + for (auto me : mes) { // Modify dirname to comply with DQM GUI format. Change: // A/B/C/plot diff --git a/DQMServices/Core/test/DQMSourceExampleConfig.cc b/DQMServices/Core/test/DQMSourceExampleConfig.cc index e31e1d82cdb60..11867095b8b00 100644 --- a/DQMServices/Core/test/DQMSourceExampleConfig.cc +++ b/DQMServices/Core/test/DQMSourceExampleConfig.cc @@ -121,7 +121,6 @@ DQMSourceExampleConfig::DQMSourceExampleConfig(const edm::ParameterSet& iConfig) iConfig.getUntrackedParameter("lDepth", 2), dbe, directories_number); - dbe->showDirStructure(); } DQMSourceExampleConfig::~DQMSourceExampleConfig() { diff --git a/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc b/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc index 85c519bd3bdde..e957c3d3d7eba 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverOnline.cc @@ -88,7 +88,7 @@ void DQMFileSaverOnline::makeSnapshot(const FileParameters& fp, bool final) cons // TODO: some parameters prepared here are now unused, and the code should // eventually be removed. LegacyIOHelper h(&*store); - h.save(tmp_root_fp, fp.run_, "RECREATE"); + h.save(tmp_root_fp, "", fp.run_, /* saveall */ true, "RECREATE"); // write metadata // format.origin: md5:d566a34b27f48d507150a332b189398b 294835 From 0a2d5fb347813514132b4c407a38840827b23687 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 12:54:15 +0100 Subject: [PATCH 074/112] Remove calls to showDirStructure(). --- Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc | 4 ---- .../plugins/TrackerOfflineValidationSummary.cc | 1 - Calibration/TkAlCaRecoProducers/test/DQMRootFileReader.cc | 3 --- DQM/EcalMonitorDbModule/plugins/EcalBarrelMonitorDbModule.cc | 3 --- DQM/SiStripMonitorHardware/src/BuildTrackerMap.cc | 1 - DQM/SiStripMonitorPedestals/src/SiStripMonitorPedestals.cc | 1 - DQM/SiStripMonitorPedestals/src/SiStripMonitorQuality.cc | 1 - DQM/SiStripMonitorPedestals/src/SiStripMonitorRawData.cc | 1 - DQM/SiStripMonitorSummary/plugins/SiStripMonitorCondData.cc | 1 - .../src/SiStripClassToMonitorCondData.cc | 1 - DQM/TrackingMonitor/src/LogMessageMonitor.cc | 1 - DQM/TrackingMonitor/src/TrackEfficiencyMonitor.cc | 1 - DQM/TrackingMonitor/src/dEdxAnalyzer.cc | 1 - DQM/TrigXMonitorClient/src/HLTScalersClient.cc | 4 ---- DQMOffline/Alignment/src/MuonAlignment.cc | 1 - DQMServices/Components/plugins/DQMFEDIntegrityClient.cc | 2 -- RecoLocalCalo/EcalRecProducers/test/EcalLocalRecoTask.cc | 5 ----- Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc | 5 ----- Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc | 5 ----- Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc | 5 ----- Validation/EcalHits/src/EcalSimHitsValidation.cc | 5 ----- Validation/GlobalHits/src/GlobalHitsProdHistStripper.cc | 5 ----- Validation/Mixing/src/TestSuite.cc | 1 - Validation/RecoMuon/src/MuonTrackResidualAnalyzer.cc | 2 -- Validation/RecoTau/plugins/DQMFileLoader.cc | 4 ---- Validation/RecoTrack/plugins/SiPixelTrackingRecHitsValid.cc | 1 - Validation/TrackerHits/src/TrackerHitAnalyzer.cc | 5 ----- 27 files changed, 70 deletions(-) diff --git a/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc b/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc index 127315dd62fd2..8fa420682dac9 100644 --- a/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc +++ b/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc @@ -1500,8 +1500,4 @@ void LaserDQM::initMonitors() { theMEBeam7TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 7 at z = -540 mm", 512, 0, 511); theMEBeam7TIBPosition6AdcCounts->setResetMe(true); - - // show directory structure - if (theDebugLevel > 3) - theDaqMonitorBEI->showDirStructure(); } diff --git a/Alignment/OfflineValidation/plugins/TrackerOfflineValidationSummary.cc b/Alignment/OfflineValidation/plugins/TrackerOfflineValidationSummary.cc index 3de109aaa0a13..d3d006c4d0f8a 100644 --- a/Alignment/OfflineValidation/plugins/TrackerOfflineValidationSummary.cc +++ b/Alignment/OfflineValidation/plugins/TrackerOfflineValidationSummary.cc @@ -236,7 +236,6 @@ void TrackerOfflineValidationSummary::endJob() { this->fillTree(*tree, mTobResiduals_, *treeMemPtr, *tkGeom_, *substructureName, tTopo); this->fillTree(*tree, mTecResiduals_, *treeMemPtr, *tkGeom_, *substructureName, tTopo); - //dbe_->showDirStructure(); //dbe_->save("dqmOut.root"); // Method for filling histograms which show summarized values (mean, rms, median ...) diff --git a/Calibration/TkAlCaRecoProducers/test/DQMRootFileReader.cc b/Calibration/TkAlCaRecoProducers/test/DQMRootFileReader.cc index c6aa82bed52bd..961cce72642f9 100644 --- a/Calibration/TkAlCaRecoProducers/test/DQMRootFileReader.cc +++ b/Calibration/TkAlCaRecoProducers/test/DQMRootFileReader.cc @@ -69,8 +69,6 @@ DQMRootFileReader::~DQMRootFileReader() { } void DQMRootFileReader::endJob(void) { - cout << "Dumping DQMStore dir structure:" << endl; - dbe->showDirStructure(); // dbe->save("test.root"); } @@ -83,7 +81,6 @@ void DQMRootFileReader::analyze(const edm::Event &iEvent, const edm::EventSetup // NOTE: this is here just because we need it after the beginRun of // MEtoEDMCoverter which calls a Reset on all MEs. dbe->open(filename, false, "", "", DQMStore::OpenRunDirs::StripRunDirs); - dbe->showDirStructure(); } // define this as a plug-in diff --git a/DQM/EcalMonitorDbModule/plugins/EcalBarrelMonitorDbModule.cc b/DQM/EcalMonitorDbModule/plugins/EcalBarrelMonitorDbModule.cc index 4d422283f9295..42f7133c94f13 100644 --- a/DQM/EcalMonitorDbModule/plugins/EcalBarrelMonitorDbModule.cc +++ b/DQM/EcalMonitorDbModule/plugins/EcalBarrelMonitorDbModule.cc @@ -50,9 +50,6 @@ EcalBarrelMonitorDbModule::EcalBarrelMonitorDbModule(const edm::ParameterSet &ps ME_Db_ = new MonitorElementsDb(ps, xmlFile_); - if (dqmStore_) - dqmStore_->showDirStructure(); - icycle_ = 0; session_ = nullptr; } diff --git a/DQM/SiStripMonitorHardware/src/BuildTrackerMap.cc b/DQM/SiStripMonitorHardware/src/BuildTrackerMap.cc index e23fea1106861..d67b7f0af66a1 100644 --- a/DQM/SiStripMonitorHardware/src/BuildTrackerMap.cc +++ b/DQM/SiStripMonitorHardware/src/BuildTrackerMap.cc @@ -153,7 +153,6 @@ void BuildTrackerMapPlugin::read(bool aMechView, } //lDqmStore->setCurrentFolder(dirName); - //lDqmStore->showDirStructure(); unsigned int nFailTot = 0; unsigned int nTotTot = 0; diff --git a/DQM/SiStripMonitorPedestals/src/SiStripMonitorPedestals.cc b/DQM/SiStripMonitorPedestals/src/SiStripMonitorPedestals.cc index 2f21cffd8ace5..489df38a03b66 100644 --- a/DQM/SiStripMonitorPedestals/src/SiStripMonitorPedestals.cc +++ b/DQM/SiStripMonitorPedestals/src/SiStripMonitorPedestals.cc @@ -452,7 +452,6 @@ void SiStripMonitorPedestals::dqmEndRun(edm::Run const &run, edm::EventSetup con bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); if (outputMEsInRootFile) { std::string outPutFileName = conf_.getParameter("OutPutFileName"); - // dqmStore_->showDirStructure(); dqmStore_->save(outPutFileName); } } diff --git a/DQM/SiStripMonitorPedestals/src/SiStripMonitorQuality.cc b/DQM/SiStripMonitorPedestals/src/SiStripMonitorQuality.cc index 012093faba5e9..cb0c9334ee2d7 100644 --- a/DQM/SiStripMonitorPedestals/src/SiStripMonitorQuality.cc +++ b/DQM/SiStripMonitorPedestals/src/SiStripMonitorQuality.cc @@ -183,7 +183,6 @@ void SiStripMonitorQuality::dqmEndRun(edm::Run const &run, edm::EventSetup const bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); std::string outputFileName = conf_.getParameter("OutputFileName"); if (outputMEsInRootFile) { - // dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } } diff --git a/DQM/SiStripMonitorPedestals/src/SiStripMonitorRawData.cc b/DQM/SiStripMonitorPedestals/src/SiStripMonitorRawData.cc index 1f7cae62c85d5..6e8fcb4018e58 100644 --- a/DQM/SiStripMonitorPedestals/src/SiStripMonitorRawData.cc +++ b/DQM/SiStripMonitorPedestals/src/SiStripMonitorRawData.cc @@ -121,7 +121,6 @@ void SiStripMonitorRawData::dqmEndRun(edm::Run const &run, edm::EventSetup const bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); std::string outputFileName = conf_.getParameter("OutputFileName"); if (outputMEsInRootFile) { - // dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } } diff --git a/DQM/SiStripMonitorSummary/plugins/SiStripMonitorCondData.cc b/DQM/SiStripMonitorSummary/plugins/SiStripMonitorCondData.cc index 68ffa5ad9e827..4723937fd5b14 100644 --- a/DQM/SiStripMonitorSummary/plugins/SiStripMonitorCondData.cc +++ b/DQM/SiStripMonitorSummary/plugins/SiStripMonitorCondData.cc @@ -218,7 +218,6 @@ void SiStripMonitorCondData::endRun(edm::Run const &run, edm::EventSetup const & DQMStore *dqmStore_ = edm::Service().operator->(); if (outputMEsInRootFile) { - dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } diff --git a/DQM/SiStripMonitorSummary/src/SiStripClassToMonitorCondData.cc b/DQM/SiStripMonitorSummary/src/SiStripClassToMonitorCondData.cc index 22883e9414be3..a8ffd61b95372 100644 --- a/DQM/SiStripMonitorSummary/src/SiStripClassToMonitorCondData.cc +++ b/DQM/SiStripMonitorSummary/src/SiStripClassToMonitorCondData.cc @@ -298,7 +298,6 @@ void SiStripClassToMonitorCondData::endRun(edm::EventSetup const &eSetup) { DQMStore *dqmStore_ = edm::Service().operator->(); if (outputMEsInRootFile) { - dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } diff --git a/DQM/TrackingMonitor/src/LogMessageMonitor.cc b/DQM/TrackingMonitor/src/LogMessageMonitor.cc index 52c8b282dee50..e8bb071d8e9b0 100644 --- a/DQM/TrackingMonitor/src/LogMessageMonitor.cc +++ b/DQM/TrackingMonitor/src/LogMessageMonitor.cc @@ -264,7 +264,6 @@ void LogMessageMonitor::endJob() { bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); std::string outputFileName = conf_.getParameter("OutputFileName"); if (outputMEsInRootFile) { - dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } } diff --git a/DQM/TrackingMonitor/src/TrackEfficiencyMonitor.cc b/DQM/TrackingMonitor/src/TrackEfficiencyMonitor.cc index f05fb26a0218e..d9df131864ec1 100644 --- a/DQM/TrackingMonitor/src/TrackEfficiencyMonitor.cc +++ b/DQM/TrackingMonitor/src/TrackEfficiencyMonitor.cc @@ -373,7 +373,6 @@ void TrackEfficiencyMonitor::endJob(void) bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); std::string outputFileName = conf_.getParameter("OutputFileName"); if (outputMEsInRootFile) { - //dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } diff --git a/DQM/TrackingMonitor/src/dEdxAnalyzer.cc b/DQM/TrackingMonitor/src/dEdxAnalyzer.cc index 7e2268842bf46..1d397579c2d29 100644 --- a/DQM/TrackingMonitor/src/dEdxAnalyzer.cc +++ b/DQM/TrackingMonitor/src/dEdxAnalyzer.cc @@ -46,7 +46,6 @@ void dEdxAnalyzer::endJob() { bool outputMEsInRootFile = conf_.getParameter("OutputMEsInRootFile"); std::string outputFileName = conf_.getParameter("OutputFileName"); if (outputMEsInRootFile) { - dqmStore_->showDirStructure(); dqmStore_->save(outputFileName); } } diff --git a/DQM/TrigXMonitorClient/src/HLTScalersClient.cc b/DQM/TrigXMonitorClient/src/HLTScalersClient.cc index 8a12b2ae12237..ba4ce0db77f3a 100644 --- a/DQM/TrigXMonitorClient/src/HLTScalersClient.cc +++ b/DQM/TrigXMonitorClient/src/HLTScalersClient.cc @@ -130,10 +130,6 @@ void HLTScalersClient::endRun(const edm::Run &run, const edm::EventSetup &c) { /// DQM Client Diagnostic should be performed here void HLTScalersClient::endLuminosityBlock(const edm::LuminosityBlock &lumiSeg, const edm::EventSetup &c) { nLumi_ = lumiSeg.id().luminosityBlock(); - // PWDEBUG - if (first_ && debug_) - dbe_->showDirStructure(); - // PWDEBUG END // get raw data std::string scalHisto = folderName_ + "/raw/hltScalers"; diff --git a/DQMOffline/Alignment/src/MuonAlignment.cc b/DQMOffline/Alignment/src/MuonAlignment.cc index 07bd37ecabef7..de4b21afb961e 100644 --- a/DQMOffline/Alignment/src/MuonAlignment.cc +++ b/DQMOffline/Alignment/src/MuonAlignment.cc @@ -913,7 +913,6 @@ void MuonAlignment::endJob(void) { } // doSummary if (outputMEsInRootFile) { - // dbe->showDirStructure(); dbe->save(outputFileName); } } diff --git a/DQMServices/Components/plugins/DQMFEDIntegrityClient.cc b/DQMServices/Components/plugins/DQMFEDIntegrityClient.cc index 90058a0c2bf57..bffbc597fa769 100644 --- a/DQMServices/Components/plugins/DQMFEDIntegrityClient.cc +++ b/DQMServices/Components/plugins/DQMFEDIntegrityClient.cc @@ -159,8 +159,6 @@ void DQMFEDIntegrityClient::endLuminosityBlock(const edm::LuminosityBlock& lumiB void DQMFEDIntegrityClient::fillHistograms() { // FED Entries - // dbe_->showDirStructure(); - std::vector entries; entries.push_back("CSC/" + fedFolderName + "/FEDEntries"); entries.push_back("DT/" + fedFolderName + "/FEDEntries"); diff --git a/RecoLocalCalo/EcalRecProducers/test/EcalLocalRecoTask.cc b/RecoLocalCalo/EcalRecProducers/test/EcalLocalRecoTask.cc index b7f47a365bb4f..9e88df25e7e3e 100644 --- a/RecoLocalCalo/EcalRecProducers/test/EcalLocalRecoTask.cc +++ b/RecoLocalCalo/EcalRecProducers/test/EcalLocalRecoTask.cc @@ -65,11 +65,6 @@ EcalLocalRecoTask::EcalLocalRecoTask(const edm::ParameterSet& ps) { // get hold of back-end interface dbe_ = edm::Service().operator->(); - if (dbe_) { - if (verbose_) - dbe_->showDirStructure(); - } - meEBUncalibRecHitMaxSampleRatio_ = 0; meEBUncalibRecHitPedestal_ = 0; meEBUncalibRecHitOccupancy_ = 0; diff --git a/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc b/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc index c30bc2459b192..8fc40f6b5615c 100644 --- a/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc @@ -29,11 +29,6 @@ EcalBarrelSimHitsValidation::EcalBarrelSimHitsValidation(const edm::ParameterSet dbe_ = nullptr; dbe_ = edm::Service().operator->(); - if (dbe_) { - if (verbose_) - dbe_->showDirStructure(); - } - menEBHits_ = nullptr; menEBCrystals_ = nullptr; meEBOccupancy_ = nullptr; diff --git a/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc b/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc index 7aed21decfca0..bcfa2b6c51f39 100644 --- a/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc @@ -29,11 +29,6 @@ EcalEndcapSimHitsValidation::EcalEndcapSimHitsValidation(const edm::ParameterSet dbe_ = nullptr; dbe_ = edm::Service().operator->(); - if (dbe_) { - if (verbose_) - dbe_->showDirStructure(); - } - meEEzpHits_ = nullptr; meEEzmHits_ = nullptr; meEEzpCrystals_ = nullptr; diff --git a/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc b/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc index 781c8a4fad5f0..f9317f89795b2 100644 --- a/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc @@ -34,11 +34,6 @@ EcalPreshowerSimHitsValidation::EcalPreshowerSimHitsValidation(const edm::Parame dbe_ = nullptr; dbe_ = edm::Service().operator->(); - if (dbe_) { - if (verbose_) - dbe_->showDirStructure(); - } - menESHits1zp_ = nullptr; menESHits2zp_ = nullptr; menESHits1zm_ = nullptr; diff --git a/Validation/EcalHits/src/EcalSimHitsValidation.cc b/Validation/EcalHits/src/EcalSimHitsValidation.cc index 199b252b4bfff..468f6cdcc67e0 100644 --- a/Validation/EcalHits/src/EcalSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalSimHitsValidation.cc @@ -45,11 +45,6 @@ EcalSimHitsValidation::EcalSimHitsValidation(const edm::ParameterSet &ps) // get hold of back-end interface dbe_ = edm::Service().operator->(); - if (dbe_) { - if (verbose_) - dbe_->showDirStructure(); - } - meGunEnergy_ = nullptr; meGunEta_ = nullptr; meGunPhi_ = nullptr; diff --git a/Validation/GlobalHits/src/GlobalHitsProdHistStripper.cc b/Validation/GlobalHits/src/GlobalHitsProdHistStripper.cc index 47b7b3837212c..6f39f5c497550 100644 --- a/Validation/GlobalHits/src/GlobalHitsProdHistStripper.cc +++ b/Validation/GlobalHits/src/GlobalHitsProdHistStripper.cc @@ -38,11 +38,6 @@ GlobalHitsProdHistStripper::GlobalHitsProdHistStripper(const edm::ParameterSet & dbe = nullptr; dbe = edm::Service().operator->(); - if (dbe) { - if (verbosity > 0) - dbe->showDirStructure(); - } - // print out Parameter Set information being used if (verbosity >= 0) { edm::LogInfo(MsgLoggerCat) << "\n===============================\n" diff --git a/Validation/Mixing/src/TestSuite.cc b/Validation/Mixing/src/TestSuite.cc index d7167ab7e56e5..719456b1e4f9a 100644 --- a/Validation/Mixing/src/TestSuite.cc +++ b/Validation/Mixing/src/TestSuite.cc @@ -48,7 +48,6 @@ TestSuite::~TestSuite() { void TestSuite::beginJob() { // get hold of back-end interface dbe_ = Service().operator->(); - dbe_->showDirStructure(); dbe_->setCurrentFolder("MixingV/Mixing"); } diff --git a/Validation/RecoMuon/src/MuonTrackResidualAnalyzer.cc b/Validation/RecoMuon/src/MuonTrackResidualAnalyzer.cc index 91f75243b812c..93d04ac292303 100644 --- a/Validation/RecoMuon/src/MuonTrackResidualAnalyzer.cc +++ b/Validation/RecoMuon/src/MuonTrackResidualAnalyzer.cc @@ -82,8 +82,6 @@ void MuonTrackResidualAnalyzer::bookHistograms(DQMStore::IBooker &ibooker, edm::EventSetup const & /* iSetup */) { LogDebug("MuonTrackResidualAnalyzer") << "Begin Run"; - //ibooker.showDirStructure(); - ibooker.cd(); InputTag algo = theMuonTrackLabel; string dirName = dirName_; diff --git a/Validation/RecoTau/plugins/DQMFileLoader.cc b/Validation/RecoTau/plugins/DQMFileLoader.cc index 6b998d36845ee..73942a2bcbbf6 100644 --- a/Validation/RecoTau/plugins/DQMFileLoader.cc +++ b/Validation/RecoTau/plugins/DQMFileLoader.cc @@ -223,8 +223,6 @@ void TauDQMFileLoader::endRun(const edm::Run& r, const edm::EventSetup& c) { std::cout << " opening inputFile = " << (*inputFileName) << std::endl; dqmStore.open(*inputFileName, true); - //if ( verbosity ) dqmStore.showDirStructure(); - //--- if dqmDirectory_store specified in configuration parameters, // move histograms from dqmRootDirectory to dqmDirectory_store // (if the histograms are not moved, the histograms get overwritten, @@ -265,8 +263,6 @@ void TauDQMFileLoader::endRun(const edm::Run& r, const edm::EventSetup& c) { } std::cout << "done." << std::endl; - if (verbosity) - dqmStore.showDirStructure(); } #include "FWCore/Framework/interface/MakerMacros.h" diff --git a/Validation/RecoTrack/plugins/SiPixelTrackingRecHitsValid.cc b/Validation/RecoTrack/plugins/SiPixelTrackingRecHitsValid.cc index ef0266009d6d4..720e92f12bcec 100644 --- a/Validation/RecoTrack/plugins/SiPixelTrackingRecHitsValid.cc +++ b/Validation/RecoTrack/plugins/SiPixelTrackingRecHitsValid.cc @@ -124,7 +124,6 @@ void SiPixelTrackingRecHitsValid::bookHistograms(DQMStore::IBooker& ibooker, const edm::EventSetup& es) { // Book histograms dbe_ = edm::Service().operator->(); - //dbe_->showDirStructure(); //float math_pi = 3.14159265; //float radtodeg = 180.0 / math_pi; diff --git a/Validation/TrackerHits/src/TrackerHitAnalyzer.cc b/Validation/TrackerHits/src/TrackerHitAnalyzer.cc index 14ca9a9b5e16b..157a932f6e222 100644 --- a/Validation/TrackerHits/src/TrackerHitAnalyzer.cc +++ b/Validation/TrackerHits/src/TrackerHitAnalyzer.cc @@ -81,11 +81,6 @@ void TrackerHitAnalyzer::bookHistograms(DQMStore::IBooker &ibooker, const edm::R Char_t hname5[50], htitle5[80]; Char_t hname6[50], htitle6[80]; - if (fDBE) { - if (verbose_) - fDBE->showDirStructure(); - } - if (fDBE != nullptr) { // fDBE->setCurrentFolder("TrackerHitsV/TrackerHitTask"); From 274b1fa7af08fc988ca05f3c80f0a54827c696ea Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 17:45:03 +0100 Subject: [PATCH 075/112] Change code to work without old APIs. Mostly untested. --- .../src/CalibrationHistograms.cc | 14 +---- .../src/SiStripCommissioningOfflineClient.cc | 17 +++--- .../plugins/EgHLTOfflineSummaryClient.cc | 9 ++- .../Components/plugins/DQMFileReader.cc | 57 ------------------- .../Components/plugins/DQMFileReader.h | 42 -------------- .../Components/plugins/DQMStoreStats.cc | 4 +- DQMServices/Components/plugins/SealModule.cc | 2 - .../Components/python/DQMStoreStats_cfi.py | 2 +- .../FileIO/plugins/DQMFileSaverBase.cc | 17 ------ DQMServices/FileIO/plugins/DQMFileSaverBase.h | 3 - .../StreamerIO/plugins/DQMProtobufReader.cc | 2 +- 11 files changed, 25 insertions(+), 144 deletions(-) delete mode 100644 DQMServices/Components/plugins/DQMFileReader.cc delete mode 100644 DQMServices/Components/plugins/DQMFileReader.h diff --git a/DQM/SiStripCommissioningClients/src/CalibrationHistograms.cc b/DQM/SiStripCommissioningClients/src/CalibrationHistograms.cc index af85a5e92a094..b2444d11f6270 100644 --- a/DQM/SiStripCommissioningClients/src/CalibrationHistograms.cc +++ b/DQM/SiStripCommissioningClients/src/CalibrationHistograms.cc @@ -176,9 +176,7 @@ void CalibrationHistograms::save(std::string& path, uint32_t run_number, std::st TFile* outputFile = TFile::Open(path.c_str(), "UPDATE"); outputFile->cd(); - // get all sub-dirs - std::vector contents; - bei()->getContents(contents); + auto contents = bei()->getAllContents(""); TMultiGraph* graph_isha = new TMultiGraph("riseTime_vs_isha", ""); TMultiGraph* graph_vfs = new TMultiGraph("decayTime_vs_vfs", ""); @@ -195,14 +193,8 @@ void CalibrationHistograms::save(std::string& path, uint32_t run_number, std::st SiStripFecKey feckey = anal->fecKey(); TString directory; - for (auto content : contents) { - std::vector tokens; - std::string token; - std::istringstream tokenStream(content); - while (std::getline(tokenStream, token, ':')) { - tokens.push_back(token); - } - directory = Form("%s", tokens.at(0).c_str()); + for (auto me : contents) { + directory = me->getPathname(); if (directory.Contains(Form("FecCrate%d", feckey.fecCrate())) and directory.Contains(Form("FecRing%d", feckey.fecRing())) and directory.Contains(Form("FecSlot%d", feckey.fecSlot())) and diff --git a/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc b/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc index 089aaed9856d0..71a48e5a05b18 100644 --- a/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc +++ b/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc @@ -157,20 +157,23 @@ void SiStripCommissioningOfflineClient::beginRun(const edm::Run& run, const edm: << " Opened " << inputFiles_.size() << " root files!"; // Retrieve list of histograms + auto allmes = bei_->getAllContents(""); std::vector contents; - bei_->getContents(contents); // If using client file, remove "source" histograms from list if (clientHistos_) { - std::vector temp; - std::vector::iterator istr = contents.begin(); - for (; istr != contents.end(); istr++) { - if (istr->find(sistrip::collate_) != std::string::npos) { - temp.push_back(*istr); + std::set temp; + for (auto me : allmes) { + auto name = me->getPathname(); + if (name.find(sistrip::collate_) != std::string::npos) { + temp.insert(name); } } contents.clear(); - contents = temp; + for (auto s : temp) { + // the old code expects a ":", but does not really need the ME names + contents.push_back(s + ":"); + } } // Some debug diff --git a/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc b/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc index 966c121f8cb70..ad28260b50092 100644 --- a/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc +++ b/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc @@ -12,6 +12,7 @@ #include "DQMOffline/Trigger/interface/EgHLTTrigTools.h" #include "HLTrigger/HLTcore/interface/HLTConfigProvider.h" #include +#include EgHLTOfflineSummaryClient::EgHLTOfflineSummaryClient(const edm::ParameterSet& iConfig) : egHLTSumHistName_("egHLTTrigSum"), isSetup_(false) { @@ -223,9 +224,15 @@ int EgHLTOfflineSummaryClient::getQTestResults_(const std::string& filterName, int nrFail = 0; int nrQTests = 0; for (auto const& pattern : patterns) { - std::vector monElems = dbe_->getMatchingContents(dirName_ + "/" + filterName + pattern); + auto filterpattern = filterName + pattern; + std::vector monElems = dbe_->getAllContents(dirName_); // std::cout <<"mon elem "<getName(); + int match = fnmatch(filterpattern.c_str(), name.c_str(), 0); + if (match == FNM_NOMATCH) + continue; + std::vector qTests = monElem->getQReports(); nrQTests += qTests.size(); // std::cout <getName()<<" "<hasError()<<" nr test "< - -DQMFileReader::DQMFileReader(const edm::ParameterSet& ps) { - pset_ = ps; - - dbe_ = edm::Service().operator->(); - - filenames_.clear(); - filenames_ = pset_.getUntrackedParameter >("FileNames"); - referenceFileName_ = pset_.getUntrackedParameter("referenceFileName", ""); -} - -DQMFileReader::~DQMFileReader() = default; - -void DQMFileReader::beginJob() { - if (!referenceFileName_.empty()) { - const std::string override = ""; - std::vector in; - in.push_back(referenceFileName_); - edm::InputFileCatalog catalog(in, override, true); - - std::string ff = catalog.fileNames()[0]; - std::cout << "DQMFileReader: reading reference file '" << ff << "'\n"; - - // now open file, quietly continuing if it does not exist - if (dbe_->open(ff, true, "", "Reference", DQMStore::StripRunDirs, false)) { - dbe_->cd(); - dbe_->setCurrentFolder("Info/ProvInfo"); - dbe_->bookString("referenceFileName", ff); - std::cout << "DQMFileReader: reference file '" << ff << "' successfully read in \n"; - } else { - dbe_->cd(); - dbe_->setCurrentFolder("Info/ProvInfo"); - dbe_->bookString("referenceFileName", "non-existent:" + ff); - std::cout << "DQMFileReader: reference file '" << ff << "' does not exist \n"; - } - dbe_->cd(); - return; - } - - dbe_->bookString("referenceFileName", "no reference file specified"); - dbe_->cd(); - - // read in files, stripping off Run Summary and Run folders - - for (auto const& filename : filenames_) { - std::cout << "DQMFileReader::beginJob: loading" << filename << std::endl; - if (dbe_) - dbe_->load(filename); - } -} - -void DQMFileReader::endJob() {} - -void DQMFileReader::analyze(const edm::Event& iEvent, const edm::EventSetup& iSetup) {} diff --git a/DQMServices/Components/plugins/DQMFileReader.h b/DQMServices/Components/plugins/DQMFileReader.h deleted file mode 100644 index fcc6198dd2741..0000000000000 --- a/DQMServices/Components/plugins/DQMFileReader.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef DQMSERVICES_COMPONENTS_DQMFileReader_H -#define DQMSERVICES_COMPONENTS_DQMFileReader_H -// -*- C++ -*- -// -// Package: DQMFileReader -// Class: DQMFileReader -// -/* - - Description: - - Implementation: - -*/ -// -// Original Author: Kenichi Hatakeyama -// -#include "FWCore/Framework/interface/EDAnalyzer.h" -#include "DQMServices/Core/interface/DQMStore.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ServiceRegistry/interface/Service.h" - -class DQMFileReader : public edm::EDAnalyzer { -public: - typedef dqm::legacy::DQMStore DQMStore; - typedef dqm::legacy::MonitorElement MonitorElement; - explicit DQMFileReader(const edm::ParameterSet &); - ~DQMFileReader() override; - -private: - void beginJob() override; - void analyze(const edm::Event &, const edm::EventSetup &) override; - void endJob() override; - - DQMStore *dbe_; - edm::ParameterSet pset_; - - std::vector filenames_; - std::string referenceFileName_; -}; - -#endif diff --git a/DQMServices/Components/plugins/DQMStoreStats.cc b/DQMServices/Components/plugins/DQMStoreStats.cc index 810f3f0956a96..aaedf2b140490 100644 --- a/DQMServices/Components/plugins/DQMStoreStats.cc +++ b/DQMServices/Components/plugins/DQMStoreStats.cc @@ -56,7 +56,7 @@ DQMStoreStats::DQMStoreStats(const edm::ParameterSet& ps) maxbinsmeglobal_(""), maxbinsmesubsys_(""), statsdepth_(1), - pathnamematch_("*"), + pathnamematch_(""), verbose_(0) { parameters_ = ps; pathnamematch_ = ps.getUntrackedParameter("pathNameMatch", pathnamematch_); @@ -168,7 +168,7 @@ int DQMStoreStats::calcstats(int mode = DQMStoreStats::considerAllME) { size_t subsysStringEnd = 0, subfolderStringBegin = 0, subfolderStringEnd = 0; std::vector melist; - melist = dbe_->getMatchingContents(pathnamematch_); + melist = dbe_->getAllContents(pathnamematch_); Folder dbeFolder("root"); DQMStoreStatsTopLevel dqmStoreStatsTopLevel; diff --git a/DQMServices/Components/plugins/SealModule.cc b/DQMServices/Components/plugins/SealModule.cc index 75fac2b3a51ef..8806b405a957b 100644 --- a/DQMServices/Components/plugins/SealModule.cc +++ b/DQMServices/Components/plugins/SealModule.cc @@ -9,8 +9,6 @@ DEFINE_FWK_MODULE(DQMStoreStats); DEFINE_FWK_MODULE(DQMMessageLogger); #include "DQMMessageLoggerClient.h" DEFINE_FWK_MODULE(DQMMessageLoggerClient); -#include "DQMFileReader.h" -DEFINE_FWK_MODULE(DQMFileReader); #include "DQMProvInfo.h" DEFINE_FWK_MODULE(DQMProvInfo); #include "DQMDcsInfo.h" diff --git a/DQMServices/Components/python/DQMStoreStats_cfi.py b/DQMServices/Components/python/DQMStoreStats_cfi.py index 2588bb95c5639..5168b2fad5606 100644 --- a/DQMServices/Components/python/DQMStoreStats_cfi.py +++ b/DQMServices/Components/python/DQMStoreStats_cfi.py @@ -3,7 +3,7 @@ dqmStoreStats = cms.EDAnalyzer("DQMStoreStats", statsDepth = cms.untracked.int32(1), - pathNameMatch = cms.untracked.string('*'), + pathNameMatch = cms.untracked.string(''), dumpMemoryHistory = cms.untracked.bool( True ), verbose = cms.untracked.int32(0), runInEventLoop = cms.untracked.bool(False), diff --git a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc index ff566494594a1..ce05a8100ae23 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverBase.cc +++ b/DQMServices/FileIO/plugins/DQMFileSaverBase.cc @@ -35,23 +35,6 @@ DQMFileSaverBase::DQMFileSaverBase(const edm::ParameterSet &ps) { fp.version_ = 1; fp.child_ = ""; - fp.saveReference_ = DQMStore::SaveWithReference; - // Check how we should save the references. - std::string refsave = ps.getUntrackedParameter("referenceHandling", "all"); - if (refsave == "skip") { - fp.saveReference_ = DQMStore::SaveWithoutReference; - } else if (refsave == "all") { - fp.saveReference_ = DQMStore::SaveWithReference; - } else if (refsave == "qtests") { - fp.saveReference_ = DQMStore::SaveWithReferenceForQTest; - } else { - //edm::LogInfo("DQMFileSaverBase") - std::cerr << "Invalid 'referenceHandling' parameter '" << refsave << "'. Expected 'skip', 'all' or 'qtests'."; - } - - // Check minimum required quality test result for which reference is saved. - fp.saveReferenceQMin_ = ps.getUntrackedParameter("referenceRequireStatus", dqm::qstatus::STATUS_OK); - std::unique_lock lck(initial_fp_lock_); initial_fp_ = fp; } diff --git a/DQMServices/FileIO/plugins/DQMFileSaverBase.h b/DQMServices/FileIO/plugins/DQMFileSaverBase.h index 874da8e97c8eb..ec2d04735421e 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverBase.h +++ b/DQMServices/FileIO/plugins/DQMFileSaverBase.h @@ -39,9 +39,6 @@ namespace dqm { long lumi_; std::string child_; // child of a fork - // other parameters - DQMStore::SaveReferenceTag saveReference_; - int saveReferenceQMin_; }; protected: diff --git a/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc b/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc index 41992044c4d8e..625e3ddd94dea 100644 --- a/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc +++ b/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc @@ -129,7 +129,7 @@ void DQMProtobufReader::beginLuminosityBlock(edm::LuminosityBlock& lb) { fiterator_.logFileAction("Initiating request to open file ", path); fiterator_.logFileAction("Successfully opened file ", path); - store->load(path); + //store->load(path); fiterator_.logFileAction("Closed file ", path); fiterator_.logLumiState(currentLumi_, "close: ok"); } else { From 76bbd1315164610790817b284ce9ad5cf4fd31e9 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 20:21:53 +0100 Subject: [PATCH 076/112] Adapt tests for new reScope default. --- DQMServices/Demo/test/run_harvesters_cfg.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/DQMServices/Demo/test/run_harvesters_cfg.py b/DQMServices/Demo/test/run_harvesters_cfg.py index 5d97f7cbc5e5d..8ff4e9d8799a4 100644 --- a/DQMServices/Demo/test/run_harvesters_cfg.py +++ b/DQMServices/Demo/test/run_harvesters_cfg.py @@ -11,6 +11,7 @@ parser.register('protobufinput', False, one, bool, "Use DQMProtobufReader for input instead of DQMIO.") parser.register('metoedminput', False, one, bool, "Use PoolInputSource and EDMtoMEConverter for input.") parser.register('outfile', "dqm.root", one, string, "Output file name.") +parser.register('reScope', "", one, string, "Type of reScoping to use.") parser.parseArguments() args = parser @@ -54,7 +55,8 @@ else: process.source = cms.Source("DQMRootSource", - fileNames = cms.untracked.vstring(*["file://" + f for f in args.inputFiles])) + fileNames = cms.untracked.vstring(*["file://" + f for f in args.inputFiles]), + reScope = cms.untracked.string(args.reScope)) process.harvest = cms.Sequence(process.testharvester) @@ -86,3 +88,6 @@ else: process.e = cms.EndPath(process.out) +# useful for debugging +#process.DQMStore.trackME = cms.untracked.string("Legacy/testlegacy/lumi/th2s0") +#process.Tracer = cms.Service("Tracer") From 6c671e881154ec7edcccafead78fe54f4094a17f Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 20:44:30 +0100 Subject: [PATCH 077/112] New ProtoBuf input code. --- .../Components/plugins/DQMStoreStats.cc | 3 +- DQMServices/Core/interface/DQMStore.h | 129 +++++++++++------ DQMServices/Core/src/DQMStore.cc | 23 ++- DQMServices/FileIO/plugins/DQMFileSaverBase.h | 1 - .../StreamerIO/plugins/DQMProtobufReader.cc | 135 +++++++++++++++++- .../StreamerIO/plugins/DQMProtobufReader.h | 1 + 6 files changed, 242 insertions(+), 50 deletions(-) diff --git a/DQMServices/Components/plugins/DQMStoreStats.cc b/DQMServices/Components/plugins/DQMStoreStats.cc index aaedf2b140490..f783664f9a820 100644 --- a/DQMServices/Components/plugins/DQMStoreStats.cc +++ b/DQMServices/Components/plugins/DQMStoreStats.cc @@ -310,7 +310,8 @@ int DQMStoreStats::calcstats(int mode = DQMStoreStats::considerAllME) { it->getNbinsX() * it->getNbinsY() * it->getNbinsZ() * sizeof(float)); break; - default: {} + default: { + } // here we have a DQM_KIND_INVALID, DQM_KIND_INT, DQM_KIND_REAL or DQM_KIND_STRING // which we don't care much about. Alternatively: diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 5f693144f200d..fda54e0ec5886 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -112,11 +112,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book1D(TString const& name, TH1F* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH1F, [=]() { - auto th1 = static_cast(object->Clone(name)); - onbooking(th1); - return th1; - }); + return bookME( + name, + MonitorElementData::Kind::TH1F, + [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -130,11 +134,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book1S(TString const& name, TH1S* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH1S, [=]() { - auto th1 = static_cast(object->Clone(name)); - onbooking(th1); - return th1; - }); + return bookME( + name, + MonitorElementData::Kind::TH1S, + [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -148,11 +156,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book1DD(TString const& name, TH1D* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH1D, [=]() { - auto th1 = static_cast(object->Clone(name)); - onbooking(th1); - return th1; - }); + return bookME( + name, + MonitorElementData::Kind::TH1D, + [=]() { + auto th1 = static_cast(object->Clone(name)); + onbooking(th1); + return th1; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -187,11 +199,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book2D(TString const& name, TH2F* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH2F, [=]() { - auto th2 = static_cast(object->Clone(name)); - onbooking(th2); - return th2; - }); + return bookME( + name, + MonitorElementData::Kind::TH2F, + [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }, + /* forceReplace */ true); } template ::value, int> = 0> MonitorElement* book2S(TString const& name, @@ -225,11 +241,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book2S(TString const& name, TH2S* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH2S, [=]() { - auto th2 = static_cast(object->Clone(name)); - onbooking(th2); - return th2; - }); + return bookME( + name, + MonitorElementData::Kind::TH2S, + [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }, + /* forceReplace */ true); } template ::value, int> = 0> MonitorElement* book2DD(TString const& name, @@ -249,11 +269,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book2DD(TString const& name, TH2D* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH2D, [=]() { - auto th2 = static_cast(object->Clone(name)); - onbooking(th2); - return th2; - }); + return bookME( + name, + MonitorElementData::Kind::TH2D, + [=]() { + auto th2 = static_cast(object->Clone(name)); + onbooking(th2); + return th2; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -277,11 +301,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* book3D(TString const& name, TH3F* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TH3F, [=]() { - auto th3 = static_cast(object->Clone(name)); - onbooking(th3); - return th3; - }); + return bookME( + name, + MonitorElementData::Kind::TH3F, + [=]() { + auto th3 = static_cast(object->Clone(name)); + onbooking(th3); + return th3; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -350,11 +378,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* bookProfile(TString const& name, TProfile* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TPROFILE, [=]() { - auto tprofile = static_cast(object->Clone(name)); - onbooking(tprofile); - return tprofile; - }); + return bookME( + name, + MonitorElementData::Kind::TPROFILE, + [=]() { + auto tprofile = static_cast(object->Clone(name)); + onbooking(tprofile); + return tprofile; + }, + /* forceReplace */ true); } template ::value, int> = 0> @@ -398,11 +430,15 @@ namespace dqm { } template ::value, int> = 0> MonitorElement* bookProfile2D(TString const& name, TProfile2D* object, FUNC onbooking = NOOP()) { - return bookME(name, MonitorElementData::Kind::TPROFILE2D, [=]() { - auto tprofile = static_cast(object->Clone(name)); - onbooking(tprofile); - return tprofile; - }); + return bookME( + name, + MonitorElementData::Kind::TPROFILE2D, + [=]() { + auto tprofile = static_cast(object->Clone(name)); + onbooking(tprofile); + return tprofile; + }, + /* forceReplace */ true); } // @@ -418,7 +454,8 @@ namespace dqm { virtual edm::LuminosityBlockID setRunLumi(edm::LuminosityBlockID runlumi); virtual MonitorElement* bookME(TString const& name, MonitorElementData::Kind kind, - std::function makeobject); + std::function makeobject, + bool forceReplace = false); DQMStore* store_ = nullptr; MonitorElementData::Scope scope_ = MonitorElementData::Scope::JOB; diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 87237c93c2430..ca09ebfee2380 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -50,7 +50,8 @@ namespace dqm::implementation { MonitorElement* IBooker::bookME(TString const& name, MonitorElementData::Kind kind, - std::function makeobject) { + std::function makeobject, + bool forceReplace /* = false */) { MonitorElementData::Path path; std::string fullpath = pwd() + std::string(name.View()); path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); @@ -93,7 +94,27 @@ namespace dqm::implementation { medata.value_.object_ = std::unique_ptr(th1); MonitorElement* me_ptr = new MonitorElement(std::move(medata)); me = store_->putME(me_ptr); + } else { + if (forceReplace) { + TH1* th1 = makeobject(); + assert(th1); + store_->debugTrackME("bookME (forceReplace)", me); + // surgically replace Histogram + // This is rather dangerous because the ME is in a global set and may + // be in use. We are protected by the booking lock here, but code + // filling in parallel might observe `isValid() == false`. + // Maybe replace this with a proper API to atomically replace the TH1. + auto medata = me->release(/* expectOwned */ true); + // Assume kind etc. matches. + // This should free the old object. + { + auto access = medata->accessMut(); + access.value.object_ = std::unique_ptr(th1); + } + me->switchData(medata); + } } + // me now points to a global ME owned by the DQMStore. assert(me); diff --git a/DQMServices/FileIO/plugins/DQMFileSaverBase.h b/DQMServices/FileIO/plugins/DQMFileSaverBase.h index ec2d04735421e..10e0ee3f97bd7 100644 --- a/DQMServices/FileIO/plugins/DQMFileSaverBase.h +++ b/DQMServices/FileIO/plugins/DQMFileSaverBase.h @@ -38,7 +38,6 @@ namespace dqm { long run_; long lumi_; std::string child_; // child of a fork - }; protected: diff --git a/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc b/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc index 625e3ddd94dea..8f9f0d05f8591 100644 --- a/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc +++ b/DQMServices/StreamerIO/plugins/DQMProtobufReader.cc @@ -2,10 +2,21 @@ #include "FWCore/MessageLogger/interface/JobReport.h" #include "DQMServices/Core/interface/DQMStore.h" +#include "DataFormats/Histograms/interface/DQMToken.h" #include "FWCore/Utilities/interface/UnixSignalHandlers.h" // #include "FWCore/Sources/interface/ProducerSourceBase.h" +#include "DQMServices/Core/src/ROOTFilePB.pb.h" +#include +#include +#include + +#include "TBufferFile.h" + +#include +#include + using namespace dqmservices; DQMProtobufReader::DQMProtobufReader(edm::ParameterSet const& pset, edm::InputSourceDescription const& desc) @@ -17,6 +28,8 @@ DQMProtobufReader::DQMProtobufReader(edm::ParameterSet const& pset, edm::InputSo produces("sourceDataPath"); produces("sourceJsonPath"); + produces("DQMGenerationRecoRun"); + produces("DQMGenerationRecoLumi"); } DQMProtobufReader::~DQMProtobufReader() {} @@ -129,7 +142,7 @@ void DQMProtobufReader::beginLuminosityBlock(edm::LuminosityBlock& lb) { fiterator_.logFileAction("Initiating request to open file ", path); fiterator_.logFileAction("Successfully opened file ", path); - //store->load(path); + load(&*store, path); fiterator_.logFileAction("Closed file ", path); fiterator_.logLumiState(currentLumi_, "close: ok"); } else { @@ -138,6 +151,126 @@ void DQMProtobufReader::beginLuminosityBlock(edm::LuminosityBlock& lb) { } } +void DQMProtobufReader::load(DQMStore* store, std::string filename) { + using google::protobuf::io::ArrayInputStream; + using google::protobuf::io::CodedInputStream; + using google::protobuf::io::FileInputStream; + using google::protobuf::io::FileOutputStream; + using google::protobuf::io::GzipInputStream; + using google::protobuf::io::GzipOutputStream; + + int filedescriptor; + if ((filedescriptor = ::open(filename.c_str(), O_RDONLY)) == -1) { + edm::LogError("DQMProtobufReader") << "File " << filename << " does not exist."; + } + + dqmstorepb::ROOTFilePB dqmstore_message; + FileInputStream fin(filedescriptor); + GzipInputStream input(&fin); + CodedInputStream input_coded(&input); + input_coded.SetTotalBytesLimit(1024 * 1024 * 1024, -1); + if (!dqmstore_message.ParseFromCodedStream(&input_coded)) { + edm::LogError("DQMProtobufReader") << "Fatal parsing file '" << filename << "'"; + } + + ::close(filedescriptor); + + for (int i = 0; i < dqmstore_message.histo_size(); ++i) { + TObject* obj = nullptr; + dqmstorepb::ROOTFilePB::Histo const& h = dqmstore_message.histo(i); + + size_t slash = h.full_pathname().rfind('/'); + size_t dirpos = (slash == std::string::npos ? 0 : slash); + size_t namepos = (slash == std::string::npos ? 0 : slash + 1); + std::string objname, dirname; + dirname.assign(h.full_pathname(), 0, dirpos); + objname.assign(h.full_pathname(), namepos, std::string::npos); + TBufferFile buf(TBufferFile::kRead, h.size(), (void*)h.streamed_histo().data(), kFALSE); + buf.Reset(); + if (buf.Length() == buf.BufferSize()) { + obj = nullptr; + } else { + buf.InitMap(); + void* ptr = buf.ReadObjectAny(nullptr); + obj = reinterpret_cast(ptr); + } + + if (!obj) { + edm::LogError("DQMProtobufReader") << "Error reading element:'" << h.full_pathname(); + } + + store->setCurrentFolder(dirname); + + if (h.flags() & DQMNet::DQM_PROP_LUMI) { + store->setScope(MonitorElementData::Scope::LUMI); + } else { + store->setScope(MonitorElementData::Scope::RUN); + } + + if (obj) { + int kind = h.flags() & DQMNet::DQM_PROP_TYPE_MASK; + if (kind == DQMNet::DQM_PROP_TYPE_INT) { + MonitorElement* me = store->bookInt(objname); + auto expression = std::string(static_cast(obj)->String().View()); + std::regex parseint{"<.*>i=(.*)"}; + std::smatch match; + bool ok = std::regex_match(expression, match, parseint); + if (!ok) { + edm::LogError("DQMProtobufReader") << "Malformed object of type INT: '" << expression << "'"; + continue; + } + int value = std::atoi(match[1].str().c_str()); + me->Fill(value); + } else if (kind == DQMNet::DQM_PROP_TYPE_REAL) { + MonitorElement* me = store->bookFloat(objname); + auto expression = std::string(static_cast(obj)->String().View()); + std::regex parsefloat{"<.*>f=(.*)"}; + std::smatch match; + bool ok = std::regex_match(expression, match, parsefloat); + if (!ok) { + edm::LogError("DQMProtobufReader") << "Malformed object of type REAL: '" << expression << "'"; + continue; + } + double value = std::atof(match[1].str().c_str()); + me->Fill(value); + } else if (kind == DQMNet::DQM_PROP_TYPE_STRING) { + auto value = static_cast(obj)->String(); + store->bookString(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH1F) { + auto value = static_cast(obj); + store->book1D(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH1S) { + auto value = static_cast(obj); + store->book1S(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH1D) { + auto value = static_cast(obj); + store->book1DD(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH2F) { + auto value = static_cast(obj); + store->book2D(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH2S) { + auto value = static_cast(obj); + store->book2S(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH2D) { + auto value = static_cast(obj); + store->book2DD(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TH3F) { + auto value = static_cast(obj); + store->book3D(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TPROF) { + auto value = static_cast(obj); + store->bookProfile(objname, value); + } else if (kind == DQMNet::DQM_PROP_TYPE_TPROF2D) { + auto value = static_cast(obj); + store->bookProfile2D(objname, value); + } else { + edm::LogError("DQMProtobufReader") << "Unknown type: " << kind; + } + delete obj; + } + } +} + void DQMProtobufReader::readEvent_(edm::EventPrincipal&){}; void DQMProtobufReader::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { diff --git a/DQMServices/StreamerIO/plugins/DQMProtobufReader.h b/DQMServices/StreamerIO/plugins/DQMProtobufReader.h index c049a53235190..c24973fd25f40 100644 --- a/DQMServices/StreamerIO/plugins/DQMProtobufReader.h +++ b/DQMServices/StreamerIO/plugins/DQMProtobufReader.h @@ -26,6 +26,7 @@ namespace dqmservices { static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); private: + void load(DQMStore* store, std::string filename); edm::InputSource::ItemType getNextItemType() override; std::shared_ptr readRunAuxiliary_() override; std::shared_ptr readLuminosityBlockAuxiliary_() override; From df607a46dd1f63e7c2940eb0627a90183a009448 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 21:00:28 +0100 Subject: [PATCH 078/112] Code-checks. --- .../src/SiStripCommissioningOfflineClient.cc | 2 +- DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc b/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc index 71a48e5a05b18..aebf7ddcd803a 100644 --- a/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc +++ b/DQM/SiStripCommissioningClients/src/SiStripCommissioningOfflineClient.cc @@ -164,7 +164,7 @@ void SiStripCommissioningOfflineClient::beginRun(const edm::Run& run, const edm: if (clientHistos_) { std::set temp; for (auto me : allmes) { - auto name = me->getPathname(); + const auto& name = me->getPathname(); if (name.find(sistrip::collate_) != std::string::npos) { temp.insert(name); } diff --git a/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc b/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc index ad28260b50092..5917b1195e35c 100644 --- a/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc +++ b/DQMOffline/Trigger/plugins/EgHLTOfflineSummaryClient.cc @@ -228,7 +228,7 @@ int EgHLTOfflineSummaryClient::getQTestResults_(const std::string& filterName, std::vector monElems = dbe_->getAllContents(dirName_); // std::cout <<"mon elem "<getName(); + const auto& name = monElem->getName(); int match = fnmatch(filterpattern.c_str(), name.c_str(), 0); if (match == FNM_NOMATCH) continue; From c4962541e016fc780acd91a27174e0f8d994a2fb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 18 Dec 2019 21:01:00 +0100 Subject: [PATCH 079/112] Improve debug reporting and fix another bug in the tests. --- DQMServices/Core/src/DQMStore.cc | 3 ++- DQMServices/Demo/test/TestDQMEDAnalyzer.cc | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index ca09ebfee2380..7d4d75063f792 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -281,7 +281,7 @@ namespace dqm::implementation { std::string name = me->getFullname(); if (name.find(this->trackME_) != std::string::npos) { edm::LogWarning("DQMStoreTrackME").log([&](auto& logger) { - logger << message << " for " << name; + logger << message << " for " << name << "(" << me << ")"; if (me->isValid()) { logger << " " << me->getRunLumi() << " scope " << scopename[me->getScope()]; if (me->kind() >= MonitorElement::Kind::TH1F) { @@ -524,6 +524,7 @@ namespace dqm::implementation { me->Reset(); auto result = prototypes.insert(me); assert(result.second); // was new insertion, else findME should succeed + debugTrackME("cleanupLumi (reset)", me); } } diff --git a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc index fc4b8353e9da1..4406ce2840149 100644 --- a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc +++ b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc @@ -16,6 +16,10 @@ class BookerFiller { BookerFiller(){}; void bookall(BOOKERLIKE& ibooker) { + mes_1D.clear(); + mes_2D.clear(); + mes_3D.clear(); + for (int i = 0; i < howmany; i++) { ibooker.setCurrentFolder(folder); auto num = std::to_string(i); From a090e789a5add0a331b7acbfec55381a3d26e612 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 10:50:25 +0100 Subject: [PATCH 080/112] Add reScope='' (no rescoping) to the FwkIO tests. --- DQMServices/Demo/test/run_analyzers_cfg.py | 2 ++ DQMServices/FwkIO/test/copy_file_multi_types_cfg.py | 1 + DQMServices/FwkIO/test/merge_file1_file2_cfg.py | 1 + DQMServices/FwkIO/test/merge_file1_file3_file2_cfg.py | 1 + .../FwkIO/test/merge_file1_file3_file2_filterOnRun1_cfg.py | 1 + DQMServices/FwkIO/test/merge_file1_file3_file4_cfg.py | 1 + DQMServices/FwkIO/test/merge_one_run_one_lumi_run_only_cfg.py | 1 + DQMServices/FwkIO/test/read_file1_file2_cfg.py | 1 + DQMServices/FwkIO/test/read_file1_file3_cfg.py | 1 + DQMServices/FwkIO/test/read_lumi_only_file_cfg.py | 1 + DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py | 1 + DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py | 1 + DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py | 1 + DQMServices/FwkIO/test/read_missing_file_cfg.py | 1 + DQMServices/FwkIO/test/read_run_lumi_file_cfg.py | 1 + DQMServices/FwkIO/test/read_run_only_file_cfg.py | 1 + .../read_write_merged_file1_file3_file2_filterOnRun1_cfg.py | 1 + DQMServices/FwkIO/test/read_write_run_lumi_file_cfg.py | 1 + 18 files changed, 19 insertions(+) diff --git a/DQMServices/Demo/test/run_analyzers_cfg.py b/DQMServices/Demo/test/run_analyzers_cfg.py index 9a22a50443017..db8c11a69c0d5 100644 --- a/DQMServices/Demo/test/run_analyzers_cfg.py +++ b/DQMServices/Demo/test/run_analyzers_cfg.py @@ -141,3 +141,5 @@ else: process.o = cms.EndPath(process.out) + +process.DQMStore.trackME = cms.untracked.string("Legacy/testlegacy/th2d0") diff --git a/DQMServices/FwkIO/test/copy_file_multi_types_cfg.py b/DQMServices/FwkIO/test/copy_file_multi_types_cfg.py index 6c6002309c564..68ab5d8e24253 100644 --- a/DQMServices/FwkIO/test/copy_file_multi_types_cfg.py +++ b/DQMServices/FwkIO/test/copy_file_multi_types_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file_multi_types.root")) process.out = cms.OutputModule("DQMRootOutputModule", diff --git a/DQMServices/FwkIO/test/merge_file1_file2_cfg.py b/DQMServices/FwkIO/test/merge_file1_file2_cfg.py index 62c65e0a0ce50..fbcf434b27fe5 100644 --- a/DQMServices/FwkIO/test/merge_file1_file2_cfg.py +++ b/DQMServices/FwkIO/test/merge_file1_file2_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root","file:dqm_file2.root")) process.out = cms.OutputModule("DQMRootOutputModule", diff --git a/DQMServices/FwkIO/test/merge_file1_file3_file2_cfg.py b/DQMServices/FwkIO/test/merge_file1_file3_file2_cfg.py index 4df413eab86a4..1ba6478729188 100644 --- a/DQMServices/FwkIO/test/merge_file1_file3_file2_cfg.py +++ b/DQMServices/FwkIO/test/merge_file1_file3_file2_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root", "file:dqm_file3.root", "file:dqm_file2.root")) diff --git a/DQMServices/FwkIO/test/merge_file1_file3_file2_filterOnRun1_cfg.py b/DQMServices/FwkIO/test/merge_file1_file3_file2_filterOnRun1_cfg.py index 80a20d2b0e825..f9629a0f8e149 100644 --- a/DQMServices/FwkIO/test/merge_file1_file3_file2_filterOnRun1_cfg.py +++ b/DQMServices/FwkIO/test/merge_file1_file3_file2_filterOnRun1_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root", "file:dqm_file3.root", "file:dqm_file2.root"), diff --git a/DQMServices/FwkIO/test/merge_file1_file3_file4_cfg.py b/DQMServices/FwkIO/test/merge_file1_file3_file4_cfg.py index c382fa191777f..97f835c5f7e8a 100644 --- a/DQMServices/FwkIO/test/merge_file1_file3_file4_cfg.py +++ b/DQMServices/FwkIO/test/merge_file1_file3_file4_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root","file:dqm_file3.root","file:dqm_file4.root")) process.out = cms.OutputModule("DQMRootOutputModule", diff --git a/DQMServices/FwkIO/test/merge_one_run_one_lumi_run_only_cfg.py b/DQMServices/FwkIO/test/merge_one_run_one_lumi_run_only_cfg.py index 0bd36191369b7..7558039e46a07 100644 --- a/DQMServices/FwkIO/test/merge_one_run_one_lumi_run_only_cfg.py +++ b/DQMServices/FwkIO/test/merge_one_run_one_lumi_run_only_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_one_run_one_lumi_run_only.root","file:dqm_one_run_one_lumi_run_only_2.root")) process.out = cms.OutputModule("DQMRootOutputModule", diff --git a/DQMServices/FwkIO/test/read_file1_file2_cfg.py b/DQMServices/FwkIO/test/read_file1_file2_cfg.py index 53985972542be..66924fa437aed 100644 --- a/DQMServices/FwkIO/test/read_file1_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_file1_file2_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root","file:dqm_file2.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_file1_file3_cfg.py b/DQMServices/FwkIO/test/read_file1_file3_cfg.py index 21b63acbd0834..48279dff1d487 100644 --- a/DQMServices/FwkIO/test/read_file1_file3_cfg.py +++ b/DQMServices/FwkIO/test/read_file1_file3_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_file1.root","file:dqm_file3.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_lumi_only_file_cfg.py b/DQMServices/FwkIO/test/read_lumi_only_file_cfg.py index c11bdfddf5119..a6831a8ce11d8 100644 --- a/DQMServices/FwkIO/test/read_lumi_only_file_cfg.py +++ b/DQMServices/FwkIO/test/read_lumi_only_file_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_lumi_only.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py index fa95038afec4b..380f05ff7c586 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file2_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_merged_file1_file2.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py index 88f8765208f8c..6ec533b977d47 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file3_file2_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_merged_file1_file3_file2.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py b/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py index 99dcc7fc1032b..7c4135a5cef88 100644 --- a/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py +++ b/DQMServices/FwkIO/test/read_merged_file1_file3_file4_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_merged_file1_file3_file4.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_missing_file_cfg.py b/DQMServices/FwkIO/test/read_missing_file_cfg.py index ad106a7c020f2..e9c4fd6aefa27 100644 --- a/DQMServices/FwkIO/test/read_missing_file_cfg.py +++ b/DQMServices/FwkIO/test/read_missing_file_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_missing.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_run_lumi_file_cfg.py b/DQMServices/FwkIO/test/read_run_lumi_file_cfg.py index ea1cbdee54730..100003fb29437 100644 --- a/DQMServices/FwkIO/test/read_run_lumi_file_cfg.py +++ b/DQMServices/FwkIO/test/read_run_lumi_file_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_run_lumi.root")) seq = cms.untracked.VEventID() diff --git a/DQMServices/FwkIO/test/read_run_only_file_cfg.py b/DQMServices/FwkIO/test/read_run_only_file_cfg.py index 63b49cf5dacb8..a7209e87b7a87 100644 --- a/DQMServices/FwkIO/test/read_run_only_file_cfg.py +++ b/DQMServices/FwkIO/test/read_run_only_file_cfg.py @@ -4,6 +4,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_run_only.root")) #NOTE: even though we only store histograms on runs, we still record the lumis that were seen diff --git a/DQMServices/FwkIO/test/read_write_merged_file1_file3_file2_filterOnRun1_cfg.py b/DQMServices/FwkIO/test/read_write_merged_file1_file3_file2_filterOnRun1_cfg.py index c92aca3aac1d1..939d02f2b41e8 100644 --- a/DQMServices/FwkIO/test/read_write_merged_file1_file3_file2_filterOnRun1_cfg.py +++ b/DQMServices/FwkIO/test/read_write_merged_file1_file3_file2_filterOnRun1_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_merged_file1_file3_file2_filterOnRun1.root")) process.out = cms.OutputModule("DQMRootOutputModule", diff --git a/DQMServices/FwkIO/test/read_write_run_lumi_file_cfg.py b/DQMServices/FwkIO/test/read_write_run_lumi_file_cfg.py index 69030fe9bb018..2c0aea9c9850b 100644 --- a/DQMServices/FwkIO/test/read_write_run_lumi_file_cfg.py +++ b/DQMServices/FwkIO/test/read_write_run_lumi_file_cfg.py @@ -3,6 +3,7 @@ process = cms.Process("READ") process.source = cms.Source("DQMRootSource", + reScope = cms.untracked.string(""), fileNames = cms.untracked.vstring("file:dqm_run_lumi.root")) process.out = cms.OutputModule("DQMRootOutputModule", From 75392b0b3b67e07d9397af0387db2a761a9a4dec Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 11:35:25 +0100 Subject: [PATCH 081/112] Properly use ROOT efficiency flag. --- DQMServices/Core/interface/MonitorElement.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index d8365eac95e5b..fcb2e223b1c91 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -271,8 +271,15 @@ namespace dqm::impl { /// this ME is meant to be an efficiency plot that must not be /// normalized when drawn in the DQM GUI. - void setEfficiencyFlag() { data_.flags |= DQMNet::DQM_PROP_EFFICIENCY_PLOT; } - bool getEfficiencyFlag() { return data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT; } + void setEfficiencyFlag() { + auto access = this->accessMut(); + if (access.value.object_) + access.value.object_->SetBit(TH1::kIsAverage); + } + bool getEfficiencyFlag() { + auto access = this->access(); + return access.value.object_ && access.value.object_->TestBit(TH1::kIsAverage); + } // A static assert to check that T actually fits in // int64_t. From 4a2d42adf17c609153498423874e61d95c4e7497 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 11:52:27 +0100 Subject: [PATCH 082/112] Use reference_wrapper to avoid a copy. Thanks @makortel for the tip --- DQMServices/Core/interface/MonitorElement.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index fcb2e223b1c91..ef95550627cb9 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -101,7 +101,9 @@ namespace dqm::impl { struct MEComparison { using is_transparent = int; // magic marker to allow C++14 heterogeneous set lookup. - auto make_tuple(MonitorElement *me) const { return std::make_tuple(me->getPathname(), me->getName()); } + auto make_tuple(MonitorElement *me) const { + return std::make_tuple(std::reference_wrapper(me->getPathname()), std::reference_wrapper(me->getName())); + } auto make_tuple(MonitorElementData::Path const &path) const { return std::make_tuple(path.getDirname(), path.getObjectname()); } From bcb43d6d812c92aef6ae574d6ef8e1e7401915aa Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 11:53:21 +0100 Subject: [PATCH 083/112] Fix cd(...) semantics. It turns out, cd actually does not cd. It is identical to setCurrentFolder. --- DQMServices/Core/interface/DQMStore.h | 2 ++ DQMServices/Core/src/DQMStore.cc | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index fda54e0ec5886..cd34d4e67f602 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -24,6 +24,8 @@ namespace dqm { class NavigatorBase { public: virtual void cd(); + // cd is identical to setCurrentFolder! + DQM_DEPRECATED virtual void cd(std::string const& dir); // This is the only method that is allowed to change cwd_ value virtual void setCurrentFolder(std::string const& fullpath); diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 7d4d75063f792..54a2e2c30022c 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -15,8 +15,8 @@ namespace dqm::implementation { std::string const& NavigatorBase::pwd() { return cwd_; } void NavigatorBase::cd() { setCurrentFolder(""); } - void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(cwd_ + dir); } - void NavigatorBase::goUp() { cd(".."); } + void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(dir); } + void NavigatorBase::goUp() { cd(cwd_ + ".."); } void NavigatorBase::setCurrentFolder(std::string const& fullpath) { MonitorElementData::Path path; path.set(fullpath, MonitorElementData::Path::Type::DIR); From 5eb454713b7229230dbc18dc48264e2f51699216 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 11:59:46 +0100 Subject: [PATCH 084/112] Remove support for read-only data in ME. It was never used here. The idea originates in product-based harvesting, where we'd copy-on-write histograms borrowed out of edm products. Since we still have the edm::Serivce, this is not required now: we can have mutable 'products'. --- DQMServices/Core/interface/MonitorElement.h | 46 ++------------------- DQMServices/Core/src/MonitorElement.cc | 4 -- 2 files changed, 3 insertions(+), 47 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index ef95550627cb9..c0597e1a6ca97 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -123,11 +123,8 @@ namespace dqm::impl { protected: DQMNet::CoreObject data_; //< Core object information. - // TODO: we only use the ::Value part so far. - // Still using the full thing to remain compatible with the new ME implementation. - std::atomic frozen_; // only set if this ME is in a product already - std::atomic mutable_; // only set if there is a mutable copy of this ME + std::atomic mutable_; // only set if this is a mutable copy of this ME bool is_owned_; // true if we are responsible for deleting the mutable object. /** * To do anything to the MEs data, one needs to obtain an access object. @@ -151,13 +148,6 @@ namespace dqm::impl { // if there is a mutable object, that is the truth, and we take a lock. return mut->access(); } // else - auto frozen = frozen_.load(); - if (frozen) { - // in case of an immutable object read from edm products, create an - // access object without lock. - return Access{std::unique_lock(), frozen->key_, frozen->value_}; - } - // else throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } @@ -171,37 +161,7 @@ namespace dqm::impl { // if there is a mutable object, that is the truth, and we take a lock. return mut->accessMut(); } // else - auto frozen = frozen_.load(); - if (!frozen) { - throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; - } - // in case of an immutable object read from edm products, attempt to - // make a clone. - MutableMonitorElementData *clone = new MutableMonitorElementData(); - clone->data_.key_ = frozen->key_; - clone->data_.value_.scalar_ = frozen->value_.scalar_; - if (frozen->value_.object_) { - // Clone() the TH1 - clone->data_.value_.object_ = std::unique_ptr(static_cast(frozen->value_.object_->Clone())); - } - - // now try to set our clone, and see if it was still needed (sb. else - // might have made a clone already!) - MutableMonitorElementData *existing = nullptr; - bool ok = mutable_.compare_exchange_strong(existing, clone); - if (!ok) { - // somebody else made a clone already, it is now in existing - delete clone; - return existing->accessMut(); - } else { - // we won the race, and our clone is the real one now. - this->is_owned_ = true; - return clone->accessMut(); - } - // in either case, if somebody destroyed the mutable object between us - // getting the pointer and us locking it, we are screwed. We have to rely - // on edm and the DQM code to make sure we only turn mutable objects into - // products once all processing is done (logically, this is safe). + throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } public: @@ -231,7 +191,7 @@ namespace dqm::impl { // check if the ME is currently backed by MEData; if false (almost) any // access will throw. - bool isValid() const { return mutable_.load() || frozen_.load(); } + bool isValid() const { return mutable_.load() != nullptr; } /// Compare monitor elements, for ordering in sets. bool operator<(const MonitorElement &x) const { return DQMNet::setOrder(data_, x.data_); } diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index bd1e221d91728..7d5c96fafcc68 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -42,7 +42,6 @@ namespace dqm::impl { MonitorElement::MonitorElement(MonitorElementData &&data) { this->mutable_ = new MutableMonitorElementData(); - this->frozen_ = nullptr; this->mutable_.load()->data_ = std::move(data); this->is_owned_ = true; syncCoreObject(); @@ -65,7 +64,6 @@ namespace dqm::impl { assert(this->is_owned_ == expectOwned); MutableMonitorElementData *data = this->mutable_.load(); this->mutable_ = nullptr; - this->frozen_ = nullptr; this->is_owned_ = false; assert(!expectOwned || data); return data; @@ -74,14 +72,12 @@ namespace dqm::impl { void MonitorElement::switchData(MonitorElement *other) { assert(other); this->mutable_ = other->mutable_.load(); - this->frozen_ = other->frozen_.load(); this->is_owned_ = false; syncCoreObject(); } void MonitorElement::switchData(MutableMonitorElementData *data) { this->mutable_ = data; - this->frozen_ = nullptr; this->is_owned_ = true; syncCoreObject(); } From e49a2b5da3275b032153f4895c51dbd0c12a8acf Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 12:29:26 +0100 Subject: [PATCH 085/112] More agressively reset Scope to JOB. In harvesting, we need to make sure verything stays at JOB Scope so things actually get saved to the legacy output file at endJob. --- DQMServices/Core/interface/DQMEDHarvester.h | 19 +++++++++++++------ DQMServices/Core/src/DQMStore.cc | 2 ++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/DQMServices/Core/interface/DQMEDHarvester.h b/DQMServices/Core/interface/DQMEDHarvester.h index 1a2ccd7b7335b..cc8e6dc50fc6a 100644 --- a/DQMServices/Core/interface/DQMEDHarvester.h +++ b/DQMServices/Core/interface/DQMEDHarvester.h @@ -103,8 +103,10 @@ class DQMEDHarvester } void accumulate(edm::Event const &ev, edm::EventSetup const &es) final { - dqmstore_->meBookerGetter( - [this, &ev, &es](DQMStore::IBooker &b, DQMStore::IGetter &g) { this->dqmAnalyze(b, g, ev, es); }); + dqmstore_->meBookerGetter([this, &ev, &es](DQMStore::IBooker &b, DQMStore::IGetter &g) { + b.setScope(MonitorElementData::Scope::JOB); + this->dqmAnalyze(b, g, ev, es); + }); } void endLuminosityBlockProduce(edm::LuminosityBlock &lumi, edm::EventSetup const &es) final { @@ -113,6 +115,7 @@ class DQMEDHarvester //lumimegetter_.fillHandles(lumi, refs); dqmstore_->meBookerGetter([this, &lumi, &es](DQMStore::IBooker &b, DQMStore::IGetter &g) { + b.setScope(MonitorElementData::Scope::JOB); this->dqmEndLuminosityBlock(b, g, lumi, es); }); @@ -122,8 +125,10 @@ class DQMEDHarvester void endLuminosityBlock(edm::LuminosityBlock const &, edm::EventSetup const &) final{}; void endRunProduce(edm::Run &run, edm::EventSetup const &es) final { - dqmstore_->meBookerGetter( - [this, &run, &es](DQMStore::IBooker &b, DQMStore::IGetter &g) { this->dqmEndRun(b, g, run, es); }); + dqmstore_->meBookerGetter([this, &run, &es](DQMStore::IBooker &b, DQMStore::IGetter &g) { + b.setScope(MonitorElementData::Scope::JOB); + this->dqmEndRun(b, g, run, es); + }); run.put(runToken_, std::make_unique()); } @@ -131,7 +136,10 @@ class DQMEDHarvester void endRun(edm::Run const &, edm::EventSetup const &) override{}; void endJob() final { - dqmstore_->meBookerGetter([this](DQMStore::IBooker &b, DQMStore::IGetter &g) { this->dqmEndJob(b, g); }); + dqmstore_->meBookerGetter([this](DQMStore::IBooker &b, DQMStore::IGetter &g) { + b.setScope(MonitorElementData::Scope::JOB); + this->dqmEndJob(b, g); + }); }; ~DQMEDHarvester() override = default; @@ -144,7 +152,6 @@ class DQMEDHarvester DQMStore::IGetter &, edm::LuminosityBlock const &, edm::EventSetup const &){}; - // DQM_EXPERIMENTAL // HARVESTING should happen in endJob (or endLumi, for online), but there can // be applications for end-run harvesting. Better to have a callback than // have unprotected DQMStore access. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 54a2e2c30022c..747386543c494 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -659,9 +659,11 @@ namespace dqm::implementation { // This is no more than a guess with concurrent runs/lumis, but should be // correct for purely sequential legacy stuff. // These transitions should only affect non-DQM*EDAnalyzer based code. + // Also reset Scope, such that legacy modules can expect it to be JOB. ar.watchPreGlobalBeginRun([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); this->enterLumi(gc.luminosityBlockID().run(), /* lumi */ 0, /* moduleID */ 0); + this->setScope(MonitorElementData::Scope::JOB); }); ar.watchPreGlobalBeginLumi([this](edm::GlobalContext const& gc) { this->setRunLumi(gc.luminosityBlockID()); From 50df2853bf4b443b7628ccfc00f99a6452370200 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 13:00:54 +0100 Subject: [PATCH 086/112] Add a reScope option also to the EDMtoMEConverter. It is equivalent to the DQMRootSource in role. The default config always sets it to JOB, since that is what *all* existing workflows expect. The not-JOB option is only used for the unit tests. An issue is that harvesting code now can no longer tell if an ME was saved per lumi or not. --- .../Components/plugins/EDMtoMEConverter.cc | 21 +++++++++++++++---- .../Components/plugins/EDMtoMEConverter.h | 1 + .../Components/python/EDMtoMEConverter_cfi.py | 2 ++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/DQMServices/Components/plugins/EDMtoMEConverter.cc b/DQMServices/Components/plugins/EDMtoMEConverter.cc index ffc3bc7bc850b..314250de7060d 100644 --- a/DQMServices/Components/plugins/EDMtoMEConverter.cc +++ b/DQMServices/Components/plugins/EDMtoMEConverter.cc @@ -248,9 +248,16 @@ namespace { }; // TODO: might need re-scoping to JOB here. - void adjustScope(DQMStore::IBooker &ibooker, const edm::Run &) { ibooker.setScope(MonitorElementData::Scope::RUN); } - void adjustScope(DQMStore::IBooker &ibooker, const edm::LuminosityBlock &) { - ibooker.setScope(MonitorElementData::Scope::LUMI); + void adjustScope(DQMStore::IBooker &ibooker, const edm::Run &, MonitorElementData::Scope reScope) { + if (reScope == MonitorElementData::Scope::JOB) { + ibooker.setScope(MonitorElementData::Scope::JOB); + } else { + ibooker.setScope(MonitorElementData::Scope::RUN); + } + } + void adjustScope(DQMStore::IBooker &ibooker, const edm::LuminosityBlock &, MonitorElementData::Scope reScope) { + // will be LUMI for no reScoping, else the expected scope. + ibooker.setScope(reScope); } } // namespace @@ -272,6 +279,12 @@ EDMtoMEConverter::EDMtoMEConverter(const edm::ParameterSet &iPSet) : verbosity(0 convertOnEndLumi = iPSet.getUntrackedParameter("convertOnEndLumi", true); convertOnEndRun = iPSet.getUntrackedParameter("convertOnEndRun", true); + auto scopeDecode = std::map{{"", MonitorElementData::Scope::LUMI}, + {"LUMI", MonitorElementData::Scope::LUMI}, + {"RUN", MonitorElementData::Scope::RUN}, + {"JOB", MonitorElementData::Scope::JOB}}; + reScope = scopeDecode[iPSet.getUntrackedParameter("reScope", "")]; + // use value of first digit to determine default output level (inclusive) // 0 is none, 1 is basic, 2 is fill output, 3 is gather output verbosity %= 10; @@ -353,7 +366,7 @@ void EDMtoMEConverter::getData(DQMStore::IBooker &iBooker, DQMStore::IGetter &iG } // define new monitor element - adjustScope(iBooker, iGetFrom); + adjustScope(iBooker, iGetFrom, reScope); AddMonitorElement::call(iBooker, iGetter, &metoedmobject[i].object, dir, name, iGetFrom); } // end loop thorugh metoedmobject diff --git a/DQMServices/Components/plugins/EDMtoMEConverter.h b/DQMServices/Components/plugins/EDMtoMEConverter.h index 7a2760f34cbf9..360c8adbfadfb 100644 --- a/DQMServices/Components/plugins/EDMtoMEConverter.h +++ b/DQMServices/Components/plugins/EDMtoMEConverter.h @@ -83,6 +83,7 @@ class EDMtoMEConverter : public edm::one::EDProducer class Tokens { diff --git a/DQMServices/Components/python/EDMtoMEConverter_cfi.py b/DQMServices/Components/python/EDMtoMEConverter_cfi.py index 13ce613fdcae2..0137bc4826141 100644 --- a/DQMServices/Components/python/EDMtoMEConverter_cfi.py +++ b/DQMServices/Components/python/EDMtoMEConverter_cfi.py @@ -7,6 +7,8 @@ Frequency = cms.untracked.int32(50), convertOnEndLumi = cms.untracked.bool(True), convertOnEndRun = cms.untracked.bool(True), + # convert everything into JOB histograms so endJob harvesting works. + reScope = cms.untracked.string("JOB"), runInputTag = cms.InputTag('MEtoEDMConverter', 'MEtoEDMConverterRun'), lumiInputTag = cms.InputTag('MEtoEDMConverter', 'MEtoEDMConverterLumi') ) From 9c2c3fab0088f23fea7cbc08086312aa911f83ff Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 15:15:09 +0100 Subject: [PATCH 087/112] Make FastTimerService book into RUN scope. --- HLTrigger/Timer/plugins/FastTimerService.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HLTrigger/Timer/plugins/FastTimerService.cc b/HLTrigger/Timer/plugins/FastTimerService.cc index 129ae435e33fd..dfb47f004892a 100644 --- a/HLTrigger/Timer/plugins/FastTimerService.cc +++ b/HLTrigger/Timer/plugins/FastTimerService.cc @@ -857,6 +857,10 @@ void FastTimerService::preGlobalBeginRun(edm::GlobalContext const& gc) { if (enable_dqm_) { // define a callback to book the MonitorElements auto bookTransactionCallback = [&, this](dqm::reco::DQMStore::IBooker& booker, dqm::reco::DQMStore::IGetter&) { + booker.setScope(MonitorElementData::Scope::RUN); + // we should really do this, but only DQMStore is allowed to touch it + // We could move to postGlobalBeginRun, then the DQMStore has sure set it up. + //booker.setRunLumi(gc.luminosityBlockID()); booker.setCurrentFolder(dqm_path_); plots_->book(booker, callgraph_, From ef545b135d1f21ee37f45b611c2fa117e319c345 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 19 Dec 2019 16:20:25 +0100 Subject: [PATCH 088/112] WIP: implement online live mode. --- DQMServices/Core/src/DQMService.cc | 62 +++++++++++++++++++++--------- DQMServices/Core/src/DQMService.h | 4 -- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/DQMServices/Core/src/DQMService.cc b/DQMServices/Core/src/DQMService.cc index 0dc837af5720d..ef24bdd41a3e5 100644 --- a/DQMServices/Core/src/DQMService.cc +++ b/DQMServices/Core/src/DQMService.cc @@ -5,7 +5,6 @@ #include "DQMServices/Core/interface/DQMScope.h" #include "DQMServices/Core/interface/MonitorElement.h" #include "FWCore/ServiceRegistry/interface/Service.h" -#include "classlib/utils/Regexp.h" #include "classlib/utils/Error.h" #include #include @@ -25,7 +24,7 @@ DQMScope::~DQMScope() { s_mutex.unlock(); } // ------------------------------------------------------------------- DQMService::DQMService(const edm::ParameterSet &pset, edm::ActivityRegistry &ar) - : store_(&*edm::Service()), net_(nullptr), filter_(nullptr), lastFlush_(0), publishFrequency_(5.0) { + : store_(&*edm::Service()), net_(nullptr), lastFlush_(0), publishFrequency_(5.0) { ar.watchPostEvent(this, &DQMService::flush); ar.watchPostStreamEndLumi(this, &DQMService::flush); @@ -33,7 +32,6 @@ DQMService::DQMService(const edm::ParameterSet &pset, edm::ActivityRegistry &ar) int port = pset.getUntrackedParameter("collectorPort", 9090); bool verbose = pset.getUntrackedParameter("verbose", false); publishFrequency_ = pset.getUntrackedParameter("publishFrequency", publishFrequency_); - std::string filter = pset.getUntrackedParameter("filter", ""); if (!host.empty() && port > 0) { net_ = new DQMBasicNet; @@ -41,20 +39,6 @@ DQMService::DQMService(const edm::ParameterSet &pset, edm::ActivityRegistry &ar) net_->updateToCollector(host, port); net_->start(); } - - if (!filter.empty()) { - try { - filter_ = new lat::Regexp(filter); - if (!filter_->valid()) - throw cms::Exception("DQMService") << "Invalid 'filter' parameter value '" << filter << "':" - << " bad regular expression syntax at character " << filter_->errorOffset() - << ": " << filter_->errorMessage(); - filter_->study(); - } catch (lat::Error &e) { - throw cms::Exception("DQMService") << "Invalid regular expression 'filter' parameter value '" << filter - << "': " << e.explain(); - } - } } DQMService::~DQMService() { shutdown(); } @@ -79,7 +63,49 @@ void DQMService::flushStandalone() { net_->lock(); bool updated = false; - // TODO: re-implement sending MEs. + auto mes = store_->getAllContents(""); + for (MonitorElement *me : mes) { + auto fullpath = me->getFullname(); + seen.insert(fullpath); + if (!me->wasUpdated()) + continue; + + o.lastreq = 0; + o.hash = DQMNet::dqmhash(fullpath.c_str(), fullpath.size()); + o.flags = me->data_.flags; + o.version = version; + o.dirname = me->data_.dirname; + o.objname = me->data_.objname; + assert(o.rawdata.empty()); + assert(o.scalar.empty()); + assert(o.qdata.empty()); + + // Pack object and reference, scalar and quality data. + + switch (me->kind()) { + case MonitorElement::Kind::INT: + case MonitorElement::Kind::REAL: + case MonitorElement::Kind::STRING: + me->packScalarData(o.scalar, ""); + break; + default: { + TBufferFile buffer(TBufferFile::kWrite); + buffer.WriteObject(me->getTH1()); + // placeholder for (no longer supported) reference + buffer.WriteObjectAny(nullptr, nullptr); + o.rawdata.resize(buffer.Length()); + memcpy(&o.rawdata[0], buffer.Buffer(), buffer.Length()); + DQMNet::packQualityData(o.qdata, me->data_.qreports); + break; + } + } + + net_->updateLocalObject(o); + DQMNet::DataBlob().swap(o.rawdata); + std::string().swap(o.scalar); + std::string().swap(o.qdata); + updated = true; + } // Find removed contents and clear the network cache. if (net_->removeLocalExcept(seen)) diff --git a/DQMServices/Core/src/DQMService.h b/DQMServices/Core/src/DQMService.h index ce61717ae5d16..0b3b31192d805 100644 --- a/DQMServices/Core/src/DQMService.h +++ b/DQMServices/Core/src/DQMService.h @@ -7,9 +7,6 @@ #include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" class DQMBasicNet; -namespace lat { - class Regexp; -} /** A bridge to udpate the DQM network layer at the end of every event. */ class DQMService { @@ -28,7 +25,6 @@ class DQMService { DQMStore *store_; DQMBasicNet *net_; - lat::Regexp *filter_; double lastFlush_; double publishFrequency_; From 3bff3932d624401f415e1aa5886babf45786c055 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 20 Dec 2019 17:16:12 +0100 Subject: [PATCH 089/112] Fix online mode. --- DQMServices/Core/src/DQMService.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMService.cc b/DQMServices/Core/src/DQMService.cc index ef24bdd41a3e5..ee6c28873abc8 100644 --- a/DQMServices/Core/src/DQMService.cc +++ b/DQMServices/Core/src/DQMService.cc @@ -74,7 +74,7 @@ void DQMService::flushStandalone() { o.hash = DQMNet::dqmhash(fullpath.c_str(), fullpath.size()); o.flags = me->data_.flags; o.version = version; - o.dirname = me->data_.dirname; + o.dirname = me->data_.dirname.substr(0, me->data_.dirname.size()-1); o.objname = me->data_.objname; assert(o.rawdata.empty()); assert(o.scalar.empty()); From 78344c99fd9935491d2935935d4d8e26c26ce54a Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 13 Jan 2020 12:43:52 +0100 Subject: [PATCH 090/112] Code-format. --- DQMServices/Core/src/DQMService.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMService.cc b/DQMServices/Core/src/DQMService.cc index ee6c28873abc8..5ae97d8bfea49 100644 --- a/DQMServices/Core/src/DQMService.cc +++ b/DQMServices/Core/src/DQMService.cc @@ -74,7 +74,7 @@ void DQMService::flushStandalone() { o.hash = DQMNet::dqmhash(fullpath.c_str(), fullpath.size()); o.flags = me->data_.flags; o.version = version; - o.dirname = me->data_.dirname.substr(0, me->data_.dirname.size()-1); + o.dirname = me->data_.dirname.substr(0, me->data_.dirname.size() - 1); o.objname = me->data_.objname; assert(o.rawdata.empty()); assert(o.scalar.empty()); From 06e26df8356e10a294c082c67c2e521a1aec1487 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Mon, 13 Jan 2020 15:33:50 +0100 Subject: [PATCH 091/112] Adjust tests. That the legacy output is not saved is arguably a bug, but it is planned like this: It could be fixed by booking into RUN scope in the test legacy modules. However, that is not what the legacy modules will do in practice. It would be nice to have an ouptut test for the Analyzer->TDirectory case which is what legacy modules will commonly do. --- DQMServices/Demo/test/run_analyzers_cfg.py | 3 ++- DQMServices/Demo/test/runtests.sh | 16 +++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/DQMServices/Demo/test/run_analyzers_cfg.py b/DQMServices/Demo/test/run_analyzers_cfg.py index db8c11a69c0d5..7656b33141b47 100644 --- a/DQMServices/Demo/test/run_analyzers_cfg.py +++ b/DQMServices/Demo/test/run_analyzers_cfg.py @@ -142,4 +142,5 @@ process.o = cms.EndPath(process.out) -process.DQMStore.trackME = cms.untracked.string("Legacy/testlegacy/th2d0") +#process.Tracer = cms.Service("Tracer") +#process.DQMStore.trackME = cms.untracked.string("th2d0") diff --git a/DQMServices/Demo/test/runtests.sh b/DQMServices/Demo/test/runtests.sh index 93b0e151767b8..dda93fbaed6d3 100755 --- a/DQMServices/Demo/test/runtests.sh +++ b/DQMServices/Demo/test/runtests.sh @@ -10,16 +10,18 @@ fi # 1. Run a very simple configuration with all module types. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=alltypes.root numberEventsInRun=100 numberEventsInLuminosityBlock=20 nEvents=100 -[ 99 = $(dqmiolistmes.py alltypes.root -r 1 | wc -l) ] +# actually we'd expect 99, but the MEs by legacy modules are booked with JOB scope and cannot be saved to DQMIO. +[ 66 = $(dqmiolistmes.py alltypes.root -r 1 | wc -l) ] [ 55 = $(dqmiolistmes.py alltypes.root -r 1 -l 1 | wc -l) ] # this is deeply related to what the analyzers actually do. -# most run histos (5 modules * 6 types) fill on every event and should have 100 entries. -# the scalar MEs should have the last lumi number (5) (7 float + 7 int) -# testonefilllumi, testlegacyfilllumi also should have 5 entries in the histograms (2*6 more) -# the two "fillrun" modules should have one entry in the histograms (2*6 total) and 0 in the scalars (4 total) -[ "0: 2, 0.0: 2, 1: 12, 100: 30, 5: 19, 5.0: 7" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 --summary)" ] +# again, the legacy modules output is not saved. +# most run histos (4 modules * 6 types) fill on every event and should have 100 entries. +# the scalar MEs should have the last lumi number (5) (5 float + 5 int) +# testonefilllumi also should have 5 entries in the histograms (6 more) +# the "fillrun" module should have one entry in the histograms (6 total) and 0 in the scalars (2 total) +[ "0: 1, 0.0: 1, 1: 6, 100: 24, 5: 11, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 --summary)" ] # per lumi we see 20 in most histograms (3*6), and the current lumi number in the scalars (5 modules * 2). -# the two fillumi modules should have one entry in each of there lumi histograms, (2*6 total) +# the two fillumi modules should have one entry in each of the lumi histograms, (2*6 total) [ "1: 17, 1.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 1 --summary)" ] [ "1: 12, 2: 5, 2.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 2 --summary)" ] [ "1: 12, 20: 18, 3: 5, 3.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 3 --summary)" ] From df9bb3245a2de628dde2ce3dfd6a74abca559375 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 14 Jan 2020 10:45:37 +0100 Subject: [PATCH 092/112] Use stable sort when reading DQMIO. This is mostly to make the tests happy: When reading pre-run scalars from split DQMIO files, the result could be the value from any file, since it dependes on processing order. We can't do anything about that in general (we might have a job processing lumi 1 and 4, and another 2 and 3, and then merge the results; which value should be used?), but for the tests we can at least make the behaviour detetministic for the test case. --- DQMServices/FwkIO/plugins/DQMRootSource.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index 381e6373d6634..be5a080a4225c 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -592,7 +592,7 @@ std::unique_ptr DQMRootSource::readFile_() { } // Sort to make sure runs and lumis appear in sequential order - std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); + std::stable_sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); // If we have lumisections without matching runs, insert dummy runs here. unsigned int run = 0; @@ -612,7 +612,7 @@ std::unique_ptr DQMRootSource::readFile_() { if (!toadd.empty()) { // rather than trying to insert at the right places, just append and sort again. m_fileMetadatas.insert(m_fileMetadatas.end(), toadd.begin(), toadd.end()); - std::sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); + std::stable_sort(m_fileMetadatas.begin(), m_fileMetadatas.end()); } //for (auto& metadata : m_fileMetadatas) From 341bf722290a77185d7102578ecbd214bf640886 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 14 Jan 2020 12:56:41 +0100 Subject: [PATCH 093/112] Update tests for new DQMStore. Apart from bug fixes in the tests, there are a few expected changes: - Output of legacy modules is JOB scope, so it will not be save to DQMIO. - DQMIO->TDirectory conversion needs reScope option. Other limitaitons existed before but are more clear now: - Values of scalar values that are not constant over a run/lumi split into JOBs are arbitrary. - Histograms filled per job or per lumi do depend on job splitting, since the number of beginRun/beginLumi transitions changes. New tests/checks cover fastHadd, concurrent lumis and in-order harvesting. The testing of actual histogram contents (correct merging, no double counting etc.) is still very limtied here; mostly these test ensure that there are no crashes and that objects make it into the respective output formats. --- DQMServices/Demo/test/dqmiodumpentries.py | 16 --- DQMServices/Demo/test/run_analyzers_cfg.py | 2 +- DQMServices/Demo/test/run_harvesters_cfg.py | 2 +- DQMServices/Demo/test/runtests.sh | 118 +++++++++++--------- 4 files changed, 66 insertions(+), 72 deletions(-) diff --git a/DQMServices/Demo/test/dqmiodumpentries.py b/DQMServices/Demo/test/dqmiodumpentries.py index 6df833caeb408..7d85ba71e48c4 100755 --- a/DQMServices/Demo/test/dqmiodumpentries.py +++ b/DQMServices/Demo/test/dqmiodumpentries.py @@ -11,19 +11,6 @@ parser.add_argument('-s', '--summary', help='Only show values and how often they appeared.', action='store_true') args = parser.parse_args() -interesting_types = { - "Ints", - "Floats", - "Strings", - "TH2Fs", - "TH1Fs", - "TH2Ds", - "TH1Ds", - "TH2Ds", - "TProfiles", - "TProfile2Ds", -} - treenames = { 0: "Ints", 1: "Floats", @@ -50,9 +37,6 @@ if run != args.run or lumi != args.lumi: continue - if not treenames[metype] in interesting_types: - continue - # inclusive range -- for 0 entries, row is left out firstidx, lastidx = idxtree.FirstIndex, idxtree.LastIndex metree = getattr(f, treenames[metype]) diff --git a/DQMServices/Demo/test/run_analyzers_cfg.py b/DQMServices/Demo/test/run_analyzers_cfg.py index 7656b33141b47..4912f8c190e7f 100644 --- a/DQMServices/Demo/test/run_analyzers_cfg.py +++ b/DQMServices/Demo/test/run_analyzers_cfg.py @@ -143,4 +143,4 @@ #process.Tracer = cms.Service("Tracer") -#process.DQMStore.trackME = cms.untracked.string("th2d0") +#process.DQMStore.trackME = cms.untracked.string("testlegacyfillrun") diff --git a/DQMServices/Demo/test/run_harvesters_cfg.py b/DQMServices/Demo/test/run_harvesters_cfg.py index 8ff4e9d8799a4..65cc3cd8fa56b 100644 --- a/DQMServices/Demo/test/run_harvesters_cfg.py +++ b/DQMServices/Demo/test/run_harvesters_cfg.py @@ -89,5 +89,5 @@ process.e = cms.EndPath(process.out) # useful for debugging -#process.DQMStore.trackME = cms.untracked.string("Legacy/testlegacy/lumi/th2s0") +#process.DQMStore.trackME = cms.untracked.string("testlegacyfillrun") #process.Tracer = cms.Service("Tracer") diff --git a/DQMServices/Demo/test/runtests.sh b/DQMServices/Demo/test/runtests.sh index dda93fbaed6d3..4ab4749c14477 100755 --- a/DQMServices/Demo/test/runtests.sh +++ b/DQMServices/Demo/test/runtests.sh @@ -15,44 +15,48 @@ cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=alltypes.root numberEventsIn [ 55 = $(dqmiolistmes.py alltypes.root -r 1 -l 1 | wc -l) ] # this is deeply related to what the analyzers actually do. # again, the legacy modules output is not saved. -# most run histos (4 modules * 6 types) fill on every event and should have 100 entries. +# most run histos (4 modules * 9 types) fill on every event and should have 100 entries. # the scalar MEs should have the last lumi number (5) (5 float + 5 int) -# testonefilllumi also should have 5 entries in the histograms (6 more) -# the "fillrun" module should have one entry in the histograms (6 total) and 0 in the scalars (2 total) -[ "0: 1, 0.0: 1, 1: 6, 100: 24, 5: 11, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 --summary)" ] -# per lumi we see 20 in most histograms (3*6), and the current lumi number in the scalars (5 modules * 2). -# the two fillumi modules should have one entry in each of the lumi histograms, (2*6 total) -[ "1: 17, 1.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 1 --summary)" ] -[ "1: 12, 2: 5, 2.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 2 --summary)" ] -[ "1: 12, 20: 18, 3: 5, 3.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 3 --summary)" ] -[ "1: 12, 20: 18, 4: 5, 4.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 4 --summary)" ] -[ "1: 12, 20: 18, 5: 5, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 5 --summary)" ] +# testonefilllumi also should have 5 entries in the histograms (9 more) +# the "fillrun" module should have one entry in the histograms (9 total) and 0 in the scalars (2 total) +[ "0: 1, 0.0: 1, 1: 9, 100: 36, 5: 14, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 --summary)" ] +# per lumi we see 20 in most histograms (3*9), and the current lumi number in the scalars (5 modules * 2). +# the two fillumi modules should have one entry in each of the lumi histograms, (2*9 total) +[ "1: 23, 1.0: 5, 20: 27" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 1 --summary)" ] +[ "1: 18, 2: 5, 2.0: 5, 20: 27" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 2 --summary)" ] +[ "1: 18, 20: 27, 3: 5, 3.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 3 --summary)" ] +[ "1: 18, 20: 27, 4: 5, 4.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 4 --summary)" ] +[ "1: 18, 20: 27, 5: 5, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 5 --summary)" ] # just make sure we are not off by one [ "" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py alltypes.root -r 1 -l 6 --summary)" ] + # 2. Run multi-threaded. First we make a baseline file without legacy modules, since they might not work. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=nolegacy.root numberEventsInRun=1000 numberEventsInLuminosityBlock=200 nEvents=1000 nolegacy=True cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=nolegacy-mt.root numberEventsInRun=1000 numberEventsInLuminosityBlock=200 nEvents=1000 nolegacy=True nThreads=10 + # 3. Try enabling concurrent lumis. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=nolegacy-cl.root numberEventsInRun=1000 numberEventsInLuminosityBlock=200 nEvents=1000 nolegacy=True nThreads=10 nConcurrent=10 # same math as above, just a few less modules, and more events. -for f in nolegacy.root nolegacy-mt.root # nolegacy-cl.root # Enabling concurrent lumis seems to mess up the output for now. +for f in nolegacy.root nolegacy-mt.root nolegacy-cl.root do - [ "0: 1, 0.0: 1, 1: 6, 1000: 18, 5: 3, 5.0: 3" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 --summary)" ] - [ "1: 1, 1.0: 1, 200: 6" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 1 --summary)" ] - [ "2: 1, 2.0: 1, 200: 6" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 2 --summary)" ] - [ "200: 6, 3: 1, 3.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 3 --summary)" ] - [ "200: 6, 4: 1, 4.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 4 --summary)" ] - [ "200: 6, 5: 1, 5.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 5 --summary)" ] + [ "0: 1, 0.0: 1, 1: 9, 1000: 27, 5: 3, 5.0: 3" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 --summary)" ] + [ "1: 1, 1.0: 1, 200: 9" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 1 --summary)" ] + [ "2: 1, 2.0: 1, 200: 9" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 2 --summary)" ] + [ "200: 9, 3: 1, 3.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 3 --summary)" ] + [ "200: 9, 4: 1, 4.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 4 --summary)" ] + [ "200: 9, 5: 1, 5.0: 1" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 5 --summary)" ] [ "" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py $f -r 1 -l 6 --summary)" ] done + # 4. Try crossing a run boundary. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=multirun.root numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=1200 dqmiodumpmetadata.py multirun.root | grep -q '4 runs, 12 lumisections' + # 5. Now, make some chopped up files to try harvesting. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=part1.root numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=50 # 1st half of 1st lumi cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=part2.root numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=50 firstEvent=50 # 2nd half of 1st lumi @@ -61,27 +65,37 @@ cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=part4.root numberEventsInRun cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=part1.root inputFiles=part2.root inputFiles=part3.root inputFiles=part4.root outfile=merged.root nomodules=True dqmiodumpmetadata.py merged.root | grep -q '4 runs, 12 lumisections' -dumproot() { root2sqlite.py -o $1.sqlite $1 ; echo '.dump' | sqlite3 $1.sqlite > $1.sqldump ; rm $1.sqlite ; } + +#dumproot() { root2sqlite.py -o $1.sqlite $1 ; echo '.dump' | sqlite3 $1.sqlite > $1.sqldump ; rm $1.sqlite ; } #dumproot multirun.root #dumproot merged.root -# these are unlikely to ever fully argee, though the histograms should. They do not, for now. -#cmp multirun.root.sqldump merged.root.sqldump +rootlist () +{ python3 -c ' +import uproot +for k in uproot.open("'"$1"'").allkeys(): print(k.decode())' +} + +# we need to exclude MEs filled on run and lumi boundaries, since the split job *does* see a different number of begin/end run/lumi transitions. +cmp <($LOCAL_TEST_DIR/dqmiodumpentries.py multirun.root -r 1 | grep -vE 'fillrun|filllumi') <($LOCAL_TEST_DIR/dqmiodumpentries.py merged.root -r 1 | grep -vE 'fillrun|filllumi') +cmp <($LOCAL_TEST_DIR/dqmiodumpentries.py multirun.root -r 3) <($LOCAL_TEST_DIR/dqmiodumpentries.py merged.root -r 3) +cmp <($LOCAL_TEST_DIR/dqmiodumpentries.py multirun.root -r 1 -l 1 | grep -v filllumi) <($LOCAL_TEST_DIR/dqmiodumpentries.py merged.root -r 1 -l 1 | grep -v filllumi) +cmp <($LOCAL_TEST_DIR/dqmiodumpentries.py multirun.root -r 1 -l 2) <($LOCAL_TEST_DIR/dqmiodumpentries.py merged.root -r 1 -l 2) # 6. A load test. ( ulimit -v 4000000 # limit available virtual memory cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=huge.root numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=600 nThreads=10 nConcurrent=2 howmany=1000 nolegacy=True ) -# 7. Try writing a TDirectory file. This is only safe for a single run for now. -cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py numberEventsInRun=100 numberEventsInLuminosityBlock=20 nEvents=100 legacyoutput=True -cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=alltypes.root nomodules=True legacyoutput=True +# 7. Try writing a TDirectory file. +cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=alltypes.root nomodules=True legacyoutput=True reScope=JOB +# this number is rather messy: we have 55 per-lumi objecs (harvested), 66 per-run objects (no legacy output), one folder for each set of 11, +# plus some higher-level folders and the ProvInfo hierarchy create by the FileSaver. +[ 149 = $(rootlist DQM_V0001_R000000001__Harvesting__DQMTests__DQMIO.root | wc -l) ] -#dumproot DQM_V0001_R000000001__EmptySource__DQMTests__DQMIO.root -#dumproot DQM_V0001_R000000001__Harvesting__DQMTests__DQMIO.root -# These disagree due to the werid handling of per-lumi MEs in the current DQMStore. -#cmp DQM_V0001_R000000001__EmptySource__DQMTests__DQMIO.root.sqldump DQM_V0001_R000000001__Harvesting__DQMTests__DQMIO.root.sqldump || true -#cmp <(grep -v lumi DQM_V0001_R000000001__EmptySource__DQMTests__DQMIO.root.sqldump) <(grep -v lumi DQM_V0001_R000000001__Harvesting__DQMTests__DQMIO.root.sqldump) +cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py numberEventsInRun=100 numberEventsInLuminosityBlock=20 nEvents=100 legacyoutput=True +# we expect only the (per-job) legacy histograms here: 3*11 objects in 3 folders, plus 9 more for ProvInfo and higher-level folders. +[ 45 = $(rootlist DQM_V0001_R000000001__EmptySource__DQMTests__DQMIO.root | wc -l) ] # 8. Try writing ProtoBuf files. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=1200 protobufoutput=True @@ -90,35 +104,41 @@ cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=./run000001 outfile=pbda [ 99 = $(dqmiolistmes.py pbdata.root -r 1 | wc -l) ] [ 55 = $(dqmiolistmes.py pbdata.root -r 1 -l 1 | wc -l) ] +# this will potentially mess up statistics (we should only fastHadd *within* a lumisection, not *across*), but should technically work. +fastHadd add -o streamDQMHistograms.pb run000001/run000001_ls*_streamDQMHistograms.pb +# the output format is different from the harvesting above, this is a not-DQM-formatted TDirectory file. +fastHadd convert -o streamDQMHistograms.root streamDQMHistograms.pb +# here we expect all (incl. legacy) MEs (99+55), plus folders (14 + 4 higher-level) +[ 172 = $(rootlist streamDQMHistograms.root | wc -l) ] -# TODO: maybe also try fastHadd. # 9. Try writing online files. This is really TDirectory files, but written via a different module. # Note that this does not really need to support multiple runs, but it appears it does. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=1200 onlineoutput=True +# here we expect full per-run output (99 objects), no per-lumi MEs, plus folders (9 + 10 higher-level). +[ 118 = $(rootlist DQM_V0001_UNKNOWN_R000000001.root | wc -l) ] +[ 118 = $(rootlist DQM_V0001_UNKNOWN_R000000002.root | wc -l) ] +[ 118 = $(rootlist DQM_V0001_UNKNOWN_R000000003.root | wc -l) ] +[ 118 = $(rootlist DQM_V0001_UNKNOWN_R000000004.root | wc -l) ] # 10. Try running some harvesting modules and check if their output makes it out. -# Note that we pass the files in order here. In the future, this should be independent of the order of input files. -cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=part1.root inputFiles=part2.root inputFiles=part3.root legacyoutput=True -rootlist () -{ python3 -c ' -import uproot -for k in uproot.open("'"$1"'").allkeys(): print(k.decode())' -} +# Note that we pass the files out-of order here; the DQMIO input should sort them. +cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=part1.root inputFiles=part3.root inputFiles=part2.root legacyoutput=True [ 2 = $(rootlist DQM_V0001_R000000001__Harvesting__DQMTests__DQMIO.root | grep -c 's=beginRun(1) endLumi(1,1) endLumi(1,2) endLumi(1,3) endRun(1) endJob() ') ] # 11. Try MEtoEDM and EDMtoME. cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=metoedm.root numberEventsInRun=100 numberEventsInLuminosityBlock=20 nEvents=100 metoedmoutput=True cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py outfile=edmtome.root inputFiles=metoedm.root nomodules=True metoedminput=True -[ 99 = $(dqmiolistmes.py edmtome.root -r 1 | wc -l) ] +[ 66 = $(dqmiolistmes.py edmtome.root -r 1 | wc -l) ] [ 55 = $(dqmiolistmes.py edmtome.root -r 1 -l 1 | wc -l) ] -[ "0: 2, 0.0: 2, 1: 12, 100: 30, 5: 19, 5.0: 7" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 --summary)" ] -[ "1: 17, 1.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 1 --summary)" ] -[ "1: 12, 2: 5, 2.0: 5, 20: 18" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 2 --summary)" ] -[ "1: 12, 20: 18, 3: 5, 3.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 3 --summary)" ] -[ "1: 12, 20: 18, 4: 5, 4.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 4 --summary)" ] -[ "1: 12, 20: 18, 5: 5, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 5 --summary)" ] +# again, no legacy module (run) output here due to JOB scope for legacy modules +[ "0: 1, 0.0: 1, 1: 9, 100: 36, 5: 14, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 --summary)" ] +[ "1: 23, 1.0: 5, 20: 27" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 1 --summary)" ] +[ "1: 18, 2: 5, 2.0: 5, 20: 27" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 2 --summary)" ] +[ "1: 18, 20: 27, 3: 5, 3.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 3 --summary)" ] +[ "1: 18, 20: 27, 4: 5, 4.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 4 --summary)" ] +[ "1: 18, 20: 27, 5: 5, 5.0: 5" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 5 --summary)" ] [ "" = "$($LOCAL_TEST_DIR/dqmiodumpentries.py edmtome.root -r 1 -l 6 --summary)" ] cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=part1_metoedm.root metoedmoutput=True numberEventsInRun=300 numberEventsInLuminosityBlock=100 nEvents=50 # 1st half of 1st lumi @@ -128,9 +148,6 @@ cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=part4_metoedm.root metoedmou cmsRun $LOCAL_TEST_DIR/run_harvesters_cfg.py inputFiles=part1_metoedm.root inputFiles=part2_metoedm.root inputFiles=part3_metoedm.root inputFiles=part4_metoedm.root outfile=metoedm_merged.root nomodules=True metoedminput=True dqmiodumpmetadata.py metoedm_merged.root | grep -q '4 runs, 12 lumisections' -#dumproot metoedm_merged.root -#cmp multirun.root.sqldump metoedm_merged.root.sqldump - # 12. Sanity checks. # this will mess up some of the files created earlier, disable for debugging. @@ -139,10 +156,3 @@ cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=empty.root howmany=0 cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=empty.root howmany=0 legacyoutput=True cmsRun $LOCAL_TEST_DIR/run_analyzers_cfg.py outfile=empty.root howmany=0 protobufoutput=True - - - - - - - From 34d729ab287f2ae64235cf9f8b67f88e8f187578 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 15 Jan 2020 09:52:31 +0100 Subject: [PATCH 094/112] Fix compareFilesFromPR.sh to have a chance of working inside cmsenv. --- DQMServices/FileIO/scripts/compareFilesFromPR.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/DQMServices/FileIO/scripts/compareFilesFromPR.sh b/DQMServices/FileIO/scripts/compareFilesFromPR.sh index 5231c185612db..c3eda4485b984 100644 --- a/DQMServices/FileIO/scripts/compareFilesFromPR.sh +++ b/DQMServices/FileIO/scripts/compareFilesFromPR.sh @@ -4,12 +4,18 @@ BASELINEURL="$2" PRNUMBER=$(date +%s) fetch() { - cern-get-sso-cookie -o cook --url $1 - for d in $(curl -L -s -k -b cook $1 | grep -oE '"[0-9]+*.[0-9]+_[^"]*"' | tr -d '"'); do - for f in $(curl -L -s -k -b cook "$1/$d" | grep -oE '"DQM.*.root"' | tr -d '"'); do - (echo "Fetching $d/$f..."; mkdir -p $d; cd $d; curl -O -L -s -k -b ../cook $1/$d/$f ) + ( + export PATH=/bin/:/usr/bin/ + export PERL5LIB= + export LD_LIBRARY_PATH= + + cern-get-sso-cookie -o cook --url $1 + for d in $(curl -L -s -k -b cook $1 | grep -oE '"[0-9]+*.[0-9]+_[^"]*"' | tr -d '"'); do + for f in $(curl -L -s -k -b cook "$1/$d" | grep -oE '"DQM.*.root"' | tr -d '"'); do + (echo "Fetching $d/$f..."; mkdir -p $d; cd $d; curl -O -L -s -k -b ../cook $1/$d/$f ) + done done - done + ) } if [[ -z $RESULTURL || -z $BASELINEURL ]]; then From f92aa2bc67fa02fa8e9b584a4fafe74439b81e0b Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 16 Jan 2020 15:36:52 +0100 Subject: [PATCH 095/112] Fix pwd() to have the old behaviour --- DQMServices/Core/interface/DQMStore.h | 5 +++-- DQMServices/Core/src/DQMStore.cc | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index cd34d4e67f602..9a168881a8c83 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -30,7 +30,8 @@ namespace dqm { // This is the only method that is allowed to change cwd_ value virtual void setCurrentFolder(std::string const& fullpath); virtual void goUp(); - virtual std::string const& pwd(); + // returns the current directory without (!) trailing slash or empty string. + virtual std::string pwd(); virtual ~NavigatorBase() {} @@ -550,7 +551,7 @@ namespace dqm { this->IGetter::setCurrentFolder(fullpath); } void goUp() override { this->IBooker::goUp(); } - std::string const& pwd() override { return this->IBooker::pwd(); } + std::string pwd() override { return this->IBooker::pwd(); } public: // internal -- figure out better protection. diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 747386543c494..30e09c3bb808f 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -13,7 +13,19 @@ namespace dqm::implementation { - std::string const& NavigatorBase::pwd() { return cwd_; } + std::string NavigatorBase::pwd() { + if (cwd_.empty()) { + return ""; + } else { + // strip trailing slash. + // This is inefficient and error prone (callers need to do the same + // branching to re-add the "/"!) but some legacy code expects it like + // that and is to complicated to change. + assert(cwd_[cwd_.size()-1] == '/'); + auto pwd = cwd_.substr(0, cwd_.size() - 1); + return pwd; + } + } void NavigatorBase::cd() { setCurrentFolder(""); } void NavigatorBase::cd(std::string const& dir) { setCurrentFolder(dir); } void NavigatorBase::goUp() { cd(cwd_ + ".."); } @@ -53,7 +65,7 @@ namespace dqm::implementation { std::function makeobject, bool forceReplace /* = false */) { MonitorElementData::Path path; - std::string fullpath = pwd() + std::string(name.View()); + std::string fullpath = cwd_ + std::string(name.View()); path.set(fullpath, MonitorElementData::Path::Type::DIR_AND_NAME); // We should check if there is a local ME for this module and name already. From e8578fbec1a3391065416120bf7675bf35ce4085 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 16 Jan 2020 18:31:13 +0100 Subject: [PATCH 096/112] Disable perLumi for the summaries, not sure if it makes sense to have it. --- DQM/EcalMonitorClient/python/SummaryClient_cfi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DQM/EcalMonitorClient/python/SummaryClient_cfi.py b/DQM/EcalMonitorClient/python/SummaryClient_cfi.py index 8d6eddb039d65..953c550b709ea 100644 --- a/DQM/EcalMonitorClient/python/SummaryClient_cfi.py +++ b/DQM/EcalMonitorClient/python/SummaryClient_cfi.py @@ -46,7 +46,7 @@ kind = cms.untracked.string('REAL'), otype = cms.untracked.string('SM'), btype = cms.untracked.string('Report'), - perLumi = cms.untracked.bool(True), + perLumi = cms.untracked.bool(False), description = cms.untracked.string('') ), GlobalSummary = cms.untracked.PSet( @@ -88,7 +88,7 @@ kind = cms.untracked.string('REAL'), otype = cms.untracked.string('Ecal'), btype = cms.untracked.string('Report'), - perLumi = cms.untracked.bool(True), + perLumi = cms.untracked.bool(False), description = cms.untracked.string('') ) ) From f9d9b11bf411fbfac344d24346f6c594d12b2c59 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 16 Jan 2020 18:31:47 +0100 Subject: [PATCH 097/112] Change default SCOPE to make legacy code work. --- DQMServices/Core/src/DQMStore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 30e09c3bb808f..427785ba77b45 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -38,7 +38,7 @@ namespace dqm::implementation { IBooker::IBooker(DQMStore* store) { store_ = store; - scope_ = MonitorElementData::Scope::RUN; + scope_ = MonitorElementData::Scope::JOB; } IBooker::~IBooker() {} From 62dd089e99418b6b6bec92b3393f5e4d1387c968 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 17 Jan 2020 11:58:34 +0100 Subject: [PATCH 098/112] Some string length issues that gcc -Og complains about. --- .../OfflineValidation/plugins/MuonAlignmentAnalyzer.cc | 2 +- Calibration/HcalCalibAlgos/plugins/Analyzer_minbias.cc | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Alignment/OfflineValidation/plugins/MuonAlignmentAnalyzer.cc b/Alignment/OfflineValidation/plugins/MuonAlignmentAnalyzer.cc index 9c9ced48646ce..97b65c1158e96 100644 --- a/Alignment/OfflineValidation/plugins/MuonAlignmentAnalyzer.cc +++ b/Alignment/OfflineValidation/plugins/MuonAlignmentAnalyzer.cc @@ -981,7 +981,7 @@ void MuonAlignmentAnalyzer::beginJob() { hprofLocalPositionRmsCSC->GetYaxis()->SetLabelSize(labelSize); hprofLocalAngleRmsCSC->GetYaxis()->SetLabelSize(labelSize); - char binLabel[15]; + char binLabel[32]; for (int i = 1; i < 15; i++) { snprintf(binLabel, sizeof(binLabel), "Sec-%d", i); hprofGlobalPositionDT->GetXaxis()->SetBinLabel(i, binLabel); diff --git a/Calibration/HcalCalibAlgos/plugins/Analyzer_minbias.cc b/Calibration/HcalCalibAlgos/plugins/Analyzer_minbias.cc index 034cf4c2b4f1a..75fcaa036851b 100644 --- a/Calibration/HcalCalibAlgos/plugins/Analyzer_minbias.cc +++ b/Calibration/HcalCalibAlgos/plugins/Analyzer_minbias.cc @@ -112,11 +112,11 @@ namespace cms { edm::LogInfo("AnalyzerMB") << " Before ordering Histos "; - char str0[15]; - char str1[15]; + char str0[32]; + char str1[32]; - char str10[15]; - char str11[15]; + char str10[32]; + char str11[32]; int k = 0; nevent = 0; From 5436ff495dab18a57374aa1945c8ac7b9dbe038e Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 17 Jan 2020 14:16:02 +0100 Subject: [PATCH 099/112] Code-format. --- DQMServices/Core/src/DQMStore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 427785ba77b45..d847ed8b823e6 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -21,7 +21,7 @@ namespace dqm::implementation { // This is inefficient and error prone (callers need to do the same // branching to re-add the "/"!) but some legacy code expects it like // that and is to complicated to change. - assert(cwd_[cwd_.size()-1] == '/'); + assert(cwd_[cwd_.size() - 1] == '/'); auto pwd = cwd_.substr(0, cwd_.size() - 1); return pwd; } From b6b5018c94637091a5fb59efc54c86ed4e6bcc55 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 22 Jan 2020 17:49:22 +0100 Subject: [PATCH 100/112] It can be useful to have a breakpoint in debugTrackME. --- DQMServices/Core/src/DQMStore.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index d847ed8b823e6..1652f33806080 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -309,6 +309,8 @@ namespace dqm::implementation { logger << " (invalid)"; } }); + // A breakpoint can be useful here. + //std::raise(SIGINT); } } } From 3f239908d7ea4fd1688e49c033f6556b18b687df Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 22 Jan 2020 17:50:58 +0100 Subject: [PATCH 101/112] Fix MuonAlignment. --- DQMOffline/Alignment/src/MuonAlignment.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/DQMOffline/Alignment/src/MuonAlignment.cc b/DQMOffline/Alignment/src/MuonAlignment.cc index de4b21afb961e..8d564cc193a62 100644 --- a/DQMOffline/Alignment/src/MuonAlignment.cc +++ b/DQMOffline/Alignment/src/MuonAlignment.cc @@ -66,6 +66,10 @@ void MuonAlignment::beginJob() { } dbe = edm::Service().operator->(); + // Since this is a legacy module which will block concurrent-anything, we can + // safely request per-run histograms here. This is required since the default + // for legacy modules (JOB histograms) cannot be saved into MEtoEDM format. + dbe->setScope(MonitorElementData::Scope::RUN); if (doSummary) { if (doDT) { From 392585e1621e77603a1af9e1aacf7af1187e9815 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 23 Jan 2020 16:14:56 +0100 Subject: [PATCH 102/112] Book EcalHitsV MEs in correct scope. This is required for all legacy modules running outside harvesting. --- Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc | 1 + Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc | 1 + Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc | 1 + Validation/EcalHits/src/EcalSimHitsValidation.cc | 1 + 4 files changed, 4 insertions(+) diff --git a/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc b/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc index 8fc40f6b5615c..15917d48bfe95 100644 --- a/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalBarrelSimHitsValidation.cc @@ -64,6 +64,7 @@ EcalBarrelSimHitsValidation::EcalBarrelSimHitsValidation(const edm::ParameterSet if (dbe_) { dbe_->setCurrentFolder("EcalHitsV/EcalSimHitsValidation"); + dbe_->setScope(MonitorElementData::Scope::RUN); sprintf(histo, "EB hits multiplicity"); menEBHits_ = dbe_->book1D(histo, histo, 50, 0., 5000.); diff --git a/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc b/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc index bcfa2b6c51f39..981deb9cd07db 100644 --- a/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalEndcapSimHitsValidation.cc @@ -67,6 +67,7 @@ EcalEndcapSimHitsValidation::EcalEndcapSimHitsValidation(const edm::ParameterSet if (dbe_) { dbe_->setCurrentFolder("EcalHitsV/EcalSimHitsValidation"); + dbe_->setScope(MonitorElementData::Scope::RUN); sprintf(histo, "EE+ hits multiplicity"); meEEzpHits_ = dbe_->book1D(histo, histo, 50, 0., 5000.); diff --git a/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc b/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc index f9317f89795b2..d429afc4e1291 100644 --- a/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalPreshowerSimHitsValidation.cc @@ -59,6 +59,7 @@ EcalPreshowerSimHitsValidation::EcalPreshowerSimHitsValidation(const edm::Parame if (dbe_) { dbe_->setCurrentFolder("EcalHitsV/EcalSimHitsValidation"); + dbe_->setScope(MonitorElementData::Scope::RUN); sprintf(histo, "ES hits layer 1 multiplicity z+"); menESHits1zp_ = dbe_->book1D(histo, histo, 50, 0., 50.); diff --git a/Validation/EcalHits/src/EcalSimHitsValidation.cc b/Validation/EcalHits/src/EcalSimHitsValidation.cc index 468f6cdcc67e0..054f97db02bdc 100644 --- a/Validation/EcalHits/src/EcalSimHitsValidation.cc +++ b/Validation/EcalHits/src/EcalSimHitsValidation.cc @@ -56,6 +56,7 @@ EcalSimHitsValidation::EcalSimHitsValidation(const edm::ParameterSet &ps) if (dbe_) { dbe_->setCurrentFolder("EcalHitsV/EcalSimHitsValidation"); + dbe_->setScope(MonitorElementData::Scope::RUN); sprintf(histo, "EcalSimHitsValidation Gun Momentum"); meGunEnergy_ = dbe_->book1D(histo, histo, 100, 0., 1000.); From af702305bc4b51f60495f25a9b5f7057fca609f5 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 24 Jan 2020 11:31:33 +0100 Subject: [PATCH 103/112] Add a RAII style guard for booking Scope. As suggested by @Dr15Jones. Also some comment cleanups. --- DQMServices/Core/interface/DQMStore.h | 12 ++++++++++++ DQMServices/Core/interface/MonitorElement.h | 10 ++-------- DQMServices/Demo/test/TestDQMEDAnalyzer.cc | 4 +--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index 9a168881a8c83..e68c8086c4d74 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -449,6 +449,18 @@ namespace dqm { // virtual MonitorElementData::Scope setScope(MonitorElementData::Scope newscope); + // RAII-Style guard to set and reset the Scope. + template + struct UseScope { + IBooker& parent; + MonitorElementData::Scope oldscope; + UseScope(IBooker& booker) : parent(booker) { oldscope = parent.setScope(SCOPE); } + ~UseScope() { parent.setScope(oldscope); } + }; + using UseLumiScope = UseScope; + using UseRunScope = UseScope; + using UseJobScope = UseScope; + ~IBooker() override; protected: diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index c0597e1a6ca97..720d43739e3df 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -131,13 +131,7 @@ namespace dqm::impl { * This object will contain the lock guard if one is needed. We differentiate * access for reading and access for mutation (denoted by `Access` or * `const Access`, however, also read-only access may need to take a lock - * if it is to a mutable object. Obtaining mutable access may involve - * creating a clone of the backing data. In this case, the pointers are - * updated using atomic operations. It can happen that reads go to the old - * object while a new object exists already, but this is fine; concurrent - * reads and writes can happen in arbitrary order. However, we need to - * protect against the case where clones happen concurrently and avoid - * leaking memory or loosing updates in this case, using atomics. + * if it is to a mutable object. * We want all of this inlined and redundant operations any copies/refs * optimized away. */ @@ -451,7 +445,7 @@ namespace dqm::impl { } // namespace dqm::impl -// These will become distinct classes in the future. +// These may become distinct classes in the future. namespace dqm::reco { typedef dqm::impl::MonitorElement MonitorElement; } diff --git a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc index 4406ce2840149..da2b5624f2374 100644 --- a/DQMServices/Demo/test/TestDQMEDAnalyzer.cc +++ b/DQMServices/Demo/test/TestDQMEDAnalyzer.cc @@ -42,7 +42,7 @@ class BookerFiller { "thprofile2d" + num, "2D Profile Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); if (DOLUMI) { - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = typename BOOKERLIKE::UseLumiScope(ibooker); ibooker.setCurrentFolder(folder + "/lumi"); mes_1D.push_back(ibooker.bookFloat("float" + num)); @@ -61,8 +61,6 @@ class BookerFiller { ibooker.book3D("th3f" + num, "3D Float Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); mes_3D.push_back(ibooker.bookProfile2D( "thprofile2d" + num, "2D Profile Histogram " + num, 101, -0.5, 100.5, 11, -0.5, 10.5, 3, -0.5, 2.5)); - - ibooker.setScope(scope); } } } From 26e822da2340fae6420430fc2e4a31c2b68ea450 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 24 Jan 2020 12:28:20 +0100 Subject: [PATCH 104/112] Use the new scope guards where appropriate. --- .../plugins/CorrPCCProducer.cc | 2 +- DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc | 23 +++++++------- DQM/CastorMonitor/src/CastorMonitorModule.cc | 3 +- DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc | 17 +++++----- DQM/DTMonitorModule/src/DTDCSByLumiTask.cc | 14 ++++----- .../src/ESIntegrityTask.cc | 3 +- DQM/HcalTasks/plugins/DigiPhase1Task.cc | 3 +- DQM/HcalTasks/plugins/DigiTask.cc | 31 ++++++++++--------- DQM/HcalTasks/plugins/RawTask.cc | 3 +- DQM/HcalTasks/plugins/RecHitTask.cc | 3 +- DQM/HcalTasks/plugins/TPTask.cc | 3 +- DQM/RPCMonitorDigi/src/RPCDcsInfo.cc | 3 +- .../src/SiPixelDigiSource.cc | 7 +++-- .../src/SiPixelRawDataErrorSource.cc | 7 +++-- .../plugins/SiStripDcsInfo.cc | 4 +-- .../src/FEDHistograms.cc | 21 +++++++------ DQM/TrackingMonitor/src/TrackingMonitor.cc | 9 ++---- DQMOffline/JetMET/src/METAnalyzer.cc | 7 +++-- DQMServices/Components/plugins/DQMDcsInfo.cc | 2 +- HLTrigger/Timer/plugins/FastTimerService.cc | 2 +- .../src/EcalSelectiveReadoutValidation.cc | 7 +++-- 21 files changed, 86 insertions(+), 88 deletions(-) diff --git a/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc b/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc index 32e33e6243d5c..771cf1b647fd9 100644 --- a/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc +++ b/Calibration/LumiAlCaRecoProducers/plugins/CorrPCCProducer.cc @@ -672,7 +672,7 @@ void CorrPCCProducer::resetBlock() { //-------------------------------------------------------------------------------------------------- void CorrPCCProducer::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& iRun, edm::EventSetup const& context) { ibooker.setCurrentFolder("AlCaReco/LumiPCC/"); - ibooker.setScope(MonitorElementData::Scope::RUN); + auto scope = DQMStore::IBooker::UseRunScope(ibooker); Type1FracMon = ibooker.book1D("type1Fraction", "Type1Fraction;Lumisection;Type 1 Fraction", maxLS, 0, maxLS); Type1ResMon = ibooker.book1D("type1Residual", "Type1Residual;Lumisection;Type 1 Residual", maxLS, 0, maxLS); Type2ResMon = ibooker.book1D("type2Residual", "Type2Residual;Lumisection;Type 2 Residual", maxLS, 0, maxLS); diff --git a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc index 8d8ebe4bd6704..e1d56a2d8d2ca 100644 --- a/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc +++ b/DQM/BeamMonitor/plugins/AlcaBeamMonitor.cc @@ -173,17 +173,18 @@ void AlcaBeamMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& } } ibooker.setCurrentFolder(monitorName_ + "Service"); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); - theValuesContainer_ = ibooker.bookProfile("hHistoLumiValues", - "Histo Lumi Values", - 3 * numberOfValuesToSave_, - 0., - 3 * numberOfValuesToSave_, - 100., - -100., - 9000., - " "); - ibooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); + theValuesContainer_ = ibooker.bookProfile("hHistoLumiValues", + "Histo Lumi Values", + 3 * numberOfValuesToSave_, + 0., + 3 * numberOfValuesToSave_, + 100., + -100., + 9000., + " "); + } // create and cd into new folder ibooker.setCurrentFolder(monitorName_ + "Validation"); diff --git a/DQM/CastorMonitor/src/CastorMonitorModule.cc b/DQM/CastorMonitor/src/CastorMonitorModule.cc index 99b6f9c018836..27c0c6e9939ee 100644 --- a/DQM/CastorMonitor/src/CastorMonitorModule.cc +++ b/DQM/CastorMonitor/src/CastorMonitorModule.cc @@ -63,9 +63,8 @@ void CastorMonitorModule::bookHistograms(DQMStore::IBooker &ibooker, const edm::EventSetup &iSetup) { if (DigiMon_) { // Run histos only since there is endRun processing. - auto scope = ibooker.setScope(MonitorElementData::Scope::RUN); + auto scope = DQMStore::IBooker::UseRunScope(ibooker); DigiMon_->bookHistograms(ibooker, iRun, iSetup); - ibooker.setScope(scope); } if (RecHitMon_) { RecHitMon_->bookHistograms(ibooker, iRun); diff --git a/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc b/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc index 9a174ce18260d..6acdb957cffa7 100644 --- a/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc +++ b/DQM/DTMonitorClient/src/DTDCSByLumiSummary.cc @@ -38,17 +38,18 @@ void DTDCSByLumiSummary::dqmEndLuminosityBlock(DQMStore::IBooker& ibooker, globalHVSummary->setAxisTitle("Sectors", 1); globalHVSummary->setAxisTitle("Wheel", 2); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); - totalDCSFraction = ibooker.bookFloat("DTDCSSummary"); - for (int wh = -2; wh <= 2; wh++) { - stringstream wheel_str; - wheel_str << wh; + { + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); + totalDCSFraction = ibooker.bookFloat("DTDCSSummary"); + for (int wh = -2; wh <= 2; wh++) { + stringstream wheel_str; + wheel_str << wh; - MonitorElement* FractionWh = ibooker.bookFloat("DT_Wheel" + wheel_str.str()); + MonitorElement* FractionWh = ibooker.bookFloat("DT_Wheel" + wheel_str.str()); - totalDCSFractionWh.push_back(FractionWh); + totalDCSFractionWh.push_back(FractionWh); + } } - ibooker.setScope(scope); globalHVSummary->Reset(); diff --git a/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc b/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc index e0b6a5808e7a6..fe52679a769c1 100644 --- a/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc +++ b/DQM/DTMonitorModule/src/DTDCSByLumiTask.cc @@ -73,13 +73,13 @@ void DTDCSByLumiTask::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& stringstream wheel_str; wheel_str << wheel; - // Set Lumi scope in order to save histo every LS - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); - MonitorElement* ME = - ibooker.book1D("hActiveUnits" + wheel_str.str(), "Active Untis x LS Wh" + wheel_str.str(), 2, 0.5, 2.5); - ibooker.setScope(scope); - - hActiveUnits.push_back(ME); + { + // Set Lumi scope in order to save histo every LS + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); + MonitorElement* ME = + ibooker.book1D("hActiveUnits" + wheel_str.str(), "Active Untis x LS Wh" + wheel_str.str(), 2, 0.5, 2.5); + hActiveUnits.push_back(ME); + } } } diff --git a/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc b/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc index 4fd2a3f180909..111ce18aecb57 100644 --- a/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc +++ b/DQM/EcalPreshowerMonitorModule/src/ESIntegrityTask.cc @@ -178,9 +178,8 @@ void ESIntegrityTask::bookHistograms(DQMStore::IBooker& iBooker, edm::Run const& if (doLumiAnalysis_) { sprintf(histo, "ES Good Channel Fraction"); - auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(iBooker); meDIFraction_ = iBooker.book2D(histo, histo, 3, 1.0, 3.0, 3, 1.0, 3.0); - iBooker.setScope(scope); } } diff --git a/DQM/HcalTasks/plugins/DigiPhase1Task.cc b/DQM/HcalTasks/plugins/DigiPhase1Task.cc index 798a797018242..898f25bc993ca 100644 --- a/DQM/HcalTasks/plugins/DigiPhase1Task.cc +++ b/DQM/HcalTasks/plugins/DigiPhase1Task.cc @@ -287,7 +287,7 @@ DigiPhase1Task::DigiPhase1Task(edm::ParameterSet const& ps) : DQTask(ps) { _dhashmap.initialize(_emap, electronicsmap::fE2DHashMap); // MARK THESE HISTOGRAMS AS LUMI BASED FOR OFFLINE PROCESSING - auto scope = ib.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ib); if (_ptype == fOffline) { //_cDigiSize_FED.setLumiFlag(); // hidefed2crate : FED stuff not available offline anymore, so this histogram doesn't make sense? _cOccupancy_depth.book(ib, _emap, _subsystem); @@ -301,7 +301,6 @@ DigiPhase1Task::DigiPhase1Task(edm::ParameterSet const& ps) : DQTask(ps) { ib.setCurrentFolder(_subsystem + "/" + _name); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); _unknownIdsPresent = false; - ib.setScope(scope); } /* virtual */ void DigiPhase1Task::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/HcalTasks/plugins/DigiTask.cc b/DQM/HcalTasks/plugins/DigiTask.cc index 874b63c4bedb0..99bc186621ce1 100644 --- a/DQM/HcalTasks/plugins/DigiTask.cc +++ b/DQM/HcalTasks/plugins/DigiTask.cc @@ -776,23 +776,24 @@ DigiTask::DigiTask(edm::ParameterSet const& ps) : DQTask(ps) { } } - // MARK THESE HISTOGRAMS AS LUMI BASED FOR OFFLINE PROCESSING - auto scope = ib.setScope(MonitorElementData::Scope::LUMI); - if (_ptype == fOffline) { - //_cDigiSize_FED.setLumiFlag(); - _cDigiSize_Crate.book(ib, _emap, _subsystem); - _cOccupancy_depth.book(ib, _emap, _subsystem); - } + { + // MARK THESE HISTOGRAMS AS LUMI BASED FOR OFFLINE PROCESSING + auto scope = DQMStore::IBooker::UseLumiScope(ib); + if (_ptype == fOffline) { + //_cDigiSize_FED.setLumiFlag(); + _cDigiSize_Crate.book(ib, _emap, _subsystem); + _cOccupancy_depth.book(ib, _emap, _subsystem); + } - // book Number of Events vs LS histogram - ib.setCurrentFolder(_subsystem + "/RunInfo"); - meNumEvents1LS = ib.book1D("NumberOfEvents", "NumberOfEvents", 1, 0, 1); + // book Number of Events vs LS histogram + ib.setCurrentFolder(_subsystem + "/RunInfo"); + meNumEvents1LS = ib.book1D("NumberOfEvents", "NumberOfEvents", 1, 0, 1); - // book the flag for unknown ids and the online guy as well - ib.setCurrentFolder(_subsystem + "/" + _name); - meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); - _unknownIdsPresent = false; - ib.setScope(scope); + // book the flag for unknown ids and the online guy as well + ib.setCurrentFolder(_subsystem + "/" + _name); + meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); + _unknownIdsPresent = false; + } } /* virtual */ void DigiTask::_resetMonitors(hcaldqm::UpdateFreq uf) { diff --git a/DQM/HcalTasks/plugins/RawTask.cc b/DQM/HcalTasks/plugins/RawTask.cc index edec13dd19ccf..85c64902a83df 100644 --- a/DQM/HcalTasks/plugins/RawTask.cc +++ b/DQM/HcalTasks/plugins/RawTask.cc @@ -182,9 +182,8 @@ RawTask::RawTask(edm::ParameterSet const& ps) : DQTask(ps) { _cBadQuality_FEDuTCA.book(ib, _emap, _filter_VME, _subsystem); } if (_ptype == fOffline) { - auto scope = ib.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ib); _cBadQuality_depth.book(ib, _emap, _subsystem); - ib.setScope(scope); } else { _cBadQuality_depth.book(ib, _emap, _subsystem); } diff --git a/DQM/HcalTasks/plugins/RecHitTask.cc b/DQM/HcalTasks/plugins/RecHitTask.cc index 2d73e8c3ac264..8d7cf644c4e5f 100644 --- a/DQM/HcalTasks/plugins/RecHitTask.cc +++ b/DQM/HcalTasks/plugins/RecHitTask.cc @@ -454,9 +454,8 @@ RecHitTask::RecHitTask(edm::ParameterSet const& ps) : DQTask(ps) { // book some mes... ib.setCurrentFolder(_subsystem + "/" + _name); - auto scope = ib.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ib); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); - ib.setScope(scope); _unknownIdsPresent = false; } diff --git a/DQM/HcalTasks/plugins/TPTask.cc b/DQM/HcalTasks/plugins/TPTask.cc index 79f520b4b17f0..764b4749f7fee 100644 --- a/DQM/HcalTasks/plugins/TPTask.cc +++ b/DQM/HcalTasks/plugins/TPTask.cc @@ -725,9 +725,8 @@ TPTask::TPTask(edm::ParameterSet const& ps) : DQTask(ps) { // book the flag for unknown ids and the online guy as well ib.setCurrentFolder(_subsystem + "/" + _name); - auto scope = ib.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ib); meUnknownIds1LS = ib.book1D("UnknownIds", "UnknownIds", 1, 0, 1); - ib.setScope(scope); _unknownIdsPresent = false; } diff --git a/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc b/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc index f37ba07994272..5ea653d453b39 100644 --- a/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc +++ b/DQM/RPCMonitorDigi/src/RPCDcsInfo.cc @@ -18,9 +18,8 @@ void RPCDcsInfo::bookHistograms(DQMStore::IBooker& ibooker, ibooker.cd(); ibooker.setCurrentFolder(subsystemname_ + "/" + dcsinfofolder_); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); DCSbyLS_ = ibooker.book1D("DCSbyLS", "DCS", 1, 0.5, 1.5); - ibooker.setScope(scope); // initialize dcs = true; diff --git a/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc b/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc index ee25f5170fc7c..cebe3479e0ff2 100644 --- a/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc +++ b/DQM/SiPixelMonitorDigi/src/SiPixelDigiSource.cc @@ -1231,9 +1231,10 @@ void SiPixelDigiSource::bookMEs(DQMStore::IBooker& iBooker, const edm::EventSetu char title8[80]; sprintf(title8, "FED Digi Occupancy (NDigis/) vs LumiSections;Lumi Section;FED"); if (modOn) { - auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); - averageDigiOccupancy = iBooker.bookProfile("averageDigiOccupancy", title7, 40, -0.5, 39.5, 0., 3.); - iBooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseLumiScope(iBooker); + averageDigiOccupancy = iBooker.bookProfile("averageDigiOccupancy", title7, 40, -0.5, 39.5, 0., 3.); + } avgfedDigiOccvsLumi = iBooker.book2D("avgfedDigiOccvsLumi", title8, 640, 0., 3200., 40, -0.5, 39.5); avgBarrelFedOccvsLumi = iBooker.book1D( "avgBarrelFedOccvsLumi", diff --git a/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc b/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc index 767a82a6244ed..91d3fb7828e2e 100644 --- a/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc +++ b/DQM/SiPixelMonitorRawData/src/SiPixelRawDataErrorSource.cc @@ -285,9 +285,10 @@ void SiPixelRawDataErrorSource::bookMEs(DQMStore::IBooker &iBooker) { iBooker.setCurrentFolder(topFolderName_ + "/AdditionalPixelErrors"); char title[80]; sprintf(title, "By-LumiSection Error counters"); - auto scope = iBooker.setScope(MonitorElementData::Scope::LUMI); - byLumiErrors = iBooker.book1D("byLumiErrors", title, 2, 0., 2.); - iBooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseLumiScope(iBooker); + byLumiErrors = iBooker.book1D("byLumiErrors", title, 2, 0., 2.); + } char title1[80]; sprintf(title1, "Errors per LumiSection;LumiSection;NErrors"); errorRate = iBooker.book1D("errorRate", title1, 5000, 0., 5000.); diff --git a/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc b/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc index fafdda66dc2b7..089743528fdc8 100644 --- a/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc +++ b/DQM/SiStripMonitorClient/plugins/SiStripDcsInfo.cc @@ -129,7 +129,8 @@ void SiStripDcsInfo::bookStatus(DQMStore& dqm_store) { else dqm_store.setCurrentFolder("SiStrip/EventInfo"); - auto scope = dqm_store.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::UseLumiScope(dqm_store); + DcsFraction_ = dqm_store.bookFloat("DCSSummary"); dqm_store.cd(); @@ -142,7 +143,6 @@ void SiStripDcsInfo::bookStatus(DQMStore& dqm_store) { subDetME.DcsFractionME = dqm_store.bookFloat(me_name); } bookedStatus_ = true; - dqm_store.setScope(scope); dqm_store.cd(); } diff --git a/DQM/SiStripMonitorHardware/src/FEDHistograms.cc b/DQM/SiStripMonitorHardware/src/FEDHistograms.cc index 8058860fd6abc..a4beb127db44f 100644 --- a/DQM/SiStripMonitorHardware/src/FEDHistograms.cc +++ b/DQM/SiStripMonitorHardware/src/FEDHistograms.cc @@ -895,16 +895,17 @@ void FEDHistograms::bookTopLevelHistograms(DQMStore::IBooker& ibooker, ibooker.setCurrentFolder(lBaseDir + "/PerLumiSection"); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); - bookHistogram(ibooker, - lumiErrorFraction_, - "lumiErrorFraction", - "Fraction of error per lumi section vs subdetector", - 6, - 0.5, - 6.5, - "SubDetId"); - ibooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); + bookHistogram(ibooker, + lumiErrorFraction_, + "lumiErrorFraction", + "Fraction of error per lumi section vs subdetector", + 6, + 0.5, + 6.5, + "SubDetId"); + } //Set special property for lumi ME if (lumiErrorFraction_.enabled && lumiErrorFraction_.monitorEle) { diff --git a/DQM/TrackingMonitor/src/TrackingMonitor.cc b/DQM/TrackingMonitor/src/TrackingMonitor.cc index 8868deba25c12..d7c136b2f1ce8 100644 --- a/DQM/TrackingMonitor/src/TrackingMonitor.cc +++ b/DQM/TrackingMonitor/src/TrackingMonitor.cc @@ -342,13 +342,12 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& if (doLumiAnalysis) { // add by Mia in order to deal with LS transitions ibooker.setCurrentFolder(MEFolderName + "/LSanalysis"); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); histname = "NumberOfTracks_lumiFlag_" + CategoryName; NumberOfTracks_lumiFlag = ibooker.book1D(histname, histname, 3 * TKNoBin, TKNoMin, (TKNoMax + 0.5) * 3. - 0.5); NumberOfTracks_lumiFlag->setAxisTitle("Number of Tracks per Event", 1); NumberOfTracks_lumiFlag->setAxisTitle("Number of Events", 2); - ibooker.setScope(scope); } // book profile plots vs LS : @@ -580,9 +579,8 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& } if (doLumiAnalysis) { - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); theTrackAnalyzer->initHisto(ibooker, iSetup, *conf); - ibooker.setScope(scope); } else { theTrackAnalyzer->initHisto(ibooker, iSetup, *conf); } @@ -607,12 +605,11 @@ void TrackingMonitor::bookHistograms(DQMStore::IBooker& ibooker, edm::Run const& if (doSeedLumiAnalysis_) { ibooker.setCurrentFolder(MEFolderName + "/LSanalysis"); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); histname = "NumberOfSeeds_lumiFlag_" + seedProducer.label() + "_" + CategoryName; NumberOfSeeds_lumiFlag = ibooker.book1D(histname, histname, TKNoSeedBin, TKNoSeedMin, TKNoSeedMax); NumberOfSeeds_lumiFlag->setAxisTitle("Number of Seeds per Event", 1); NumberOfSeeds_lumiFlag->setAxisTitle("Number of Events", 2); - ibooker.setScope(scope); } } diff --git a/DQMOffline/JetMET/src/METAnalyzer.cc b/DQMOffline/JetMET/src/METAnalyzer.cc index f1d47af9e480c..bf8e82a0dfa09 100644 --- a/DQMOffline/JetMET/src/METAnalyzer.cc +++ b/DQMOffline/JetMET/src/METAnalyzer.cc @@ -338,9 +338,10 @@ void METAnalyzer::bookMonitorElement(std::string DirName, hMET_2 = ibooker.book1D("MET_2", "MET Range 2", 200, 0, 2000); hSumET = ibooker.book1D("SumET", "SumET", 400, 0, 4000); - auto scope = ibooker.setScope(MonitorElementData::Scope::LUMI); - hMETSig = ibooker.book1D("METSig", "METSig", 51, 0, 51); - ibooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); + hMETSig = ibooker.book1D("METSig", "METSig", 51, 0, 51); + } hMETPhi = ibooker.book1D("METPhi", "METPhi", 60, -M_PI, M_PI); hMET_logx = ibooker.book1D("MET_logx", "MET_logx", 40, -1, 9); diff --git a/DQMServices/Components/plugins/DQMDcsInfo.cc b/DQMServices/Components/plugins/DQMDcsInfo.cc index 289ef6e2ebc87..53e3b083e033a 100644 --- a/DQMServices/Components/plugins/DQMDcsInfo.cc +++ b/DQMServices/Components/plugins/DQMDcsInfo.cc @@ -45,7 +45,7 @@ void DQMDcsInfo::bookHistograms(DQMStore::IBooker& ibooker, ibooker.cd(); ibooker.setCurrentFolder(subsystemname_ + "/" + dcsinfofolder_); - ibooker.setScope(MonitorElementData::Scope::LUMI); + auto scope = DQMStore::IBooker::UseLumiScope(ibooker); DCSbyLS_ = ibooker.book1D("DCSbyLS", "DCS", 25, 0., 25.); // initialize diff --git a/HLTrigger/Timer/plugins/FastTimerService.cc b/HLTrigger/Timer/plugins/FastTimerService.cc index dfb47f004892a..e72a6085496ff 100644 --- a/HLTrigger/Timer/plugins/FastTimerService.cc +++ b/HLTrigger/Timer/plugins/FastTimerService.cc @@ -857,7 +857,7 @@ void FastTimerService::preGlobalBeginRun(edm::GlobalContext const& gc) { if (enable_dqm_) { // define a callback to book the MonitorElements auto bookTransactionCallback = [&, this](dqm::reco::DQMStore::IBooker& booker, dqm::reco::DQMStore::IGetter&) { - booker.setScope(MonitorElementData::Scope::RUN); + auto scope = dqm::reco::DQMStore::IBooker::UseRunScope(booker); // we should really do this, but only DQMStore is allowed to touch it // We could move to postGlobalBeginRun, then the DQMStore has sure set it up. //booker.setRunLumi(gc.luminosityBlockID()); diff --git a/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc b/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc index 91ad29b44e36c..92e8da2475eda 100644 --- a/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc +++ b/Validation/EcalDigis/src/EcalSelectiveReadoutValidation.cc @@ -857,9 +857,10 @@ void EcalSelectiveReadoutValidation::bookHistograms(DQMStore::IBooker& ibooker, edm::EventSetup const&) { ibooker.setCurrentFolder("EcalDigisV/SelectiveReadout"); - auto scope = ibooker.setScope(MonitorElementData::Scope::RUN); - meL1aRate_ = bookFloat(ibooker, "l1aRate_"); - ibooker.setScope(scope); + { + auto scope = DQMStore::IBooker::UseRunScope(ibooker); + meL1aRate_ = bookFloat(ibooker, "l1aRate_"); + } meDccVol_ = bookProfile(ibooker, "hDccVol", //"EcalDccEventSizeComputed", From 280e50fe029fa8522aea9b54952fa775d66a1a54 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 29 Jan 2020 18:58:49 +0100 Subject: [PATCH 105/112] Improve debug output. --- DQMServices/Core/interface/DQMStore.h | 2 +- DQMServices/Core/src/DQMStore.cc | 83 +++++++++++++++------------ 2 files changed, 48 insertions(+), 37 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index e68c8086c4d74..cb2485a9bea51 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -626,7 +626,7 @@ namespace dqm { // Log a backtrace on booking. void printTrace(std::string const& message); // print a log message if ME matches trackME_. - void debugTrackME(const char* message, MonitorElement* me) const; + void debugTrackME(const char* message, MonitorElement* me_local, MonitorElement* me_global) const; private: // MEComparison is a name-only comparison on MEs and Paths, allowing diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 1652f33806080..96cde25dbb1d1 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -110,7 +110,7 @@ namespace dqm::implementation { if (forceReplace) { TH1* th1 = makeobject(); assert(th1); - store_->debugTrackME("bookME (forceReplace)", me); + store_->debugTrackME("bookME (forceReplace)", nullptr, me); // surgically replace Histogram // This is rather dangerous because the ME is in a global set and may // be in use. We are protected by the booking lock here, but code @@ -147,11 +147,11 @@ namespace dqm::implementation { // assertLegacySafe option). // We still created a local ME, so we can drive the lumi-changing for // legacy modules in watchPreGlobalBeginLumi. - store_->debugTrackME("bookME (legacy)", me); + store_->debugTrackME("bookME (legacy)", localme, me); return me; } else { // the normal case. - store_->debugTrackME("bookME (normal)", me); + store_->debugTrackME("bookME (normal)", localme, me); return localme; } } @@ -162,7 +162,7 @@ namespace dqm::implementation { auto existing_new = globalMEs_[me->getRunLumi()].insert(me); if (existing_new.second == true) { // successfully inserted, return new object - debugTrackME("putME (global)", me); + debugTrackME("putME (global)", nullptr, me); return me; } else { // already present, return old object @@ -183,7 +183,7 @@ namespace dqm::implementation { auto existing_new = localmes.insert(local_me); // successfully inserted, return new object assert(existing_new.second == true); // insert successful - debugTrackME("putME (local, new)", me); + debugTrackME("putME (local, new)", local_me, me); return local_me; } else { // already present, return old object @@ -198,7 +198,7 @@ namespace dqm::implementation { if (!local_me->isValid()) { local_me->switchData(me); } - debugTrackME("putME (local, existing)", me); + debugTrackME("putME (local, existing)", local_me, me); return local_me; } } @@ -209,7 +209,7 @@ namespace dqm::implementation { for (auto& [runlumi, meset] : this->globalMEs_) { auto it = meset.find(path); if (it != meset.end()) { - debugTrackME("findME (found)", *it); + debugTrackME("findME (found)", nullptr, *it); // no guarantee on which ME we return here -- only that clone'ing this // would give a valid ME for that path. return *it; @@ -287,26 +287,36 @@ namespace dqm::implementation { }); } - void DQMStore::debugTrackME(const char* message, MonitorElement* me) const { + void DQMStore::debugTrackME(const char* message, MonitorElement* me_local, MonitorElement* me_global) const { const char* scopename[] = {"INVALID", "JOB", "RUN", "LUMI"}; - if (!this->trackME_.empty() && me) { - std::string name = me->getFullname(); + if (!this->trackME_.empty() && (me_local || me_global)) { + std::string name = me_global ? me_global->getFullname() : me_local->getFullname(); if (name.find(this->trackME_) != std::string::npos) { edm::LogWarning("DQMStoreTrackME").log([&](auto& logger) { - logger << message << " for " << name << "(" << me << ")"; - if (me->isValid()) { - logger << " " << me->getRunLumi() << " scope " << scopename[me->getScope()]; - if (me->kind() >= MonitorElement::Kind::TH1F) { - logger << " entries " << me->getEntries(); - } else if (me->kind() == MonitorElement::Kind::STRING) { - logger << " value " << me->getStringValue(); - } else if (me->kind() == MonitorElement::Kind::REAL) { - logger << " value " << me->getFloatValue(); - } else if (me->kind() == MonitorElement::Kind::INT) { - logger << " value " << me->getIntValue(); + logger << message << " for " << name << "(" << me_local << "," << me_global << ")"; + auto writeme = [&](MonitorElement* me) { + if (me->isValid()) { + logger << " " << me->getRunLumi() << " scope " << scopename[me->getScope()]; + if (me->kind() >= MonitorElement::Kind::TH1F) { + logger << " entries " << me->getEntries(); + } else if (me->kind() == MonitorElement::Kind::STRING) { + logger << " value " << me->getStringValue(); + } else if (me->kind() == MonitorElement::Kind::REAL) { + logger << " value " << me->getFloatValue(); + } else if (me->kind() == MonitorElement::Kind::INT) { + logger << " value " << me->getIntValue(); + } + } else { + logger << " (invalid)"; } - } else { - logger << " (invalid)"; + }; + if (me_local) { + logger << " local:"; + writeme(me_local); + } + if (me_global) { + logger << " global:"; + writeme(me_global); } }); // A breakpoint can be useful here. @@ -325,7 +335,7 @@ namespace dqm::implementation { auto existing = this->get(key); if (existing) { // exactly matching ME found, needs merging with the new data. - debugTrackME("findOrRecycle (found)", existing); + debugTrackME("findOrRecycle (found)", nullptr, existing); return existing; } // else @@ -351,7 +361,7 @@ namespace dqm::implementation { auto newme = *result.first; // iterator to new ME assert(oldme == newme); // recycling! // newme is reset and ready to accept data. - debugTrackME("findOrRecycle (recycled)", newme); + debugTrackME("findOrRecycle (recycled)", nullptr, newme); return newme; } // else @@ -387,7 +397,7 @@ namespace dqm::implementation { auto target = targetset.find(me); // lookup by path, thanks to MEComparison if (target != targetset.end()) { // we already have a ME, just use it! - debugTrackME("enterLumi (existing)", *target); + debugTrackME("enterLumi (existing)", nullptr, *target); } else { // look for a prototype to reuse. auto proto = prototypes.find(me); @@ -411,7 +421,7 @@ namespace dqm::implementation { auto result = targetset.insert(oldme); assert(result.second); // was new insertion target = result.first; // iterator to new ME - debugTrackME("enterLumi (reused)", *target); + debugTrackME("enterLumi (reused)", nullptr, *target); } else { // no prototype available. That means we have concurrent Lumis/Runs, // and need to make a clone now. @@ -431,13 +441,14 @@ namespace dqm::implementation { auto result = targetset.insert(newme); assert(result.second); // was new insertion target = result.first; // iterator to new ME - debugTrackME("enterLumi (allocated)", *target); + debugTrackME("enterLumi (allocated)", nullptr, *target); } } // now we have the proper global ME in the right place, point the local there. // This is only safe if the name is exactly the same -- else it might corrupt // the tree structure of the set! me->switchData(*target); + debugTrackME("enterLumi (switchdata)", me, *target); } } @@ -469,7 +480,7 @@ namespace dqm::implementation { // we have to be very careful with the ME here, it might not be backed by data at all. if (me->isValid() && checkScope(me->getScope()) == true) { // if we left the scope, simply release the data. - debugTrackME("leaveLumi (release)", me); + debugTrackME("leaveLumi (release)", me, nullptr); me->release(/* expectOwned */ false); } } @@ -515,12 +526,12 @@ namespace dqm::implementation { auto other = this->findME(me); if (other) { // we still have a global one, so we can just remove this. - debugTrackME("cleanupLumi (delete)", me); + debugTrackME("cleanupLumi (delete)", nullptr, me); delete me; } else { // we will modify the ME, so it needs to be out of the set. // use a temporary vector to be save. - debugTrackME("cleanupLumi (recycle)", me); + debugTrackME("cleanupLumi (recycle)", nullptr, me); torecycle.push_back(me); } } @@ -538,7 +549,7 @@ namespace dqm::implementation { me->Reset(); auto result = prototypes.insert(me); assert(result.second); // was new insertion, else findME should succeed - debugTrackME("cleanupLumi (reset)", me); + debugTrackME("cleanupLumi (reset)", nullptr, me); } } @@ -550,7 +561,7 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getPathname() == path.getDirname()) { - store_->debugTrackME("getContents (match)", *it); + store_->debugTrackME("getContents (match)", nullptr, *it); out.push_back(*it); ++it; } @@ -571,7 +582,7 @@ namespace dqm::implementation { if (runlumi == edm::LuminosityBlockID() && (*it)->getScope() != MonitorElementData::Scope::JOB) { // skip prototypes } else { - store_->debugTrackME("getAllContents (match)", *it); + store_->debugTrackME("getAllContents (match)", nullptr, *it); out.push_back(*it); } ++it; @@ -591,7 +602,7 @@ namespace dqm::implementation { auto it = meset.lower_bound(path); // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getFullname().rfind(path_str, 0) == 0) { - store_->debugTrackME("getAllContents (run/lumi match)", *it); + store_->debugTrackME("getAllContents (run/lumi match)", nullptr, *it); out.push_back(*it); ++it; } @@ -611,7 +622,7 @@ namespace dqm::implementation { auto it = meset.find(key.path_); if (it != meset.end()) { assert((*it)->getScope() == key.scope_); - store_->debugTrackME("get (key found)", *it); + store_->debugTrackME("get (key found)", nullptr, *it); return *it; } return nullptr; From 587359343ea9024e25a6165ed064eef5cbe3270e Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Wed, 29 Jan 2020 18:59:29 +0100 Subject: [PATCH 106/112] Add a rather crucial line. That does not fix the unit test failure, though. --- DQMServices/Core/src/DQMStore.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index 96cde25dbb1d1..c9dfb76201014 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -436,6 +436,7 @@ namespace dqm::implementation { assert(!assertLegacySafe_); MonitorElementData newdata = anyme->cloneMEData(); + newdata.key_.id_ = edm::LuminosityBlockID(run, lumi); auto newme = new MonitorElement(std::move(newdata)); newme->Reset(); // we cloned a ME in use, not an empty prototype auto result = targetset.insert(newme); From 3b06832de8c23d4c6e76b7665056207c6f0649bf Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 30 Jan 2020 15:51:13 +0100 Subject: [PATCH 107/112] Clean up the DQMStore header a bit. --- DQMServices/Core/interface/DQMStore.h | 66 +++++++++++++++++---------- DQMServices/Core/src/DQMStore.cc | 4 +- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index cb2485a9bea51..f4fc53651ec2f 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -42,6 +42,9 @@ namespace dqm { class IBooker : public dqm::implementation::NavigatorBase { public: + // DQMStore configures the IBooker in bookTransaction. + friend class DQMStore; + // functor to be passed as a default argument that does not do anything. struct NOOP { void operator()(TH1*) const {}; @@ -463,7 +466,7 @@ namespace dqm { ~IBooker() override; - protected: + private: IBooker(DQMStore* store); virtual uint64_t setModuleID(uint64_t moduleID); virtual edm::LuminosityBlockID setRunLumi(edm::LuminosityBlockID runlumi); @@ -515,6 +518,9 @@ namespace dqm { virtual MonitorElement* getElement(std::string const& path) const; // return sub-directories of current directory + // Deprecated because the current implementation is very slow and barely + // used, use getAllContents instead. + DQM_DEPRECATED virtual std::vector getSubdirs() const; // return element names of direct children of current directory virtual std::vector getMEs() const; @@ -536,6 +542,9 @@ namespace dqm { enum OpenRunDirs { KeepRunDirs, StripRunDirs }; DQMStore(edm::ParameterSet const& pset, edm::ActivityRegistry&); + DQMStore(DQMStore const&) = delete; + DQMStore(DQMStore&&) = delete; + DQMStore& operator=(DQMStore const&) = delete; ~DQMStore() override; // ------------------------------------------------------------------------ @@ -550,26 +559,37 @@ namespace dqm { OpenRunDirs stripdirs = KeepRunDirs, bool fileMustExist = true); - DQMStore(DQMStore const&) = delete; - DQMStore& operator=(DQMStore const&) = delete; - // ------------------------------------------------------------------------ // ------------ IBooker/IGetter overrides to prevent ambiguity ------------ void cd() override { this->IBooker::cd(); } void cd(std::string const& dir) override { this->IBooker::cd(dir); } + void goUp() override { this->IBooker::goUp(); } + std::string pwd() override { return this->IBooker::pwd(); } + void setCurrentFolder(std::string const& fullpath) override { // only here we keep the two in sync -- all the others call this in the end! this->IBooker::setCurrentFolder(fullpath); this->IGetter::setCurrentFolder(fullpath); } - void goUp() override { this->IBooker::goUp(); } - std::string pwd() override { return this->IBooker::pwd(); } public: // internal -- figure out better protection. template void bookTransaction(iFunc f, uint64_t moduleId, bool canSaveByLumi) { + // taking the lock here only to protect the single, global IBooker (as + // base class of DQMStore). We could avoid taking this lock by making a + // new IBooker instance for each transaction, and the DQMStore itself + // takes the lock before touching any data structures. + // There is a race in bookME when we don't take this lock, where two + // modules might prepare a global ME for the same name at the same time + // and only one of them succeeds in putME: this is is safe, but we need + // to remove the assertion over there and subsystem code has to be + // aware that the booking callback *can* run multiple times. + // Additionally, this lock is what keeps usage of getTH1() safe during + // booking... all code needs to be migrated to callbacks before this can + // be removed. auto lock = std::scoped_lock(this->booking_mutex_); + IBooker& booker = *this; auto newscope = MonitorElementData::Scope::RUN; if (canSaveByLumi && this->doSaveByLumi_) { @@ -577,11 +597,10 @@ namespace dqm { } auto oldscope = booker.setScope(newscope); assert(moduleId != 0 || !"moduleID must be set for normal booking transaction"); - // Access via this-> to allow access to protected member - auto oldmoduleid = this->setModuleID(moduleId); + auto oldmoduleid = booker.setModuleID(moduleId); assert(oldmoduleid == 0 || !"Nested booking transaction?"); // always book prototypes (except for Scope::JOB, where we can use these directly). - auto oldrunlumi = this->setRunLumi(edm::LuminosityBlockID()); + auto oldrunlumi = booker.setRunLumi(edm::LuminosityBlockID()); f(booker); @@ -603,11 +622,13 @@ namespace dqm { // For input modules: trigger recycling without local ME/enterLumi/moduleID. MonitorElement* findOrRecycle(MonitorElementData::Key const&); + // modules are expected to call these callbacks when they change run/lumi. // The DQMStore then updates the module's MEs, potentially cloning them // if there are concurrent runs/lumis. void enterLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); void leaveLumi(edm::RunNumber_t run, edm::LuminosityBlockNumber_t lumi, uint64_t moduleID); + // this is triggered by a framework hook to remove/recycle MEs after a // run/lumi is saved. We do this via the edm::Service interface to make // sure it runs after *all* output modules, even if there are multiple. @@ -637,9 +658,7 @@ namespace dqm { // ME objects must never appear more than once in these sets. ME objects // in localMEs_ cannot be deleted, since the module might hold pointers. // MEs in globalMEs_ can be deleted/recycled at the end of their scope, - // if there are no MEs left that share the data -- for non-legacy modules - // that should hold by construction, for legacy MEs (moduleID == 0) it - // needs an explicit check. + // if there are no local MEs left that share the data. // MEs can be _protoype MEs_ if their scope is not yet known (after // booking, after leaveLumi). A prototype is kept if and only if there is // no other global instance of the same ME. Prototype MEs have @@ -651,19 +670,19 @@ namespace dqm { // pointers, this may be possible (not everything is const), but it could // still corrupt the datastructure. std::map> globalMEs_; - // Key is (moduleID [, run]), run is only needed for edm::global. + // Key is (moduleID [, run | , stream]), run is only needed for + // edm::global, stream only for edm::stream. // Legacy MEs have moduleID 0. std::map> localMEs_; - // Whenever modifying these sets, take tihs mutex. It's recursive, so we + // Whenever modifying these sets, take this mutex. It's recursive, so we // can be liberal -- lock on any access, but also lock on the full booking - // transaction. The former is required since also the MEComparison is not - // really thread safe, the latter since booking still uses a single, - // shared IBooker instance (`this`!), and the transaction needs to be - // atomic. + // transaction. std::recursive_mutex booking_mutex_; - // universal verbose flag. + // Universal verbose flag. + // Only very few usages remain, the main debugging tool is trackME_. int verbose_; + // If set to true, error out whenever things happen that are not safe for // legacy modules. bool assertLegacySafe_; @@ -681,16 +700,17 @@ namespace dqm { namespace legacy { class DQMStore : public dqm::implementation::DQMStore { public: - typedef dqm::implementation::IBooker IBooker; - typedef dqm::implementation::IGetter IGetter; + using IBooker = dqm::implementation::IBooker; + using IGetter = dqm::implementation::IGetter; + // import constructors. using dqm::implementation::DQMStore::DQMStore; }; } // namespace legacy namespace reco { - typedef dqm::legacy::DQMStore DQMStore; + using DQMStore = dqm::legacy::DQMStore; } // namespace reco namespace harvesting { - typedef dqm::legacy::DQMStore DQMStore; + using DQMStore = dqm::legacy::DQMStore; } // namespace harvesting } // namespace dqm diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index c9dfb76201014..dd9303d7613c1 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -555,12 +555,12 @@ namespace dqm::implementation { } std::vector IGetter::getContents(std::string const& pathname) const { + auto lock = std::scoped_lock(store_->booking_mutex_); std::vector out; MonitorElementData::Path path; path.set(pathname, MonitorElementData::Path::Type::DIR); for (auto& [runlumi, meset] : store_->globalMEs_) { auto it = meset.lower_bound(path); - // rfind can be used as a prefix match. while (it != meset.end() && (*it)->getPathname() == path.getDirname()) { store_->debugTrackME("getContents (match)", nullptr, *it); out.push_back(*it); @@ -571,6 +571,7 @@ namespace dqm::implementation { } std::vector IGetter::getAllContents(std::string const& pathname) const { + auto lock = std::scoped_lock(store_->booking_mutex_); std::vector out; MonitorElementData::Path path; path.set(pathname, MonitorElementData::Path::Type::DIR); @@ -594,6 +595,7 @@ namespace dqm::implementation { std::vector IGetter::getAllContents(std::string const& pathname, uint32_t runNumber, uint32_t lumi) const { + auto lock = std::scoped_lock(store_->booking_mutex_); std::vector out; MonitorElementData::Path path; path.set(pathname, MonitorElementData::Path::Type::DIR); From d5ad6bb5a7716f61409aabfac64c1f9c1686d0a6 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 30 Jan 2020 17:44:50 +0100 Subject: [PATCH 108/112] Clean up MonitorElement.h. - Remove unused methods (also their bodies). - Restrict access, use friends (this needed also some changes in other places) - Provide a switchObject() to prevent a race when using forceReplace when booking. - Change mutable_ to a normal pointer. - Reorder declarations to make a bit more sense. Still many things to be improved (the API of MonitorElement is huge and messy) but better than before. --- DQMServices/Core/interface/MonitorElement.h | 174 ++++++++++---------- DQMServices/Core/interface/QTest.h | 2 - DQMServices/Core/src/DQMStore.cc | 13 +- DQMServices/Core/src/MonitorElement.cc | 147 ++--------------- DQMServices/FwkIO/plugins/DQMRootSource.cc | 9 +- 5 files changed, 106 insertions(+), 239 deletions(-) diff --git a/DQMServices/Core/interface/MonitorElement.h b/DQMServices/Core/interface/MonitorElement.h index 720d43739e3df..042610e2308c6 100644 --- a/DQMServices/Core/interface/MonitorElement.h +++ b/DQMServices/Core/interface/MonitorElement.h @@ -59,10 +59,17 @@ namespace dqm { } } // namespace dqm -class DQMService; -namespace dqm::dqmstoreimpl { +// forward declarations for all our friends +namespace dqm::implementation { class DQMStore; + class IBooker; +} // namespace dqm::implementation +struct DQMTTreeIO; +namespace dqm { + class DQMFileSaverPB; } +class DQMService; +class QualityTester; namespace dqm::impl { @@ -89,8 +96,15 @@ namespace dqm::impl { /** The base class for all MonitorElements (ME) */ class MonitorElement { - friend dqm::dqmstoreimpl::DQMStore; - friend DQMService; + // these need to create and destroy MEs. + friend dqm::implementation::DQMStore; + friend dqm::implementation::IBooker; + // these need to access some of the IO related methods. + friend ::DQMTTreeIO; // declared in DQMRootSource + friend ::dqm::DQMFileSaverPB; + friend ::DQMService; + // this one only needs syncCoreObject. + friend ::QualityTester; public: using Scalar = MonitorElementData::Scalar; @@ -121,26 +135,24 @@ namespace dqm::impl { } }; - protected: - DQMNet::CoreObject data_; //< Core object information. - - std::atomic mutable_; // only set if this is a mutable copy of this ME - bool is_owned_; // true if we are responsible for deleting the mutable object. + private: + MutableMonitorElementData *mutable_; // only set if this is a mutable copy of this ME + // there are no immutable MEs at this time, but we might need them in the future. + bool is_owned_; // true if we are responsible for deleting the mutable object. /** * To do anything to the MEs data, one needs to obtain an access object. * This object will contain the lock guard if one is needed. We differentiate * access for reading and access for mutation (denoted by `Access` or - * `const Access`, however, also read-only access may need to take a lock + * `AccessMut`, however, also read-only access may need to take a lock * if it is to a mutable object. * We want all of this inlined and redundant operations any copies/refs * optimized away. */ const Access access() const { // First, check if there is a mutable object - auto mut = mutable_.load(); - if (mut) { + if (mutable_) { // if there is a mutable object, that is the truth, and we take a lock. - return mut->access(); + return mutable_->access(); } // else throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } @@ -150,15 +162,16 @@ namespace dqm::impl { this->update(); // First, check if there is a mutable object - auto mut = mutable_.load(); - if (mut) { + if (mutable_) { // if there is a mutable object, that is the truth, and we take a lock. - return mut->accessMut(); + return mutable_->accessMut(); } // else throw cms::Exception("LogicError") << "MonitorElement " << getName() << " not backed by any data!"; } - public: + private: + // but internal -- only for DQMStore etc. + // Create ME using this data. A ROOT object pointer may be moved into the // new ME. The new ME will own this data. MonitorElement(MonitorElementData &&data); @@ -166,39 +179,73 @@ namespace dqm::impl { MonitorElement(MutableMonitorElementData *data); // Create a new ME sharing data with this existing ME. MonitorElement(MonitorElement *me); - MonitorElement &operator=(const MonitorElement &) = delete; - MonitorElement &operator=(MonitorElement &&) = delete; + // return a new clone of the data of this ME. Calls ->Clone(), new object // is owned by the returned value. MonitorElementData cloneMEData(); + // Remove access and ownership to the data. The flag is used for a sanity check. MutableMonitorElementData *release(bool expectOwned); + // re-initialize this ME as a shared copy of the other. void switchData(MonitorElement *other); // re-initialize taking ownership of this data. void switchData(MutableMonitorElementData *data); + + // Replace the ROOT object in this ME's data with the new object, taking + // ownership. The old object is deleted. + void switchObject(std::unique_ptr &&newobject); + // copy applicable fileds into the DQMNet core object for compatibility. // In a few places these flags are also still used by the ME. void syncCoreObject(); void syncCoreObject(AccessMut &access); - virtual ~MonitorElement(); // check if the ME is currently backed by MEData; if false (almost) any // access will throw. - bool isValid() const { return mutable_.load() != nullptr; } + bool isValid() const { return mutable_ != nullptr; } + + // used to implement getQErrors et. al. + template + std::vector filterQReports(FILTER filter) const; + + // legacy interfaces, there are no alternatives but they should not be used /// Compare monitor elements, for ordering in sets. bool operator<(const MonitorElement &x) const { return DQMNet::setOrder(data_, x.data_); } - /// Check the consistency of the axis labels static bool CheckBinLabels(const TAxis *a1, const TAxis *a2); + /// Get the object flags. + uint32_t flags() const { return data_.flags; } + /// Mark the object updated. + void update() { data_.flags |= DQMNet::DQM_PROP_NEW; } + + // mostly used for IO, should be private. + std::string valueString() const; + std::string tagString() const; + std::string tagLabelString() const; + std::string effLabelString() const; + std::string qualityTagString(const DQMNet::QValue &qv) const; + + // kept for DQMService. data_ is also used for MEComparison, without it + // we'd need to keep a copy od the name somewhere else. + /// true if ME was updated in last monitoring cycle + bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } + void packScalarData(std::string &into, const char *prefix) const; + void packQualityData(std::string &into) const; + DQMNet::CoreObject data_; //< Core object information. + + public: + MonitorElement &operator=(const MonitorElement &) = delete; + MonitorElement &operator=(MonitorElement &&) = delete; + virtual ~MonitorElement(); + + public: + // good to be used in subsystem code /// Get the type of the monitor element. Kind kind() const { return Kind(data_.flags & DQMNet::DQM_PROP_TYPE_MASK); } - /// Get the object flags. - uint32_t flags() const { return data_.flags; } - /// get name of ME const std::string &getName() const { return this->data_.objname; } @@ -212,16 +259,6 @@ namespace dqm::impl { MonitorElementData::Scope getScope() { return access().key.scope_; } - /// true if ME was updated in last monitoring cycle - bool wasUpdated() const { return data_.flags & DQMNet::DQM_PROP_NEW; } - - /// Mark the object updated. - void update() { data_.flags |= DQMNet::DQM_PROP_NEW; } - - /// specify whether ME should be reset at end of monitoring cycle (default:false); - /// (typically called by Sources that control the original ME) - void setResetMe(bool /* flag */) { data_.flags |= DQMNet::DQM_PROP_RESET; } - /// true if ME is meant to be stored for each luminosity section bool getLumiFlag() const { return access().key.scope_ == MonitorElementData::Scope::LUMI; } @@ -237,6 +274,7 @@ namespace dqm::impl { return access.value.object_ && access.value.object_->TestBit(TH1::kIsAverage); } + private: // A static assert to check that T actually fits in // int64_t. template @@ -244,10 +282,11 @@ namespace dqm::impl { int checkArray[sizeof(int64_t) - sizeof(T) + 1]; }; - protected: void doFill(int64_t x); public: + // filling API. + void Fill(long long x) { fits_in_int64_t(); doFill(static_cast(x)); @@ -299,14 +338,11 @@ namespace dqm::impl { DQM_DEPRECATED void ShiftFillLast(double y, double ye = 0., int32_t xscale = 1); - virtual void Reset(); + public: + // additional APIs, mainly for harvesting. - // mostly used for IO, should be private. - std::string valueString() const; - std::string tagString() const; - std::string tagLabelString() const; - std::string effLabelString() const; - std::string qualityTagString(const DQMNet::QValue &qv) const; + /// Remove all data from the ME, keept the empty histogram with all its settings. + virtual void Reset(); /// true if at least of one of the quality tests returned an error bool hasError() const { return data_.flags & DQMNet::DQM_PROP_REPORT_ERROR; } @@ -317,14 +353,6 @@ namespace dqm::impl { /// true if at least of one of the tests returned some other (non-ok) status bool hasOtherReport() const { return data_.flags & DQMNet::DQM_PROP_REPORT_OTHER; } - /// true if the plot has been marked as an efficiency plot, which - /// will not be normalized when rendered within the DQM GUI. - bool isEfficiency() const { return data_.flags & DQMNet::DQM_PROP_EFFICIENCY_PLOT; } - - // used to implement getQErrors et. al. - template - std::vector filterQReports(FILTER filter) const; - /// get QReport corresponding to (null pointer if QReport does not exist) const MonitorElementData::QReport *getQReport(const std::string &qtname) const; /// get map of QReports @@ -360,6 +388,10 @@ namespace dqm::impl { virtual double getEntries() const; virtual double getBinEntries(int bin) const; + virtual int64_t getIntValue() const; + virtual double getFloatValue() const; + virtual const std::string &getStringValue() const; + // non-const -- thread safety and semantical issues virtual void setBinContent(int binx, double content); virtual void setBinContent(int binx, int biny, double content); @@ -375,14 +407,13 @@ namespace dqm::impl { virtual void setAxisTimeDisplay(int value, int axis = 1); virtual void setAxisTimeFormat(const char *format = "", int axis = 1); virtual void setTitle(const std::string &title); - // --- Operations that origianted in ConcurrentME --- + + // additional operations mainly for booking virtual void setXTitle(std::string const &title); virtual void setYTitle(std::string const &title); virtual void enableSumw2(); virtual void disableAlphanumeric(); virtual void setOption(const char *option); - - // new operations to reduce usage of getTH* virtual double getAxisMin(int axis = 1) const; virtual double getAxisMax(int axis = 1) const; // We should avoid extending histograms in general, and if the behaviour @@ -407,47 +438,20 @@ namespace dqm::impl { virtual TProfile *getTProfile(); virtual TProfile2D *getTProfile2D(); - public: - virtual int64_t getIntValue() const; - virtual double getFloatValue() const; - virtual const std::string &getStringValue() const; - void packScalarData(std::string &into, const char *prefix) const; - void packQualityData(std::string &into) const; - - protected: + private: void incompatible(const char *func) const; TH1 const *accessRootObject(Access const &access, const char *func, int reqdim) const; TH1 *accessRootObject(AccessMut const &, const char *func, int reqdim) const; - void setAxisTimeOffset(double toffset, const char *option = "local", int axis = 1); - - /// true if ME is marked for deletion - bool markedToDelete() const { return data_.flags & DQMNet::DQM_PROP_MARKTODELETE; } - - /// Mark the object for deletion. - /// NB: make sure that the following method is not called simultaneously for the same ME - void markToDelete() { data_.flags |= DQMNet::DQM_PROP_MARKTODELETE; } - - /// reset "was updated" flag - void resetUpdate() { data_.flags &= ~DQMNet::DQM_PROP_NEW; } - - /// true if ME should be reset at end of monitoring cycle - bool resetMe() const { return data_.flags & DQMNet::DQM_PROP_RESET; } - TAxis const *getAxis(Access const &access, const char *func, int axis) const; TAxis *getAxis(AccessMut const &access, const char *func, int axis) const; - - void addProfiles(TProfile *h1, TProfile *h2, TProfile *sum, float c1, float c2); - void addProfiles(TProfile2D *h1, TProfile2D *h2, TProfile2D *sum, float c1, float c2); - void copyFunctions(TH1 *from, TH1 *to); - void copyFrom(TH1 *from); }; } // namespace dqm::impl // These may become distinct classes in the future. namespace dqm::reco { - typedef dqm::impl::MonitorElement MonitorElement; + using MonitorElement = dqm::impl::MonitorElement; } namespace dqm::legacy { class MonitorElement : public dqm::reco::MonitorElement { @@ -455,6 +459,8 @@ namespace dqm::legacy { // import constructors using dqm::reco::MonitorElement::MonitorElement; + // Add ROOT object accessors without cost here so that harvesting code can + // still freely use getTH1() and friends. using dqm::reco::MonitorElement::getRootObject; TObject *getRootObject() const override { return const_cast( @@ -503,7 +509,7 @@ namespace dqm::legacy { }; } // namespace dqm::legacy namespace dqm::harvesting { - typedef dqm::legacy::MonitorElement MonitorElement; + using MonitorElement = dqm::legacy::MonitorElement; } #endif // DQMSERVICES_CORE_MONITOR_ELEMENT_H diff --git a/DQMServices/Core/interface/QTest.h b/DQMServices/Core/interface/QTest.h index 5f14b59a074a3..6b67efbc7d359 100644 --- a/DQMServices/Core/interface/QTest.h +++ b/DQMServices/Core/interface/QTest.h @@ -110,8 +110,6 @@ class QCriterion { int verbose_; private: - /// for creating and deleting class instances - friend class dqm::dqmstoreimpl::DQMStore; /// for running the test friend class dqm::legacy::MonitorElement; friend class dqm::impl::MonitorElement; diff --git a/DQMServices/Core/src/DQMStore.cc b/DQMServices/Core/src/DQMStore.cc index dd9303d7613c1..095094b7c3820 100644 --- a/DQMServices/Core/src/DQMStore.cc +++ b/DQMServices/Core/src/DQMStore.cc @@ -112,18 +112,7 @@ namespace dqm::implementation { assert(th1); store_->debugTrackME("bookME (forceReplace)", nullptr, me); // surgically replace Histogram - // This is rather dangerous because the ME is in a global set and may - // be in use. We are protected by the booking lock here, but code - // filling in parallel might observe `isValid() == false`. - // Maybe replace this with a proper API to atomically replace the TH1. - auto medata = me->release(/* expectOwned */ true); - // Assume kind etc. matches. - // This should free the old object. - { - auto access = medata->accessMut(); - access.value.object_ = std::unique_ptr(th1); - } - me->switchData(medata); + me->switchObject(std::unique_ptr(th1)); } } diff --git a/DQMServices/Core/src/MonitorElement.cc b/DQMServices/Core/src/MonitorElement.cc index 7d5c96fafcc68..b7ca5d7b71497 100644 --- a/DQMServices/Core/src/MonitorElement.cc +++ b/DQMServices/Core/src/MonitorElement.cc @@ -42,7 +42,7 @@ namespace dqm::impl { MonitorElement::MonitorElement(MonitorElementData &&data) { this->mutable_ = new MutableMonitorElementData(); - this->mutable_.load()->data_ = std::move(data); + this->mutable_->data_ = std::move(data); this->is_owned_ = true; syncCoreObject(); } @@ -62,7 +62,7 @@ namespace dqm::impl { MutableMonitorElementData *MonitorElement::release(bool expectOwned) { assert(this->is_owned_ == expectOwned); - MutableMonitorElementData *data = this->mutable_.load(); + MutableMonitorElementData *data = this->mutable_; this->mutable_ = nullptr; this->is_owned_ = false; assert(!expectOwned || data); @@ -71,7 +71,7 @@ namespace dqm::impl { void MonitorElement::switchData(MonitorElement *other) { assert(other); - this->mutable_ = other->mutable_.load(); + this->mutable_ = other->mutable_; this->is_owned_ = false; syncCoreObject(); } @@ -82,6 +82,13 @@ namespace dqm::impl { syncCoreObject(); } + void MonitorElement::switchObject(std::unique_ptr &&newobject) { + auto access = this->accessMut(); + // Assume kind etc. matches. + // This should free the old object. + access.value.object_ = std::move(newobject); + } + void MonitorElement::syncCoreObject() { auto access = this->accessMut(); syncCoreObject(access); @@ -753,12 +760,6 @@ namespace dqm::impl { getAxis(access, __PRETTY_FUNCTION__, axis)->SetTimeFormat(format); } - /// set the time offset, if option = "gmt" then the offset is treated as a GMT time - void MonitorElement::setAxisTimeOffset(double toffset, const char *option /* ="local" */, int axis /* = 1 */) { - auto access = this->accessMut(); - getAxis(access, __PRETTY_FUNCTION__, axis)->SetTimeOffset(toffset, option); - } - /// set (ie. change) histogram/profile title void MonitorElement::setTitle(const std::string &title) { auto access = this->accessMut(); @@ -872,134 +873,6 @@ namespace dqm::impl { return access.value.scalar_.str; } - // implementation: Giuseppe.Della-Ricca@ts.infn.it - // Can be called with sum = h1 or sum = h2 - void MonitorElement::addProfiles(TProfile *h1, TProfile *h2, TProfile *sum, float c1, float c2) { - assert(h1); - assert(h2); - assert(sum); - - static const Int_t NUM_STAT = 6; - Double_t stats1[NUM_STAT]; - Double_t stats2[NUM_STAT]; - Double_t stats3[NUM_STAT]; - - bool isRebinOn = sum->CanExtendAllAxes(); - sum->SetCanExtend(TH1::kNoAxis); - - for (Int_t i = 0; i < NUM_STAT; ++i) - stats1[i] = stats2[i] = stats3[i] = 0; - - h1->GetStats(stats1); - h2->GetStats(stats2); - - for (Int_t i = 0; i < NUM_STAT; ++i) - stats3[i] = c1 * stats1[i] + c2 * stats2[i]; - - stats3[1] = c1 * TMath::Abs(c1) * stats1[1] + c2 * TMath::Abs(c2) * stats2[1]; - - Double_t entries = c1 * h1->GetEntries() + c2 * h2->GetEntries(); - TArrayD *h1sumw2 = h1->GetSumw2(); - TArrayD *h2sumw2 = h2->GetSumw2(); - for (Int_t bin = 0, nbin = sum->GetNbinsX() + 1; bin <= nbin; ++bin) { - Double_t entries = c1 * h1->GetBinEntries(bin) + c2 * h2->GetBinEntries(bin); - Double_t content = - c1 * h1->GetBinEntries(bin) * h1->GetBinContent(bin) + c2 * h2->GetBinEntries(bin) * h2->GetBinContent(bin); - Double_t error = - TMath::Sqrt(c1 * TMath::Abs(c1) * h1sumw2->fArray[bin] + c2 * TMath::Abs(c2) * h2sumw2->fArray[bin]); - sum->SetBinContent(bin, content); - sum->SetBinError(bin, error); - sum->SetBinEntries(bin, entries); - } - - sum->SetEntries(entries); - sum->PutStats(stats3); - if (isRebinOn) - sum->SetCanExtend(TH1::kAllAxes); - } - - // implementation: Giuseppe.Della-Ricca@ts.infn.it - // Can be called with sum = h1 or sum = h2 - void MonitorElement::addProfiles(TProfile2D *h1, TProfile2D *h2, TProfile2D *sum, float c1, float c2) { - assert(h1); - assert(h2); - assert(sum); - - static const Int_t NUM_STAT = 9; - Double_t stats1[NUM_STAT]; - Double_t stats2[NUM_STAT]; - Double_t stats3[NUM_STAT]; - - bool isRebinOn = sum->CanExtendAllAxes(); - sum->SetCanExtend(TH1::kNoAxis); - - for (Int_t i = 0; i < NUM_STAT; ++i) - stats1[i] = stats2[i] = stats3[i] = 0; - - h1->GetStats(stats1); - h2->GetStats(stats2); - - for (Int_t i = 0; i < NUM_STAT; i++) - stats3[i] = c1 * stats1[i] + c2 * stats2[i]; - - stats3[1] = c1 * TMath::Abs(c1) * stats1[1] + c2 * TMath::Abs(c2) * stats2[1]; - - Double_t entries = c1 * h1->GetEntries() + c2 * h2->GetEntries(); - TArrayD *h1sumw2 = h1->GetSumw2(); - TArrayD *h2sumw2 = h2->GetSumw2(); - for (Int_t xbin = 0, nxbin = sum->GetNbinsX() + 1; xbin <= nxbin; ++xbin) - for (Int_t ybin = 0, nybin = sum->GetNbinsY() + 1; ybin <= nybin; ++ybin) { - Int_t bin = sum->GetBin(xbin, ybin); - Double_t entries = c1 * h1->GetBinEntries(bin) + c2 * h2->GetBinEntries(bin); - Double_t content = - c1 * h1->GetBinEntries(bin) * h1->GetBinContent(bin) + c2 * h2->GetBinEntries(bin) * h2->GetBinContent(bin); - Double_t error = - TMath::Sqrt(c1 * TMath::Abs(c1) * h1sumw2->fArray[bin] + c2 * TMath::Abs(c2) * h2sumw2->fArray[bin]); - - sum->SetBinContent(bin, content); - sum->SetBinError(bin, error); - sum->SetBinEntries(bin, entries); - } - sum->SetEntries(entries); - sum->PutStats(stats3); - if (isRebinOn) - sum->SetCanExtend(TH1::kAllAxes); - } - - void MonitorElement::copyFunctions(TH1 *from, TH1 *to) { - update(); - TList *fromf = from->GetListOfFunctions(); - TList *tof = to->GetListOfFunctions(); - for (int i = 0, nfuncs = fromf ? fromf->GetSize() : 0; i < nfuncs; ++i) { - TObject *obj = fromf->At(i); - // not interested in statistics - if (!strcmp(obj->IsA()->GetName(), "TPaveStats")) - continue; - - if (auto *fn = dynamic_cast(obj)) - tof->Add(new TF1(*fn)); - //else if (dynamic_cast(obj)) - // ; // FIXME? tof->Add(new TPaveStats(*stats)); - else - raiseDQMError("MonitorElement", - "Cannot extract function '%s' of type" - " '%s' from monitor element '%s' for a copy", - obj->GetName(), - obj->IsA()->GetName(), - data_.objname.c_str()); - } - } - - void MonitorElement::copyFrom(TH1 *from) { - TH1 *orig = getTH1(); - if (orig->GetTitle() != from->GetTitle()) - orig->SetTitle(from->GetTitle()); - - orig->Add(from); - - copyFunctions(from, orig); - } - void MonitorElement::getQReport(bool create, const std::string &qtname, MonitorElementData::QReport *&qr, diff --git a/DQMServices/FwkIO/plugins/DQMRootSource.cc b/DQMServices/FwkIO/plugins/DQMRootSource.cc index be5a080a4225c..9b810c9f6d750 100644 --- a/DQMServices/FwkIO/plugins/DQMRootSource.cc +++ b/DQMServices/FwkIO/plugins/DQMRootSource.cc @@ -49,7 +49,9 @@ #include "format.h" -namespace { +// class rather than namespace so we can make this a friend of the +// MonitorElement to get access to constructors etc. +struct DQMTTreeIO { typedef dqm::harvesting::MonitorElement MonitorElement; typedef dqm::harvesting::DQMStore DQMStore; @@ -302,10 +304,9 @@ namespace { T m_buffer = 0; uint32_t m_tag = 0; }; +}; -} // namespace - -class DQMRootSource : public edm::PuttableSourceBase { +class DQMRootSource : public edm::PuttableSourceBase, DQMTTreeIO { public: DQMRootSource(edm::ParameterSet const&, const edm::InputSourceDescription&); ~DQMRootSource() override; From 44a7d3d24108ba8b98468faedc515404643724ae Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 30 Jan 2020 17:54:11 +0100 Subject: [PATCH 109/112] Remove setResetME from the historical modules still using it. --- .../LaserDQM/plugins/LaserDQMInitMonitors.cc | 434 ------------------ DQM/HLXMonitor/src/HLXMonitor.cc | 12 - 2 files changed, 446 deletions(-) diff --git a/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc b/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc index 8fa420682dac9..dfe38a796ff82 100644 --- a/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc +++ b/Alignment/LaserDQM/plugins/LaserDQMInitMonitors.cc @@ -14,1132 +14,794 @@ void LaserDQM::initMonitors() { theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam0"); theMEBeam0Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam0Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 1 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam1"); theMEBeam1Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam1Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc9PosAdcCounts->setResetMe(true); // plots for TEC2TEC beam 1 theMEBeam1Ring4Disc1PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc1PosTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc2PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc2PosTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc3PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc3PosTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc4PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc4PosTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc5PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc5PosTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 2 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam2"); theMEBeam2Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam2Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc9PosAdcCounts->setResetMe(true); // plots for TEC2TEC beam 2 theMEBeam2Ring4Disc1PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc1PosTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc2PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc2PosTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc3PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc3PosTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc4PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc4PosTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc5PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc5PosTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 3 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam3"); theMEBeam3Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam3Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 4 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam4"); theMEBeam4Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam4Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc9PosAdcCounts->setResetMe(true); // plots for TEC2TEC beam 4 theMEBeam4Ring4Disc1PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc1PosTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc2PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc2PosTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc3PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc3PosTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc4PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc4PosTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc5PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc5PosTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 5 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam5"); theMEBeam5Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam5Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 6 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam6"); theMEBeam6Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam6Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc9PosAdcCounts->setResetMe(true); // plots for TEC2TEC beam 6 theMEBeam6Ring4Disc1PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc1PosTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc2PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc2PosTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc3PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc3PosTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc4PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc4PosTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc5PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc5PosTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 7 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring4/Beam7"); theMEBeam7Ring4Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc1PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc2PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc3PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc4PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc5PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc6PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc7PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc8PosAdcCounts->setResetMe(true); theMEBeam7Ring4Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc9PosAdcCounts->setResetMe(true); // plots for TEC2TEC beam 7 theMEBeam7Ring4Disc1PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc1PosTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc2PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc2PosTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc3PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc3PosTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc4PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc4PosTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc5PosTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc5PosTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 0 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam0"); theMEBeam0Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam0Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 1 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam1"); theMEBeam1Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam1Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 2 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam2"); theMEBeam2Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam2Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 3 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam3"); theMEBeam3Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam3Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 4 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam4"); theMEBeam4Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam4Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 5 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam5"); theMEBeam5Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam5Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 6 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam6"); theMEBeam6Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam6Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc9PosAdcCounts->setResetMe(true); // ----- Adc counts for Beam 7 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/PosTEC/Ring6/Beam7"); theMEBeam7Ring6Disc1PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc1PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc2PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc2PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc3PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc3PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc4PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc4PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc5PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc5PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc6PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc6PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc7PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc7PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc8PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc8PosAdcCounts->setResetMe(true); theMEBeam7Ring6Disc9PosAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc9PosAdcCounts->setResetMe(true); /* LaserBeams in the TEC- */ // ----- Adc counts for Beam 0 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam0"); theMEBeam0Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam0Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 0 in Ring 4", 512, 0, 511); - theMEBeam0Ring4Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 1 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam1"); theMEBeam1Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam1Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc9NegAdcCounts->setResetMe(true); // plots for TEC2TEC beam 1 theMEBeam1Ring4Disc1NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc1NegTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc2NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc2NegTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc3NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc3NegTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc4NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc4NegTEC2TECAdcCounts->setResetMe(true); theMEBeam1Ring4Disc5NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 1 in Ring 4", 512, 0, 511); - theMEBeam1Ring4Disc5NegTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 2 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam2"); theMEBeam2Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam2Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc9NegAdcCounts->setResetMe(true); // plots for TEC2TEC beam 2 theMEBeam2Ring4Disc1NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc1NegTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc2NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc2NegTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc3NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc3NegTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc4NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc4NegTEC2TECAdcCounts->setResetMe(true); theMEBeam2Ring4Disc5NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 2 in Ring 4", 512, 0, 511); - theMEBeam2Ring4Disc5NegTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 3 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam3"); theMEBeam3Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam3Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 3 in Ring 4", 512, 0, 511); - theMEBeam3Ring4Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 4 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam4"); theMEBeam4Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam4Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc9NegAdcCounts->setResetMe(true); // plots for TEC2TEC beam 4 theMEBeam4Ring4Disc1NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc1NegTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc2NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc2NegTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc3NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc3NegTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc4NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc4NegTEC2TECAdcCounts->setResetMe(true); theMEBeam4Ring4Disc5NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 4 in Ring 4", 512, 0, 511); - theMEBeam4Ring4Disc5NegTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 5 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam5"); theMEBeam5Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam5Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 5 in Ring 4", 512, 0, 511); - theMEBeam5Ring4Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 6 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam6"); theMEBeam6Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam6Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc9NegAdcCounts->setResetMe(true); // plots for TEC2TEC beam 6 theMEBeam6Ring4Disc1NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc1NegTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc2NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc2NegTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc3NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc3NegTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc4NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc4NegTEC2TECAdcCounts->setResetMe(true); theMEBeam6Ring4Disc5NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 6 in Ring 4", 512, 0, 511); - theMEBeam6Ring4Disc5NegTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 7 in Ring 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring4/Beam7"); theMEBeam7Ring4Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc1NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc2NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc3NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc4NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc5NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc6NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc7NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc8NegAdcCounts->setResetMe(true); theMEBeam7Ring4Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc9NegAdcCounts->setResetMe(true); // plots for TEC2TEC beam 7 theMEBeam7Ring4Disc1NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1TEC2TEC", "Adc counts on Disc 1 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc1NegTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc2NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2TEC2TEC", "Adc counts on Disc 2 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc2NegTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc3NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3TEC2TEC", "Adc counts on Disc 3 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc3NegTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc4NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4TEC2TEC", "Adc counts on Disc 4 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc4NegTEC2TECAdcCounts->setResetMe(true); theMEBeam7Ring4Disc5NegTEC2TECAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5TEC2TEC", "Adc counts on Disc 5 for Beam 7 in Ring 4", 512, 0, 511); - theMEBeam7Ring4Disc5NegTEC2TECAdcCounts->setResetMe(true); // ----- Adc counts for Beam 0 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam0"); theMEBeam0Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam0Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 0 in Ring 6", 512, 0, 511); - theMEBeam0Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 1 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam1"); theMEBeam1Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam1Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 1 in Ring 6", 512, 0, 511); - theMEBeam1Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 2 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam2"); theMEBeam2Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam2Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 2 in Ring 6", 512, 0, 511); - theMEBeam2Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 3 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam3"); theMEBeam3Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam3Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 3 in Ring 6", 512, 0, 511); - theMEBeam3Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 4 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam4"); theMEBeam4Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam4Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 4 in Ring 6", 512, 0, 511); - theMEBeam4Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 5 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam5"); theMEBeam5Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam5Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 5 in Ring 6", 512, 0, 511); - theMEBeam5Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 6 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam6"); theMEBeam6Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam6Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 6 in Ring 6", 512, 0, 511); - theMEBeam6Ring6Disc9NegAdcCounts->setResetMe(true); // ----- Adc counts for Beam 7 in Ring 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/NegTEC/Ring6/Beam7"); theMEBeam7Ring6Disc1NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc1", "Adc counts on Disc 1 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc1NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc2NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc2", "Adc counts on Disc 2 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc2NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc3NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc3", "Adc counts on Disc 3 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc3NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc4NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc4", "Adc counts on Disc 4 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc4NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc5NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc5", "Adc counts on Disc 5 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc5NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc6NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc6", "Adc counts on Disc 6 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc6NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc7NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc7", "Adc counts on Disc 7 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc7NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc8NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc8", "Adc counts on Disc 8 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc8NegAdcCounts->setResetMe(true); theMEBeam7Ring6Disc9NegAdcCounts = theDaqMonitorBEI->book1D("AdcCountsDisc9", "Adc counts on Disc 9 for Beam 7 in Ring 6", 512, 0, 511); - theMEBeam7Ring6Disc9NegAdcCounts->setResetMe(true); /* LaserBeams in the TOB */ /**************************************** @@ -1157,169 +819,121 @@ void LaserDQM::initMonitors() { theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam0"); theMEBeam0TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 0 at z = 1040 mm", 512, 0, 511); - theMEBeam0TOBPosition1AdcCounts->setResetMe(true); theMEBeam0TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 0 at z = 580 mm", 512, 0, 511); - theMEBeam0TOBPosition2AdcCounts->setResetMe(true); theMEBeam0TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 0 at z = 220 mm", 512, 0, 511); - theMEBeam0TOBPosition3AdcCounts->setResetMe(true); theMEBeam0TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 0 at z = -140 mm", 512, 0, 511); - theMEBeam0TOBPosition4AdcCounts->setResetMe(true); theMEBeam0TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 0 at z = -500 mm", 512, 0, 511); - theMEBeam0TOBPosition5AdcCounts->setResetMe(true); theMEBeam0TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 0 at z = -860 mm", 512, 0, 511); - theMEBeam0TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 1 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam1"); theMEBeam1TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 1 at z = 1040 mm", 512, 0, 511); - theMEBeam1TOBPosition1AdcCounts->setResetMe(true); theMEBeam1TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 1 at z = 580 mm", 512, 0, 511); - theMEBeam1TOBPosition2AdcCounts->setResetMe(true); theMEBeam1TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 1 at z = 220 mm", 512, 0, 511); - theMEBeam1TOBPosition3AdcCounts->setResetMe(true); theMEBeam1TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 1 at z = -140 mm", 512, 0, 511); - theMEBeam1TOBPosition4AdcCounts->setResetMe(true); theMEBeam1TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 1 at z = -500 mm", 512, 0, 511); - theMEBeam1TOBPosition5AdcCounts->setResetMe(true); theMEBeam1TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 1 at z = -860 mm", 512, 0, 511); - theMEBeam1TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 2 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam2"); theMEBeam2TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 2 at z = 1040 mm", 512, 0, 511); - theMEBeam2TOBPosition1AdcCounts->setResetMe(true); theMEBeam2TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 2 at z = 580 mm", 512, 0, 511); - theMEBeam2TOBPosition2AdcCounts->setResetMe(true); theMEBeam2TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 2 at z = 220 mm", 512, 0, 511); - theMEBeam2TOBPosition3AdcCounts->setResetMe(true); theMEBeam2TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 2 at z = -140 mm", 512, 0, 511); - theMEBeam2TOBPosition4AdcCounts->setResetMe(true); theMEBeam2TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 2 at z = -500 mm", 512, 0, 511); - theMEBeam2TOBPosition5AdcCounts->setResetMe(true); theMEBeam2TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 2 at z = -860 mm", 512, 0, 511); - theMEBeam2TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 3 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam3"); theMEBeam3TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 3 at z = 1040 mm", 512, 0, 511); - theMEBeam3TOBPosition1AdcCounts->setResetMe(true); theMEBeam3TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 3 at z = 580 mm", 512, 0, 511); - theMEBeam3TOBPosition2AdcCounts->setResetMe(true); theMEBeam3TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 3 at z = 220 mm", 512, 0, 511); - theMEBeam3TOBPosition3AdcCounts->setResetMe(true); theMEBeam3TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 3 at z = -140 mm", 512, 0, 511); - theMEBeam3TOBPosition4AdcCounts->setResetMe(true); theMEBeam3TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 3 at z = -500 mm", 512, 0, 511); - theMEBeam3TOBPosition5AdcCounts->setResetMe(true); theMEBeam3TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 3 at z = -860 mm", 512, 0, 511); - theMEBeam3TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam4"); theMEBeam4TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 4 at z = 1040 mm", 512, 0, 511); - theMEBeam4TOBPosition1AdcCounts->setResetMe(true); theMEBeam4TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 4 at z = 580 mm", 512, 0, 511); - theMEBeam4TOBPosition2AdcCounts->setResetMe(true); theMEBeam4TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 4 at z = 220 mm", 512, 0, 511); - theMEBeam4TOBPosition3AdcCounts->setResetMe(true); theMEBeam4TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 4 at z = -140 mm", 512, 0, 511); - theMEBeam4TOBPosition4AdcCounts->setResetMe(true); theMEBeam4TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 4 at z = -500 mm", 512, 0, 511); - theMEBeam4TOBPosition5AdcCounts->setResetMe(true); theMEBeam4TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 4 at z = -860 mm", 512, 0, 511); - theMEBeam4TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 5 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam5"); theMEBeam5TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 5 at z = 1040 mm", 512, 0, 511); - theMEBeam5TOBPosition1AdcCounts->setResetMe(true); theMEBeam5TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 5 at z = 580 mm", 512, 0, 511); - theMEBeam5TOBPosition2AdcCounts->setResetMe(true); theMEBeam5TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 5 at z = 220 mm", 512, 0, 511); - theMEBeam5TOBPosition3AdcCounts->setResetMe(true); theMEBeam5TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 5 at z = -140 mm", 512, 0, 511); - theMEBeam5TOBPosition4AdcCounts->setResetMe(true); theMEBeam5TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 5 at z = -500 mm", 512, 0, 511); - theMEBeam5TOBPosition5AdcCounts->setResetMe(true); theMEBeam5TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 5 at z = -860 mm", 512, 0, 511); - theMEBeam5TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam6"); theMEBeam6TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 6 at z = 1040 mm", 512, 0, 511); - theMEBeam6TOBPosition1AdcCounts->setResetMe(true); theMEBeam6TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 6 at z = 580 mm", 512, 0, 511); - theMEBeam6TOBPosition2AdcCounts->setResetMe(true); theMEBeam6TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 6 at z = 220 mm", 512, 0, 511); - theMEBeam6TOBPosition3AdcCounts->setResetMe(true); theMEBeam6TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 6 at z = -140 mm", 512, 0, 511); - theMEBeam6TOBPosition4AdcCounts->setResetMe(true); theMEBeam6TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 6 at z = -500 mm", 512, 0, 511); - theMEBeam6TOBPosition5AdcCounts->setResetMe(true); theMEBeam6TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 6 at z = -860 mm", 512, 0, 511); - theMEBeam6TOBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 7 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TOB/Beam7"); theMEBeam7TOBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=1040mm", "Adc counts for Beam 7 at z = 1040 mm", 512, 0, 511); - theMEBeam7TOBPosition1AdcCounts->setResetMe(true); theMEBeam7TOBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=580mm", "Adc counts for Beam 7 at z = 580 mm", 512, 0, 511); - theMEBeam7TOBPosition2AdcCounts->setResetMe(true); theMEBeam7TOBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=220mm", "Adc counts for Beam 7 at z = 220 mm", 512, 0, 511); - theMEBeam7TOBPosition3AdcCounts->setResetMe(true); theMEBeam7TOBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-140mm", "Adc counts for Beam 7 at z = -140 mm", 512, 0, 511); - theMEBeam7TOBPosition4AdcCounts->setResetMe(true); theMEBeam7TOBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-500mm", "Adc counts for Beam 7 at z = -500 mm", 512, 0, 511); - theMEBeam7TOBPosition5AdcCounts->setResetMe(true); theMEBeam7TOBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-860mm", "Adc counts for Beam 7 at z = -860 mm", 512, 0, 511); - theMEBeam7TOBPosition6AdcCounts->setResetMe(true); /* LaserBeams in the TIB */ /**************************************** @@ -1337,167 +951,119 @@ void LaserDQM::initMonitors() { theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam0"); theMEBeam0TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 0 at z = 620 mm", 512, 0, 511); - theMEBeam0TIBPosition1AdcCounts->setResetMe(true); theMEBeam0TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 0 at z = 380 mm", 512, 0, 511); - theMEBeam0TIBPosition2AdcCounts->setResetMe(true); theMEBeam0TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 0 at z = 180 mm", 512, 0, 511); - theMEBeam0TIBPosition3AdcCounts->setResetMe(true); theMEBeam0TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 0 at z = -100 mm", 512, 0, 511); - theMEBeam0TIBPosition4AdcCounts->setResetMe(true); theMEBeam0TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 0 at z = -340 mm", 512, 0, 511); - theMEBeam0TIBPosition5AdcCounts->setResetMe(true); theMEBeam0TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 0 at z = -540 mm", 512, 0, 511); - theMEBeam0TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 1 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam1"); theMEBeam1TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 1 at z = 620 mm", 512, 0, 511); - theMEBeam1TIBPosition1AdcCounts->setResetMe(true); theMEBeam1TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 1 at z = 380 mm", 512, 0, 511); - theMEBeam1TIBPosition2AdcCounts->setResetMe(true); theMEBeam1TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 1 at z = 180 mm", 512, 0, 511); - theMEBeam1TIBPosition3AdcCounts->setResetMe(true); theMEBeam1TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 1 at z = -100 mm", 512, 0, 511); - theMEBeam1TIBPosition4AdcCounts->setResetMe(true); theMEBeam1TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 1 at z = -340 mm", 512, 0, 511); - theMEBeam1TIBPosition5AdcCounts->setResetMe(true); theMEBeam1TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 1 at z = -540 mm", 512, 0, 511); - theMEBeam1TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 2 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam2"); theMEBeam2TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 2 at z = 620 mm", 512, 0, 511); - theMEBeam2TIBPosition1AdcCounts->setResetMe(true); theMEBeam2TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 2 at z = 380 mm", 512, 0, 511); - theMEBeam2TIBPosition2AdcCounts->setResetMe(true); theMEBeam2TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 2 at z = 180 mm", 512, 0, 511); - theMEBeam2TIBPosition3AdcCounts->setResetMe(true); theMEBeam2TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 2 at z = -100 mm", 512, 0, 511); - theMEBeam2TIBPosition4AdcCounts->setResetMe(true); theMEBeam2TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 2 at z = -340 mm", 512, 0, 511); - theMEBeam2TIBPosition5AdcCounts->setResetMe(true); theMEBeam2TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 2 at z = -540 mm", 512, 0, 511); - theMEBeam2TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 3 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam3"); theMEBeam3TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 3 at z = 620 mm", 512, 0, 511); - theMEBeam3TIBPosition1AdcCounts->setResetMe(true); theMEBeam3TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 3 at z = 380 mm", 512, 0, 511); - theMEBeam3TIBPosition2AdcCounts->setResetMe(true); theMEBeam3TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 3 at z = 180 mm", 512, 0, 511); - theMEBeam3TIBPosition3AdcCounts->setResetMe(true); theMEBeam3TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 3 at z = -100 mm", 512, 0, 511); - theMEBeam3TIBPosition4AdcCounts->setResetMe(true); theMEBeam3TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 3 at z = -340 mm", 512, 0, 511); - theMEBeam3TIBPosition5AdcCounts->setResetMe(true); theMEBeam3TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 3 at z = -540 mm", 512, 0, 511); - theMEBeam3TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 4 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam4"); theMEBeam4TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 4 at z = 620 mm", 512, 0, 511); - theMEBeam4TIBPosition1AdcCounts->setResetMe(true); theMEBeam4TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 4 at z = 380 mm", 512, 0, 511); - theMEBeam4TIBPosition2AdcCounts->setResetMe(true); theMEBeam4TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 4 at z = 180 mm", 512, 0, 511); - theMEBeam4TIBPosition3AdcCounts->setResetMe(true); theMEBeam4TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 4 at z = -100 mm", 512, 0, 511); - theMEBeam4TIBPosition4AdcCounts->setResetMe(true); theMEBeam4TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 4 at z = -340 mm", 512, 0, 511); - theMEBeam4TIBPosition5AdcCounts->setResetMe(true); theMEBeam4TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 4 at z = -540 mm", 512, 0, 511); - theMEBeam4TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 5 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam5"); theMEBeam5TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 5 at z = 620 mm", 512, 0, 511); - theMEBeam5TIBPosition1AdcCounts->setResetMe(true); theMEBeam5TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 5 at z = 380 mm", 512, 0, 511); - theMEBeam5TIBPosition2AdcCounts->setResetMe(true); theMEBeam5TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 5 at z = 180 mm", 512, 0, 511); - theMEBeam5TIBPosition3AdcCounts->setResetMe(true); theMEBeam5TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 5 at z = -100 mm", 512, 0, 511); - theMEBeam5TIBPosition4AdcCounts->setResetMe(true); theMEBeam5TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 5 at z = -340 mm", 512, 0, 511); - theMEBeam5TIBPosition5AdcCounts->setResetMe(true); theMEBeam5TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 5 at z = -540 mm", 512, 0, 511); - theMEBeam5TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 6 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam6"); theMEBeam6TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 6 at z = 620 mm", 512, 0, 511); - theMEBeam6TIBPosition1AdcCounts->setResetMe(true); theMEBeam6TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 6 at z = 380 mm", 512, 0, 511); - theMEBeam6TIBPosition2AdcCounts->setResetMe(true); theMEBeam6TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 6 at z = 180 mm", 512, 0, 511); - theMEBeam6TIBPosition3AdcCounts->setResetMe(true); theMEBeam6TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 6 at z = -100 mm", 512, 0, 511); - theMEBeam6TIBPosition4AdcCounts->setResetMe(true); theMEBeam6TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 6 at z = -340 mm", 512, 0, 511); - theMEBeam6TIBPosition5AdcCounts->setResetMe(true); theMEBeam6TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 6 at z = -540 mm", 512, 0, 511); - theMEBeam6TIBPosition6AdcCounts->setResetMe(true); // ----- Adc Counts in Beam 7 theDaqMonitorBEI->setCurrentFolder("LaserAlignment/TIB/Beam7"); theMEBeam7TIBPosition1AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=620mm", "Adc counts for Beam 7 at z = 620 mm", 512, 0, 511); - theMEBeam7TIBPosition1AdcCounts->setResetMe(true); theMEBeam7TIBPosition2AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=380mm", "Adc counts for Beam 7 at z = 380 mm", 512, 0, 511); - theMEBeam7TIBPosition2AdcCounts->setResetMe(true); theMEBeam7TIBPosition3AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=180mm", "Adc counts for Beam 7 at z = 180 mm", 512, 0, 511); - theMEBeam7TIBPosition3AdcCounts->setResetMe(true); theMEBeam7TIBPosition4AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-100mm", "Adc counts for Beam 7 at z = -100 mm", 512, 0, 511); - theMEBeam7TIBPosition4AdcCounts->setResetMe(true); theMEBeam7TIBPosition5AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-340mm", "Adc counts for Beam 7 at z = -340 mm", 512, 0, 511); - theMEBeam7TIBPosition5AdcCounts->setResetMe(true); theMEBeam7TIBPosition6AdcCounts = theDaqMonitorBEI->book1D("AdcCountsZ=-540mm", "Adc counts for Beam 7 at z = -540 mm", 512, 0, 511); - theMEBeam7TIBPosition6AdcCounts->setResetMe(true); } diff --git a/DQM/HLXMonitor/src/HLXMonitor.cc b/DQM/HLXMonitor/src/HLXMonitor.cc index eb1179a652d56..b30f2b1cdd15d 100644 --- a/DQM/HLXMonitor/src/HLXMonitor.cc +++ b/DQM/HLXMonitor/src/HLXMonitor.cc @@ -197,18 +197,6 @@ void HLXMonitor::SetupHists(DQMStore::IBooker &iBooker) { } } - if (!Accumulate) { - for (unsigned int iWedge = 0; iWedge < NUM_HLX; ++iWedge) { - Set1Below[iWedge]->setResetMe(true); - Set1Between[iWedge]->setResetMe(true); - Set1Above[iWedge]->setResetMe(true); - Set2Below[iWedge]->setResetMe(true); - Set2Between[iWedge]->setResetMe(true); - Set2Above[iWedge]->setResetMe(true); - ETSum[iWedge]->setResetMe(true); - } - } - if (Style == "BX") { OccXAxisTitle = "Bunch Crossing"; OccYAxisTitle = "Tower Occupancy"; From b26295def7aa71535177eda50cce8738d0171987 Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Thu, 30 Jan 2020 18:33:15 +0100 Subject: [PATCH 110/112] Remove some other usages of removed ME APIs. Not sure it the update() in beammonitor was really unnecessary, but getTH1() should also mark the ME as dirty. --- DQM/BeamMonitor/plugins/BeamMonitor.cc | 2 -- DQM/SiStripMonitorClient/src/SiStripUtility.cc | 9 ++++++--- DQM/TrackingMonitorClient/src/TrackingUtility.cc | 9 ++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/DQM/BeamMonitor/plugins/BeamMonitor.cc b/DQM/BeamMonitor/plugins/BeamMonitor.cc index 39363b6b8ac1e..2be6bd823d3be 100644 --- a/DQM/BeamMonitor/plugins/BeamMonitor.cc +++ b/DQM/BeamMonitor/plugins/BeamMonitor.cc @@ -1022,7 +1022,6 @@ void BeamMonitor::FitAndFill(const LuminosityBlock& lumiSeg, int& lastlumi, int& auto tmphisto = h_PVy[0]->getTH1F(); h_PVy[1]->getTH1()->SetBins( tmphisto->GetNbinsX(), tmphisto->GetXaxis()->GetXmin(), tmphisto->GetXaxis()->GetXmax()); - h_PVy[1]->update(); h_PVy[1]->Reset(); h_PVy[1]->getTH1()->Add(tmphisto); h_PVy[1]->getTH1()->Fit(fgaus.get(), "QLM"); @@ -1053,7 +1052,6 @@ void BeamMonitor::FitAndFill(const LuminosityBlock& lumiSeg, int& lastlumi, int& auto tmphisto = h_PVz[0]->getTH1F(); h_PVz[1]->getTH1()->SetBins( tmphisto->GetNbinsX(), tmphisto->GetXaxis()->GetXmin(), tmphisto->GetXaxis()->GetXmax()); - h_PVz[1]->update(); h_PVz[1]->Reset(); h_PVz[1]->getTH1()->Add(tmphisto); h_PVz[1]->getTH1()->Fit(fgaus.get(), "QLM"); diff --git a/DQM/SiStripMonitorClient/src/SiStripUtility.cc b/DQM/SiStripMonitorClient/src/SiStripUtility.cc index 231c53a8164cc..231968a8164e1 100644 --- a/DQM/SiStripMonitorClient/src/SiStripUtility.cc +++ b/DQM/SiStripMonitorClient/src/SiStripUtility.cc @@ -238,9 +238,12 @@ int SiStripUtility::getMEStatus(MonitorElement const* me, int& bad_channels) { // void SiStripUtility::getMEValue(MonitorElement const* me, std::string& val) { val = ""; - if (me && (me->kind() == MonitorElement::Kind::REAL || me->kind() == MonitorElement::Kind::INT)) { - val = me->valueString(); - val = val.substr(val.find("=") + 1); + if (me) { + if (me->kind() == MonitorElement::Kind::REAL) { + val = std::to_string(me->getFloatValue()); + } else if (me->kind() == MonitorElement::Kind::INT) { + val = std::to_string(me->getIntValue()); + } } } // diff --git a/DQM/TrackingMonitorClient/src/TrackingUtility.cc b/DQM/TrackingMonitorClient/src/TrackingUtility.cc index d788b26959bc4..10e46312b707e 100644 --- a/DQM/TrackingMonitorClient/src/TrackingUtility.cc +++ b/DQM/TrackingMonitorClient/src/TrackingUtility.cc @@ -179,9 +179,12 @@ int TrackingUtility::getMEStatus(MonitorElement* me, int& bad_channels) { // void TrackingUtility::getMEValue(MonitorElement* me, std::string& val) { val = ""; - if (me && (me->kind() == MonitorElement::Kind::REAL || me->kind() == MonitorElement::Kind::INT)) { - val = me->valueString(); - val = val.substr(val.find("=") + 1); + if (me) { + if (me->kind() == MonitorElement::Kind::REAL) { + val = std::to_string(me->getFloatValue()); + } else if (me->kind() == MonitorElement::Kind::INT) { + val = std::to_string(me->getIntValue()); + } } } // From 6ebd8f523c807bf5fb093839999f20631077f27c Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Fri, 31 Jan 2020 12:34:23 +0100 Subject: [PATCH 111/112] Properly cleanup in case of exceptions. This is not functionally required, but makes debugging much easier, in case a different thread continues using the IBooker while the first thread handles the exception. --- DQMServices/Core/interface/DQMStore.h | 53 ++++++++++++++++++--------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/DQMServices/Core/interface/DQMStore.h b/DQMServices/Core/interface/DQMStore.h index f4fc53651ec2f..67d84b711804c 100644 --- a/DQMServices/Core/interface/DQMStore.h +++ b/DQMServices/Core/interface/DQMStore.h @@ -590,23 +590,42 @@ namespace dqm { // be removed. auto lock = std::scoped_lock(this->booking_mutex_); - IBooker& booker = *this; - auto newscope = MonitorElementData::Scope::RUN; - if (canSaveByLumi && this->doSaveByLumi_) { - newscope = MonitorElementData::Scope::LUMI; - } - auto oldscope = booker.setScope(newscope); - assert(moduleId != 0 || !"moduleID must be set for normal booking transaction"); - auto oldmoduleid = booker.setModuleID(moduleId); - assert(oldmoduleid == 0 || !"Nested booking transaction?"); - // always book prototypes (except for Scope::JOB, where we can use these directly). - auto oldrunlumi = booker.setRunLumi(edm::LuminosityBlockID()); - - f(booker); - - booker.setScope(oldscope); - this->setModuleID(oldmoduleid); - this->setRunLumi(oldrunlumi); + // This is to make sure everything gets reset in case of an exception. + // That is not really required (an exception here will crash the job + // anyways, and it is technically not required to reset everything), but + // it prevents misleading error messages in other threads. + struct ModuleIdScope { + IBooker& booker_; + uint64_t oldid_; + MonitorElementData::Scope oldscope_; + edm::LuminosityBlockID oldrunlumi_; + ModuleIdScope(IBooker& booker, + uint64_t newid, + MonitorElementData::Scope newscope, + edm::LuminosityBlockID newrunlumi) + : booker_(booker) { + oldid_ = booker_.setModuleID(newid); + oldscope_ = booker_.setScope(newscope); + oldrunlumi_ = booker_.setRunLumi(newrunlumi); + assert(newid != 0 || !"moduleID must be set for normal booking transaction"); + assert(oldid_ == 0 || !"Nested booking transaction?"); + } + ~ModuleIdScope() { + booker_.setModuleID(oldid_); + booker_.setScope(oldscope_); + booker_.setRunLumi(oldrunlumi_); + } + }; + + ModuleIdScope booker( + *this, + moduleId, + // enable per-lumi-by-default here + canSaveByLumi && this->doSaveByLumi_ ? MonitorElementData::Scope::LUMI : MonitorElementData::Scope::RUN, + // always book prototypes + edm::LuminosityBlockID()); + + f(booker.booker_); }; template From 2f51722c107e3813da767d6ce5adbf49bc818cbb Mon Sep 17 00:00:00 2001 From: Marcel Andre Schneider Date: Tue, 4 Feb 2020 09:31:40 +0100 Subject: [PATCH 112/112] Change std::cout to edm::LogInfo. --- DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc b/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc index a095d636002f0..35928e57ca74e 100644 --- a/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc +++ b/DQMServices/Demo/plugins/DemoNormalDQMEDAnalyzer.cc @@ -70,7 +70,8 @@ void DemoNormalDQMEDAnalyzer::bookHistograms(DQMStore::IBooker& ibook, edm::EventSetup const& iSetup) { ibook.setCurrentFolder(folder_); - example_ = ibook.book1D("EXAMPLE", "Example 1D", 20, 0., 10., [](TH1*) { std::cout << "booked!\n"; }); + example_ = ibook.book1D( + "EXAMPLE", "Example 1D", 20, 0., 10., [](TH1*) { edm::LogInfo("DemoNormalDQMEDAnalyzer") << "booked!\n"; }); example2D_ = ibook.book2D("EXAMPLE_2D", "Example 2D", 20, 0, 20, 15, 0, 15); example3D_ = ibook.book3D("EXAMPLE_3D", "Example 3D", 20, 0, 20, 15, 0, 15, 25, 0, 25); exampleTProfile_ = ibook.bookProfile("EXAMPLE_TPROFILE", "Example TProfile", 20, 0, 20, 15, 0, 15);