Merge pull request #34111 from cvuosalo/dd4hep-payloads-3

[DD4hep] Tool for creation of DD4hep geometry DB payloads and for migration validation

CondTools/Geometry/plugins/BuildFile.xml

@@ -2,6 +2,7 @@
 <use name="CondCore/DBOutputService"/>
 <use name="CondFormats/Common"/>
 <use name="CondFormats/GeometryObjects"/>
+<use name="DataFormats/Math"/>
 <use name="DetectorDescription/Core"/>
 <use name="DetectorDescription/DDCMS"/>
 <use name="FWCore/Framework"/>

CondTools/Geometry/plugins/PGeometricDetBuilder.cc

@@ -8,6 +8,7 @@
 #include "FWCore/Framework/interface/ESHandle.h"
 #include "FWCore/Framework/interface/MakerMacros.h"
 #include "CondFormats/GeometryObjects/interface/PGeometricDet.h"
+#include "DataFormats/Math/interface/Rounding.h"
 #include "Geometry/Records/interface/IdealGeometryRecord.h"
 #include "Geometry/TrackerNumberingBuilder/interface/GeometricDet.h"
 #include "DetectorDescription/DDCMS/interface/DDCompactView.h"
@@ -136,20 +137,24 @@ void PGeometricDetBuilder::putOne(const GeometricDet* gd, PGeometricDet* pgd, in
   item._name = gd->name();
   item._ns = std::string();
   item._level = lev;
-  item._x = tran.X();
-  item._y = tran.Y();
-  item._z = tran.Z();
+  using cms_rounding::roundIfNear0;
+  const double tol = 1.e-10;
+  // Round very small calculated values to 0 to avoid discrepancies
+  // between +0 and -0 in comparisons.
+  item._x = roundIfNear0(tran.X(), tol);
+  item._y = roundIfNear0(tran.Y(), tol);
+  item._z = roundIfNear0(tran.Z(), tol);
   item._phi = gd->phi();
   item._rho = gd->rho();
-  item._a11 = x.X();
-  item._a12 = y.X();
-  item._a13 = z.X();
-  item._a21 = x.Y();
-  item._a22 = y.Y();
-  item._a23 = z.Y();
-  item._a31 = x.Z();
-  item._a32 = y.Z();
-  item._a33 = z.Z();
+  item._a11 = roundIfNear0(x.X(), tol);
+  item._a12 = roundIfNear0(y.X(), tol);
+  item._a13 = roundIfNear0(z.X(), tol);
+  item._a21 = roundIfNear0(x.Y(), tol);
+  item._a22 = roundIfNear0(y.Y(), tol);
+  item._a23 = roundIfNear0(z.Y(), tol);
+  item._a31 = roundIfNear0(x.Z(), tol);
+  item._a32 = roundIfNear0(y.Z(), tol);
+  item._a33 = roundIfNear0(z.Z(), tol);
   item._shape = static_cast<int>(gd->shape_dd4hep());
   item._type = gd->type();
   if (gd->shape_dd4hep() == cms::DDSolidShape::ddbox) {

CondTools/Geometry/test/writehelpers/createExtended2021DD4hepPayloads.sh

@@ -0,0 +1,93 @@
+#!/bin/sh
+
+
+if [ $# -ne 1 ]
+then
+  echo Error: createExtended2021Payloads.sh requires exactly one argument which is the tag
+  exit 1
+fi
+mytag=$1
+echo ${mytag}
+
+# Set the tag in all the scripts and the metadata text files
+sed -i {s/TagXX/${mytag}/g} *.py
+compgen -G "*.txt" > /dev/null && sed -i {s/TagXX/${mytag}/g} *.txt
+sed -i {s/TagXX/${mytag}/g} splitExtended2021Database.sh
+
+# First read in the little XML files and create the
+# large XML file for the Phase1_R30F12_HCal Ideal scenario.
+# Input cff                                             Output file
+# GeometryExtended2021_cff                       geSingleBigFile.xml
+cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# Now convert the content of the large XML file into
+# a "blob" and write it to the database.
+# Also reads in the little XML files again and fills
+# the DDCompactView. From the DDCompactView the
+# reco parts of the database are also filled.
+cmsRun geometryExtended2021DD4hep_writer.py
+
+# Now put the other scenarios into the database.
+# Input the many XML files referenced by the cff file and
+# output a single big XML file.
+# This is repeated several times below.  The sed commands
+# serve to give the following sequence of input and output
+# files
+#
+# Input cff                    Output file
+# GeometryIdeal_cff            giSingleBigFile.xml
+#
+# sed -i '{s/Extended2021/Extended2021ZeroMaterial/g}' geometryExtended2021DD4hep_xmlwriter.py
+# sed -i '{s/\/ge/\/gez/g}' geometryExtended2021DD4hep_xmlwriter.py
+# cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# sed -i '{s/Extended2021ZeroMaterial/Extended2021FlatMinus05Percent/g}' geometryExtended2021DD4hep_xmlwriter.py
+# sed -i '{s/\/gez/\/geFM05/g}' geometryExtended2021DD4hep_xmlwriter.py
+# cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# sed -i '{s/Extended2021FlatMinus05Percent/Extended2021FlatMinus10Percent/g}' geometryExtended2021DD4hep_xmlwriter.py
+# sed -i '{s/\/geFM05/\/geFM10/g}' geometryExtended2021DD4hep_xmlwriter.py
+# cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# sed -i '{s/Extended2021FlatMinus10Percent/Extended2021FlatPlus05Percent/g}' geometryExtended2021DD4hep_xmlwriter.py
+# sed -i '{s/\/geFM10/\/geFP05/g}' geometryExtended2021DD4hep_xmlwriter.py
+# cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# sed -i '{s/Extended2021FlatPlus05Percent/Extended2021FlatPlus10Percent/g}' geometryExtended2021DD4hep_xmlwriter.py
+# sed -i '{s/\/geFP05/\/geFP10/g}' geometryExtended2021DD4hep_xmlwriter.py
+# cmsRun geometryExtended2021DD4hep_xmlwriter.py
+
+# Read the one big XML file and output a record to the
+# database with the an identifying tag
+# This is repeated several times below.  The sed commands
+# serve to give the following sequence of input file and output
+# tag
+#
+# Input file                Output tag
+# gezSingleBigFile.xml      XMLFILE_Geometry_${mytag}_Extended2021ZeroMaterial_mc
+#
+# sed -i '{s/Extended/Extended2021ZeroMaterial/g}' xmlgeometrywriter.py
+# sed -i '{s/\/ge/\/gez/g}' xmlgeometrywriter.py
+# cmsRun xmlgeometrywriter.py
+
+# sed -i '{s/Extended2021ZeroMaterial/Extended2021FlatMinus05Percent/g}' xmlgeometrywriter.py
+# sed -i '{s/\/gez/\/geFM05/g}' xmlgeometrywriter.py
+# cmsRun xmlgeometrywriter.py
+
+# sed -i '{s/Extended2021FlatMinus05Percent/Extended2021FlatMinus10Percent/g}' xmlgeometrywriter.py
+# sed -i '{s/\/geFM05/\/geFM10/g}' xmlgeometrywriter.py
+# cmsRun xmlgeometrywriter.py
+
+# sed -i '{s/Extended2021FlatMinus10Percent/Extended2021FlatPlus05Percent/g}' xmlgeometrywriter.py
+# sed -i '{s/\/geFM10/\/geFP05/g}' xmlgeometrywriter.py
+# cmsRun xmlgeometrywriter.py
+
+# sed -i '{s/Extended2021FlatPlus05Percent/Extended2021FlatPlus10Percent/g}' xmlgeometrywriter.py
+# sed -i '{s/\/geFP05/\/geFP10/g}' xmlgeometrywriter.py
+# cmsRun xmlgeometrywriter.py
+
+# All the database objects were written into one database
+# (myfile.db) in the steps above.  Extract the different
+# pieces into separate database files.  These are the payloads
+# that get uploaded to the dropbox.  There is one for each tag
+./splitExtended2021Database.sh

CondTools/Geometry/test/writehelpers/geometryExtended2021DD4hep_writer.py

@@ -0,0 +1,80 @@
+import FWCore.ParameterSet.Config as cms
+from Configuration.Eras.Era_Run3_dd4hep_cff import Run3_dd4hep
+
+process = cms.Process("GeometryWriter", Run3_dd4hep)
+
+from Configuration.ProcessModifiers.dd4hep_cff import dd4hep
+
+process.load('CondCore.CondDB.CondDB_cfi')
+
+process.load('Configuration.Geometry.GeometryDD4hepExtended2021_cff')
+process.load('Geometry.CaloEventSetup.CaloGeometryDBWriter_cfi')
+process.load('CondTools.Geometry.HcalParametersWriter_cff')
+process.load("Geometry.MuonNumbering.muonGeometryConstants_cff")
+
+process.CaloGeometryBuilder = cms.ESProducer("CaloGeometryBuilder",
+    SelectedCalos = cms.vstring(
+        'HCAL',
+        'ZDC',
+        'EcalBarrel',
+        'EcalEndcap',
+        'EcalPreshower',
+        'TOWER'
+    )
+)
+
+process.source = cms.Source("EmptyIOVSource",
+                            lastValue = cms.uint64(1),
+                            timetype = cms.string('runnumber'),
+                            firstValue = cms.uint64(1),
+                            interval = cms.uint64(1)
+                            )
+
+# This reads the big XML file and the only way to fill the
+# nonreco part of the database is to read this file.
+process.XMLGeometryWriter = cms.EDAnalyzer("XMLGeometryBuilder",
+                                           XMLFileName = cms.untracked.string("./geSingleBigFile.xml"),
+                                           ZIP = cms.untracked.bool(True)
+                                           )
+
+process.TrackerGeometryWriter = cms.EDAnalyzer("PGeometricDetBuilder",fromDD4hep=cms.bool(True))
+process.TrackerParametersWriter = cms.EDAnalyzer("PTrackerParametersDBBuilder",fromDD4hep=cms.bool(True))
+
+process.CaloGeometryWriter = cms.EDAnalyzer("PCaloGeometryBuilder",fromDD4Hep = cms.untracked.bool(True))
+
+process.CSCGeometryWriter = cms.EDAnalyzer("CSCRecoIdealDBLoader",fromDD4Hep = cms.untracked.bool(True))
+
+process.DTGeometryWriter = cms.EDAnalyzer("DTRecoIdealDBLoader",fromDD4Hep = cms.untracked.bool(True))
+
+process.RPCGeometryWriter = cms.EDAnalyzer("RPCRecoIdealDBLoader",fromDD4Hep = cms.untracked.bool(True))
+
+process.GEMGeometryWriter = cms.EDAnalyzer("GEMRecoIdealDBLoader",fromDD4Hep = cms.untracked.bool(True))
+
+process.CondDB.timetype = cms.untracked.string('runnumber')
+process.CondDB.connect = cms.string('sqlite_file:myfile.db')
+process.PoolDBOutputService = cms.Service("PoolDBOutputService",
+                                          process.CondDB,
+                                          toPut = cms.VPSet(cms.PSet(record = cms.string('GeometryFileRcd'),tag = cms.string('XMLFILE_Geometry_TagXX_Extended2021_mc')),
+                                                            cms.PSet(record = cms.string('IdealGeometryRecord'),tag = cms.string('TKRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PTrackerParametersRcd'),tag = cms.string('TKParameters_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PEcalBarrelRcd'),   tag = cms.string('EBRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PEcalEndcapRcd'),   tag = cms.string('EERECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PEcalPreshowerRcd'),tag = cms.string('EPRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PHcalRcd'),         tag = cms.string('HCALRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('HcalParametersRcd'), tag = cms.string('HCALParameters_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PCaloTowerRcd'),    tag = cms.string('CTRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PZdcRcd'),          tag = cms.string('ZDCRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('PCastorRcd'),       tag = cms.string('CASTORRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('CSCRecoGeometryRcd'),tag = cms.string('CSCRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('CSCRecoDigiParametersRcd'),tag = cms.string('CSCRECODIGI_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('DTRecoGeometryRcd'),tag = cms.string('DTRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('RPCRecoGeometryRcd'),tag = cms.string('RPCRECO_Geometry_TagXX')),
+                                                            cms.PSet(record = cms.string('GEMRecoGeometryRcd'),tag = cms.string('GEMRECO_Geometry_TagXX'))
+                                                        )
+                                    )
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(1)
+    )
+
+process.p1 = cms.Path(process.XMLGeometryWriter+process.TrackerGeometryWriter+process.TrackerParametersWriter+process.CaloGeometryWriter+process.HcalParametersWriter+process.CSCGeometryWriter+process.DTGeometryWriter+process.RPCGeometryWriter+process.GEMGeometryWriter)

CondTools/Geometry/test/writehelpers/geometryExtended2021DD4hep_xmlwriter.py

@@ -0,0 +1,31 @@
+import FWCore.ParameterSet.Config as cms
+
+process = cms.Process("GeometryXMLWriter")
+
+process.source = cms.Source("EmptyIOVSource",
+                            lastValue = cms.uint64(1),
+                            timetype = cms.string('runnumber'),
+                            firstValue = cms.uint64(1),
+                            interval = cms.uint64(1)
+                            )
+
+process.DDDetectorESProducer = cms.ESSource("DDDetectorESProducer",
+                                            confGeomXMLFiles = cms.FileInPath('Geometry/CMSCommonData/data/dd4hep/cmsExtendedGeometry2021.xml'),
+                                            appendToDataLabel = cms.string('make-payload')
+                                           )
+
+process.DDCompactViewESProducer = cms.ESProducer("DDCompactViewESProducer",
+                                                  appendToDataLabel = cms.string('make-payload')
+                                                )
+
+process.BigXMLWriter = cms.EDAnalyzer("OutputDD4hepToDDL",
+                              fileName = cms.untracked.string("./geSingleBigFile.xml")
+                              )
+
+
+process.maxEvents = cms.untracked.PSet(
+    input = cms.untracked.int32(1)
+    )
+
+process.p1 = cms.Path(process.BigXMLWriter)
+

DetectorDescription/DDCMS/interface/DDDetector.h

@@ -2,7 +2,6 @@
 #define DetectorDescription_DDCMS_DDDetector_h
 
 #include "DetectorDescription/DDCMS/interface/DDVectorRegistry.h"
-#include "DetectorDescription/DDCMS/interface/DDParsingContext.h"
 #include <DD4hep/Detector.h>
 #include <DD4hep/SpecParRegistry.h>
 #include <string>
@@ -12,7 +11,7 @@ class TGeoManager;
 namespace cms {
   class DDDetector {
   public:
-    explicit DDDetector(const std::string&, const std::string&, bool bigXML = false, bool makePayload = false);
+    explicit DDDetector(const std::string&, const std::string&, bool bigXML = false);
     DDDetector() = delete;
 
     cms::DDVectorsMap const& vectors() const { return m_vectors; }
@@ -35,8 +34,6 @@ namespace cms {
 
     dd4hep::Detector const* description() const { return m_description; }
 
-    DDParsingContext* m_context;
-
   private:
     void process(const std::string&);
     void processXML(const std::string&);

DetectorDescription/DDCMS/interface/DDNamespace.h

@@ -62,12 +62,11 @@ namespace cms {
     dd4hep::Solid addSolid(const std::string& name, dd4hep::Solid solid) const;
     dd4hep::Solid addSolidNS(const std::string& name, dd4hep::Solid solid) const;
 
-    dd4hep::Assembly assembly(const std::string& name) const;
+    dd4hep::Assembly assembly(const std::string& name, bool exception = true) const;
     dd4hep::Assembly addAssembly(dd4hep::Assembly asmb, bool addSolid = true) const;
     dd4hep::Assembly addAssemblySolid(dd4hep::Assembly assembly) const;
 
     dd4hep::Volume volume(const std::string& name, bool exc = true) const;
-    dd4hep::Volume* getVolPtr(const std::string& name) const;
     dd4hep::Volume addVolume(dd4hep::Volume vol) const;
     dd4hep::Volume addVolumeNS(dd4hep::Volume vol) const;
 

DetectorDescription/DDCMS/interface/DDParsingContext.h

@@ -20,7 +20,6 @@ namespace cms {
       rotations.reserve(3000);
       shapes.reserve(4000);
       volumes.reserve(3000);
-      volPtrs.reserve(3000);
       unresolvedMaterials.reserve(300);
       unresolvedVectors.reserve(300);
       unresolvedShapes.reserve(1000);
@@ -82,7 +81,6 @@ namespace cms {
     std::unordered_map<std::string, std::string> rotRevMap;
     std::unordered_map<std::string, dd4hep::Solid> shapes;
     std::unordered_map<std::string, dd4hep::Volume> volumes;
-    std::unordered_map<std::string, dd4hep::Volume*> volPtrs;
     std::vector<std::string> namespaces;
 
     std::unordered_map<std::string, std::vector<CompositeMaterial>> unresolvedMaterials;

DetectorDescription/DDCMS/plugins/DDDetectorESProducer.cc

@@ -59,7 +59,6 @@ class DDDetectorESProducer : public ESProducer, public EventSetupRecordIntervalF
   const string confGeomXMLFiles_;
   const string rootDDName_;
   const string label_;
-  const bool makePayload_;
   edm::ESGetToken<FileBlob, MFGeometryFileRcd> mfToken_;
   edm::ESGetToken<FileBlob, GeometryFileRcd> geomToken_;
 };
@@ -69,8 +68,7 @@ DDDetectorESProducer::DDDetectorESProducer(const ParameterSet& iConfig)
       appendToDataLabel_(iConfig.getParameter<string>("appendToDataLabel")),
       confGeomXMLFiles_(fromDB_ ? "none" : iConfig.getParameter<FileInPath>("confGeomXMLFiles").fullPath()),
       rootDDName_(iConfig.getParameter<string>("rootDDName")),
-      label_(iConfig.getParameter<string>("label")),
-      makePayload_(iConfig.getParameter<bool>("makePayload")) {
+      label_(iConfig.getParameter<string>("label")) {
   usesResources({edm::ESSharedResourceNames::kDD4Hep});
   if (rootDDName_ == "MagneticFieldVolumes:MAGF" || rootDDName_ == "cmsMagneticField:MAGF") {
     auto c = setWhatProduced(this,
@@ -98,14 +96,12 @@ void DDDetectorESProducer::fillDescriptions(ConfigurationDescriptions& descripti
   desc.add<string>("rootDDName", "cms:OCMS");
   desc.add<string>("label", "");
   desc.add<bool>("fromDB", false);
-  desc.add<bool>("makePayload", false);
   descriptions.add("DDDetectorESProducer", desc);
 
   edm::ParameterSetDescription descDB;
   descDB.add<string>("rootDDName", "cms:OCMS");
   descDB.add<string>("label", "Extended");
   descDB.add<bool>("fromDB", true);
-  descDB.add<bool>("makePayload", false);
   descriptions.add("DDDetectorESProducerFromDB", descDB);
 }
 
@@ -135,13 +131,13 @@ DDDetectorESProducer::ReturnType DDDetectorESProducer::produceGeom(const IdealGe
 
     return make_unique<cms::DDDetector>(label_, string(tb->begin(), tb->end()), true);
   } else {
-    return make_unique<DDDetector>(appendToDataLabel_, confGeomXMLFiles_, false, makePayload_);
+    return make_unique<DDDetector>(appendToDataLabel_, confGeomXMLFiles_, false);
   }
 }
 
 DDDetectorESProducer::ReturnType DDDetectorESProducer::produce() {
   LogVerbatim("Geometry") << "DDDetectorESProducer::Produce " << appendToDataLabel_;
-  return make_unique<DDDetector>(appendToDataLabel_, confGeomXMLFiles_, false, makePayload_);
+  return make_unique<DDDetector>(appendToDataLabel_, confGeomXMLFiles_, false);
 }
 
 DEFINE_FWK_EVENTSETUP_SOURCE(DDDetectorESProducer);