update hit-based training (#184)

* update hit-based training

* remove referencepoint

* lumi training

.github/workflows/test.yml

@@ -43,6 +43,18 @@ jobs:
       - run: pip install -r requirements.txt
       - run: ./scripts/local_test_clic_pipeline.sh
 
+  tf-clic-hits-pipeline:
+    runs-on: ubuntu-20.04
+    needs: [deps]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8.10'
+          cache: 'pip'
+      - run: pip install -r requirements.txt
+      - run: ./scripts/local_test_clic_hits_pipeline.sh
+
   tf-delphes-pipeline:
     runs-on: ubuntu-20.04
     needs: [deps]

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 default_language_version:
     python: python3
-exclude: ^(delphes/tev14_pythia8_*.)|^(images/)|^(clic/dumper_hepsim.py)|^(mlpf/pyg/__init__.py)|^(fcc/.*)
+exclude: ^(delphes/tev14_pythia8_*.)|^(images/)|^(clic/dumper_hepsim.py)|^(mlpf/pyg/__init__.py)|^(fcc/clicRec_e4h_input.py)|^(fcc/clic_steer.py)|^(fcc/pythia.py)|^(fcc/PandoraSettings)
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks

fcc/check_files.py

@@ -1,28 +1,51 @@
 import os
 
-#check for file presence in this path
-outpath = "/local/joosep/clic_edm4hep_2023_02_27/"
+# check for file presence in this path
+outpath = "/local/joosep/clic_edm4hep_2023_05_05/"
 
-#pythia card, start seed, end seed
+# pythia card, start seed, end seed
 samples = [
-    ("p8_ee_tt_ecm380",              1,  10011),
-    ("p8_ee_qq_ecm380",         100001, 120011),
+    ("p8_ee_tt_ecm380", 1, 10011),
+    ("p8_ee_qq_ecm380", 100001, 120011),
     ("p8_ee_ZH_Htautau_ecm380", 200001, 210011),
     ("p8_ee_WW_fullhad_ecm380", 300001, 310011),
 ]
 
 samples_pu = [
-    ("p8_ee_tt_ecm380",              1,  10001),
+    ("p8_ee_tt_ecm380", 1, 10001),
+]
+
+samples_gun = [
+    ("neutron", 1, 101),
+    ("kaon0L", 1, 101),
+    ("pi-", 1, 101),
+    ("pi+", 1, 101),
+    ("pi0", 1, 101),
+    ("mu-", 1, 101),
+    ("mu+", 1, 101),
+    ("e-", 1, 101),
+    ("e+", 1, 101),
+    ("gamma", 1, 101),
 ]
 
 if __name__ == "__main__":
-    #for sname, seed0, seed1 in samples:
+    # basic samples
+    # for sname, seed0, seed1 in samples:
     #    for seed in range(seed0, seed1):
     #        #check if output file exists, and print out batch submission if it doesn't
     #        if not os.path.isfile("{}/{}/reco_{}_{}.root".format(outpath, sname, sname, seed)):
-    #            print("sbatch run_sim.sh {} {}".format(seed, sname)) 
-    for sname, seed0, seed1 in samples_pu:
+    #            print("sbatch run_sim.sh {} {}".format(seed, sname))
+
+    # PU
+    # for sname, seed0, seed1 in samples_pu:
+    #    for seed in range(seed0, seed1):
+    #        # check if output file exists, and print out batch submission if it doesn't
+    #        if not os.path.isfile("{}/{}_PU10/reco_{}_{}.root".format(outpath, sname, sname, seed)):
+    #            print("sbatch run_sim_pu.sh {} {} p8_ee_gg_ecm380".format(seed, sname))
+
+    # gun
+    for sname, seed0, seed1 in samples_gun:
         for seed in range(seed0, seed1):
-            #check if output file exists, and print out batch submission if it doesn't
-            if not os.path.isfile("{}/{}_PU10/reco_{}_{}.root".format(outpath, sname, sname, seed)):
-                print("sbatch run_sim_pu.sh {} {} p8_ee_gg_ecm380".format(seed, sname)) 
+            # check if output file exists, and print out batch submission if it doesn't
+            if not os.path.isfile("{}/{}/reco_{}_{}.root".format(outpath, sname, sname, seed)):
+                print("sbatch run_sim_gun.sh {} {}".format(seed, sname))

fcc/clicRec_e4h_input.py

@@ -2360,4 +2360,23 @@
 
 from Configurables import ApplicationMgr
 
-ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=WARNING)
+SequencerTimerTool().OutputLevel = INFO
+TIMER = TimingAuditor("TIMER")
+TIMER.addTool(SequencerTimerTool, name="TIMER")
+TIMER.TIMER.HistoProduce = True
+TIMER.TIMER.OutputLevel = INFO
+
+toolsvc = ToolSvc()
+auditorsvc = AuditorSvc()
+auditorsvc.Auditors += [TIMER]
+RootHistSvc().OutputFile = "timing_histos.root"
+
+ApplicationMgr(
+    TopAlg=algList,
+    EvtSel="NONE",
+    EvtMax=3,
+    ExtSvc=[evtsvc, toolsvc, auditorsvc],
+    OutputLevel=WARNING,
+    AuditAlgorithms=True,
+    HistogramPersistency = "ROOT")
+

fcc/clic_steer.py

@@ -139,7 +139,7 @@
 ##     Setting a distribution will set isotrop = True
 ##     
 SIM.gun.distribution = None
-SIM.gun.energy = 10000.0
+SIM.gun.energy = None
 
 ##  isotropic distribution for the particle gun
 ## 

fcc/main19.cc

@@ -54,15 +54,15 @@ int main(int argc, char *argv[]) {
     std::cerr << "./main SEED NPU" << std::endl;
     return 1;
   }
-  
+
   std::string seedStr = std::string("Random:seed = ").append(std::string(argv[1]));
 
   // Average number of pileup events per signal event.
   double nPileupAvg = atoi(argv[2]);
-  
+
   // Shift each PU event by this time delta in time to mimic ee overlay
   double timeDelta = 0.5;
-  
+
   Pythia8ToHepMC ToHepMC;
   ToHepMC.setNewFile("pythia.hepmc");
 
@@ -84,18 +84,18 @@ int main(int argc, char *argv[]) {
 
     // Select the number of pileup events to generate.
     int nPileup = poisson(nPileupAvg, pythiaPileup.rndm);
-    
+
     // create a random index permutation from [0, nPileup)
     std::vector<int> puVectorInds;
     for (int npu=0; npu<nPileup; npu++) {
-      puVectorInds.push_back(npu); 
+      puVectorInds.push_back(npu);
     }
     pythiaPileup.rndm.shuffle(puVectorInds);
 
     // Generate a number of pileup events. Add them to sumEvent.
     for (int iPileup = 0; iPileup < nPileup; ++iPileup) {
-     
-      //generate a signal event if the permutation value is 0, otherwise generate a pileup event 
+
+      //generate a signal event if the permutation value is 0, otherwise generate a pileup event
       auto& pythiaSigOrPU = (puVectorInds[iPileup] == 0) ? pythiaSignal : pythiaPileup;
       pythiaSigOrPU.next();
 

fcc/p8_ee_ZZ_fullhad_ecm365.cmd

@@ -22,4 +22,3 @@ WeakDoubleBoson:ffbar2gmZgmZ = on
 ! 4) Settings for the event generation process in the Pythia8 library.
 PartonLevel:ISR = on               ! no initial-state radiation
 PartonLevel:FSR = on               ! no final-state radiation
-

fcc/p8_ee_qcd_ecm380.cmd

@@ -16,7 +16,7 @@ Stat:showPartonLevel = off
 Beams:idA = 11                   ! first beam, e- = 11
 Beams:idB = -11                  ! second beam, e+ = -11
 
-! s-channel gamma/Z 
+! s-channel gamma/Z
 Beams:eCM = 380               ! CM energy of collision
 HardQCD:all = on
 WeakSingleBoson:ffbar2ffbar(s:gmZ) = on

fcc/p8_ee_qq_ecm365.cmd

@@ -17,7 +17,7 @@ Beams:eCM = 365                    ! CM energy of collision
 PartonLevel:ISR = on
 PartonLevel:FSR = on
 
-! s-channel gamma/Z 
+! s-channel gamma/Z
 WeakSingleBoson:ffbar2ffbar(s:gmZ) = on
 23:onMode   = off
 23:onIfAny = 1 2 3 4 5 6

fcc/p8_ee_qq_ecm380.cmd

@@ -17,7 +17,7 @@ Beams:eCM = 380                    ! CM energy of collision
 PartonLevel:ISR = on
 PartonLevel:FSR = on
 
-! s-channel gamma/Z 
+! s-channel gamma/Z
 WeakSingleBoson:ffbar2ffbar(s:gmZ) = on
 23:onMode   = off
 23:onIfAny = 1 2 3 4 5