diff --git a/RecoLuminosity/LumiDB/doc/norm_PIXELV2.cfg b/RecoLuminosity/LumiDB/doc/norm_PIXELV2.cfg new file mode 100644 index 0000000000000..6dddfcba05bba --- /dev/null +++ b/RecoLuminosity/LumiDB/doc/norm_PIXELV2.cfg @@ -0,0 +1,21 @@ +[NormDefinition] +name=PIXELV2 +istypedefault=0 +lumitype=PIXEL +comment= + +[NormData 160430] +since=160430 +amodetag=PROTPHYS +egev=3500 +corrector=fPolyScheme:a0 +a0=1.0 +comment=2011 + +[NormData 189924] +since=189924 +amodetag=PROTPHYS +egev=4000 +corrector=fPoly:afterglow +afterglow=[(35,0.999),(60,0.992),(72,0.991),(249,0.990),(471,0.984),(618,0.983),(1051,0.980),(1368,0.977)] +comment=2012full \ No newline at end of file diff --git a/RecoLuminosity/LumiDB/plotdata/create_public_lumi_plots.py b/RecoLuminosity/LumiDB/plotdata/create_public_lumi_plots.py index d87d510f18996..13c732f8643ca 100755 --- a/RecoLuminosity/LumiDB/plotdata/create_public_lumi_plots.py +++ b/RecoLuminosity/LumiDB/plotdata/create_public_lumi_plots.py @@ -14,6 +14,7 @@ import math import optparse import ConfigParser +import cjson import numpy as np @@ -68,13 +69,15 @@ class LumiDataPoint(object): """Holds info from one line of lumiCalc lumibyls output.""" - def __init__(self, line): + def __init__(self, line, json_file_name=None): # Decode the comma-separated line from lumiCalc. line_split = line.split(",") tmp = line_split[0].split(":") self.run_number = int(tmp[0]) self.fill_number = int(tmp[1]) + tmp = line_split[1].split(":") + self.ls = int(tmp[0]) tmp = line_split[2] self.timestamp = datetime.datetime.strptime(tmp, DATE_FMT_STR_LUMICALC) # NOTE: Convert from ub^{-1} to b^{-1}. @@ -82,6 +85,16 @@ def __init__(self, line): self.lum_del = scale_factor * float(line_split[5]) self.lum_rec = scale_factor * float(line_split[6]) + # Adding lum_cert for the data certification information + if json_file_name: + addcertls = bool(checkCertification(self.run_number, self.ls)) + if addcertls: + self.lum_cert = scale_factor * float(line_split[6]) + else: + self.lum_cert = 0. + else: + self.lum_cert = 0. + # End of __init__(). # End of class LumiDataPoint. @@ -150,6 +163,12 @@ def lum_rec_tot(self, units="b^{-1}"): # End of lum_rec_tot(). return res + def lum_cert_tot(self, units="b^{-1}"): + res = sum([i.lum_cert for i in self.data_points]) + res *= LumiDataBlock.scale_factors[units] + # End of lum_cert_tot(). + return res + def max_inst_lum(self, units="Hz/b"): res = 0. if len(self.data_points): @@ -235,6 +254,11 @@ def lum_rec(self, units="b^{-1}"): # End of lum_rec(). return res + def lum_cert(self, units="b^{-1}"): + res = [i.lum_cert_tot(units) for i in self.data_blocks] + # End of lum_cert(). + return res + def lum_del_tot(self, units="b^{-1}"): # End of lum_del(). return sum(self.lum_del(units)) @@ -243,6 +267,10 @@ def lum_rec_tot(self, units="b^{-1}"): # End of lum_rec(). return sum(self.lum_rec(units)) + def lum_cert_tot(self, units="b^{-1}"): + # End of lum_cert(). + return sum(self.lum_cert(units)) + def lum_inst_max(self, units="Hz/b"): res = [i.max_inst_lum(units) for i in self.data_blocks] # End of lum_inst_max(). @@ -436,6 +464,39 @@ def TweakPlot(fig, ax, (time_begin, time_end), ###################################################################### +def checkCertification(run_number, ls): + """Check if this run and LS are certified as good and return a boolean parameter.""" + try: + ls_ranges = certification_data[run_number] + for ranges in ls_ranges: + if (ls >= ranges[0]) and (ls <= ranges[1]): + return True + except KeyError: + return False + + return False + +###################################################################### + +def loadCertificationJSON(json_file_name): + + full_file = open(json_file_name, "r") + full_file_content = ["".join(l) for l in full_file.readlines()] + full_object = cjson.decode(full_file_content[0]) + + # Now turn this into a dictionary for easier handling. + tmp = full_object.keys() + tmp = [int(i) for i in tmp] + run_list = sorted(tmp) + certification_data = {} + for run in run_list: + ls_ranges = full_object.get(str(run), None) + certification_data[run] = ls_ranges + + return certification_data + +###################################################################### + if __name__ == "__main__": desc_str = "This script creates the official CMS luminosity plots " \ @@ -460,7 +521,8 @@ def TweakPlot(fig, ax, (time_begin, time_end), "beam_energy" : None, "beam_fluctuation" : None, "verbose" : False, - "oracle_connection" : None + "oracle_connection" : None, + "json_file" : None } cfg_parser = ConfigParser.SafeConfigParser(cfg_defaults) if not os.path.exists(config_file_name): @@ -549,6 +611,27 @@ def TweakPlot(fig, ax, (time_begin, time_end), oracle_connection_string = cfg_parser.get("general", "oracle_connection") use_oracle = (len(oracle_connection_string) != 0) + # If a JSON file is specified, use the JSON file to add in the + # plot data certified as good for physics. + json_file_name = cfg_parser.get("general", "json_file") + if len(json_file_name) < 1: + json_file_name = None + if json_file_name: + if not os.path.exists(json_file_name): + print >> sys.stderr, \ + "ERROR Requested JSON file '%s' is not available" % json_file_name + sys.exit(1) + print "Using JSON file '%s' for certified data" % json_file_name + else: + if verbose: + print "No JSON file specified, filling only standard lumi plot." + + ########## + + certification_data = None + if json_file_name: + certification_data = loadCertificationJSON(json_file_name) + ########## # Map accelerator modes (as fed to lumiCalc) to particle type @@ -680,7 +763,6 @@ def TweakPlot(fig, ax, (time_begin, time_end), print "Running lumiCalc for all requested days" for day in days: print " %s" % day.isoformat() - print "DEBUG JGH %s" % day.isoformat() use_cache = (not ignore_cache) and (day <= last_day_from_cache) cache_file_path = CacheFilePath(cache_file_dir, day) cache_file_tmp = cache_file_path.replace(".csv", "_tmp.csv") @@ -753,14 +835,6 @@ def TweakPlot(fig, ax, (time_begin, time_end), "ERROR Problem running lumiCalc: %s" % output sys.exit(1) - # BUG BUG BUG - # Trying to track down where these empty files come - # from... - num_lines_0 = len(open(cache_file_tmp).readlines()) - print "DEBUG JGH Straight after calling lumiCalc: " \ - "line count in cache file is %d" % num_lines_0 - # BUG BUG BUG end - # BUG BUG BUG # This works around a bug in lumiCalc where sometimes not # all data for a given day is returned. The work-around is @@ -776,14 +850,6 @@ def TweakPlot(fig, ax, (time_begin, time_end), newfile.close() # BUG BUG BUG end - # BUG BUG BUG - # Trying to track down where these empty files come - # from... - num_lines_1 = len(open(cache_file_path).readlines()) - print "DEBUG JGH After filtering: " \ - "line count in cache file is %d" % num_lines_1 - # BUG BUG BUG end - if verbose: print " CSV file for the day written to %s" % \ cache_file_path @@ -791,90 +857,6 @@ def TweakPlot(fig, ax, (time_begin, time_end), if verbose: print " cache file for %s exists" % day.isoformat() - # BUG BUG BUG - # Trying to track down where these empty files come - # from... - num_lines_2 = len(open(cache_file_path).readlines()) - print "DEBUG JGH From existing cache file: " \ - "line count in cache file is %d" % num_lines_2 - - # BUG BUG BUG - # Still trying to track down where these empty files come from. - date_begin_str = day.strftime(DATE_FMT_STR_LUMICALC) - date_begin_day_str = day.strftime(DATE_FMT_STR_LUMICALC_DAY) - date_end_str = (day + datetime.timedelta(days=1)).strftime(DATE_FMT_STR_LUMICALC) - date_previous_str = (day - datetime.timedelta(days=1)).strftime(DATE_FMT_STR_LUMICALC) - if not beam_energy_from_cfg: - year = day.isocalendar()[0] - beam_energy = beam_energy_defaults[accel_mode][year] - if not beam_fluctuation_from_cfg: - year = day.isocalendar()[0] - beam_fluctuation = beam_fluctuation_defaults[accel_mode][year] - # WORKAROUND WORKAROUND WORKAROUND - # Same as above. - if amodetag_bug_workaround: - lumicalc_flags = "%s --without-checkforupdate " \ - "--beamenergy %.1f " \ - "--beamfluctuation %.2f " \ - "lumibyls" % \ - (lumicalc_flags_from_cfg, - beam_energy, beam_fluctuation) - else: - lumicalc_flags = "%s --without-checkforupdate " \ - "--beamenergy %.1f " \ - "--beamfluctuation %.2f " \ - "--amodetag %s " \ - "lumibyls" % \ - (lumicalc_flags_from_cfg, - beam_energy, beam_fluctuation, - accel_mode) - # WORKAROUND WORKAROUND WORKAROUND end - lumicalc_flags = lumicalc_flags.strip() - lumicalc_cmd = "%s %s" % (lumicalc_script, lumicalc_flags) - cache_file_dbg = cache_file_path.replace(".csv", "_dbg.csv") - cmd = "%s --begin '%s' --end '%s' -o %s" % \ - (lumicalc_cmd, date_begin_str, date_end_str, cache_file_dbg) - if use_oracle: - lumicalc_cmd = "%s %s" % (lumicalc_cmd, oracle_connection_string) - print "DEBUG JGH Re-running lumicalc for a single day as '%s'" % cmd - (status, output) = commands.getstatusoutput(cmd) - # BUG BUG BUG - # Trying to track down the bad-cache problem. - output_1 = copy.deepcopy(output) - # BUG BUG BUG end - if status != 0: - # This means 'no qualified data found'. - if ((status >> 8) == 13 or (status >> 8) == 14): - dummy_file = open(cache_file_dbg, "w") - dummy_file.write("Run:Fill,LS,UTCTime,Beam Status,E(GeV),Delivered(/ub),Recorded(/ub),avgPU\r\n") - dummy_file.close() - else: - print >> sys.stderr, \ - "ERROR Problem running lumiCalc: %s" % output - sys.exit(1) - num_lines_3 = len(open(cache_file_path).readlines()) - num_lines_4 = len(open(cache_file_dbg).readlines()) - print "DEBUG JGH Line count in data file: " \ - "%d" % num_lines_3 - print "DEBUG JGH Line count in cross-check: " \ - "%d" % num_lines_4 - if num_lines_4 != num_lines_3: - print "DEBUG JGH !!! Line count mismatch!!!" - if num_lines_4 < num_lines_3: - print "DEBUG JGH !!! --> found an occurrence of the lumiCalc stoptime bug!!!" - else: - print "DEBUG JGH !!! --> found a problem !!!" -# if output_0: -# print "DEBUG JGH ----------" -# print "DEBUG JGH output from first lumiCalc run:" -# print "DEBUG JGH ----------" -# print output_0 -# print "DEBUG JGH ----------" -# print "DEBUG JGH output from second lumiCalc run:" -# print "DEBUG JGH ----------" -# print output_1 - # BUG BUG BUG end - # Now read back all lumiCalc results. print "Reading back lumiCalc results" lumi_data_by_day = {} @@ -893,7 +875,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), assert lines[0] == "Run:Fill,LS,UTCTime,Beam Status,E(GeV),Delivered(/ub),Recorded(/ub),avgPU\r\n" # DEBUG DEBUG DEBUG end for line in lines[1:]: - lumi_data_day.add(LumiDataPoint(line)) + lumi_data_day.add(LumiDataPoint(line, json_file_name)) in_file.close() except IOError, err: print >> sys.stderr, \ @@ -1057,6 +1039,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), units = GetUnits(year, accel_mode, "cum_year") weights_del_for_cum = lumi_data.lum_del(units) weights_rec_for_cum = lumi_data.lum_rec(units) + weights_cert_for_cum = lumi_data.lum_cert(units) # Maximum instantaneous delivered luminosity per day. units = GetUnits(year, accel_mode, "max_inst") weights_del_inst = lumi_data.lum_inst_max(units) @@ -1080,14 +1063,14 @@ def TweakPlot(fig, ax, (time_begin, time_end), # Loop over all color schemes. for color_scheme_name in color_scheme_names: - print " color scheme '%s'" % color_scheme_name - color_scheme = ColorScheme(color_scheme_name) color_fill_del = color_scheme.color_fill_del color_fill_rec = color_scheme.color_fill_rec + color_fill_cert = color_scheme.color_fill_cert color_fill_peak = color_scheme.color_fill_peak color_line_del = color_scheme.color_line_del color_line_rec = color_scheme.color_line_rec + color_line_cert = color_scheme.color_line_cert color_line_peak = color_scheme.color_line_peak logo_name = color_scheme.logo_name file_suffix = color_scheme.file_suffix @@ -1112,7 +1095,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), # Figure out the maximum instantaneous luminosity. max_inst = max(weights_del_inst) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del_inst, histtype="stepfilled", @@ -1172,7 +1155,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), max_del = max(weights_del) max_rec = max(weights_rec) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del, histtype="stepfilled", @@ -1223,6 +1206,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), min_del = min(weights_del_for_cum) tot_del = sum(weights_del_for_cum) tot_rec = sum(weights_rec_for_cum) + tot_cert = sum(weights_cert_for_cum) for type in ["lin", "log"]: is_log = (type == "log") @@ -1235,7 +1219,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), fig.clear() ax = fig.add_subplot(111) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del_for_cum, histtype="stepfilled", cumulative=True, @@ -1249,8 +1233,16 @@ def TweakPlot(fig, ax, (time_begin, time_end), facecolor=color_fill_rec, edgecolor=color_line_rec, label="CMS Recorded: %.2f %s" % \ (tot_rec, LatexifyUnits(units))) - leg = ax.legend(loc="upper left", bbox_to_anchor=(0.125, 0., 1., 1.01), - frameon=False) + if sum(weights_cert_for_cum) > 0.: + ax.hist(times, bin_edges, weights=weights_cert_for_cum, + histtype="stepfilled", cumulative=True, + log=log_setting, + facecolor=color_fill_cert, edgecolor=color_line_cert, + label="CMS Validated: %.2f %s" % \ + (tot_cert, LatexifyUnits(units))) + leg = ax.legend(loc="upper left", + bbox_to_anchor=(0.125, 0., 1., 1.01), + frameon=False) for t in leg.get_texts(): t.set_font_properties(FONT_PROPS_TICK_LABEL) @@ -1267,6 +1259,12 @@ def TweakPlot(fig, ax, (time_begin, time_end), LatexifyUnits(units), fontproperties=FONT_PROPS_AX_TITLE) + # Add "CMS Preliminary" to the plot. + if json_file_name: + ax.text(0.05, 0.7, "CMS Preliminary", + verticalalignment="center", horizontalalignment="left", + transform = ax.transAxes, fontsize=15) + # Add the logo. AddLogo(logo_name, ax) TweakPlot(fig, ax, (time_begin, time_end), @@ -1376,7 +1374,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), # Figure out the maximum instantaneous luminosity. max_inst = max(weights_del_inst) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del_inst, histtype="stepfilled", @@ -1437,7 +1435,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), max_del = max(weights_del) max_rec = max(weights_rec) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del, histtype="stepfilled", @@ -1501,7 +1499,7 @@ def TweakPlot(fig, ax, (time_begin, time_end), fig.clear() ax = fig.add_subplot(111) - if sum(weights_del) > 0: + if sum(weights_del) > 0.: ax.hist(times, bin_edges, weights=weights_del_for_cum, histtype="stepfilled", cumulative=True, diff --git a/RecoLuminosity/LumiDB/plotdata/create_public_peakpu_plots.py b/RecoLuminosity/LumiDB/plotdata/create_public_peakpu_plots.py new file mode 100644 index 0000000000000..2c8d48963fdd1 --- /dev/null +++ b/RecoLuminosity/LumiDB/plotdata/create_public_peakpu_plots.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python + +###################################################################### +## File: create_public_lumi_plots.py +###################################################################### + +import sys +import csv +import os +import commands +import time +import datetime +import calendar +import copy +import math +import optparse +import ConfigParser + +import numpy as np + +import matplotlib +matplotlib.use('Agg') +from matplotlib import pyplot as plt +# FIX FIX FIX +# This fixes a well-know bug with stepfilled logarithmic histograms in +# Matplotlib. +from RecoLuminosity.LumiDB.mpl_axes_hist_fix import hist +if matplotlib.__version__ != '1.0.1': + print >> sys.stderr, \ + "ERROR The %s script contains a hard-coded bug-fix " \ + "for Matplotlib 1.0.1. The Matplotlib version loaded " \ + "is %s" % (__file__, matplotlib.__version__) + sys.exit(1) +matplotlib.axes.Axes.hist = hist +# FIX FIX FIX end + +from RecoLuminosity.LumiDB.public_plots_tools import ColorScheme +from RecoLuminosity.LumiDB.public_plots_tools import LatexifyUnits +from RecoLuminosity.LumiDB.public_plots_tools import AddLogo +from RecoLuminosity.LumiDB.public_plots_tools import InitMatplotlib +from RecoLuminosity.LumiDB.public_plots_tools import SavePlot +from RecoLuminosity.LumiDB.public_plots_tools import FONT_PROPS_SUPTITLE +from RecoLuminosity.LumiDB.public_plots_tools import FONT_PROPS_TITLE +from RecoLuminosity.LumiDB.public_plots_tools import FONT_PROPS_AX_TITLE +from RecoLuminosity.LumiDB.public_plots_tools import FONT_PROPS_TICK_LABEL + +try: + import debug_hook + import pdb +except ImportError: + pass + + +###################################################################### + +# Some global constants. Not nice, but okay. +DATE_FMT_STR_LUMICALC = "%m/%d/%y %H:%M:%S" +DATE_FMT_STR_LUMICALC_DAY = "%m/%d/%y" +DATE_FMT_STR_OUT = "%Y-%m-%d %H:%M" +DATE_FMT_STR_AXES = "%-d %b" +DATE_FMT_STR_CFG = "%Y-%m-%d" +NUM_SEC_IN_LS = 2**18 / 11246. + +KNOWN_ACCEL_MODES = ["PROTPHYS", "IONPHYS", "PAPHYS", + "2013_amode_bug_workaround"] +LEAD_SCALE_FACTOR = 82. / 208. + +def processdata(years): + results={}#{year:[[str,str],[pu,pu]]} + for y in years: + fills=[]#[fillnum] + maxputimelist=[] + maxpulist=[] + results[y]=[maxputimelist,maxpulist] + f=None + try: + lumifilename=str(y)+'lumibyls.csv' + f=open(lumifilename,'rb') + except IOError: + print 'failed to open file ',lumifilename + return result + freader=csv.reader(f,delimiter=',') + idx=0 + for row in freader: + if idx==0: + idx=1 + continue + if row[0].find('#')==1: + continue + [runnum,fillnum]=map(lambda i:int(i),row[0].split(':')) + avgpu=float(row[7]) + if avgpu>50: continue + putime=row[2] + max_avgpu=avgpu + max_putime=putime + if fillnum not in fills:#new fill + fills.append(fillnum) + results[y][0].append(max_putime) + results[y][1].append(max_avgpu) + if avgpu>max_avgpu: + results[y][0][-1]=max_putime + results[y][1][-1]=max_avgpu + print results + return results + + +###################################################################### + +def CacheFilePath(cache_file_dir, day=None): + cache_file_path = os.path.abspath(cache_file_dir) + if day: + cache_file_name = "lumicalc_cache_%s.csv" % day.isoformat() + cache_file_path = os.path.join(cache_file_path, cache_file_name) + return cache_file_path + + +def GetXLocator(ax): + """Pick a DateLocator based on the range of the x-axis.""" + (x_lo, x_hi) = ax.get_xlim() + num_days = x_hi - x_lo + min_num_ticks = min(num_days, 5) + locator = matplotlib.dates.AutoDateLocator(minticks=min_num_ticks, + maxticks=None) + # End of GetLocator(). + return locator + +###################################################################### + +def TweakPlot(fig, ax, (time_begin, time_end), + add_extra_head_room=False): + + # Fiddle with axes ranges etc. + ax.relim() + ax.autoscale_view(False, True, True) + for label in ax.get_xticklabels(): + label.set_ha("right") + label.set_rotation(30.) + + # Bit of magic here: increase vertical scale by one tick to make + # room for the legend. + if add_extra_head_room: + y_ticks = ax.get_yticks() + (y_min, y_max) = ax.get_ylim() + is_log = (ax.get_yscale() == "log") + y_max_new = y_max + if is_log: + tmp = y_ticks[-1] / y_ticks[-2] + y_max_new = y_max * math.pow(tmp, add_extra_head_room) + else: + tmp = y_ticks[-1] - y_ticks[-2] + y_max_new = y_max + add_extra_head_room * tmp + ax.set_ylim(y_min, y_max_new) + + # Add a second vertical axis on the right-hand side. + ax_sec = ax.twinx() + ax_sec.set_ylim(ax.get_ylim()) + ax_sec.set_yscale(ax.get_yscale()) + + for ax_tmp in fig.axes: + for sub_ax in [ax_tmp.xaxis, ax_tmp.yaxis]: + for label in sub_ax.get_ticklabels(): + label.set_font_properties(FONT_PROPS_TICK_LABEL) + + ax.set_xlim(time_begin, time_end) + + locator = GetXLocator(ax) + minorXlocator=matplotlib.ticker.AutoMinorLocator() + #ax.yaxis.set_major_locator(matplotlib.ticker.MultipleLocator(5)) + #ax.yaxis.set_minor_locator(matplotlib.ticker.MultipleLocator(5)) + ax.xaxis.set_major_locator(locator) + ax.xaxis.set_minor_locator(minorXlocator) + + formatter = matplotlib.dates.DateFormatter(DATE_FMT_STR_AXES) + ax.xaxis.set_major_formatter(formatter) + + fig.subplots_adjust(top=.85, bottom=.14, left=.13, right=.91) + # End of TweakPlot(). + +###################################################################### + +if __name__ == "__main__": + + desc_str = "test pu" \ + + arg_parser = optparse.OptionParser(description=desc_str) + arg_parser.add_option("--ignore-cache", action="store_true", + help="Ignore all cached lumiCalc results " \ + "and re-query lumiCalc. " \ + "(Rebuilds the cache as well.)") + (options, args) = arg_parser.parse_args() + if len(args) != 1: + print >> sys.stderr, \ + "ERROR Need exactly one argument: a config file name" + sys.exit(1) + config_file_name = args[0] + ignore_cache = options.ignore_cache + + cfg_defaults = { + "lumicalc_flags" : "", + "date_end" : None, + "color_schemes" : "Joe, Greg", + "beam_energy" : None, + "beam_fluctuation" : None, + "verbose" : False, + "oracle_connection" : None + } + cfg_parser = ConfigParser.SafeConfigParser(cfg_defaults) + if not os.path.exists(config_file_name): + print >> sys.stderr, \ + "ERROR Config file '%s' does not exist" % config_file_name + sys.exit(1) + cfg_parser.read(config_file_name) + + # Which color scheme to use for drawing the plots. + color_scheme_names_tmp = cfg_parser.get("general", "color_schemes") + color_scheme_names = [i.strip() for i in color_scheme_names_tmp.split(",")] + # Where to store cache files containing the lumiCalc output. + cache_file_dir = cfg_parser.get("general", "cache_dir") + # Flag to turn on verbose output. + verbose = cfg_parser.getboolean("general", "verbose") + + # Some details on how to invoke lumiCalc. + lumicalc_script = cfg_parser.get("general", "lumicalc_script") + lumicalc_flags_from_cfg = cfg_parser.get("general", "lumicalc_flags") + accel_mode = cfg_parser.get("general", "accel_mode") + # Check if we know about this accelerator mode. + if not accel_mode in KNOWN_ACCEL_MODES: + print >> sys.stderr, \ + "ERROR Unknown accelerator mode '%s'" % \ + accel_mode + + # WORKAROUND WORKAROUND WORKAROUND + amodetag_bug_workaround = False + if accel_mode == "2013_amode_bug_workaround": + amodetag_bug_workaround = True + accel_mode = "PAPHYS" + # WORKAROUND WORKAROUND WORKAROUND end + + beam_energy_tmp = cfg_parser.get("general", "beam_energy") + # If no beam energy specified, use the default(s) for this + # accelerator mode. + beam_energy = None + beam_energy_from_cfg = None + if not beam_energy_tmp: + print "No beam energy specified --> using defaults for '%s'" % \ + accel_mode + beam_energy_from_cfg = False + else: + beam_energy_from_cfg = True + beam_energy = float(beam_energy_tmp) + + beam_fluctuation_tmp = cfg_parser.get("general", "beam_fluctuation") + # If no beam energy fluctuation specified, use the default for + # this accelerator mode. + beam_fluctuation = None + beam_fluctuation_from_cfg = None + if not beam_fluctuation_tmp: + print "No beam energy fluctuation specified --> using the defaults to '%s'" % \ + accel_mode + beam_fluctuation_from_cfg = False + else: + beam_fluctuation_from_cfg = True + beam_fluctuation = float(beam_fluctuation_tmp) + + # Overall begin and end dates of all data to include. + tmp = cfg_parser.get("general", "date_begin") + date_begin = datetime.datetime.strptime(tmp, DATE_FMT_STR_CFG).date() + tmp = cfg_parser.get("general", "date_end") + date_end = None + if tmp: + date_end = datetime.datetime.strptime(tmp, DATE_FMT_STR_CFG).date() + # If no end date is given, use today. + today = datetime.datetime.utcnow().date() + if not date_end: + print "No end date given --> using today" + date_end = today + # If end date lies in the future, truncate at today. + if date_end > today: + print "End date lies in the future --> using today instead" + date_end = today + # If end date is before start date, give up. + if date_end < date_begin: + print >> sys.stderr, \ + "ERROR End date before begin date (%s < %s)" % \ + (date_end.isoformat(), date_begin.isoformat()) + sys.exit(1) + + # If an Oracle connection string is specified, use direct Oracle + # access. Otherwise access passes through the Frontier + # cache. (Fine, but much slower to receive the data.) + oracle_connection_string = cfg_parser.get("general", "oracle_connection") + use_oracle = (len(oracle_connection_string) != 0) + + ########## + + # Map accelerator modes (as fed to lumiCalc) to particle type + # strings to be used in plot titles etc. + particle_type_strings = { + "PROTPHYS" : "pp", + "IONPHYS" : "PbPb", + "PAPHYS" : "pPb" + } + particle_type_str = particle_type_strings[accel_mode] + + beam_energy_defaults = { + "PROTPHYS" : {2010 : 3500., + 2011 : 3500., + 2012 : 4000., + 2013 : 1380.1}, + "IONPHYS" : {2010 : 3500., + 2011 : 3500.}, + "PAPHYS" : {2013 : 4000.} + } + beam_fluctuation_defaults = { + "PROTPHYS" : {2010 : .15, + 2011 : .15, + 2012 : .15, + 2013 : .15}, + "IONPHYS" : {2010 : .15, + 2011 : .15}, + "PAPHYS" : {2013 : .15} + } + + ########## + + # Environment parameter for access to the Oracle DB. + if use_oracle: + os.putenv("TNS_ADMIN", "/afs/cern.ch/cms/lumi/DB") + + ########## + + # Tell the user what's going to happen. + print "Using configuration from file '%s'" % config_file_name + if ignore_cache: + print "Ignoring all cached lumiCalc results (and rebuilding the cache)" + else: + print "Using cached lumiCalc results from %s" % \ + CacheFilePath(cache_file_dir) + print "Using color schemes '%s'" % ", ".join(color_scheme_names) + print "Using lumiCalc script '%s'" % lumicalc_script + print "Using additional lumiCalc flags from configuration: '%s'" % \ + lumicalc_flags_from_cfg + print "Selecting data for accelerator mode '%s'" % accel_mode + if beam_energy_from_cfg: + print "Selecting data for beam energy %.0f GeV" % beam_energy + else: + print "Selecting data for default beam energy for '%s' from:" % accel_mode + for (key, val) in beam_energy_defaults[accel_mode].iteritems(): + print " %d : %.1f GeV" % (key, val) + if beam_fluctuation_from_cfg: + print "Using beam energy fluctuation of +/- %.0f%%" % \ + (100. * beam_fluctuation) + else: + print "Using default beam energy fluctuation for '%s' from:" % accel_mode + for (key, val) in beam_fluctuation_defaults[accel_mode].iteritems(): + print " %d : +/- %.0f%%" % (key, 100. * val) + if use_oracle: + print "Using direct access to the Oracle luminosity database" + else: + print "Using access to the luminosity database through the Frontier cache" + + ########## + + # See if the cache file dir exists, otherwise try to create it. + path_name = CacheFilePath(cache_file_dir) + if not os.path.exists(path_name): + if verbose: + print "Cache file path does not exist: creating it" + try: + os.makedirs(path_name) + except Exception, err: + print >> sys.stderr, \ + "ERROR Could not create cache dir: %s" % path_name + sys.exit(1) + + ########## + + + years=[2010,2011,2012] + + lumi_data_by_fill_per_year={} #{year:[[timestamp,timestamp],[max_pu,max_pu]]} + lumi_data_by_fill_per_year=processdata(years) + + #lumi_data_by_fill_per_year[2010]=[['05/05/10 05:59:58','06/02/10 10:47:25','07/02/10 12:47:25','08/02/10 11:47:25'],[10.0,2.5,11.3,4.5]] + #lumi_data_by_fill_per_year[2011]=[['05/05/11 05:59:58','06/02/11 10:47:25','07/02/11 12:47:25','08/02/11 11:47:25','09/05/11 05:59:58','10/02/11 10:47:25','11/02/11 12:47:25'],[20.0,27.4,30.5,40.,22.,15.,45.]] + #lumi_data_by_fill_per_year[2012]=[['05/05/12 05:59:58','06/02/12 10:47:25','07/02/12 12:47:25','08/02/12 11:47:25','09/05/12 05:59:58','10/02/12 10:47:25','11/02/12 12:47:25','11/02/12 12:47:25'],[10.0,17.4,20.5,30.,32.,25.,33.,42.]] + + + InitMatplotlib() + + ########## + + year_begin = date_begin.isocalendar()[0] + year_end = date_end.isocalendar()[0] + # DEBUG DEBUG DEBUG + assert year_end >= year_begin + ########## + + # And this is where the plotting starts. + print "Drawing things..." + ColorScheme.InitColors() + + #---------- + + if len(years) > 1: + print " peak interactions for %s together" % ", ".join([str(i) for i in years]) + + def PlotPeakPUAllYears(lumi_data_by_fill_per_year): + """Mode 1: years side-by-side""" + + # Loop over all color schemes and plot. + for color_scheme_name in color_scheme_names: + print " color scheme '%s'" % color_scheme_name + color_scheme = ColorScheme(color_scheme_name) + color_by_year = color_scheme.color_by_year + logo_name = color_scheme.logo_name + file_suffix = color_scheme.file_suffix + + for type in ["lin", "log"]: + is_log = (type == "log") + aspect_ratio = matplotlib.figure.figaspect(1. / 2.5) + fig = plt.figure(figsize=aspect_ratio) + ax = fig.add_subplot(111) + + time_begin_ultimate = datetime.datetime.strptime(lumi_data_by_fill_per_year[years[0]][0][0],DATE_FMT_STR_LUMICALC).date() + str_begin_ultimate = time_begin_ultimate.strftime(DATE_FMT_STR_OUT) + for (year_index, year) in enumerate(years): + + lumi_data = lumi_data_by_fill_per_year[year] + times_tmp = [datetime.datetime.strptime(tmp,DATE_FMT_STR_LUMICALC).date() for tmp in lumi_data[0]] + times = [matplotlib.dates.date2num(i) for i in times_tmp] #x_axis + maxpus = lumi_data[1] # y_axis + + # NOTE: Special case for 2010. + label = None + if year == 2010 or year == 2011 : + label = r"%d, %s" % \ + (year,r'7TeV $\sigma$=71.5mb') + else: + label = r"%d, %s" % \ + (year,r'8TeV $\sigma$=73mb') + ax.plot(times,maxpus, + color=color_by_year[year], +# marker="none", linestyle="solid", + marker="o", linestyle='none', + linewidth=4, + label=label) + if is_log: + ax.set_yscale("log") + + time_begin = datetime.datetime(years[0], 1, 1, 0, 0, 0) + time_end = datetime.datetime(years[-1], 12, 16, 20, 50,9) + str_begin = time_begin.strftime(DATE_FMT_STR_OUT) + str_end = time_end.strftime(DATE_FMT_STR_OUT) + + num_cols = None + num_cols = len(years) + tmp_x = 0.095 + tmp_y = .95 + + leg = ax.legend(loc="upper left", bbox_to_anchor=(tmp_x, 0., 1., tmp_y), + frameon=False, ncol=num_cols) + for t in leg.get_texts(): + t.set_font_properties(FONT_PROPS_TICK_LABEL) + + # Set titles and labels. + fig.suptitle(r"CMS peak interactions per crossing, %s" % particle_type_str, + fontproperties=FONT_PROPS_SUPTITLE) + ax.set_title("Data included from %s to %s UTC \n" % \ + (str_begin_ultimate, str_end), + fontproperties=FONT_PROPS_TITLE) + ax.set_xlabel(r"Date (UTC)", fontproperties=FONT_PROPS_AX_TITLE) + ax.set_ylabel(r"Peak interactions per crossing",\ + fontproperties=FONT_PROPS_AX_TITLE) + + # Add the logo. + #zoom = 1.7 + zoom = .95 + AddLogo(logo_name, ax, zoom=zoom) + extra_head_room = 0 + if is_log: + #if mode == 1: + extra_head_room = 1 + #elif mode == 2: + # extra_head_room = 2 + TweakPlot(fig, ax, (time_begin, time_end), +# TweakPlot(fig, ax, (time_begin_ultimate, time_end), + add_extra_head_room=True) + + log_suffix = "" + if is_log: + log_suffix = "_log" + SavePlot(fig, "peak_pu_%s_%s%s" % \ + (particle_type_str.lower(), + log_suffix, file_suffix)) + PlotPeakPUAllYears(lumi_data_by_fill_per_year) + + plt.close() + + print "Done" + +###################################################################### diff --git a/RecoLuminosity/LumiDB/plotdata/public_lumi_plots_pp_2012.cfg b/RecoLuminosity/LumiDB/plotdata/public_lumi_plots_pp_2012.cfg index 2b7747d4852c1..928eba983fd7f 100644 --- a/RecoLuminosity/LumiDB/plotdata/public_lumi_plots_pp_2012.cfg +++ b/RecoLuminosity/LumiDB/plotdata/public_lumi_plots_pp_2012.cfg @@ -20,3 +20,7 @@ accel_mode = PROTPHYS date_begin = 2012-04-04 # This was the last day with non-zero delivered luminosity in 2012. date_end = 2012-12-17 + +# If a JSON file is specified, corresponding data will be shown in +# addition to delivered and recorded data. +# json_file = /afs/cern.ch/user/b/borrell/public/Cert_190456-208686_8TeV_22Jan2013ReReco_Collisions12_JSON.txt diff --git a/RecoLuminosity/LumiDB/python/dataDML.py b/RecoLuminosity/LumiDB/python/dataDML.py index f131eb5a59101..81339bb36ef50 100644 --- a/RecoLuminosity/LumiDB/python/dataDML.py +++ b/RecoLuminosity/LumiDB/python/dataDML.py @@ -239,7 +239,7 @@ def fillrunMap(schema,fillnum=None,runmin=None,runmax=None,startT=None,stopT=Non def runList(schema,datatagid,runmin=None,runmax=None,fillmin=None,fillmax=None,startT=None,stopT=None,l1keyPattern=None,hltkeyPattern=None,amodetag=None,nominalEnergy=None,energyFlut=0.2,requiretrg=True,requirehlt=True,preselectedruns=None,lumitype=None): ''' - select r.runnum,l.starttime,l.stoptime,l.data_id,t.data_id,h.data_id from cmsrunsummary r,tagruns tag,lumidata l,trgdata t,hltdata h where l.runnum=tag.runnum and r.runnum=l.runnum and l.runnum=t.runnum and t.runnum=h.runnum and r.fillnum>=:fillmin and r.fillnum<=fillmax and r.runnum>:runmin and r.runnum<:runmax and r.amodetag=:amodetag and regexp_like(r.l1key,:l1keypattern) and regexp_like(hltkey,:hltkeypattern) and l.nominalEnergy>=:nominalEnergy*(1-energyFlut) and l.nominalEnergy<=:nominalEnergy*(1+energyFlut) and tag.tagid<=:tagid and l.starttime is not null and l.stoptime is not null + select r.runnum,l.starttime,l.stoptime,l.data_id,tag.trgdataid,tag.hltdataid from cmsrunsummary r,tagruns tag,lumidata l,trgdata t,hltdata h where l.runnum=tag.runnum and r.runnum=l.runnum (and l.runnum=t.runnum and t.runnum=h.runnum) and r.fillnum>=:fillmin and r.fillnum<=fillmax and r.runnum>:runmin and r.runnum<:runmax and r.amodetag=:amodetag (and regexp_like(r.l1key,:l1keypattern) and regexp_like(hltkey,:hltkeypattern) ) and l.nominalEnergy>=:nominalEnergy*(1-energyFlut) and l.nominalEnergy<=:nominalEnergy*(1+energyFlut) and tag.tagid<=:tagid and l.starttime is not null and l.stoptime is not null output: {runnum:[lumiid,trgid,hltid]} ''' #print datatagid,runmin,runmax,fillmin,fillmax,preselectedruns @@ -272,12 +272,12 @@ def runList(schema,datatagid,runmin=None,runmax=None,fillmin=None,fillmax=None,s qCondition=coral.AttributeList() qCondition.extend('tagid','unsigned long long') qCondition['tagid'].setData(datatagid) - if requiretrg: - qHandle.addToTableList(t) - qConditionStr+=' and '+l+'.RUNNUM='+t+'.RUNNUM' - if requirehlt: - qHandle.addToTableList(h) - qConditionStr+=' and '+l+'.RUNNUM='+h+'.RUNNUM' + #if requiretrg: + # qHandle.addToTableList(t) + # qConditionStr+=' and '+l+'.RUNNUM='+t+'.RUNNUM' + #if requirehlt: + # qHandle.addToTableList(h) + # qConditionStr+=' and '+l+'.RUNNUM='+h+'.RUNNUM' if runmin and runmax : if runmin==runmax: qConditionStr+=' AND '+r+'.RUNNUM=:runmin' @@ -319,11 +319,13 @@ def runList(schema,datatagid,runmin=None,runmax=None,fillmin=None,fillmax=None,s qCondition.extend('amodetag','string') qCondition['amodetag'].setData(amodetag) if l1keyPattern: - qConditionStr+=' AND regexp_like('+r+'.L1KEY,:l1keypattern)' + qHandle.addToTableList(t) + qConditionStr+=' AND regexp_like('+r+'.L1KEY,:l1keypattern )'+' AND '+l+'.RUNNUM='+t+'.RUNNUM' qCondition.extend('l1keypattern','string') qCondition['l1keypattern'].setData(l1keyPattern) if hltkeyPattern: - qConditionStr+=' AND regexp_like('+r+'.HLTKEY,:hltkeypattern)' + qHandle.addToTableList(h) + qConditionStr+=' AND regexp_like('+r+'.HLTKEY,:hltkeypattern)'+' AND '+l+'.RUNNUM='+h+'.RUNNUM' qCondition.extend('hltkeypattern','string') qCondition['hltkeypattern'].setData(hltkeyPattern) if nominalEnergy: @@ -348,10 +350,14 @@ def runList(schema,datatagid,runmin=None,runmax=None,fillmin=None,fillmax=None,s qHandle.addToOutputList('TO_CHAR('+l+'.STARTTIME,\'MM/DD/YY HH24:MI:SS\')','starttime') qHandle.addToOutputList('TO_CHAR('+l+'.STOPTIME,\'MM/DD/YY HH24:MI:SS\')','stoptime') qHandle.addToOutputList(l+'.DATA_ID','lumiid') + #if requiretrg: + # qHandle.addToOutputList(t+'.DATA_ID','trgid') + #if requirehlt: + # qHandle.addToOutputList(h+'.DATA_ID','hltid') if requiretrg: - qHandle.addToOutputList(t+'.DATA_ID','trgid') + qHandle.addToOutputList(tag+'.TRGDATAID','trgid') if requirehlt: - qHandle.addToOutputList(h+'.DATA_ID','hltid') + qHandle.addToOutputList(tag+'.HLTDATAID','hltid') qHandle.defineOutput(qResult) cursor=qHandle.execute() lumiid=0 @@ -394,8 +400,11 @@ def runList(schema,datatagid,runmin=None,runmax=None,fillmin=None,fillmax=None,s result[runnum][0]=lumiid else: result[runnum]=[lumiid,0,0] - if requiretrg and not cursor.currentRow()['trgid'].isNull(): - trgid=cursor.currentRow()['trgid'].data() + if requiretrg : + if cursor.currentRow()['trgid'].isNull(): + trgid=0 + else: + trgid=cursor.currentRow()['trgid'].data() if result.has_key(runnum): if trgid>result[runnum][1]: result[runnum][1]=trgid diff --git a/RecoLuminosity/LumiDB/python/lumiTime.py b/RecoLuminosity/LumiDB/python/lumiTime.py index 1485fded58cab..5e3d864f67b3a 100644 --- a/RecoLuminosity/LumiDB/python/lumiTime.py +++ b/RecoLuminosity/LumiDB/python/lumiTime.py @@ -6,6 +6,7 @@ def __init__(self): self.coraltimefm='MM/DD/YY HH24:MI:SS' self.pydatetimefm='%m/%d/%y %H:%M:%S' self.nbx=3564 + self.orbits_per_ls=262144 # a lumisection is 2**18 orbits self.bunchspace_us=0.02495 #in microseconds self.bunchspace_s=24.95e-09 #in seconds @@ -13,22 +14,24 @@ def __init__(self): ##return datetime.fromtimestamp(ts,tz=pytz.utc) #return datetime.utcfromtimestamp(ts) - def LSDuration(self,norbits): + def LSDuration(self): + return self.OrbitDuration(self.orbits_per_ls) + + def OrbitDuration(self, norbits = 1): return timedelta(microseconds=(self.nbx*norbits*self.bunchspace_us)) - def OrbitDuration(self): - return timedelta(microseconds=(self.nbx*self.bunchspace_us)) - def OrbitToTimeStr(self,begStrTime,orbitnumber,begorbit=0,customfm=''): ''' given a orbit number, return its corresponding time. Assuming begin time has orbit=0 ''' - return self.DatetimeToStr(self.StrToDatetime(begStrTime)+(orbitnumber-begorbit)*self.OrbitDuration(),customfm=customfm) + return self.DatetimeToStr(self.StrToDatetime(begStrTime)+self.OrbitDuration(orbitnumber-begorbit),customfm=customfm) + def OrbitToTime(self,begStrTime,orbitnumber,begorbit=0,customfm=''): ''' given a orbit number, return its corresponding time. Default run begin time counting from orbit=0 ''' - return self.StrToDatetime(begStrTime,customfm=customfm)+(orbitnumber-begorbit)*self.OrbitDuration() + return self.StrToDatetime(begStrTime,customfm=customfm)+self.OrbitDuration(orbitnumber-begorbit) + def OrbitToLocalTimestamp(self,begStrTime,orbitnumber,begorbit=0,customfm=''): ''' given a orbit number, return its corresponding unixtimestamp. Default run begin time counting from orbit=0 diff --git a/RecoLuminosity/LumiDB/python/public_plots_tools.py b/RecoLuminosity/LumiDB/python/public_plots_tools.py index a7c9ea940ea36..2586f1f4b3cfe 100644 --- a/RecoLuminosity/LumiDB/python/public_plots_tools.py +++ b/RecoLuminosity/LumiDB/python/public_plots_tools.py @@ -139,6 +139,7 @@ def InitColors(cls): ColorScheme.cms_purple = (125./255., 16./255., 123./255.) ColorScheme.cms_green = (60./255., 177./255., 110./255.) ColorScheme.cms_orange2 = (227./255., 136./255., 36./255.) + ColorScheme.cms_lightyellow = (255./255., 235./255., 215./255.) # End of InitColors(). @@ -149,9 +150,11 @@ def __init__(self, name): # Some defaults. self.color_fill_del = "black" self.color_fill_rec = "white" + self.color_fill_cert = "red" self.color_fill_peak = "black" self.color_line_del = DarkenColor(self.color_fill_del) self.color_line_rec = DarkenColor(self.color_fill_rec) + self.color_line_cert = DarkenColor(self.color_fill_cert) self.color_line_peak = DarkenColor(self.color_fill_peak) self.color_by_year = { 2010 : "green", @@ -168,9 +171,11 @@ def __init__(self, name): # Color scheme 'Greg'. self.color_fill_del = ColorScheme.cms_blue self.color_fill_rec = ColorScheme.cms_orange + self.color_fill_cert = ColorScheme.cms_lightyellow self.color_fill_peak = ColorScheme.cms_orange self.color_line_del = DarkenColor(self.color_fill_del) self.color_line_rec = DarkenColor(self.color_fill_rec) + self.color_line_cert = DarkenColor(self.color_fill_cert) self.color_line_peak = DarkenColor(self.color_fill_peak) self.color_line_pileup = "black" self.color_fill_pileup = ColorScheme.cms_blue @@ -180,9 +185,11 @@ def __init__(self, name): # Color scheme 'Joe'. self.color_fill_del = ColorScheme.cms_yellow self.color_fill_rec = ColorScheme.cms_red + self.color_fill_cert = ColorScheme.cms_orange self.color_fill_peak = ColorScheme.cms_red self.color_line_del = DarkenColor(self.color_fill_del) self.color_line_rec = DarkenColor(self.color_fill_rec) + self.color_line_cert = DarkenColor(self.color_fill_cert) self.color_line_peak = DarkenColor(self.color_fill_peak) self.color_line_pileup = "black" self.color_fill_pileup = ColorScheme.cms_yellow diff --git a/RecoLuminosity/LumiDB/scripts/estimatePileup_makeJSON.py b/RecoLuminosity/LumiDB/scripts/estimatePileup_makeJSON.py index eae12a075f650..854132c3dd36f 100755 --- a/RecoLuminosity/LumiDB/scripts/estimatePileup_makeJSON.py +++ b/RecoLuminosity/LumiDB/scripts/estimatePileup_makeJSON.py @@ -151,11 +151,12 @@ def CalcPileup (deadTable, parameters, mode='deadtable'): runLumiDict = {} csvDict = {} pieces = sepRE.split (line.strip()) - if len (pieces) < 8: # means we are missing data; keep track of LS, lumi + + if len (pieces) < 10: # means we are missing data; keep track of LS, lumi InGap = 1 try: run, lumi = int ( pieces[0] ), int ( pieces[2] ) - delivered, recorded = float( pieces[7] ), float( pieces[8] ) + delivered, recorded = float( pieces[8] ), float( pieces[9] ) except: if pieces[0] != 'run': print " cannot parse csv file " @@ -168,13 +169,13 @@ def CalcPileup (deadTable, parameters, mode='deadtable'): # continue try: run, lumi = int ( pieces[0] ), int ( pieces[2] ) - delivered, recorded = float( pieces[7] ), float( pieces[8] ) + delivered, recorded = float( pieces[8] ), float( pieces[9] ) xingInstLumiArray = [( int(orbit), float(lum) ) \ - for orbit, lum in zip( pieces[9::2], - pieces[10::2] ) ] + for orbit, lum in zip( pieces[10::2], + pieces[11::2] ) ] except: print " Bad Parsing: Check if the input format has changed" - print pieces[0],pieces[1],pieces[2],pieces[3],pieces[4],pieces[5],pieces[6] + print pieces[0],pieces[1],pieces[2],pieces[3],pieces[4],pieces[5],pieces[6],pieces[7],pieces[8],pieces[9] continue csvDict.setdefault (run, {})[lumi] = \ diff --git a/RecoLuminosity/LumiDB/scripts/lumiCalc2.py b/RecoLuminosity/LumiDB/scripts/lumiCalc2.py index 4d980318f849d..3e3bf08e8c66e 100755 --- a/RecoLuminosity/LumiDB/scripts/lumiCalc2.py +++ b/RecoLuminosity/LumiDB/scripts/lumiCalc2.py @@ -334,10 +334,10 @@ def parseInputFiles(inputfilename): print '[INFO] No qualified lumi data found for run, ',irun if reqTrg and not tid: print '[INFO] No qualified trg data found for run ',irun - continue + # continue if reqHlt and not hid: print '[INFO] No qualified hlt data found for run ',irun - continue + # continue rruns.append(irun) if not irunlsdict: #no file irunlsdict=dict(zip(rruns,[None]*len(rruns))) diff --git a/RecoLuminosity/LumiDB/scripts/pixelLumiCalc.py b/RecoLuminosity/LumiDB/scripts/pixelLumiCalc.py index 139ef1457d91c..b0d204bc23d19 100755 --- a/RecoLuminosity/LumiDB/scripts/pixelLumiCalc.py +++ b/RecoLuminosity/LumiDB/scripts/pixelLumiCalc.py @@ -167,6 +167,8 @@ def parseInputFiles(inputfilename): reqHlt=False if options.action=='overview' or options.action=='lumibyls': reqTrg=True + if options.action=='lumibyls' and options.hltpath: + reqHlt=True if options.action=='recorded': reqTrg=True reqHlt=True @@ -259,9 +261,9 @@ def parseInputFiles(inputfilename): # ############################################################# datatagname=options.datatag if not datatagname: - (datatagid,datatagname)=revisionDML.currentDataTag(session.nominalSchema()) + (datatagid,datatagname)=revisionDML.currentDataTag(session.nominalSchema(),lumitype='PIXEL') else: - datatagid=revisionDML.getDataTagId(session.nominalSchema(),datatagname) + datatagid=revisionDML.getDataTagId(session.nominalSchema(),datatagname,lumitype='PIXEL') dataidmap=lumiCalcAPI.runList(session.nominalSchema(),datatagid,runmin=reqrunmin,runmax=reqrunmax,fillmin=reqfillmin,fillmax=reqfillmax,startT=reqtimemin,stopT=reqtimemax,l1keyPattern=None,hltkeyPattern=None,amodetag=None,nominalEnergy=None,energyFlut=None,requiretrg=reqTrg,requirehlt=reqHlt,preselectedruns=filerunlist,lumitype='PIXEL') if not dataidmap: @@ -273,10 +275,10 @@ def parseInputFiles(inputfilename): print '[INFO] No qualified lumi data found for run, ',irun if reqTrg and not tid: print '[INFO] No qualified trg data found for run ',irun - continue + # continue if reqHlt and not hid: print '[INFO] No qualified hlt data found for run ',irun - continue + # continue rruns.append(irun) if not irunlsdict: #no file irunlsdict=dict(zip(rruns,[None]*len(rruns))) diff --git a/RecoLuminosity/LumiDB/test/pixelLumiLoader.py b/RecoLuminosity/LumiDB/test/pixelLumiLoader.py index 56e5303865d60..2333e451a4685 100644 --- a/RecoLuminosity/LumiDB/test/pixelLumiLoader.py +++ b/RecoLuminosity/LumiDB/test/pixelLumiLoader.py @@ -5,25 +5,31 @@ # Author: Zhen Xie # ################################################################### -import os,sys,time,json -from RecoLuminosity.LumiDB import sessionManager,argparse,nameDealer,revisionDML,dataDML,lumiParameters +import os,sys,time,json,coral +from datetime import datetime +from RecoLuminosity.LumiDB import sessionManager,argparse,nameDealer,revisionDML,dataDML,lumiParameters,lumiTime def generateLumiRundata(filename,runsummaryData,runlist): ''' - input: runsummaryData {runnum:[]} - output: {runnum:[datasource,nominalegev,ncollidingbunches]} + input: runsummaryData {runnum: (datasource(0),nominalegev(1),ncollidingbunches(2),starttime(3),stoptime(4)} + output: {runnum: [nominalenergy,ncollidingbunches,starttime,stoptime,nls] } ''' result={} + t=lumiTime.lumiTime() for run in runlist: summary=runsummaryData[run] - result[run]=[filename,summary[1],summary[2]] + start=datetime.strptime(summary[3],'%m/%d/%y %H:%M:%S') + stop=datetime.strptime(summary[4],'%m/%d/%y %H:%M:%S') + starttime=coral.TimeStamp(start.year,start.month,start.day,start.hour,start.minute,start.second,0) + stoptime=coral.TimeStamp(stop.year,stop.month,stop.day,stop.hour,stop.minute,stop.second,0) + result[run]=[filename,summary[1],summary[2],starttime,stoptime,0] return result def generateLumiLSdataForRun(lsdata,lumirundata,beamsta): ''' input: lsdata: [(cmslsnum,instlumi),...] - lumirundata: [datasource,nominalegev,ncollidingbunches] + lumirundata: [datasource,nominalegev,ncollidingbunches,ncollidingbunches,starttime,stoptime,o] beamstatus {cmslsnum:beamstatus} output: i.e. bulkInsertLumiLSSummary expected input: {lumilsnum:[cmslsnum,instlumi,instlumierror,instlumiquality,beamstatus,beamenergy,numorbit,startorbit]} @@ -138,9 +144,9 @@ def parseInputFile(filename,singlerun=None): hfdataidmap=revisionDML.dataIdsByTagId(session.nominalSchema(),hf_tagid,runlist,withcomment=False,lumitype='HF') beamstatusdata=dataDML.beamstatusByIds(session.nominalSchema(),hfdataidmap) #print 'beamstatusdata ',beamstatusdata - lumirundata=dataDML.lumiRunByIds(session.nominalSchema(),hfdataidmap,lumitype='HF') - #{runnum: (datasource(1),nominalegev(2),ncollidingbunches(3)} + lumirundata=dataDML.lumiRunByIds(session.nominalSchema(),hfdataidmap,lumitype='HF')#{runnum: (nominalegev(0),ncollidingbunches(1),starttime(2),stoptime(3),nls(4)} session.transaction().commit() + #print 'lumirundata ', lumirundata alllumirundata=generateLumiRundata(inputfilename,lumirundata,runlist) alllumilsdata={} for runnum,perrundata in parseresult.items(): @@ -154,7 +160,7 @@ def parseInputFile(filename,singlerun=None): beamsta={} if beamstatusdata.has_key(runnum): beamsta=beamstatusdata[runnum] - alllumilsdata[runnum]=generateLumiLSdataForRun(perrundata,alllumirundata[runnum],beamsta) + alllumilsdata[runnum]=generateLumiLSdataForRun(perrundata,alllumirundata[runnum],beamsta)#lumirundata [datasource,nominalenergy,ncollidingbunches,starttime,stoptime,nls] pixellumirundata=alllumirundata[runnum] (pixellumirevid,pixellumientryid,pixellumidataid)=dataDML.addLumiRunDataToBranch(session.nominalSchema(),runnum,pixellumirundata,pixellumibranchinfo,nameDealer.pixellumidataTableName()) pixellumilsdata=alllumilsdata[runnum] diff --git a/RecoLuminosity/LumiDB/test/testNew.py b/RecoLuminosity/LumiDB/test/testNew.py index 3fa61805f230e..8baf2432136fe 100644 --- a/RecoLuminosity/LumiDB/test/testNew.py +++ b/RecoLuminosity/LumiDB/test/testNew.py @@ -47,4 +47,4 @@ print 'pathname, hltprescales ',pathname,hltprescales[pidx] session.transaction().commit() del session - + print ''