Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improvements to k4run #93

Merged
merged 7 commits into from
Apr 17, 2023
+82 −92
Merged

Improvements to k4run #93

Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
4d7213e
Clean up code and print options in the same order always
jmcarcell Jan 5, 2023
0c06471
Use the python logging module
jmcarcell Jan 11, 2023
cbd7c8f
Use argparse to parse from the files
jmcarcell Jan 12, 2023
9e8e2c8
Fix tests
jmcarcell Jan 14, 2023
f6ce074
Address some comments in the PR
jmcarcell Jan 18, 2023
15f8313
Clean up logging
jmcarcell Jan 18, 2023
00d5723
Remove brackets
jmcarcell Jan 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
170 changes: 85 additions & 85 deletions k4FWCore/scripts/k4run
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ from __future__ import print_function
import os
import sys
import argparse
from multiprocessing import cpu_count
import logging


# these default properties are filtered as otherwise they will clutter the argument list
__filterGaudiProps = [ "ContextService", "Cardinality", "Context", "CounterList", "EfficiencyRowFormat",
FILTER_GAUDI_PROPS = [ "ContextService", "Cardinality", "Context", "CounterList", "EfficiencyRowFormat",
"Enable", "ErrorCount", "ErrorMax", "ErrorsPrint", "ExtraInputs", "ExtraOutputs",
"FilterCircularDependencies", "StatEntityList", "IsIOBound", "MonitorService", "NeededResources",
"PropertiesPrint", "RegisterForContextService", "RegularRowFormat", "RequireObjects", "RootInTES",
Expand All @@ -18,35 +20,83 @@ __filterGaudiProps = [ "ContextService", "Cardinality", "Context", "CounterList"
]

#---------------------------------------------------------------------

seen_files = set()
option_db = {}


class LoadFromFile(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
for wrapper in values:
if wrapper.name in seen_files:
return
seen_files.add(wrapper.name)
exec(open(wrapper.name).read(), globals())
# loop over all components that were configured in the options file
# use dict.fromkeys so that the confs are sorted (Python >= 3.7)
for conf in dict.fromkeys(ApplicationMgr.allConfigurables.values()):
# skip public tools and the applicationmgr itself
if "ToolSvc" in conf.name() or "ApplicationMgr" in conf.name():
continue
props = conf.getPropertiesWithDescription() #dict propertyname: (propertyvalue, propertydescription)
for prop in props:
# only add arguments for relevant properties
if prop in FILTER_GAUDI_PROPS or "Audit" in prop or hasattr(props[prop][0], '__slots__'):
continue
propvalue = props[prop][0]

# if it is set to "no value" it hasn't been touched in the options file
if propvalue == conf.propertyNoValue:
propvalue = conf.getDefaultProperty(prop) # thus get the default value
proptype = type(props[prop][0])
# if the property is a list of something, we need to set argparse nargs to '+'
propnargs = "?"
if proptype == list:
# tricky edgecase: if the default is an empty list there is no way to get the type
if len(propvalue) == 0:
# just skip for now
#print("Warning: argparse cannot deduce type for property %s of %s. Needs to be set in options file." % (prop, conf.name()))
continue
else:
# deduce type from first item of the list
proptype = type(propvalue[0])
propnargs = "+"

# add the argument twice, once as "--PodioOutput.filename"
# and once as "--filename.PodioOutput"
propName = conf.name() + '.' + prop
propNameReversed = prop + '.' + conf.name()
option_db[propName] = (conf, propName)
parser.add_argument( "--%s" % propName, "--%s" % propNameReversed, type=proptype, help=props[prop][1],
nargs=propnargs,
default=propvalue)


if __name__ == "__main__":
# ensure that we (and the subprocesses) use the C standard localization
if os.environ.get('LC_ALL') != 'C':
os.environ['LC_ALL'] = 'C'

# run all the configuration files
# we generate arguments from them, so we need to load them
# before parsing the arguments
# to avoid parsing them twice, just assume the positional arguments
# come first and as soon as we encounter "-" we stop
run_with_options_file = False
for f in sys.argv[1:]:
if f[0] != '-':
exec(open(f).read())
run_with_options_file = True
else:
break
os.environ['LC_ALL'] = 'C'

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
# Don't propagate to the root logger, otherwise the output will be doubled
andresailer marked this conversation as resolved.
Show resolved Hide resolved
logger.propagate = 0
# formatter = logging.Formatter('[%(asctime)s %(levelname)s] %(message)s', datefmt='%Y-%b-%d %H:%M:%S')
formatter = logging.Formatter('[k4run] %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)

parser = argparse.ArgumentParser(description="Run job in the Key4HEP framework")
parser.add_argument("config_files", nargs="*",
help="Gaudi config (python) files describing the job")
parser.add_argument("config_files", type=open, action=LoadFromFile, nargs="*",
help="Gaudi config (python) files describing the job")
parser.add_argument("--dry-run", action="store_true",
help="Do not actually run the job, just parse the config files")
help="Do not actually run the job, just parse the config files")
parser.add_argument("-v", "--verbose", action="store_true",
help="Run job with verbose output")
help="Run job with verbose output")
parser.add_argument("-n", "--num-events", type=int,
help="Number of events to run")
help="Number of events to run")
parser.add_argument("-l", "--list", action="store_true",
help="Print all the configurable components available in the framework and exit")
help="Print all the configurable components available in the framework and exit")
parser.add_argument("--gdb", action="store_true", help="Attach gdb debugger")

# TODO: Don't want to advertise this until tested.
Expand All @@ -55,68 +105,25 @@ if __name__ == "__main__":
# -1 : All available cpus
# 0 : Serial Mode (traditional gaudirun)
# n>0 : parallel with n cpus (n <= sys_cpus)
parser.add_argument("--ncpus", type=int, default=0,
help="Start Gaudi in parallel mode using NCPUS processes. "
"0 => serial mode (default), -1 => use all CPUs")

parser.add_argument("--ncpus", type=int, default=0,
help="Start Gaudi in parallel mode using NCPUS processes. "
"0 => serial mode (default), -1 => use all CPUs")

# Once to parse the files and another time to have the new parameters
# in the namespace since parsing of the file happens when the namespace
# has already been filled
opts = parser.parse_known_args()
opts = parser.parse_args()

# print a doc line showing the configured algorithms
if run_with_options_file:
for alg in ApplicationMgr().TopAlg:
print(' --> ', alg.name(), end='')
print("",'\n')

option_db = {}

# loop over all components that were configured in the options file
for conf in set(list(ApplicationMgr.allConfigurables.values())):
# skip public tools and the applicationmgr itself
if "ToolSvc" not in conf.name() and "ApplicationMgr" not in conf.name():
props = conf.getPropertiesWithDescription() #dict propertyname: (propertyvalue, propertydescription)
for prop in props:
# only add arguments for relevant properties
if not prop in __filterGaudiProps:
if not "Audit" in prop:
# do not want to deal with other components / datahandles for now
if not hasattr(props[prop][0], '__slots__'):
propvalue = props[prop][0]

# if it is set to "no value" it hasn't been touched in the options file

if propvalue == conf.propertyNoValue:
propvalue = conf.getDefaultProperty(prop) # thus get the default value
proptype = type(props[prop][0])
# if the property is a list of something, we need to set argparse nargs to '+'
propnargs = "?"
if proptype == list:
# tricky edgecase: if the default is an empty list there is no way to get the type
if len(propvalue) == 0:
# just skip for now
#print("Warning: argparse cannot deduce type for property %s of %s. Needs to be set in options file." % (prop, conf.name()))
continue
else:
# deduce type from first item of the list
proptype = type(propvalue[0])
propnargs = "+"

# add the argument twice, once as "--PodioOutput.filename"
# and once as "--filename.PodioOutput"
propName = conf.name() + '.' + prop
propNameReversed = prop + '.' + conf.name()
option_db[propName] = (conf, propName)
parser.add_argument( "--%s" % propName, "--%s" % propNameReversed, type=proptype, help=props[prop][1],
nargs=propnargs,
default=propvalue)

opts=parser.parse_args()
logger.info(' --> '.join([alg.name() for alg in ApplicationMgr().TopAlg]))
jmcarcell marked this conversation as resolved.
Show resolved Hide resolved

if opts.list:
from Gaudi import Configuration
cfgDb = Configuration.cfgDb
print("Available components:\n%s" % (21 * "="))
logger.info("Available components:\n%s" % (21 * "="))
jmcarcell marked this conversation as resolved.
Show resolved Hide resolved
for item in sorted(cfgDb):
if True: # another option could filter Gaudi components here
if True: # another option could filter Gaudi components here
try:
path_to_component = __import__(cfgDb[item]["module"]).__file__
except ImportError:
Expand All @@ -127,13 +134,9 @@ if __name__ == "__main__":
print(" %s (from %s)" % (item, cfgDb[item]["lib"]))
sys.exit()

if not run_with_options_file:
print("usage: " + os.path.basename(__file__) + " [gaudi_config.py ...] Specify a gaudi options file to run")
sys.exit()

opts_dict = vars(opts)
for optionName, propTuple in option_db.items():
print("Option name:", propTuple[1], optionName, opts_dict[optionName])
logger.info("Option name: " + str(propTuple[1]) + " " + optionName + " " + str(opts_dict[optionName]))
jmcarcell marked this conversation as resolved.
Show resolved Hide resolved
propTuple[0].setProp(propTuple[1].rsplit(".",1)[1], opts_dict[optionName])

if opts.verbose:
Expand All @@ -145,7 +148,6 @@ if __name__ == "__main__":

# Parallel Option ---------------------------------------------------------
if opts.ncpus:
from multiprocessing import cpu_count
sys_cpus = cpu_count()
if opts.ncpus > sys_cpus:
s = "Invalid value : --ncpus : only %i cpus available" % sys_cpus
Expand All @@ -155,9 +157,7 @@ if __name__ == "__main__":
parser.error(s)

# configure the logging
import logging
from GaudiKernel.ProcessJobOptions import (InstallRootLoggingHandler,
PrintOff)
from GaudiKernel.ProcessJobOptions import InstallRootLoggingHandler
andresailer marked this conversation as resolved.
Show resolved Hide resolved

prefix = "# "
level = logging.INFO
Expand Down