22 __all__ = [
"ArgumentParser",
"ConfigFileAction",
"ConfigValueAction",
"DataIdContainer",
23 "DatasetArgument",
"ConfigDatasetType",
"InputOnlyArgumentParser"]
39 import lsst.pex.config
as pexConfig
40 import lsst.pex.config.history
44 DEFAULT_INPUT_NAME =
"PIPE_INPUT_ROOT" 45 DEFAULT_CALIB_NAME =
"PIPE_CALIB_ROOT" 46 DEFAULT_OUTPUT_NAME =
"PIPE_OUTPUT_ROOT" 49 def _fixPath(defName, path):
50 """Apply environment variable as default root, if present, and abspath. 55 Name of environment variable containing default root path; 56 if the environment variable does not exist 57 then the path is relative to the current working directory 59 Path relative to default root path. 64 Path that has been expanded, or `None` if the environment variable 65 does not exist and path is `None`. 67 defRoot = os.environ.get(defName)
71 return os.path.abspath(path)
72 return os.path.abspath(os.path.join(defRoot, path
or ""))
76 """Container for data IDs and associated data references. 81 The lowest hierarchy level to descend to for this dataset type, 82 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`. 83 Use `""` to use the mapper's default for the dataset type. 84 This class does not support `None`, but if it did, `None` 85 would mean the level should not be restricted. 89 Override this class for data IDs that require special handling to be 90 converted to ``data references``, and specify the override class 91 as ``ContainerClass`` for ``add_id_argument``. 93 If you don't want the argument parser to compute data references, 94 specify ``doMakeDataRefList=False`` in ``add_id_argument``. 99 """Dataset type of the data references (`str`). 102 """See parameter ``level`` (`str`). 105 """List of data IDs specified on the command line for the 106 appropriate data ID argument (`list` of `dict`). 109 """List of data references for the data IDs in ``idList`` 110 (`list` of `lsst.daf.persistence.ButlerDataRef`). 111 Elements will be omitted if the corresponding data is not found. 112 The list will be empty when returned by ``parse_args`` if 113 ``doMakeDataRefList=False`` was specified in ``add_id_argument``. 117 """Set actual dataset type, once it is known. 126 The reason ``datasetType`` is not a constructor argument is that 127 some subclasses do not know the dataset type until the command 128 is parsed. Thus, to reduce special cases in the code, 129 ``datasetType`` is always set after the command is parsed. 134 """Validate data IDs and cast them to the correct type 135 (modify idList in place). 137 This code casts the values in the data IDs dicts in `dataIdList` 138 to the type required by the butler. Data IDs are read from the 139 command line as `str`, but the butler requires some values to be 140 other types. For example "visit" values should be `int`. 144 butler : `lsst.daf.persistence.Butler` 148 raise RuntimeError(
"Must call setDatasetType first")
150 idKeyTypeDict = butler.getKeys(datasetType=self.
datasetType, level=self.
level)
151 except KeyError
as e:
152 msg =
"Cannot get keys for datasetType %s at level %s" % (self.
datasetType, self.
level)
153 raise KeyError(msg)
from e
155 for dataDict
in self.
idList:
156 for key, strVal
in dataDict.items():
158 keyType = idKeyTypeDict[key]
163 log = lsstLog.Log.getDefaultLogger()
164 log.warn(
"Unexpected ID %s; guessing type is \"%s\"" %
165 (key,
'str' if keyType == str
else keyType))
166 idKeyTypeDict[key] = keyType
170 castVal = keyType(strVal)
172 raise TypeError(
"Cannot cast value %r to %s for ID key %r" % (strVal, keyType, key,))
173 dataDict[key] = castVal
176 """Compute refList based on idList. 180 namespace : `argparse.Namespace` 181 Results of parsing command-line. The ``butler`` and ``log`` 182 elements must be set. 186 Not called if ``add_id_argument`` was called with 187 ``doMakeDataRefList=False``. 190 raise RuntimeError(
"Must call setDatasetType first")
191 butler = namespace.butler
192 for dataId
in self.
idList:
193 refList = dafPersist.searchDataRefs(butler, datasetType=self.
datasetType,
194 level=self.
level, dataId=dataId)
196 namespace.log.warn(
"No data found for dataId=%s", dataId)
202 """data ID argument, used by `ArgumentParser.add_id_argument`. 207 Name of identifier (argument name without dashes). 209 Type of dataset; specify a string for a fixed dataset type 210 or a `DatasetArgument` for a dynamic dataset type (e.g. 211 one specified by a command-line argument). 213 The lowest hierarchy level to descend to for this dataset type, 214 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`. 215 Use `""` to use the mapper's default for the dataset type. 216 Some container classes may also support `None`, which means 217 the level should not be restricted; however the default class, 218 `DataIdContainer`, does not support `None`. 219 doMakeDataRefList : `bool`, optional 220 If `True` (default), construct data references. 221 ContainerClass : `class`, optional 222 Class to contain data IDs and data references; the default class 223 `DataIdContainer` will work for many, but not all, cases. 224 For example if the dataset type is specified on the command line 225 then use `DynamicDatasetType`. 228 def __init__(self, name, datasetType, level, doMakeDataRefList=True, ContainerClass=DataIdContainer):
229 if name.startswith(
"-"):
230 raise RuntimeError(
"Name %s must not start with -" % (name,))
240 """`True` if the dataset type is dynamic (that is, specified 241 on the command line). 243 return isinstance(self.
datasetType, DynamicDatasetType)
246 """Get the dataset type as a string. 265 """Abstract base class for a dataset type determined from parsed 266 command-line arguments. 270 """Add a command-line argument to specify dataset type name, 275 parser : `ArgumentParser` 276 Argument parser to add the argument to. 278 Name of data ID argument, without the leading ``"--"``, 283 The default implementation does nothing 289 """Get the dataset type as a string, based on parsed command-line 297 raise NotImplementedError(
"Subclasses must override")
301 """Dataset type specified by a command-line argument. 305 name : `str`, optional 306 Name of command-line argument (including leading "--", 307 if appropriate) whose value is the dataset type. 308 If `None`, uses ``--idName_dstype`` where idName 309 is the name of the data ID argument (e.g. "id"). 310 help : `str`, optional 311 Help string for the command-line argument. 312 default : `object`, optional 313 Default value. If `None`, then the command-line option is required. 314 This argument isignored if the command-line argument is positional 315 (name does not start with "-") because positional arguments do 316 not support default values. 321 help="dataset type to process from input data repository",
324 DynamicDatasetType.__init__(self)
330 """Get the dataset type as a string, from the appropriate 331 command-line argument. 343 argName = self.
name.lstrip(
"-")
344 return getattr(namespace, argName)
347 """Add a command-line argument to specify the dataset type name. 351 parser : `ArgumentParser` 358 Also sets the `name` attribute if it is currently `None`. 360 help = self.
help if self.
help else "dataset type for %s" % (idName,)
361 if self.
name is None:
362 self.
name =
"--%s_dstype" % (idName,)
363 requiredDict = dict()
364 if self.
name.startswith(
"-"):
365 requiredDict = dict(required=self.
default is None)
374 """Dataset type specified by a config parameter. 379 Name of config option whose value is the dataset type. 383 DynamicDatasetType.__init__(self)
387 """Return the dataset type as a string, from the appropriate 392 namespace : `argparse.Namespace` 397 keyList = self.
name.split(
".")
398 value = namespace.config
401 value = getattr(value, key)
403 raise RuntimeError(
"Cannot find config parameter %r" % (self.
name,))
408 """Argument parser for command-line tasks that is based on 409 `argparse.ArgumentParser`. 414 Name of top-level task; used to identify camera-specific override 416 usage : `str`, optional 417 Command-line usage signature. 419 Additional keyword arguments for `argparse.ArgumentParser`. 423 Users may wish to add additional arguments before calling `parse_args`. 432 """Require an output directory to be specified (`bool`).""" 434 def __init__(self, name, usage="%(prog)s input [options]
", **kwargs): 437 argparse.ArgumentParser.__init__(self,
439 fromfile_prefix_chars=
'@',
440 epilog=textwrap.dedent(
"""Notes: 441 * --config, --configfile, --id, --loglevel and @file may appear multiple times; 442 all values are used, in order left to right 443 * @file reads command-line options from the specified file: 444 * data may be distributed among multiple lines (e.g. one option per line) 445 * data after # is treated as a comment and ignored 446 * blank lines and lines starting with # are ignored 447 * To specify multiple values for an option, do not use = after the option name: 448 * right: --configfile foo bar 449 * wrong: --configfile=foo bar 451 formatter_class=argparse.RawDescriptionHelpFormatter,
453 self.add_argument(metavar=
'input', dest=
"rawInput",
454 help=
"path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME,))
455 self.add_argument(
"--calib", dest=
"rawCalib",
456 help=
"path to input calibration repository, relative to $%s" %
457 (DEFAULT_CALIB_NAME,))
458 self.add_argument(
"--output", dest=
"rawOutput",
459 help=
"path to output data repository (need not exist), relative to $%s" %
460 (DEFAULT_OUTPUT_NAME,))
461 self.add_argument(
"--rerun", dest=
"rawRerun", metavar=
"[INPUT:]OUTPUT",
462 help=
"rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; " 463 "optionally sets ROOT to ROOT/rerun/INPUT")
464 self.add_argument(
"-c",
"--config", nargs=
"*", action=ConfigValueAction,
465 help=
"config override(s), e.g. -c foo=newfoo bar.baz=3", metavar=
"NAME=VALUE")
466 self.add_argument(
"-C",
"--configfile", dest=
"configfile", nargs=
"*", action=ConfigFileAction,
467 help=
"config override file(s)")
468 self.add_argument(
"-L",
"--loglevel", nargs=
"*", action=LogLevelAction,
469 help=
"logging level; supported levels are [trace|debug|info|warn|error|fatal]",
470 metavar=
"LEVEL|COMPONENT=LEVEL")
471 self.add_argument(
"--longlog", action=
"store_true", help=
"use a more verbose format for the logging")
472 self.add_argument(
"--debug", action=
"store_true", help=
"enable debugging output?")
473 self.add_argument(
"--doraise", action=
"store_true",
474 help=
"raise an exception on error (else log a message and continue)?")
475 self.add_argument(
"--noExit", action=
"store_true",
476 help=
"Do not exit even upon failure (i.e. return a struct to the calling script)")
477 self.add_argument(
"--profile", help=
"Dump cProfile statistics to filename")
478 self.add_argument(
"--show", nargs=
"+", default=(),
479 help=
"display the specified information to stdout and quit " 480 "(unless run is specified); information is " 481 "(config[=PATTERN]|history=PATTERN|tasks|data|run)")
482 self.add_argument(
"-j",
"--processes", type=int, default=1, help=
"Number of processes to use")
483 self.add_argument(
"-t",
"--timeout", type=float,
484 help=
"Timeout for multiprocessing; maximum wall time (sec)")
485 self.add_argument(
"--clobber-output", action=
"store_true", dest=
"clobberOutput", default=
False,
486 help=(
"remove and re-create the output directory if it already exists " 487 "(safe with -j, but not all other forms of parallel execution)"))
488 self.add_argument(
"--clobber-config", action=
"store_true", dest=
"clobberConfig", default=
False,
489 help=(
"backup and then overwrite existing config files instead of checking them " 490 "(safe with -j, but not all other forms of parallel execution)"))
491 self.add_argument(
"--no-backup-config", action=
"store_true", dest=
"noBackupConfig", default=
False,
492 help=
"Don't copy config to file~N backup.")
493 self.add_argument(
"--clobber-versions", action=
"store_true", dest=
"clobberVersions", default=
False,
494 help=(
"backup and then overwrite existing package versions instead of checking" 495 "them (safe with -j, but not all other forms of parallel execution)"))
496 self.add_argument(
"--no-versions", action=
"store_true", dest=
"noVersions", default=
False,
497 help=
"don't check package versions; useful for development")
498 lsstLog.configure_prop(
""" 499 log4j.rootLogger=INFO, A1 500 log4j.appender.A1=ConsoleAppender 501 log4j.appender.A1.Target=System.out 502 log4j.appender.A1.layout=PatternLayout 503 log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n 507 lgr = logging.getLogger()
508 lgr.setLevel(logging.INFO)
509 lgr.addHandler(lsstLog.LogHandler())
511 def add_id_argument(self, name, datasetType, help, level=None, doMakeDataRefList=True,
512 ContainerClass=DataIdContainer):
513 """Add a data ID argument. 519 Data ID argument (including leading dashes, if wanted). 520 datasetType : `str` or `DynamicDatasetType`-type 521 Type of dataset. Supply a string for a fixed dataset type. 522 For a dynamically determined dataset type, supply 523 a `DynamicDatasetType`, such a `DatasetArgument`. 525 Help string for the argument. 527 The lowest hierarchy level to descend to for this dataset type, 528 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`. 529 Use `""` to use the mapper's default for the dataset type. 530 Some container classes may also support `None`, which means 531 the level should not be restricted; however the default class, 532 `DataIdContainer`, does not support `None`. 533 doMakeDataRefList : bool, optional 534 If `True` (default), construct data references. 535 ContainerClass : `class`, optional 536 Class to contain data IDs and data references; the default class 537 `DataIdContainer` will work for many, but not all, cases. 538 For example if the dataset type is specified on the command line 539 then use `DynamicDatasetType`. 543 If ``datasetType`` is an instance of `DatasetArgument`, 544 then add a second argument to specify the dataset type. 546 The associated data is put into ``namespace.<dataIdArgument.name>`` 547 as an instance of `ContainerClass`; the container includes fields: 549 - ``idList``: a list of data ID dicts. 550 - ``refList``: a list of `~lsst.daf.persistence.Butler` 551 data references (empty if ``doMakeDataRefList`` is `False`). 553 argName = name.lstrip(
"-")
556 raise RuntimeError(
"Data ID argument %s already exists" % (name,))
557 if argName
in set((
"camera",
"config",
"butler",
"log",
"obsPkg")):
558 raise RuntimeError(
"Data ID argument %s is a reserved name" % (name,))
560 self.add_argument(name, nargs=
"*", action=IdValueAction, help=help,
561 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
565 datasetType=datasetType,
567 doMakeDataRefList=doMakeDataRefList,
568 ContainerClass=ContainerClass,
571 if dataIdArgument.isDynamicDatasetType:
572 datasetType.addArgument(parser=self, idName=argName)
576 def parse_args(self, config, args=None, log=None, override=None):
577 """Parse arguments for a command-line task. 581 config : `lsst.pex.config.Config` 582 Config for the task being run. 583 args : `list`, optional 584 Argument list; if `None` then ``sys.argv[1:]`` is used. 585 log : `lsst.log.Log`, optional 586 `~lsst.log.Log` instance; if `None` use the default log. 587 override : callable, optional 588 A config override function. It must take the root config object 589 as its only argument and must modify the config in place. 590 This function is called after camera-specific overrides files 591 are applied, and before command-line config overrides 592 are applied (thus allowing the user the final word). 596 namespace : `argparse.Namespace` 597 A `~argparse.Namespace` instance containing fields: 599 - ``camera``: camera name. 600 - ``config``: the supplied config with all overrides applied, 601 validated and frozen. 602 - ``butler``: a `lsst.daf.persistence.Butler` for the data. 603 - An entry for each of the data ID arguments registered by 604 `add_id_argument`, of the type passed to its ``ContainerClass`` 605 keyword (`~lsst.pipe.base.DataIdContainer` by default). It 606 includes public elements ``idList`` and ``refList``. 607 - ``log``: a `lsst.log` Log. 608 - An entry for each command-line argument, 609 with the following exceptions: 611 - config is the supplied config, suitably updated. 612 - configfile, id and loglevel are all missing. 613 - ``obsPkg``: name of the ``obs_`` package for this camera. 618 if len(args) < 1
or args[0].startswith(
"-")
or args[0].startswith(
"@"):
620 if len(args) == 1
and args[0]
in (
"-h",
"--help"):
623 self.exit(
"%s: error: Must specify input as first argument" % self.prog)
627 namespace = argparse.Namespace()
628 namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
629 if not os.path.isdir(namespace.input):
630 self.error(
"Error: input=%r not found" % (namespace.input,))
632 namespace.config = config
633 namespace.log = log
if log
is not None else lsstLog.Log.getDefaultLogger()
634 mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
635 if mapperClass
is None:
636 self.error(
"Error: no mapper specified for input repo %r" % (namespace.input,))
638 namespace.camera = mapperClass.getCameraName()
639 namespace.obsPkg = mapperClass.getPackageName()
644 if override
is not None:
645 override(namespace.config)
649 setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
651 namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
652 del namespace.configfile
656 if namespace.clobberOutput:
657 if namespace.output
is None:
658 self.error(
"--clobber-output is only valid with --output or --rerun")
659 elif namespace.output == namespace.input:
660 self.error(
"--clobber-output is not valid when the output and input repos are the same")
661 if os.path.exists(namespace.output):
662 namespace.log.info(
"Removing output repo %s for --clobber-output", namespace.output)
663 shutil.rmtree(namespace.output)
665 namespace.log.debug(
"input=%s", namespace.input)
666 namespace.log.debug(
"calib=%s", namespace.calib)
667 namespace.log.debug(
"output=%s", namespace.output)
672 if self.
requireOutput and namespace.output
is None and namespace.rerun
is None:
673 self.error(
"no output directory specified.\n" 674 "An output directory must be specified with the --output or --rerun\n" 675 "command-line arguments.\n")
679 butlerArgs = {
'mapperArgs': {
'calibRoot': namespace.calib}}
681 outputs = {
'root': namespace.output,
'mode':
'rw'}
682 inputs = {
'root': namespace.input}
683 inputs.update(butlerArgs)
684 outputs.update(butlerArgs)
685 namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
687 outputs = {
'root': namespace.input,
'mode':
'rw'}
688 outputs.update(butlerArgs)
689 namespace.butler = dafPersist.Butler(outputs=outputs)
696 if "data" in namespace.show:
698 for dataRef
in getattr(namespace, dataIdName).refList:
699 print(
"%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId))
701 if namespace.show
and "run" not in namespace.show:
709 sys.stderr.write(
"Warning: no 'debug' module found\n")
710 namespace.debug =
False 712 del namespace.loglevel
714 if namespace.longlog:
715 lsstLog.configure_prop(
""" 716 log4j.rootLogger=INFO, A1 717 log4j.appender.A1=ConsoleAppender 718 log4j.appender.A1.Target=System.out 719 log4j.appender.A1.layout=PatternLayout 720 log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n 722 del namespace.longlog
724 namespace.config.validate()
725 namespace.config.freeze()
729 def _parseDirectories(self, namespace):
730 """Parse input, output and calib directories 732 This allows for hacking the directories, e.g., to include a 734 Modifications are made to the 'namespace' object in-place. 736 mapperClass = dafPersist.Butler.getMapperClass(_fixPath(DEFAULT_INPUT_NAME, namespace.rawInput))
737 namespace.calib = _fixPath(DEFAULT_CALIB_NAME, namespace.rawCalib)
740 if namespace.rawOutput:
741 namespace.output = _fixPath(DEFAULT_OUTPUT_NAME, namespace.rawOutput)
743 namespace.output =
None 751 if namespace.rawRerun:
753 self.error(
"Error: cannot specify both --output and --rerun")
754 namespace.rerun = namespace.rawRerun.split(
":")
755 rerunDir = [os.path.join(namespace.input,
"rerun", dd)
for dd
in namespace.rerun]
756 modifiedInput =
False 757 if len(rerunDir) == 2:
758 namespace.input, namespace.output = rerunDir
760 elif len(rerunDir) == 1:
761 namespace.output = rerunDir[0]
762 if os.path.exists(os.path.join(namespace.output,
"_parent")):
763 namespace.input = os.path.realpath(os.path.join(namespace.output,
"_parent"))
766 self.error(
"Error: invalid argument for --rerun: %s" % namespace.rerun)
767 if modifiedInput
and dafPersist.Butler.getMapperClass(namespace.input) != mapperClass:
768 self.error(
"Error: input directory specified by --rerun must have the same mapper as INPUT")
770 namespace.rerun =
None 771 del namespace.rawInput
772 del namespace.rawCalib
773 del namespace.rawOutput
774 del namespace.rawRerun
776 def _processDataIds(self, namespace):
777 """Process the parsed data for each data ID argument in an 778 `~argparse.Namespace`. 782 - Validate data ID keys. 783 - Cast the data ID values to the correct type. 784 - Compute data references from data IDs. 788 namespace : `argparse.Namespace` 789 Parsed namespace. These attributes are read: 793 - ``config``, if any dynamic dataset types are set by 795 - Dataset type arguments (e.g. ``id_dstype``), if any dynamic 796 dataset types are specified by such 798 These attributes are modified: 800 - ``<name>`` for each data ID argument registered using 801 `add_id_argument` with name ``<name>``. 804 dataIdContainer = getattr(namespace, dataIdArgument.name)
805 dataIdContainer.setDatasetType(dataIdArgument.getDatasetType(namespace))
806 if dataIdArgument.doMakeDataRefList:
808 dataIdContainer.castDataIds(butler=namespace.butler)
809 except (KeyError, TypeError)
as e:
815 dataIdContainer.makeDataRefList(namespace)
817 def _applyInitialOverrides(self, namespace):
818 """Apply obs-package-specific and camera-specific config 819 override files, if found 823 namespace : `argparse.Namespace` 824 Parsed namespace. These attributes are read: 828 Look in the package namespace.obsPkg for files: 830 - ``config/<task_name>.py`` 831 - ``config/<camera_name>/<task_name>.py`` and load if found. 834 fileName = self.
_name +
".py" 836 os.path.join(obsPkgDir,
"config", fileName),
837 os.path.join(obsPkgDir,
"config", namespace.camera, fileName),
839 if os.path.exists(filePath):
840 namespace.log.info(
"Loading config overrride file %r", filePath)
841 namespace.config.load(filePath)
843 namespace.log.debug(
"Config override file does not exist: %r", filePath)
846 """Perform camera-specific operations before parsing the command-line. 850 namespace : `argparse.Namespace` 851 Namespace (an ) with the following fields: 853 - ``camera``: the camera name. 854 - ``config``: the config passed to parse_args, with no overrides applied. 855 - ``obsPkg``: the ``obs_`` package for this camera. 856 - ``log``: a `lsst.log` Log. 860 The default implementation does nothing. 865 """Allow files of arguments referenced by ``@<path>`` to contain 866 multiple values on each line. 871 Line of text read from an argument file. 873 arg_line = arg_line.strip()
874 if not arg_line
or arg_line.startswith(
"#"):
876 for arg
in shlex.split(arg_line, comments=
True, posix=
True):
882 """Add a "--reuse-outputs-from SUBTASK" option to the argument 885 CmdLineTasks that can be restarted at an intermediate step using 886 outputs from earlier (but still internal) steps should use this 887 method to allow the user to control whether that happens when 888 outputs from earlier steps are present. 893 A sequence of string names (by convention, top-level subtasks) 894 that identify the steps that could be skipped when their 895 outputs are already present. The list is ordered, so when the 896 user specifies one step on the command line, all previous steps 897 may be skipped as well. In addition to the choices provided, 898 users may pass "all" to indicate that all steps may be thus 901 When this method is called, the ``namespace`` object returned by 902 ``parse_args`` will contain a ``reuse`` attribute containing 903 a list of all steps that should be skipped if their outputs 905 If no steps should be skipped, the ``reuse`` will be an empty list. 907 choices = list(choices)
908 choices.append(
"all")
909 self.add_argument(
"--reuse-outputs-from", dest=
"reuse", choices=choices,
910 default=[], action=ReuseAction,
911 help=(
"Skip the given subtask and its predecessors and reuse their outputs " 912 "if those outputs already exist. Use 'all' to specify all subtasks."))
916 """`ArgumentParser` for command-line tasks that don't write any output. 919 requireOutput =
False 923 """Get a dictionary of task info for all subtasks in a config 927 config : `lsst.pex.config.Config` 928 Configuration to process. 929 taskDict : `dict`, optional 930 Users should not specify this argument. Supports recursion. 931 If provided, taskDict is updated in place, else a new `dict` 933 baseName : `str`, optional 934 Users should not specify this argument. It is only used for 935 recursion: if a non-empty string then a period is appended 936 and the result is used as a prefix for additional entries 937 in taskDict; otherwise no prefix is used. 942 Keys are config field names, values are task names. 946 This function is designed to be called recursively. 947 The user should call with only a config (leaving taskDict and baseName 948 at their default values). 952 for fieldName, field
in config.items():
953 if hasattr(field,
"value")
and hasattr(field,
"target"):
954 subConfig = field.value
955 if isinstance(subConfig, pexConfig.Config):
956 subBaseName =
"%s.%s" % (baseName, fieldName)
if baseName
else fieldName
958 taskName =
"%s.%s" % (field.target.__module__, field.target.__name__)
960 taskName = repr(field.target)
961 taskDict[subBaseName] = taskName
962 getTaskDict(config=subConfig, taskDict=taskDict, baseName=subBaseName)
967 """Process arguments specified with ``--show`` (but ignores 972 showOpts : `list` of `str` 973 List of options passed to ``--show``. 976 exit : bool, optional 977 Exit if ``"run"`` isn't included in ``showOpts``. 981 Supports the following options in showOpts: 983 - ``config[=PAT]``. Dump all the config entries, or just the ones that 984 match the glob pattern. 985 - ``history=PAT``. Show where the config entries that match the glob 987 - ``tasks``. Show task hierarchy. 988 - ``data``. Ignored; to be processed by caller. 989 - ``run``. Keep going (the default behaviour is to exit if 990 ``--show`` is specified). 992 Calls ``sys.exit(1)`` if any other option found. 997 for what
in showOpts:
998 showCommand, showArgs = what.split(
"=", 1)
if "=" in what
else (what,
"")
1000 if showCommand ==
"config":
1001 matConfig = re.search(
r"^(?:config.)?(.+)?", showArgs)
1002 pattern = matConfig.group(1)
1004 class FilteredStream:
1005 """A file object that only prints lines 1006 that match the glob "pattern". 1008 N.b. Newlines are silently discarded and reinserted; 1009 crude but effective. 1012 def __init__(self, pattern):
1014 mat = re.search(
r"(.*):NOIGNORECASE$", pattern)
1017 pattern = mat.group(1)
1018 self._pattern = re.compile(fnmatch.translate(pattern))
1020 if pattern != pattern.lower():
1021 print(
u"Matching \"%s\" without regard to case " 1022 "(append :NOIGNORECASE to prevent this)" % (pattern,), file=sys.stdout)
1023 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
1025 def write(self, showStr):
1026 showStr = showStr.rstrip()
1029 matchStr = showStr.split(
"\n")[-1].split(
"=")[0]
1030 if self._pattern.search(matchStr):
1031 print(
u"\n" + showStr)
1033 fd = FilteredStream(pattern)
1037 config.saveToStream(fd,
"config")
1038 elif showCommand ==
"history":
1039 matHistory = re.search(
r"^(?:config.)?(.+)?", showArgs)
1040 globPattern = matHistory.group(1)
1042 print(
"Please provide a value with --show history (e.g. history=*.doXXX)", file=sys.stderr)
1046 for i, pattern
in enumerate(fnmatch.filter(config.names(), globPattern)):
1050 pattern = pattern.split(
".")
1051 cpath, cname = pattern[:-1], pattern[-1]
1053 for i, cpt
in enumerate(cpath):
1055 hconfig = getattr(hconfig, cpt)
1056 except AttributeError:
1057 print(
"Error: configuration %s has no subconfig %s" %
1058 (
".".join([
"config"] + cpath[:i]), cpt), file=sys.stderr)
1062 print(pexConfig.history.format(hconfig, cname))
1064 print(
"Error: %s has no field %s" % (
".".join([
"config"] + cpath), cname),
1071 elif showCommand ==
"data":
1073 elif showCommand ==
"run":
1075 elif showCommand ==
"tasks":
1078 print(
u"Unknown value for show: %s (choose from '%s')" %
1079 (what,
"', '".join(
"config[=XXX] data history=XXX tasks run".split())), file=sys.stderr)
1082 if exit
and "run" not in showOpts:
1087 """Print task hierarchy to stdout. 1091 config : `lsst.pex.config.Config` 1092 Configuration to process. 1097 fieldNameList = sorted(taskDict.keys())
1098 for fieldName
in fieldNameList:
1099 taskName = taskDict[fieldName]
1100 print(
u"%s: %s" % (fieldName, taskName))
1104 """argparse action callback to override config parameters using 1105 name=value pairs from the command-line. 1108 def __call__(self, parser, namespace, values, option_string):
1109 """Override one or more config name value pairs. 1113 parser : `argparse.ArgumentParser` 1115 namespace : `argparse.Namespace` 1116 Parsed command. The ``namespace.config`` attribute is updated. 1118 A list of ``configItemName=value`` pairs. 1119 option_string : `str` 1120 Option value specified by the user. 1122 if namespace.config
is None:
1124 for nameValue
in values:
1125 name, sep, valueStr = nameValue.partition(
"=")
1127 parser.error(
"%s value %s must be in form name=value" % (option_string, nameValue))
1132 except AttributeError:
1133 parser.error(
"no config field: %s" % (name,))
1136 value = eval(valueStr, {})
1138 parser.error(
"cannot parse %r as a value for %s" % (valueStr, name))
1141 except Exception
as e:
1142 parser.error(
"cannot set config.%s=%r: %s" % (name, value, e))
1146 """argparse action to load config overrides from one or more files. 1149 def __call__(self, parser, namespace, values, option_string=None):
1150 """Load one or more files of config overrides. 1154 parser : `argparse.ArgumentParser` 1156 namespace : `argparse.Namespace` 1157 Parsed command. The following attributes are updated by this 1158 method: ``namespace.config``. 1160 A list of data config file paths. 1161 option_string : `str`, optional 1162 Option value specified by the user. 1164 if namespace.config
is None:
1166 for configfile
in values:
1168 namespace.config.load(configfile)
1169 except Exception
as e:
1170 parser.error(
"cannot load config file %r: %s" % (configfile, e))
1174 """argparse action callback to process a data ID into a dict. 1177 def __call__(self, parser, namespace, values, option_string):
1178 """Parse ``--id`` data and append results to 1179 ``namespace.<argument>.idList``. 1183 parser : `ArgumentParser` 1185 namespace : `argparse.Namespace` 1186 Parsed command (an instance of argparse.Namespace). 1187 The following attributes are updated: 1189 - ``<idName>.idList``, where ``<idName>`` is the name of the 1190 ID argument, for instance ``"id"`` for ID argument ``--id``. 1192 A list of data IDs; see Notes below. 1193 option_string : `str` 1194 Option value specified by the user. 1198 The data format is:: 1200 key1=value1_1[^value1_2[^value1_3...] 1201 key2=value2_1[^value2_2[^value2_3...]... 1203 The values (e.g. ``value1_1``) may either be a string, 1204 or of the form ``"int..int"`` (e.g. ``"1..3"``) which is 1205 interpreted as ``"1^2^3"`` (inclusive, unlike a python range). 1206 So ``"0^2..4^7..9"`` is equivalent to ``"0^2^3^4^7^8^9"``. 1207 You may also specify a stride: ``"1..5:2"`` is ``"1^3^5"``. 1209 The cross product is computed for keys with multiple values. 1212 --id visit 1^2 ccd 1,1^2,2 1214 results in the following data ID dicts being appended to 1215 ``namespace.<argument>.idList``: 1217 {"visit":1, "ccd":"1,1"} 1218 {"visit":2, "ccd":"1,1"} 1219 {"visit":1, "ccd":"2,2"} 1220 {"visit":2, "ccd":"2,2"} 1222 if namespace.config
is None:
1224 idDict = collections.OrderedDict()
1225 for nameValue
in values:
1226 name, sep, valueStr = nameValue.partition(
"=")
1228 parser.error(
"%s appears multiple times in one ID argument: %s" % (name, option_string))
1230 for v
in valueStr.split(
"^"):
1231 mat = re.search(
r"^(\d+)\.\.(\d+)(?::(\d+))?$", v)
1233 v1 = int(mat.group(1))
1234 v2 = int(mat.group(2))
1236 v3 = int(v3)
if v3
else 1
1237 for v
in range(v1, v2 + 1, v3):
1238 idDict[name].append(str(v))
1240 idDict[name].append(v)
1242 iterList = [idDict[key]
for key
in idDict.keys()]
1243 idDictList = [collections.OrderedDict(zip(idDict.keys(), valList))
1244 for valList
in itertools.product(*iterList)]
1246 argName = option_string.lstrip(
"-")
1247 ident = getattr(namespace, argName)
1248 ident.idList += idDictList
1252 """argparse action to set log level. 1255 def __call__(self, parser, namespace, values, option_string):
1260 parser : `ArgumentParser` 1262 namespace : `argparse.Namespace` 1263 Parsed command. This argument is not used. 1265 List of trace levels; each item must be of the form 1266 ``component_name=level`` or ``level``, where ``level`` 1267 is a keyword (not case sensitive) or an integer. 1268 option_string : `str` 1269 Option value specified by the user. 1271 permittedLevelList = (
'TRACE',
'DEBUG',
'INFO',
'WARN',
'ERROR',
'FATAL')
1272 permittedLevelSet = set(permittedLevelList)
1273 for componentLevel
in values:
1274 component, sep, levelStr = componentLevel.partition(
"=")
1276 levelStr, component = component,
None 1277 logLevelUpr = levelStr.upper()
1278 if logLevelUpr
in permittedLevelSet:
1279 logLevel = getattr(lsstLog.Log, logLevelUpr)
1281 parser.error(
"loglevel=%r not one of %s" % (levelStr, permittedLevelList))
1282 if component
is None:
1283 namespace.log.setLevel(logLevel)
1285 lsstLog.Log.getLogger(component).setLevel(logLevel)
1287 pyLevel = lsstLog.LevelTranslator.lsstLog2logging(logLevel)
1288 logging.getLogger(component).setLevel(pyLevel)
1292 """argparse action associated with ArgumentPraser.addReuseOption.""" 1294 def __call__(self, parser, namespace, value, option_string):
1296 value = self.choices[-2]
1297 index = self.choices.index(value)
1298 namespace.reuse = self.choices[:index + 1]
1302 """Set an instance attribute (like `setattr` but accepting 1303 hierarchical names such as ``foo.bar.baz``). 1308 Object whose attribute is to be set. 1310 Name of attribute to set. 1312 New value for the attribute. 1316 For example if name is ``foo.bar.baz`` then ``item.foo.bar.baz`` 1317 is set to the specified value. 1320 subnameList = name.split(
".")
1321 for subname
in subnameList[:-1]:
1322 subitem = getattr(subitem, subname)
1323 setattr(subitem, subnameList[-1], value)
1327 """Get an attribute (like `getattr` but accepts hierarchical names 1328 such as ``foo.bar.baz``). 1333 Object whose attribute is to be returned. 1335 Name of the attribute to get. 1340 If name is ``foo.bar.baz then the return value is 1341 ``item.foo.bar.baz``. 1344 for subname
in name.split(
"."):
1345 subitem = getattr(subitem, subname)
def addArgument(self, parser, idName)
def getTaskDict(config, taskDict=None, baseName="")
def __init__(self, level=None)
def getDottedAttr(item, name)
def __init__(self, name=None, help="dataset type to process from input data repository", default=None)
def getDatasetType(self, namespace)
def addArgument(self, parser, idName)
def makeDataRefList(self, namespace)
def setDottedAttr(item, name, value)
def __call__(self, parser, namespace, values, option_string=None)
def getDatasetType(self, namespace)
def obeyShowArgument(showOpts, config=None, exit=False)
def addReuseOption(self, choices)
def getDatasetType(self, namespace)
std::string getPackageDir(std::string const &packageName)
def _processDataIds(self, namespace)
def getDatasetType(self, namespace)
def convert_arg_line_to_args(self, arg_line)
def isDynamicDatasetType(self)
def handleCamera(self, namespace)
def __init__(self, name, datasetType, level, doMakeDataRefList=True, ContainerClass=DataIdContainer)
def __call__(self, parser, namespace, value, option_string)
def _applyInitialOverrides(self, namespace)
def castDataIds(self, butler)
def __init__(self, name, usage="%(prog)s input [options]", kwargs)
def __call__(self, parser, namespace, values, option_string)
def setDatasetType(self, datasetType)
def showTaskHierarchy(config)
def __call__(self, parser, namespace, values, option_string)
def __call__(self, parser, namespace, values, option_string)
def parse_args(self, config, args=None, log=None, override=None)
def _parseDirectories(self, namespace)
def add_id_argument(self, name, datasetType, help, level=None, doMakeDataRefList=True, ContainerClass=DataIdContainer)