22 __all__ = [
"ArgumentParser",
"ConfigFileAction",
"ConfigValueAction",
"DataIdContainer",
23 "DatasetArgument",
"ConfigDatasetType",
"InputOnlyArgumentParser"]
39 import lsst.pex.config
as pexConfig
40 import lsst.pex.config.history
44 DEFAULT_INPUT_NAME =
"PIPE_INPUT_ROOT"
45 DEFAULT_CALIB_NAME =
"PIPE_CALIB_ROOT"
46 DEFAULT_OUTPUT_NAME =
"PIPE_OUTPUT_ROOT"
49 def _fixPath(defName, path):
50 """Apply environment variable as default root, if present, and abspath.
55 Name of environment variable containing default root path;
56 if the environment variable does not exist
57 then the path is relative to the current working directory
59 Path relative to default root path.
64 Path that has been expanded, or `None` if the environment variable
65 does not exist and path is `None`.
67 defRoot = os.environ.get(defName)
71 return os.path.abspath(path)
72 return os.path.abspath(os.path.join(defRoot, path
or ""))
76 """Container for data IDs and associated data references.
81 The lowest hierarchy level to descend to for this dataset type,
82 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
83 Use `""` to use the mapper's default for the dataset type.
84 This class does not support `None`, but if it did, `None`
85 would mean the level should not be restricted.
89 Override this class for data IDs that require special handling to be
90 converted to ``data references``, and specify the override class
91 as ``ContainerClass`` for ``add_id_argument``.
93 If you don't want the argument parser to compute data references,
94 specify ``doMakeDataRefList=False`` in ``add_id_argument``.
99 """Dataset type of the data references (`str`).
102 """See parameter ``level`` (`str`).
105 """List of data IDs specified on the command line for the
106 appropriate data ID argument (`list` of `dict`).
109 """List of data references for the data IDs in ``idList``
110 (`list` of `lsst.daf.persistence.ButlerDataRef`).
111 Elements will be omitted if the corresponding data is not found.
112 The list will be empty when returned by ``parse_args`` if
113 ``doMakeDataRefList=False`` was specified in ``add_id_argument``.
117 """Set actual dataset type, once it is known.
126 The reason ``datasetType`` is not a constructor argument is that
127 some subclasses do not know the dataset type until the command
128 is parsed. Thus, to reduce special cases in the code,
129 ``datasetType`` is always set after the command is parsed.
134 """Validate data IDs and cast them to the correct type
135 (modify idList in place).
137 This code casts the values in the data IDs dicts in `dataIdList`
138 to the type required by the butler. Data IDs are read from the
139 command line as `str`, but the butler requires some values to be
140 other types. For example "visit" values should be `int`.
144 butler : `lsst.daf.persistence.Butler`
148 raise RuntimeError(
"Must call setDatasetType first")
150 idKeyTypeDict = butler.getKeys(datasetType=self.
datasetType, level=self.
level)
151 except KeyError
as e:
152 msg =
"Cannot get keys for datasetType %s at level %s" % (self.
datasetType, self.
level)
153 raise KeyError(msg)
from e
155 for dataDict
in self.
idList:
156 for key, strVal
in dataDict.items():
158 keyType = idKeyTypeDict[key]
163 log = lsstLog.Log.getDefaultLogger()
164 log.warn(
"Unexpected ID %s; guessing type is \"%s\"" %
165 (key,
'str' if keyType == str
else keyType))
166 idKeyTypeDict[key] = keyType
170 castVal = keyType(strVal)
172 raise TypeError(
"Cannot cast value %r to %s for ID key %r" % (strVal, keyType, key,))
173 dataDict[key] = castVal
176 """Compute refList based on idList.
180 namespace : `argparse.Namespace`
181 Results of parsing command-line. The ``butler`` and ``log``
182 elements must be set.
186 Not called if ``add_id_argument`` was called with
187 ``doMakeDataRefList=False``.
190 raise RuntimeError(
"Must call setDatasetType first")
191 butler = namespace.butler
192 for dataId
in self.
idList:
193 refList = dafPersist.searchDataRefs(butler, datasetType=self.
datasetType,
194 level=self.
level, dataId=dataId)
196 namespace.log.warn(
"No data found for dataId=%s", dataId)
202 """data ID argument, used by `ArgumentParser.add_id_argument`.
207 Name of identifier (argument name without dashes).
209 Type of dataset; specify a string for a fixed dataset type
210 or a `DatasetArgument` for a dynamic dataset type (e.g.
211 one specified by a command-line argument).
213 The lowest hierarchy level to descend to for this dataset type,
214 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
215 Use `""` to use the mapper's default for the dataset type.
216 Some container classes may also support `None`, which means
217 the level should not be restricted; however the default class,
218 `DataIdContainer`, does not support `None`.
219 doMakeDataRefList : `bool`, optional
220 If `True` (default), construct data references.
221 ContainerClass : `class`, optional
222 Class to contain data IDs and data references; the default class
223 `DataIdContainer` will work for many, but not all, cases.
224 For example if the dataset type is specified on the command line
225 then use `DynamicDatasetType`.
228 def __init__(self, name, datasetType, level, doMakeDataRefList=True, ContainerClass=DataIdContainer):
229 if name.startswith(
"-"):
230 raise RuntimeError(
"Name %s must not start with -" % (name,))
240 """`True` if the dataset type is dynamic (that is, specified
241 on the command line).
243 return isinstance(self.
datasetType, DynamicDatasetType)
246 """Get the dataset type as a string.
265 """Abstract base class for a dataset type determined from parsed
266 command-line arguments.
270 """Add a command-line argument to specify dataset type name,
275 parser : `ArgumentParser`
276 Argument parser to add the argument to.
278 Name of data ID argument, without the leading ``"--"``,
283 The default implementation does nothing
289 """Get the dataset type as a string, based on parsed command-line
297 raise NotImplementedError(
"Subclasses must override")
301 """Dataset type specified by a command-line argument.
305 name : `str`, optional
306 Name of command-line argument (including leading "--",
307 if appropriate) whose value is the dataset type.
308 If `None`, uses ``--idName_dstype`` where idName
309 is the name of the data ID argument (e.g. "id").
310 help : `str`, optional
311 Help string for the command-line argument.
312 default : `object`, optional
313 Default value. If `None`, then the command-line option is required.
314 This argument isignored if the command-line argument is positional
315 (name does not start with "-") because positional arguments do
316 not support default values.
321 help="dataset type to process from input data repository",
324 DynamicDatasetType.__init__(self)
330 """Get the dataset type as a string, from the appropriate
331 command-line argument.
343 argName = self.
name.lstrip(
"-")
344 return getattr(namespace, argName)
347 """Add a command-line argument to specify the dataset type name.
351 parser : `ArgumentParser`
358 Also sets the `name` attribute if it is currently `None`.
360 help = self.
help if self.
help else "dataset type for %s" % (idName,)
361 if self.
name is None:
362 self.
name =
"--%s_dstype" % (idName,)
363 requiredDict = dict()
364 if self.
name.startswith(
"-"):
365 requiredDict = dict(required=self.
default is None)
374 """Dataset type specified by a config parameter.
379 Name of config option whose value is the dataset type.
383 DynamicDatasetType.__init__(self)
387 """Return the dataset type as a string, from the appropriate
392 namespace : `argparse.Namespace`
397 keyList = self.
name.split(
".")
398 value = namespace.config
401 value = getattr(value, key)
403 raise RuntimeError(
"Cannot find config parameter %r" % (self.
name,))
408 """Argument parser for command-line tasks that is based on
409 `argparse.ArgumentParser`.
414 Name of top-level task; used to identify camera-specific override
416 usage : `str`, optional
417 Command-line usage signature.
419 Additional keyword arguments for `argparse.ArgumentParser`.
423 Users may wish to add additional arguments before calling `parse_args`.
432 """Require an output directory to be specified (`bool`)."""
434 def __init__(self, name, usage="%(prog)s input [options]
", **kwargs):
437 argparse.ArgumentParser.__init__(self,
439 fromfile_prefix_chars=
'@',
440 epilog=textwrap.dedent(
"""Notes:
441 * --config, --configfile, --id, --loglevel and @file may appear multiple times;
442 all values are used, in order left to right
443 * @file reads command-line options from the specified file:
444 * data may be distributed among multiple lines (e.g. one option per line)
445 * data after # is treated as a comment and ignored
446 * blank lines and lines starting with # are ignored
447 * To specify multiple values for an option, do not use = after the option name:
448 * right: --configfile foo bar
449 * wrong: --configfile=foo bar
451 formatter_class=argparse.RawDescriptionHelpFormatter,
453 self.add_argument(metavar=
'input', dest=
"rawInput",
454 help=
"path to input data repository, relative to $%s" % (DEFAULT_INPUT_NAME,))
455 self.add_argument(
"--calib", dest=
"rawCalib",
456 help=
"path to input calibration repository, relative to $%s" %
457 (DEFAULT_CALIB_NAME,))
458 self.add_argument(
"--output", dest=
"rawOutput",
459 help=
"path to output data repository (need not exist), relative to $%s" %
460 (DEFAULT_OUTPUT_NAME,))
461 self.add_argument(
"--rerun", dest=
"rawRerun", metavar=
"[INPUT:]OUTPUT",
462 help=
"rerun name: sets OUTPUT to ROOT/rerun/OUTPUT; "
463 "optionally sets ROOT to ROOT/rerun/INPUT")
464 self.add_argument(
"-c",
"--config", nargs=
"*", action=ConfigValueAction,
465 help=
"config override(s), e.g. -c foo=newfoo bar.baz=3", metavar=
"NAME=VALUE")
466 self.add_argument(
"-C",
"--configfile", dest=
"configfile", nargs=
"*", action=ConfigFileAction,
467 help=
"config override file(s)")
468 self.add_argument(
"-L",
"--loglevel", nargs=
"*", action=LogLevelAction,
469 help=
"logging level; supported levels are [trace|debug|info|warn|error|fatal]",
470 metavar=
"LEVEL|COMPONENT=LEVEL")
471 self.add_argument(
"--longlog", action=
"store_true", help=
"use a more verbose format for the logging")
472 self.add_argument(
"--debug", action=
"store_true", help=
"enable debugging output?")
473 self.add_argument(
"--doraise", action=
"store_true",
474 help=
"raise an exception on error (else log a message and continue)?")
475 self.add_argument(
"--noExit", action=
"store_true",
476 help=
"Do not exit even upon failure (i.e. return a struct to the calling script)")
477 self.add_argument(
"--profile", help=
"Dump cProfile statistics to filename")
478 self.add_argument(
"--show", nargs=
"+", default=(),
479 help=
"display the specified information to stdout and quit "
480 "(unless run is specified); information is "
481 "(config[=PATTERN]|history=PATTERN|tasks|data|run)")
482 self.add_argument(
"-j",
"--processes", type=int, default=1, help=
"Number of processes to use")
483 self.add_argument(
"-t",
"--timeout", type=float,
484 help=
"Timeout for multiprocessing; maximum wall time (sec)")
485 self.add_argument(
"--clobber-output", action=
"store_true", dest=
"clobberOutput", default=
False,
486 help=(
"remove and re-create the output directory if it already exists "
487 "(safe with -j, but not all other forms of parallel execution)"))
488 self.add_argument(
"--clobber-config", action=
"store_true", dest=
"clobberConfig", default=
False,
489 help=(
"backup and then overwrite existing config files instead of checking them "
490 "(safe with -j, but not all other forms of parallel execution)"))
491 self.add_argument(
"--no-backup-config", action=
"store_true", dest=
"noBackupConfig", default=
False,
492 help=
"Don't copy config to file~N backup.")
493 self.add_argument(
"--clobber-versions", action=
"store_true", dest=
"clobberVersions", default=
False,
494 help=(
"backup and then overwrite existing package versions instead of checking"
495 "them (safe with -j, but not all other forms of parallel execution)"))
496 self.add_argument(
"--no-versions", action=
"store_true", dest=
"noVersions", default=
False,
497 help=
"don't check package versions; useful for development")
498 lsstLog.configure_prop(
"""
499 log4j.rootLogger=INFO, A1
500 log4j.appender.A1=ConsoleAppender
501 log4j.appender.A1.Target=System.out
502 log4j.appender.A1.layout=PatternLayout
503 log4j.appender.A1.layout.ConversionPattern=%c %p: %m%n
507 lgr = logging.getLogger()
508 lgr.setLevel(logging.INFO)
509 lgr.addHandler(lsstLog.LogHandler())
511 def add_id_argument(self, name, datasetType, help, level=None, doMakeDataRefList=True,
512 ContainerClass=DataIdContainer):
513 """Add a data ID argument.
519 Data ID argument (including leading dashes, if wanted).
520 datasetType : `str` or `DynamicDatasetType`-type
521 Type of dataset. Supply a string for a fixed dataset type.
522 For a dynamically determined dataset type, supply
523 a `DynamicDatasetType`, such a `DatasetArgument`.
525 Help string for the argument.
527 The lowest hierarchy level to descend to for this dataset type,
528 for example `"amp"` for `"raw"` or `"ccd"` for `"calexp"`.
529 Use `""` to use the mapper's default for the dataset type.
530 Some container classes may also support `None`, which means
531 the level should not be restricted; however the default class,
532 `DataIdContainer`, does not support `None`.
533 doMakeDataRefList : bool, optional
534 If `True` (default), construct data references.
535 ContainerClass : `class`, optional
536 Class to contain data IDs and data references; the default class
537 `DataIdContainer` will work for many, but not all, cases.
538 For example if the dataset type is specified on the command line
539 then use `DynamicDatasetType`.
543 If ``datasetType`` is an instance of `DatasetArgument`,
544 then add a second argument to specify the dataset type.
546 The associated data is put into ``namespace.<dataIdArgument.name>``
547 as an instance of `ContainerClass`; the container includes fields:
549 - ``idList``: a list of data ID dicts.
550 - ``refList``: a list of `~lsst.daf.persistence.Butler`
551 data references (empty if ``doMakeDataRefList`` is `False`).
553 argName = name.lstrip(
"-")
556 raise RuntimeError(
"Data ID argument %s already exists" % (name,))
557 if argName
in set((
"camera",
"config",
"butler",
"log",
"obsPkg")):
558 raise RuntimeError(
"Data ID argument %s is a reserved name" % (name,))
560 self.add_argument(name, nargs=
"*", action=IdValueAction, help=help,
561 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
565 datasetType=datasetType,
567 doMakeDataRefList=doMakeDataRefList,
568 ContainerClass=ContainerClass,
571 if dataIdArgument.isDynamicDatasetType:
572 datasetType.addArgument(parser=self, idName=argName)
576 def parse_args(self, config, args=None, log=None, override=None):
577 """Parse arguments for a command-line task.
581 config : `lsst.pex.config.Config`
582 Config for the task being run.
583 args : `list`, optional
584 Argument list; if `None` then ``sys.argv[1:]`` is used.
585 log : `lsst.log.Log`, optional
586 `~lsst.log.Log` instance; if `None` use the default log.
587 override : callable, optional
588 A config override function. It must take the root config object
589 as its only argument and must modify the config in place.
590 This function is called after camera-specific overrides files
591 are applied, and before command-line config overrides
592 are applied (thus allowing the user the final word).
596 namespace : `argparse.Namespace`
597 A `~argparse.Namespace` instance containing fields:
599 - ``camera``: camera name.
600 - ``config``: the supplied config with all overrides applied,
601 validated and frozen.
602 - ``butler``: a `lsst.daf.persistence.Butler` for the data.
603 - An entry for each of the data ID arguments registered by
604 `add_id_argument`, of the type passed to its ``ContainerClass``
605 keyword (`~lsst.pipe.base.DataIdContainer` by default). It
606 includes public elements ``idList`` and ``refList``.
607 - ``log``: a `lsst.log` Log.
608 - An entry for each command-line argument,
609 with the following exceptions:
611 - config is the supplied config, suitably updated.
612 - configfile, id and loglevel are all missing.
613 - ``obsPkg``: name of the ``obs_`` package for this camera.
618 if len(args) < 1
or args[0].startswith(
"-")
or args[0].startswith(
"@"):
620 if len(args) == 1
and args[0]
in (
"-h",
"--help"):
623 self.exit(
"%s: error: Must specify input as first argument" % self.prog)
627 namespace = argparse.Namespace()
628 namespace.input = _fixPath(DEFAULT_INPUT_NAME, args[0])
629 if not os.path.isdir(namespace.input):
630 self.error(
"Error: input=%r not found" % (namespace.input,))
632 namespace.config = config
633 namespace.log = log
if log
is not None else lsstLog.Log.getDefaultLogger()
634 mapperClass = dafPersist.Butler.getMapperClass(namespace.input)
635 if mapperClass
is None:
636 self.error(
"Error: no mapper specified for input repo %r" % (namespace.input,))
638 namespace.camera = mapperClass.getCameraName()
639 namespace.obsPkg = mapperClass.getPackageName()
644 if override
is not None:
645 override(namespace.config)
649 setattr(namespace, dataIdArgument.name, dataIdArgument.ContainerClass(level=dataIdArgument.level))
651 namespace = argparse.ArgumentParser.parse_args(self, args=args, namespace=namespace)
652 del namespace.configfile
656 if namespace.clobberOutput:
657 if namespace.output
is None:
658 self.error(
"--clobber-output is only valid with --output or --rerun")
659 elif namespace.output == namespace.input:
660 self.error(
"--clobber-output is not valid when the output and input repos are the same")
661 if os.path.exists(namespace.output):
662 namespace.log.info(
"Removing output repo %s for --clobber-output", namespace.output)
663 shutil.rmtree(namespace.output)
665 namespace.log.debug(
"input=%s", namespace.input)
666 namespace.log.debug(
"calib=%s", namespace.calib)
667 namespace.log.debug(
"output=%s", namespace.output)
672 if self.
requireOutput and namespace.output
is None and namespace.rerun
is None:
673 self.error(
"no output directory specified.\n"
674 "An output directory must be specified with the --output or --rerun\n"
675 "command-line arguments.\n")
679 butlerArgs = {
'mapperArgs': {
'calibRoot': namespace.calib}}
681 outputs = {
'root': namespace.output,
'mode':
'rw'}
682 inputs = {
'root': namespace.input}
683 inputs.update(butlerArgs)
684 outputs.update(butlerArgs)
685 namespace.butler = dafPersist.Butler(inputs=inputs, outputs=outputs)
687 outputs = {
'root': namespace.input,
'mode':
'rw'}
688 outputs.update(butlerArgs)
689 namespace.butler = dafPersist.Butler(outputs=outputs)
696 if "data" in namespace.show:
698 for dataRef
in getattr(namespace, dataIdName).refList:
699 print(
"%s dataRef.dataId = %s" % (dataIdName, dataRef.dataId))
701 if namespace.show
and "run" not in namespace.show:
709 sys.stderr.write(
"Warning: no 'debug' module found\n")
710 namespace.debug =
False
712 del namespace.loglevel
714 if namespace.longlog:
715 lsstLog.configure_prop(
"""
716 log4j.rootLogger=INFO, A1
717 log4j.appender.A1=ConsoleAppender
718 log4j.appender.A1.Target=System.out
719 log4j.appender.A1.layout=PatternLayout
720 log4j.appender.A1.layout.ConversionPattern=%-5p %d{yyyy-MM-ddTHH:mm:ss.SSSZ} %c (%X{LABEL})(%F:%L)- %m%n
722 del namespace.longlog
724 namespace.config.validate()
725 namespace.config.freeze()
729 def _parseDirectories(self, namespace):
730 """Parse input, output and calib directories
732 This allows for hacking the directories, e.g., to include a
734 Modifications are made to the 'namespace' object in-place.
736 mapperClass = dafPersist.Butler.getMapperClass(_fixPath(DEFAULT_INPUT_NAME, namespace.rawInput))
737 namespace.calib = _fixPath(DEFAULT_CALIB_NAME, namespace.rawCalib)
740 if namespace.rawOutput:
741 namespace.output = _fixPath(DEFAULT_OUTPUT_NAME, namespace.rawOutput)
743 namespace.output =
None
751 if namespace.rawRerun:
753 self.error(
"Error: cannot specify both --output and --rerun")
754 namespace.rerun = namespace.rawRerun.split(
":")
755 rerunDir = [os.path.join(namespace.input,
"rerun", dd)
for dd
in namespace.rerun]
756 modifiedInput =
False
757 if len(rerunDir) == 2:
758 namespace.input, namespace.output = rerunDir
760 elif len(rerunDir) == 1:
761 namespace.output = rerunDir[0]
762 if os.path.exists(os.path.join(namespace.output,
"_parent")):
763 namespace.input = os.path.realpath(os.path.join(namespace.output,
"_parent"))
766 self.error(
"Error: invalid argument for --rerun: %s" % namespace.rerun)
767 if modifiedInput
and dafPersist.Butler.getMapperClass(namespace.input) != mapperClass:
768 self.error(
"Error: input directory specified by --rerun must have the same mapper as INPUT")
770 namespace.rerun =
None
771 del namespace.rawInput
772 del namespace.rawCalib
773 del namespace.rawOutput
774 del namespace.rawRerun
776 def _processDataIds(self, namespace):
777 """Process the parsed data for each data ID argument in an
778 `~argparse.Namespace`.
782 - Validate data ID keys.
783 - Cast the data ID values to the correct type.
784 - Compute data references from data IDs.
788 namespace : `argparse.Namespace`
789 Parsed namespace. These attributes are read:
793 - ``config``, if any dynamic dataset types are set by
795 - Dataset type arguments (e.g. ``id_dstype``), if any dynamic
796 dataset types are specified by such
798 These attributes are modified:
800 - ``<name>`` for each data ID argument registered using
801 `add_id_argument` with name ``<name>``.
804 dataIdContainer = getattr(namespace, dataIdArgument.name)
805 dataIdContainer.setDatasetType(dataIdArgument.getDatasetType(namespace))
806 if dataIdArgument.doMakeDataRefList:
808 dataIdContainer.castDataIds(butler=namespace.butler)
809 except (KeyError, TypeError)
as e:
815 dataIdContainer.makeDataRefList(namespace)
817 def _applyInitialOverrides(self, namespace):
818 """Apply obs-package-specific and camera-specific config
819 override files, if found
823 namespace : `argparse.Namespace`
824 Parsed namespace. These attributes are read:
828 Look in the package namespace.obsPkg for files:
830 - ``config/<task_name>.py``
831 - ``config/<camera_name>/<task_name>.py`` and load if found.
834 fileName = self.
_name +
".py"
836 os.path.join(obsPkgDir,
"config", fileName),
837 os.path.join(obsPkgDir,
"config", namespace.camera, fileName),
839 if os.path.exists(filePath):
840 namespace.log.info(
"Loading config overrride file %r", filePath)
841 namespace.config.load(filePath)
843 namespace.log.debug(
"Config override file does not exist: %r", filePath)
846 """Perform camera-specific operations before parsing the command-line.
850 namespace : `argparse.Namespace`
851 Namespace (an ) with the following fields:
853 - ``camera``: the camera name.
854 - ``config``: the config passed to parse_args, with no overrides applied.
855 - ``obsPkg``: the ``obs_`` package for this camera.
856 - ``log``: a `lsst.log` Log.
860 The default implementation does nothing.
865 """Allow files of arguments referenced by ``@<path>`` to contain
866 multiple values on each line.
871 Line of text read from an argument file.
873 arg_line = arg_line.strip()
874 if not arg_line
or arg_line.startswith(
"#"):
876 for arg
in shlex.split(arg_line, comments=
True, posix=
True):
882 """Add a "--reuse-outputs-from SUBTASK" option to the argument
885 CmdLineTasks that can be restarted at an intermediate step using
886 outputs from earlier (but still internal) steps should use this
887 method to allow the user to control whether that happens when
888 outputs from earlier steps are present.
893 A sequence of string names (by convention, top-level subtasks)
894 that identify the steps that could be skipped when their
895 outputs are already present. The list is ordered, so when the
896 user specifies one step on the command line, all previous steps
897 may be skipped as well. In addition to the choices provided,
898 users may pass "all" to indicate that all steps may be thus
901 When this method is called, the ``namespace`` object returned by
902 ``parse_args`` will contain a ``reuse`` attribute containing
903 a list of all steps that should be skipped if their outputs
905 If no steps should be skipped, the ``reuse`` will be an empty list.
907 choices = list(choices)
908 choices.append(
"all")
909 self.add_argument(
"--reuse-outputs-from", dest=
"reuse", choices=choices,
910 default=[], action=ReuseAction,
911 help=(
"Skip the given subtask and its predecessors and reuse their outputs "
912 "if those outputs already exist. Use 'all' to specify all subtasks."))
916 """`ArgumentParser` for command-line tasks that don't write any output.
919 requireOutput =
False
923 """Get a dictionary of task info for all subtasks in a config
927 config : `lsst.pex.config.Config`
928 Configuration to process.
929 taskDict : `dict`, optional
930 Users should not specify this argument. Supports recursion.
931 If provided, taskDict is updated in place, else a new `dict`
933 baseName : `str`, optional
934 Users should not specify this argument. It is only used for
935 recursion: if a non-empty string then a period is appended
936 and the result is used as a prefix for additional entries
937 in taskDict; otherwise no prefix is used.
942 Keys are config field names, values are task names.
946 This function is designed to be called recursively.
947 The user should call with only a config (leaving taskDict and baseName
948 at their default values).
952 for fieldName, field
in config.items():
953 if hasattr(field,
"value")
and hasattr(field,
"target"):
954 subConfig = field.value
955 if isinstance(subConfig, pexConfig.Config):
956 subBaseName =
"%s.%s" % (baseName, fieldName)
if baseName
else fieldName
958 taskName =
"%s.%s" % (field.target.__module__, field.target.__name__)
960 taskName = repr(field.target)
961 taskDict[subBaseName] = taskName
962 getTaskDict(config=subConfig, taskDict=taskDict, baseName=subBaseName)
967 """Process arguments specified with ``--show`` (but ignores
972 showOpts : `list` of `str`
973 List of options passed to ``--show``.
976 exit : bool, optional
977 Exit if ``"run"`` isn't included in ``showOpts``.
981 Supports the following options in showOpts:
983 - ``config[=PAT]``. Dump all the config entries, or just the ones that
984 match the glob pattern.
985 - ``history=PAT``. Show where the config entries that match the glob
987 - ``tasks``. Show task hierarchy.
988 - ``data``. Ignored; to be processed by caller.
989 - ``run``. Keep going (the default behaviour is to exit if
990 ``--show`` is specified).
992 Calls ``sys.exit(1)`` if any other option found.
997 for what
in showOpts:
998 showCommand, showArgs = what.split(
"=", 1)
if "=" in what
else (what,
"")
1000 if showCommand ==
"config":
1001 matConfig = re.search(
r"^(?:config.)?(.+)?", showArgs)
1002 pattern = matConfig.group(1)
1004 class FilteredStream:
1005 """A file object that only prints lines
1006 that match the glob "pattern".
1008 N.b. Newlines are silently discarded and reinserted;
1009 crude but effective.
1012 def __init__(self, pattern):
1014 mat = re.search(
r"(.*):NOIGNORECASE$", pattern)
1017 pattern = mat.group(1)
1018 self._pattern = re.compile(fnmatch.translate(pattern))
1020 if pattern != pattern.lower():
1021 print(
u"Matching \"%s\" without regard to case "
1022 "(append :NOIGNORECASE to prevent this)" % (pattern,), file=sys.stdout)
1023 self._pattern = re.compile(fnmatch.translate(pattern), re.IGNORECASE)
1025 def write(self, showStr):
1026 showStr = showStr.rstrip()
1029 matchStr = showStr.split(
"\n")[-1].split(
"=")[0]
1030 if self._pattern.search(matchStr):
1031 print(
u"\n" + showStr)
1033 fd = FilteredStream(pattern)
1037 config.saveToStream(fd,
"config")
1038 elif showCommand ==
"history":
1039 matHistory = re.search(
r"^(?:config.)?(.+)?", showArgs)
1040 globPattern = matHistory.group(1)
1042 print(
"Please provide a value with --show history (e.g. history=*.doXXX)", file=sys.stderr)
1046 for i, pattern
in enumerate(fnmatch.filter(config.names(), globPattern)):
1050 pattern = pattern.split(
".")
1051 cpath, cname = pattern[:-1], pattern[-1]
1053 for i, cpt
in enumerate(cpath):
1055 hconfig = getattr(hconfig, cpt)
1056 except AttributeError:
1057 print(
"Error: configuration %s has no subconfig %s" %
1058 (
".".join([
"config"] + cpath[:i]), cpt), file=sys.stderr)
1062 print(pexConfig.history.format(hconfig, cname))
1064 print(
"Error: %s has no field %s" % (
".".join([
"config"] + cpath), cname),
1071 elif showCommand ==
"data":
1073 elif showCommand ==
"run":
1075 elif showCommand ==
"tasks":
1078 print(
u"Unknown value for show: %s (choose from '%s')" %
1079 (what,
"', '".join(
"config[=XXX] data history=XXX tasks run".split())), file=sys.stderr)
1082 if exit
and "run" not in showOpts:
1087 """Print task hierarchy to stdout.
1091 config : `lsst.pex.config.Config`
1092 Configuration to process.
1097 fieldNameList = sorted(taskDict.keys())
1098 for fieldName
in fieldNameList:
1099 taskName = taskDict[fieldName]
1100 print(
u"%s: %s" % (fieldName, taskName))
1104 """argparse action callback to override config parameters using
1105 name=value pairs from the command-line.
1108 def __call__(self, parser, namespace, values, option_string):
1109 """Override one or more config name value pairs.
1113 parser : `argparse.ArgumentParser`
1115 namespace : `argparse.Namespace`
1116 Parsed command. The ``namespace.config`` attribute is updated.
1118 A list of ``configItemName=value`` pairs.
1119 option_string : `str`
1120 Option value specified by the user.
1122 if namespace.config
is None:
1124 for nameValue
in values:
1125 name, sep, valueStr = nameValue.partition(
"=")
1127 parser.error(
"%s value %s must be in form name=value" % (option_string, nameValue))
1132 except AttributeError:
1133 parser.error(
"no config field: %s" % (name,))
1136 value = eval(valueStr, {})
1138 parser.error(
"cannot parse %r as a value for %s" % (valueStr, name))
1141 except Exception
as e:
1142 parser.error(
"cannot set config.%s=%r: %s" % (name, value, e))
1146 """argparse action to load config overrides from one or more files.
1149 def __call__(self, parser, namespace, values, option_string=None):
1150 """Load one or more files of config overrides.
1154 parser : `argparse.ArgumentParser`
1156 namespace : `argparse.Namespace`
1157 Parsed command. The following attributes are updated by this
1158 method: ``namespace.config``.
1160 A list of data config file paths.
1161 option_string : `str`, optional
1162 Option value specified by the user.
1164 if namespace.config
is None:
1166 for configfile
in values:
1168 namespace.config.load(configfile)
1169 except Exception
as e:
1170 parser.error(
"cannot load config file %r: %s" % (configfile, e))
1174 """argparse action callback to process a data ID into a dict.
1177 def __call__(self, parser, namespace, values, option_string):
1178 """Parse ``--id`` data and append results to
1179 ``namespace.<argument>.idList``.
1183 parser : `ArgumentParser`
1185 namespace : `argparse.Namespace`
1186 Parsed command (an instance of argparse.Namespace).
1187 The following attributes are updated:
1189 - ``<idName>.idList``, where ``<idName>`` is the name of the
1190 ID argument, for instance ``"id"`` for ID argument ``--id``.
1192 A list of data IDs; see Notes below.
1193 option_string : `str`
1194 Option value specified by the user.
1198 The data format is::
1200 key1=value1_1[^value1_2[^value1_3...]
1201 key2=value2_1[^value2_2[^value2_3...]...
1203 The values (e.g. ``value1_1``) may either be a string,
1204 or of the form ``"int..int"`` (e.g. ``"1..3"``) which is
1205 interpreted as ``"1^2^3"`` (inclusive, unlike a python range).
1206 So ``"0^2..4^7..9"`` is equivalent to ``"0^2^3^4^7^8^9"``.
1207 You may also specify a stride: ``"1..5:2"`` is ``"1^3^5"``.
1209 The cross product is computed for keys with multiple values.
1212 --id visit 1^2 ccd 1,1^2,2
1214 results in the following data ID dicts being appended to
1215 ``namespace.<argument>.idList``:
1217 {"visit":1, "ccd":"1,1"}
1218 {"visit":2, "ccd":"1,1"}
1219 {"visit":1, "ccd":"2,2"}
1220 {"visit":2, "ccd":"2,2"}
1222 if namespace.config
is None:
1224 idDict = collections.OrderedDict()
1225 for nameValue
in values:
1226 name, sep, valueStr = nameValue.partition(
"=")
1228 parser.error(
"%s appears multiple times in one ID argument: %s" % (name, option_string))
1230 for v
in valueStr.split(
"^"):
1231 mat = re.search(
r"^(\d+)\.\.(\d+)(?::(\d+))?$", v)
1233 v1 = int(mat.group(1))
1234 v2 = int(mat.group(2))
1236 v3 = int(v3)
if v3
else 1
1237 for v
in range(v1, v2 + 1, v3):
1238 idDict[name].append(str(v))
1240 idDict[name].append(v)
1242 iterList = [idDict[key]
for key
in idDict.keys()]
1243 idDictList = [collections.OrderedDict(zip(idDict.keys(), valList))
1244 for valList
in itertools.product(*iterList)]
1246 argName = option_string.lstrip(
"-")
1247 ident = getattr(namespace, argName)
1248 ident.idList += idDictList
1252 """argparse action to set log level.
1255 def __call__(self, parser, namespace, values, option_string):
1260 parser : `ArgumentParser`
1262 namespace : `argparse.Namespace`
1263 Parsed command. This argument is not used.
1265 List of trace levels; each item must be of the form
1266 ``component_name=level`` or ``level``, where ``level``
1267 is a keyword (not case sensitive) or an integer.
1268 option_string : `str`
1269 Option value specified by the user.
1271 permittedLevelList = (
'TRACE',
'DEBUG',
'INFO',
'WARN',
'ERROR',
'FATAL')
1272 permittedLevelSet = set(permittedLevelList)
1273 for componentLevel
in values:
1274 component, sep, levelStr = componentLevel.partition(
"=")
1276 levelStr, component = component,
None
1277 logLevelUpr = levelStr.upper()
1278 if logLevelUpr
in permittedLevelSet:
1279 logLevel = getattr(lsstLog.Log, logLevelUpr)
1281 parser.error(
"loglevel=%r not one of %s" % (levelStr, permittedLevelList))
1282 if component
is None:
1283 namespace.log.setLevel(logLevel)
1285 lsstLog.Log.getLogger(component).setLevel(logLevel)
1287 pyLevel = lsstLog.LevelTranslator.lsstLog2logging(logLevel)
1288 logging.getLogger(component).setLevel(pyLevel)
1292 """argparse action associated with ArgumentPraser.addReuseOption."""
1294 def __call__(self, parser, namespace, value, option_string):
1296 value = self.choices[-2]
1297 index = self.choices.index(value)
1298 namespace.reuse = self.choices[:index + 1]
1302 """Set an instance attribute (like `setattr` but accepting
1303 hierarchical names such as ``foo.bar.baz``).
1308 Object whose attribute is to be set.
1310 Name of attribute to set.
1312 New value for the attribute.
1316 For example if name is ``foo.bar.baz`` then ``item.foo.bar.baz``
1317 is set to the specified value.
1320 subnameList = name.split(
".")
1321 for subname
in subnameList[:-1]:
1322 subitem = getattr(subitem, subname)
1323 setattr(subitem, subnameList[-1], value)
1327 """Get an attribute (like `getattr` but accepts hierarchical names
1328 such as ``foo.bar.baz``).
1333 Object whose attribute is to be returned.
1335 Name of the attribute to get.
1340 If name is ``foo.bar.baz then the return value is
1341 ``item.foo.bar.baz``.
1344 for subname
in name.split(
"."):
1345 subitem = getattr(subitem, subname)