22 """Module defining config classes for PipelineTask. 25 __all__ = [
"InputDatasetConfig",
"InputDatasetField",
26 "OutputDatasetConfig",
"OutputDatasetField",
27 "InitInputDatasetConfig",
"InitInputDatasetField",
28 "InitOutputDatasetConfig",
"InitOutputDatasetField",
29 "ResourceConfig",
"QuantumConfig",
"PipelineTaskConfig"]
34 from textwrap
import dedent, indent
49 PIPELINETASK_CONFIG_TEMPLATE_DICT = {}
52 def _makeDatasetField(name, dtype):
53 """ Function to make callables which produce ConfigField objects 55 This is factory function which produces factory functions. The factories 56 returned by this function are used to simplify the process of creating 57 pex config ConfigFields which have dtypes derived from either 58 _DatasetTypeConfig, or _GlobalDatasetTypeConfig. These functions can 59 then be used in a mannor similar to other ConfigField constructors. 61 Below is a flow diagram to explain the use of this function visually, 62 where arrows indicate processing flow. 64 Make a ConfigField factory: 65 _makeDatasetField() -> return wrappedFunc -> assign to variable corresponding 68 Use a ConfigField factory: 69 name() -> factory() -> return pexConfig instance 73 FooField = _makeDatasetField("FooField", FooConfig) 74 fooFieldInstance = FooField("An example Foo ConfigField", 82 The name to use as the final output Field constructor 83 dtype : Configuration Object 84 This is the python type to set as the dtype in the ConfigField 90 Python callable function which can be used to produce instances of 91 ConfigFields of type dtype. 96 Possibly raises a TypeError if attempting to create a factory function 97 from an incompatible type 100 def factory(**kwargs):
101 """ This is the innermost function in the closure, and does the work 102 of actually producing the ConfigField 109 return pexConfig.ConfigField(doc=kwargs[
'doc'],
112 **{k: v
for k, v
in kwargs.items()
113 if k
not in (
'doc',
'check')}),
114 check=kwargs[
'check'])
119 if issubclass(dtype, _GlobalDatasetTypeConfig):
122 def wrappedFunc(*, doc, storageClass, check=None, name="", nameTemplate=''):
123 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
128 elif issubclass(dtype, _DatasetTypeConfig):
130 def wrappedFunc(*, doc, dimensions, storageClass, name="", scalar=False, check=None, nameTemplate=''):
131 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
135 dimensions : iterable of `str` 136 Iterable of Dimensions for this `~lsst.daf.butler.DatasetType` 137 scalar : `bool`, optional 138 If set to True then only a single dataset is expected on input or 139 produced on output. In that case list of objects/DataIds will be 140 unpacked before calling task methods, returned data is expected 141 to contain single objects as well. 142 nameTemplate : `str`, optional 143 Template for the `name` field which is specified as a python formattable 144 string. The template is formatted during the configuration of a Config 145 class with a user defined string. Defaults to empty string, in which 146 case no formatting is done.""" 150 extraFields =
", dimensions, scalar, nameTemplate" 154 raise TypeError(f
"Cannot create a factory for dtype {dtype}")
157 docstring = f
""" Factory function to create `~lsst.pex.config.Config` class instances 158 of `{dtype.__name__}` 160 This function servers as syntactic sugar for creating Configurable fields 161 which are `{dtype.__name__}`. The naming of this function violates the 162 normal convention of a lowercase first letter in the function name, as 163 this function is intended to sit in the same place as 164 `~lsst.pex.config.ConfigField` classes, and consistency in declaration 167 The input arguments for this class are a combination of the arguments for 168 `~lsst.pex.config.ConfigField` and `{dtype.__name__}`. The arguments 169 doc and check come from `~lsst.pex.config.ConfigField`, while name{extraFields} 170 and storageClass come from `{dtype.__name__}`. 175 Documentation string for the `{dtype.__name__}` 177 Name of the `~lsst.daf.butler.DatasetType` in the returned 178 `{dtype.__name__}`{indent(dedent(extraDoc), " " * 4)} 180 Name of the `~lsst.daf.butler.StorageClass` in the `{dtype.__name__}` 182 A callable to be called with the field value that returns 183 False if the value is invalid. 187 result : `~lsst.pex.config.ConfigField` 188 Instance of a `~lsst.pex.config.ConfigField` with `InputDatasetConfig` as a dtype 191 wrappedFunc.__name__ = name
194 wrappedFunc.__doc__ = dedent(docstring)
199 """Configuration class which defines PipelineTask quanta dimensions. 201 In addition to a list of dataUnit names this also includes optional list of 202 SQL statements to be executed against Registry database. Exact meaning and 203 format of SQL will be determined at later point. 205 dimensions = pexConfig.ListField(dtype=str,
206 doc=
"list of Dimensions which define quantum")
207 sql = pexConfig.ListField(dtype=str,
208 doc=
"sequence of SQL statements",
213 """Intermediate base class for dataset type configuration in PipelineTask. 215 name = pexConfig.Field(dtype=str,
216 doc=
"name of the DatasetType")
217 storageClass = pexConfig.Field(dtype=str,
218 doc=
"name of the StorageClass")
219 nameTemplate = pexConfig.Field(dtype=str,
222 doc=(
"Templated name of string, used to set name " 223 "field according to a shared substring when " 224 "formatTemplateNames is called"))
228 """Configuration class which defines dataset type used by PipelineTask. 230 Consists of DatasetType name, list of Dimension names and StorageCass name. 231 PipelineTasks typically define one or more input and output datasets. This 232 class should not be used directly, instead one of `InputDatasetConfig` or 233 `OutputDatasetConfig` should be used in PipelineTask config. 235 dimensions = pexConfig.ListField(dtype=str,
236 doc=
"list of Dimensions for this DatasetType")
237 scalar = pexConfig.Field(dtype=bool,
240 doc=(
"If set to True then only a single dataset is expected " 241 "on input or produced on output. In that case list of " 242 "objects/DataIds will be unpacked before calling task " 243 "methods, returned data is expected to contain single " 251 class OutputDatasetConfig(_DatasetTypeConfig):
256 """Configuration class which defines dataset types used in PipelineTask 259 Consists of DatasetType name and StorageCass name, with a read-only 260 ``dimensions`` property that returns an empty tuple, enforcing the 261 constraint that datasets used in initialization are not associated with 262 any Dimensions. This class should not be used directly, instead one of 263 `InitInputDatasetConfig` or `InitOutputDatasetConfig` should be used in 268 """Dimensions associated with this DatasetType (always empty).""" 276 class InitOutputDatasetConfig(_GlobalDatasetTypeConfig):
281 """Configuration for resource requirements. 283 This configuration class will be used by some activators to estimate 284 resource use by pipeline. Additionally some tasks could use it to adjust 285 their resource use (e.g. reduce the number of threads). 287 For some resources their limit can be estimated by corresponding task, 288 in that case task could set the field value. For many fields defined in 289 this class their associated resource used by a task will depend on the 290 size of the data and is not known in advance. For these resources their 291 value will be configured through overrides based on some external 294 minMemoryMB = pexConfig.Field(dtype=int, default=
None, optional=
True,
295 doc=
"Minimal memory needed by task, can be None if estimate is unknown.")
296 minNumCores = pexConfig.Field(dtype=int, default=1,
297 doc=
"Minimal number of cores needed by task.")
301 """Base class for all PipelineTask configurations. 303 This class defines fields that must be defined for every PipelineTask. 304 It will be used as a base class for all PipelineTask configurations instead 305 of `pex.config.Config`. 307 quantum = pexConfig.ConfigField(dtype=QuantumConfig,
308 doc=
"configuration for PipelineTask quantum")
313 storedParamsDict = PIPELINETASK_CONFIG_TEMPLATE_DICT.setdefault(id(self), {})
314 storedParamsDict.update(templateParamsDict)
315 for key, value
in self.items():
316 if isinstance(value, _BaseDatasetTypeConfig)
and value.nameTemplate !=
'':
317 value.name = value.nameTemplate.format(**storedParamsDict)
320 InputDatasetField = _makeDatasetField(
"InputDatasetField", InputDatasetConfig)
321 OutputDatasetField = _makeDatasetField(
"OutputDatasetField", OutputDatasetConfig)
322 InitInputDatasetField = _makeDatasetField(
"InitInputDatasetField", InitInputDatasetConfig)
323 InitOutputDatasetField = _makeDatasetField(
"InitOutputDatasetField", InitOutputDatasetConfig)
def formatTemplateNames(self, templateParamsDict)