22 """Module defining config classes for PipelineTask. 25 __all__ = [
"InputDatasetConfig",
"InputDatasetField",
26 "OutputDatasetConfig",
"OutputDatasetField",
27 "InitInputDatasetConfig",
"InitInputDatasetField",
28 "InitOutputDatasetConfig",
"InitOutputDatasetField",
29 "ResourceConfig",
"QuantumConfig",
"PipelineTaskConfig"]
34 from textwrap
import dedent, indent
49 PIPELINETASK_CONFIG_TEMPLATE_DICT = {}
52 def _makeDatasetField(name, dtype):
53 """ Function to make callables which produce ConfigField objects 55 This is factory function which produces factory functions. The factories 56 returned by this function are used to simplify the process of creating 57 `lsst.pex.config.ConfigField` which have dtypes derived from either 58 `_DatasetTypeConfig`, or `_GlobalDatasetTypeConfig`. These functions can 59 then be used in a mannor similar to other `~lsst.pex.config.ConfigField` 62 Below is a flow diagram to explain the use of this function visually, 63 where arrows indicate processing flow. 65 Make a ConfigField factory: 66 _makeDatasetField() -> return wrappedFunc -> assign to variable corresponding 69 Use a ConfigField factory: 70 name() -> factory() -> return pexConfig instance 74 FooField = _makeDatasetField("FooField", FooConfig) 75 fooFieldInstance = FooField("An example Foo ConfigField", 83 The name to use as the final output `~lsst.pex.config.Field` 85 dtype : Configuration Object 86 This is the python type to set as the dtype in the ConfigField 92 Python callable function which can be used to produce instances of 93 `~lsst.pex.config.ConfigField` of type dtype. 98 Possibly raises a TypeError if attempting to create a factory function 99 from an incompatible type 102 def factory(**kwargs):
103 """ This is the innermost function in the closure, and does the work 104 of actually producing the ConfigField 111 return pexConfig.ConfigField(doc=kwargs[
'doc'],
114 **{k: v
for k, v
in kwargs.items()
115 if k
not in (
'doc',
'check')}),
116 check=kwargs[
'check'])
121 if issubclass(dtype, _GlobalDatasetTypeConfig):
124 def wrappedFunc(*, doc, storageClass, check=None, name="", nameTemplate=''):
125 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
130 elif issubclass(dtype, _DatasetTypeConfig):
132 def wrappedFunc(*, doc, dimensions, storageClass, name="", scalar=False, check=None, nameTemplate=''):
133 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
137 dimensions : iterable of `str` 138 Iterable of Dimensions for this `~lsst.daf.butler.DatasetType` 139 scalar : `bool`, optional 140 If set to True then only a single dataset is expected on input or 141 produced on output. In that case list of objects/DataIds will be 142 unpacked before calling task methods, returned data is expected 143 to contain single objects as well. 144 nameTemplate : `str`, optional 145 Template for the `name` field which is specified as a python formattable 146 string. The template is formatted during the configuration of a Config 147 class with a user defined string. Defaults to empty string, in which 148 case no formatting is done.""" 152 extraFields =
", dimensions, scalar, nameTemplate" 156 raise TypeError(f
"Cannot create a factory for dtype {dtype}")
159 docstring = f
""" Factory function to create `~lsst.pex.config.Config` class instances 160 of `{dtype.__name__}` 162 This function servers as syntactic sugar for creating 163 `~lsst.pex.config.ConfigField` which are `{dtype.__name__}`. The naming of 164 this function violates the normal convention of a lowercase first letter 165 in the function name, as this function is intended to sit in the same 166 place as `~lsst.pex.config.ConfigField` classes, and consistency in 167 declaration syntax is important. 169 The input arguments for this class are a combination of the arguments for 170 `~lsst.pex.config.ConfigField` and `{dtype.__name__}`. The arguments 171 doc and check come from `~lsst.pex.config.ConfigField`, while name{extraFields} 172 and storageClass come from `{dtype.__name__}`. 177 Documentation string for the `{dtype.__name__}` 179 Name of the `~lsst.daf.butler.DatasetType` in the returned 180 `{dtype.__name__}`{indent(dedent(extraDoc), " " * 4)} 182 Name of the `~lsst.daf.butler.StorageClass` in the `{dtype.__name__}` 184 A callable to be called with the field value that returns 185 False if the value is invalid. 189 result : `~lsst.pex.config.ConfigField` 190 Instance of a `~lsst.pex.config.ConfigField` with `InputDatasetConfig` as a dtype 193 wrappedFunc.__name__ = name
196 wrappedFunc.__doc__ = dedent(docstring)
201 """Configuration class which defines PipelineTask quanta dimensions. 203 In addition to a list of dataUnit names this also includes optional list of 204 SQL statements to be executed against Registry database. Exact meaning and 205 format of SQL will be determined at later point. 207 dimensions = pexConfig.ListField(dtype=str,
208 doc=
"list of Dimensions which define quantum")
209 sql = pexConfig.ListField(dtype=str,
210 doc=
"sequence of SQL statements",
215 """Intermediate base class for dataset type configuration in PipelineTask. 217 name = pexConfig.Field(dtype=str,
218 doc=
"name of the DatasetType")
219 storageClass = pexConfig.Field(dtype=str,
220 doc=
"name of the StorageClass")
221 nameTemplate = pexConfig.Field(dtype=str,
224 doc=(
"Templated name of string, used to set name " 225 "field according to a shared substring when " 226 "`~PipelineTaskConfig.formatTemplateNames` " 231 """Configuration class which defines dataset type used by PipelineTask. 233 Consists of DatasetType name, list of Dimension names and StorageCass name. 234 PipelineTasks typically define one or more input and output datasets. This 235 class should not be used directly, instead one of `InputDatasetConfig` or 236 `OutputDatasetConfig` should be used in PipelineTask config. 238 dimensions = pexConfig.ListField(dtype=str,
239 doc=
"list of Dimensions for this DatasetType")
240 scalar = pexConfig.Field(dtype=bool,
243 doc=(
"If set to True then only a single dataset is expected " 244 "on input or produced on output. In that case list of " 245 "objects/DataIds will be unpacked before calling task " 246 "methods, returned data is expected to contain single " 254 class OutputDatasetConfig(_DatasetTypeConfig):
259 """Configuration class which defines dataset types used in PipelineTask 262 Consists of DatasetType name and StorageCass name, with a read-only 263 ``dimensions`` property that returns an empty tuple, enforcing the 264 constraint that datasets used in initialization are not associated with 265 any Dimensions. This class should not be used directly, instead one of 266 `InitInputDatasetConfig` or `InitOutputDatasetConfig` should be used in 271 """Dimensions associated with this DatasetType (always empty).""" 279 class InitOutputDatasetConfig(_GlobalDatasetTypeConfig):
284 """Configuration for resource requirements. 286 This configuration class will be used by some activators to estimate 287 resource use by pipeline. Additionally some tasks could use it to adjust 288 their resource use (e.g. reduce the number of threads). 290 For some resources their limit can be estimated by corresponding task, 291 in that case task could set the field value. For many fields defined in 292 this class their associated resource used by a task will depend on the 293 size of the data and is not known in advance. For these resources their 294 value will be configured through overrides based on some external 297 minMemoryMB = pexConfig.Field(dtype=int, default=
None, optional=
True,
298 doc=
"Minimal memory needed by task, can be None if estimate is unknown.")
299 minNumCores = pexConfig.Field(dtype=int, default=1,
300 doc=
"Minimal number of cores needed by task.")
304 """Base class for all PipelineTask configurations. 306 This class defines fields that must be defined for every 307 `~lsst.pipe.base.PipelineTask`. It will be used as a base class for all 308 `~lsst.pipe.base.PipelineTask` configurations instead of 311 quantum = pexConfig.ConfigField(dtype=QuantumConfig,
312 doc=
"configuration for PipelineTask quantum")
317 storedParamsDict = PIPELINETASK_CONFIG_TEMPLATE_DICT.setdefault(id(self), {})
318 storedParamsDict.update(templateParamsDict)
319 for key, value
in self.items():
320 if isinstance(value, _BaseDatasetTypeConfig)
and value.nameTemplate !=
'':
321 value.name = value.nameTemplate.format(**storedParamsDict)
324 InputDatasetField = _makeDatasetField(
"InputDatasetField", InputDatasetConfig)
325 OutputDatasetField = _makeDatasetField(
"OutputDatasetField", OutputDatasetConfig)
326 InitInputDatasetField = _makeDatasetField(
"InitInputDatasetField", InitInputDatasetConfig)
327 InitOutputDatasetField = _makeDatasetField(
"InitOutputDatasetField", InitOutputDatasetConfig)
def formatTemplateNames(self, templateParamsDict)