22 """Module defining config classes for PipelineTask. 25 __all__ = [
"InputDatasetConfig",
"InputDatasetField",
26 "OutputDatasetConfig",
"OutputDatasetField",
27 "InitInputDatasetConfig",
"InitInputDatasetField",
28 "InitOutputDatasetConfig",
"InitOutputDatasetField",
29 "ResourceConfig",
"QuantumConfig",
"PipelineTaskConfig"]
34 from textwrap
import dedent, indent
50 def _makeDatasetField(name, dtype):
51 """ Function to make callables which produce ConfigField objects 53 This is factory function which produces factory functions. The factories 54 returned by this function are used to simplify the process of creating 55 pex config ConfigFields which have dtypes derived from either 56 _DatasetTypeConfig, or _GlobalDatasetTypeConfig. These functions can 57 then be used in a mannor similar to other ConfigField constructors. 59 Below is a flow diagram to explain the use of this function visually, 60 where arrows indicate processing flow. 62 Make a ConfigField factory: 63 _makeDatasetField() -> return wrappedFunc -> assign to variable corresponding 66 Use a ConfigField factory: 67 name() -> factory() -> return pexConfig instance 71 FooField = _makeDatasetField("FooField", FooConfig) 72 fooFieldInstance = FooField("An example Foo ConfigField", 80 The name to use as the final output Field constructor 81 dtype : Configuration Object 82 This is the python type to set as the dtype in the ConfigField 88 Python callable function which can be used to produce instances of 89 ConfigFields of type dtype. 94 Possibly raises a TypeError if attempting to create a factory function 95 from an incompatible type 98 def factory(**kwargs):
99 """ This is the innermost function in the closure, and does the work 100 of actually producing the ConfigField 107 return pexConfig.ConfigField(doc=kwargs[
'doc'],
110 **{k: v
for k, v
in kwargs.items()
111 if k
not in (
'doc',
'check')}),
112 check=kwargs[
'check'])
117 if issubclass(dtype, _GlobalDatasetTypeConfig):
120 def wrappedFunc(doc, name, storageClass, check=None):
121 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
126 elif issubclass(dtype, _DatasetTypeConfig):
128 def wrappedFunc(doc, name, dimensions, storageClass, scalar=False, check=None):
129 return factory(**{k: v
for k, v
in locals().items()
if k !=
'factory'})
133 dimensions : iterable of `str` 134 Iterable of Dimensions for this `~lsst.daf.butler.DatasetType` 135 scalar : `bool`, optional 136 If set to True then only a single dataset is expected on input or 137 produced on output. In that case list of objects/DataIds will be 138 unpacked before calling task methods, returned data is expected 139 to contain single objects as well.""" 143 extraFields =
", dimensions, scalar," 147 raise TypeError(f
"Cannot create a factory for dtype {dtype}")
150 docstring = f
""" Factory function to create `~lsst.pex.config.Config` class instances 151 of `{dtype.__name__}` 153 This function servers as syntactic sugar for creating Configurable fields 154 which are `{dtype.__name__}`. The naming of this function violates the 155 normal convention of a lowercase first letter in the function name, as 156 this function is intended to sit in the same place as 157 `~lsst.pex.config.ConfigField` classes, and consistency in declaration 160 The input arguments for this class are a combination of the arguments for 161 `~lsst.pex.config.ConfigField` and `{dtype.__name__}`. The arguments 162 doc and check come from `~lsst.pex.config.ConfigField`, while name{extraFields} 163 and storageClass come from `{dtype.__name__}`. 168 Documentation string for the `{dtype.__name__}` 170 Name of the `~lsst.daf.butler.DatasetType` in the returned 171 `{dtype.__name__}`{indent(dedent(extraDoc), " " * 4)} 173 Name of the `~lsst.daf.butler.StorageClass` in the `{dtype.__name__}` 175 A callable to be called with the field value that returns 176 False if the value is invalid. 180 result : `~lsst.pex.config.ConfigField` 181 Instance of a `~lsst.pex.config.ConfigField` with `InputDatasetConfig` as a dtype 184 wrappedFunc.__name__ = name
187 wrappedFunc.__doc__ = dedent(docstring)
192 """Configuration class which defines PipelineTask quanta dimensions. 194 In addition to a list of dataUnit names this also includes optional list of 195 SQL statements to be executed against Registry database. Exact meaning and 196 format of SQL will be determined at later point. 198 dimensions = pexConfig.ListField(dtype=str,
199 doc=
"list of Dimensions which define quantum")
200 sql = pexConfig.ListField(dtype=str,
201 doc=
"sequence of SQL statements",
206 """Intermediate base class for dataset type configuration in PipelineTask. 208 name = pexConfig.Field(dtype=str,
209 doc=
"name of the DatasetType")
210 storageClass = pexConfig.Field(dtype=str,
211 doc=
"name of the StorageClass")
215 """Configuration class which defines dataset type used by PipelineTask. 217 Consists of DatasetType name, list of Dimension names and StorageCass name. 218 PipelineTasks typically define one or more input and output datasets. This 219 class should not be used directly, instead one of `InputDatasetConfig` or 220 `OutputDatasetConfig` should be used in PipelineTask config. 222 dimensions = pexConfig.ListField(dtype=str,
223 doc=
"list of Dimensions for this DatasetType")
224 scalar = pexConfig.Field(dtype=bool,
227 doc=(
"If set to True then only a single dataset is expected " 228 "on input or produced on output. In that case list of " 229 "objects/DataIds will be unpacked before calling task " 230 "methods, returned data is expected to contain single " 238 class OutputDatasetConfig(_DatasetTypeConfig):
243 """Configuration class which defines dataset types used in PipelineTask 246 Consists of DatasetType name and StorageCass name, with a read-only 247 ``dimensions`` property that returns an empty tuple, enforcing the 248 constraint that datasets used in initialization are not associated with 249 any Dimensions. This class should not be used directly, instead one of 250 `InitInputDatasetConfig` or `InitOutputDatasetConfig` should be used in 255 """Dimensions associated with this DatasetType (always empty).""" 263 class InitOutputDatasetConfig(_GlobalDatasetTypeConfig):
268 """Configuration for resource requirements. 270 This configuration class will be used by some activators to estimate 271 resource use by pipeline. Additionally some tasks could use it to adjust 272 their resource use (e.g. reduce the number of threads). 274 For some resources their limit can be estimated by corresponding task, 275 in that case task could set the field value. For many fields defined in 276 this class their associated resource used by a task will depend on the 277 size of the data and is not known in advance. For these resources their 278 value will be configured through overrides based on some external 281 minMemoryMB = pexConfig.Field(dtype=int, default=
None, optional=
True,
282 doc=
"Minimal memory needed by task, can be None if estimate is unknown.")
283 minNumCores = pexConfig.Field(dtype=int, default=1,
284 doc=
"Minimal number of cores needed by task.")
288 """Base class for all PipelineTask configurations. 290 This class defines fields that must be defined for every PipelineTask. 291 It will be used as a base class for all PipelineTask configurations instead 292 of `pex.config.Config`. 294 quantum = pexConfig.ConfigField(dtype=QuantumConfig,
295 doc=
"configuration for PipelineTask quantum")
298 InputDatasetField = _makeDatasetField(
"InputDatasetField", InputDatasetConfig)
299 OutputDatasetField = _makeDatasetField(
"OutputDatasetField", OutputDatasetConfig)
300 InitInputDatasetField = _makeDatasetField(
"InitInputDatasetField", InitInputDatasetConfig)
301 InitOutputDatasetField = _makeDatasetField(
"InitOutputDatasetField", InitOutputDatasetConfig)