Coverage for python/lsst/pipe/base/config.py: 48%
81 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 03:12 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 03:12 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining config classes for PipelineTask.
23"""
25from __future__ import annotations
27__all__ = ["ResourceConfig", "PipelineTaskConfig"]
29# -------------------------------
30# Imports of standard modules --
31# -------------------------------
32import os
33from collections.abc import Iterable
34from numbers import Number
35from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar
37# -----------------------------
38# Imports for other modules --
39# -----------------------------
40import lsst.pex.config as pexConfig
42from ._instrument import Instrument
43from .configOverrides import ConfigOverrides
44from .connections import PipelineTaskConnections
45from .pipelineIR import ConfigIR, ParametersIR
47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true
48 from lsst.pex.config.callStack import StackFrame
50# ----------------------------------
51# Local non-exported definitions --
52# ----------------------------------
54_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
56# ------------------------
57# Exported definitions --
58# ------------------------
61class TemplateField(pexConfig.Field):
62 """This Field is specialized for use with connection templates.
63 Specifically it treats strings or numbers as valid input, as occasionally
64 numbers are used as a cycle counter in templates.
66 The reason for the specialized field, is that when numbers are involved
67 with the config override system through pipelines or from the command line,
68 sometimes the quoting to get appropriate values as strings gets
69 complicated. This will simplify the process greatly.
70 """
72 def _validateValue(self, value: Any) -> None:
73 if value is None:
74 return
76 if not (isinstance(value, str) or isinstance(value, Number)):
77 raise TypeError(
78 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
79 " Expected type str or a number"
80 )
81 if self.check is not None and not self.check(value):
82 ValueError("Value {value} is not a valid value")
84 def __set__(
85 self,
86 instance: pexConfig.Config,
87 value: Any,
88 at: Optional[StackFrame] = None,
89 label: str = "assignment",
90 ) -> None:
91 # validate first, even though validate will be called in super
92 self._validateValue(value)
93 # now, explicitly make it into a string
94 value = str(value)
95 super().__set__(instance, value, at, label)
98class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
99 """Metaclass used in the creation of PipelineTaskConfig classes
101 This metaclass ensures a `PipelineTaskConnections` class is specified in
102 the class construction parameters with a parameter name of
103 pipelineConnections. Using the supplied connection class, this metaclass
104 constructs a `lsst.pex.config.Config` instance which can be used to
105 configure the connections class. This config is added to the config class
106 under declaration with the name "connections" used as an identifier. The
107 connections config also has a reference to the connections class used in
108 its construction associated with an atttribute named `ConnectionsClass`.
109 Finally the newly constructed config class (not an instance of it) is
110 assigned to the Config class under construction with the attribute name
111 `ConnectionsConfigClass`.
112 """
114 def __new__(
115 cls: Type[_S],
116 name: str,
117 bases: Tuple[type[PipelineTaskConfig], ...],
118 dct: Dict[str, Any],
119 **kwargs: Any,
120 ) -> _S:
121 if name != "PipelineTaskConfig":
122 # Verify that a connection class was specified and the argument is
123 # an instance of PipelineTaskConfig
124 if "pipelineConnections" not in kwargs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true
125 for base in bases:
126 if hasattr(base, "connections"):
127 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
128 break
129 if "pipelineConnections" not in kwargs: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true
130 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
131 connectionsClass = kwargs["pipelineConnections"]
132 if not issubclass(connectionsClass, PipelineTaskConnections): 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true
133 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
135 # Create all the fields that will be used in the newly created sub
136 # config (under the attribute name "connections")
137 configConnectionsNamespace: dict[str, pexConfig.Field] = {}
138 for fieldName, obj in connectionsClass.allConnections.items():
139 configConnectionsNamespace[fieldName] = pexConfig.Field[str](
140 doc=f"name for connection {fieldName}", default=obj.name
141 )
142 # If there are default templates also add them as fields to
143 # configure the template values
144 if hasattr(connectionsClass, "defaultTemplates"): 144 ↛ 152line 144 didn't jump to line 152, because the condition on line 144 was never false
145 docString = "Template parameter used to format corresponding field template parameter"
146 for templateName, default in connectionsClass.defaultTemplates.items():
147 configConnectionsNamespace[templateName] = TemplateField(
148 dtype=str, doc=docString, default=default
149 )
150 # add a reference to the connection class used to create this sub
151 # config
152 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
154 # Create a new config class with the fields defined above
155 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace)
156 # add it to the Config class that is currently being declared
157 dct["connections"] = pexConfig.ConfigField(
158 dtype=Connections,
159 doc="Configurations describing the connections of the PipelineTask to datatypes",
160 )
161 dct["ConnectionsConfigClass"] = Connections
162 dct["ConnectionsClass"] = connectionsClass
163 inst = super().__new__(cls, name, bases, dct)
164 return inst
166 def __init__(
167 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any
168 ):
169 # This overrides the default init to drop the kwargs argument. Python
170 # metaclasses will have this argument set if any kwargs are passes at
171 # class construction time, but should be consumed before calling
172 # __init__ on the type metaclass. This is in accordance with python
173 # documentation on metaclasses
174 super().__init__(name, bases, dct)
177class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
178 """Configuration class for `PipelineTask`
180 This Configuration class functions in largely the same manner as any other
181 derived from `lsst.pex.config.Config`. The only difference is in how it is
182 declared. `PipelineTaskConfig` children need to be declared with a
183 pipelineConnections argument. This argument should specify a child class of
184 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
185 a config class is created with information from the supplied connections
186 class to allow configuration of the connections class. This dynamically
187 created config class is then attached to the `PipelineTaskConfig` via a
188 `~lsst.pex.config.ConfigField` with the attribute name `connections`.
189 """
191 connections: pexConfig.ConfigField
192 """Field which refers to a dynamically added configuration class which is
193 based on a PipelineTaskConnections class.
194 """
196 saveMetadata = pexConfig.Field[bool](
197 default=True,
198 optional=False,
199 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
200 )
201 saveLogOutput = pexConfig.Field[bool](
202 default=True,
203 optional=False,
204 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
205 )
207 def applyConfigOverrides(
208 self,
209 instrument: Instrument | None,
210 taskDefaultName: str,
211 pipelineConfigs: Iterable[ConfigIR] | None,
212 parameters: ParametersIR,
213 label: str,
214 ) -> None:
215 r"""Apply config overrides to this config instance.
217 Parameters
218 ---------
219 instrument : `Instrument` or `None`
220 An instance of the `Instrument` specified in a pipeline.
221 If `None` then the pipeline did not specify and instrument.
222 taskDefaultName : `str`
223 The default name associated with the `Task` class. This
224 may be used with instrumental overrides.
225 pipelineConfigs : `Iterable` of `ConfigIR`
226 An iterable of `ConfigIR` objects that contain overrides
227 to apply to this config instance.
228 parameters : `ParametersIR`
229 Parameters defined in a Pipeline which are used in formatting
230 of config values across multiple `Task`\ s in a pipeline.
231 label : `str`
232 The label associated with this class's Task in a pipeline.
233 """
234 overrides = ConfigOverrides()
235 if instrument is not None:
236 overrides.addInstrumentOverride(instrument, taskDefaultName)
237 if pipelineConfigs is not None:
238 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs):
239 if subConfig.dataId is not None:
240 raise NotImplementedError(
241 "Specializing a config on a partial data id is not yet "
242 "supported in Pipeline definition"
243 )
244 # only apply override if it applies to everything
245 if subConfig.dataId is None:
246 if subConfig.file:
247 for configFile in subConfig.file:
248 overrides.addFileOverride(os.path.expandvars(configFile))
249 if subConfig.python is not None:
250 overrides.addPythonOverride(subConfig.python)
251 for key, value in subConfig.rest.items():
252 overrides.addValueOverride(key, value)
253 overrides.applyTo(self)
256class ResourceConfig(pexConfig.Config):
257 """Configuration for resource requirements.
259 This configuration class will be used by some activators to estimate
260 resource use by pipeline. Additionally some tasks could use it to adjust
261 their resource use (e.g. reduce the number of threads).
263 For some resources their limit can be estimated by corresponding task,
264 in that case task could set the field value. For many fields defined in
265 this class their associated resource used by a task will depend on the
266 size of the data and is not known in advance. For these resources their
267 value will be configured through overrides based on some external
268 estimates.
269 """
271 minMemoryMB = pexConfig.Field[int](
272 default=None,
273 optional=True,
274 doc="Minimal memory needed by task, can be None if estimate is unknown.",
275 )
276 minNumCores = pexConfig.Field[int](default=1, doc="Minimal number of cores needed by task.")