Coverage for python/lsst/pipe/base/config.py: 48%
83 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-18 02:12 -0700
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-18 02:12 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining config classes for PipelineTask.
23"""
25from __future__ import annotations
27__all__ = ["ResourceConfig", "PipelineTaskConfig"]
29# -------------------------------
30# Imports of standard modules --
31# -------------------------------
32import os
33from collections.abc import Iterable
34from numbers import Number
35from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar
37# -----------------------------
38# Imports for other modules --
39# -----------------------------
40import lsst.pex.config as pexConfig
42from ._instrument import Instrument
43from .configOverrides import ConfigOverrides
44from .connections import PipelineTaskConnections
45from .pipelineIR import ConfigIR, ParametersIR
47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true
48 from lsst.pex.config.callStack import StackFrame
50# ----------------------------------
51# Local non-exported definitions --
52# ----------------------------------
54_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
56# ------------------------
57# Exported definitions --
58# ------------------------
61class TemplateField(pexConfig.Field):
62 """This Field is specialized for use with connection templates.
63 Specifically it treats strings or numbers as valid input, as occasionally
64 numbers are used as a cycle counter in templates.
66 The reason for the specialized field, is that when numbers are involved
67 with the config override system through pipelines or from the command line,
68 sometimes the quoting to get appropriate values as strings gets
69 complicated. This will simplify the process greatly.
70 """
72 def _validateValue(self, value: Any) -> None:
73 if value is None:
74 return
76 if not (isinstance(value, str) or isinstance(value, Number)):
77 raise TypeError(
78 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
79 " Expected type str or a number"
80 )
81 if self.check is not None and not self.check(value):
82 ValueError("Value {value} is not a valid value")
84 def __set__(
85 self,
86 instance: pexConfig.Config,
87 value: Any,
88 at: Optional[StackFrame] = None,
89 label: str = "assignment",
90 ) -> None:
91 # validate first, even though validate will be called in super
92 self._validateValue(value)
93 # now, explicitly make it into a string
94 value = str(value)
95 super().__set__(instance, value, at, label)
98class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
99 """Metaclass used in the creation of PipelineTaskConfig classes
101 This metaclass ensures a `PipelineTaskConnections` class is specified in
102 the class construction parameters with a parameter name of
103 pipelineConnections. Using the supplied connection class, this metaclass
104 constructs a `lsst.pex.config.Config` instance which can be used to
105 configure the connections class. This config is added to the config class
106 under declaration with the name "connections" used as an identifier. The
107 connections config also has a reference to the connections class used in
108 its construction associated with an atttribute named `ConnectionsClass`.
109 Finally the newly constructed config class (not an instance of it) is
110 assigned to the Config class under construction with the attribute name
111 `ConnectionsConfigClass`.
112 """
114 def __new__(
115 cls: Type[_S],
116 name: str,
117 bases: Tuple[type[PipelineTaskConfig], ...],
118 dct: Dict[str, Any],
119 **kwargs: Any,
120 ) -> _S:
121 if name != "PipelineTaskConfig":
122 # Verify that a connection class was specified and the argument is
123 # an instance of PipelineTaskConfig
124 if "pipelineConnections" not in kwargs: 124 ↛ 125line 124 didn't jump to line 125, because the condition on line 124 was never true
125 for base in bases:
126 if hasattr(base, "connections"):
127 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
128 break
129 if "pipelineConnections" not in kwargs: 129 ↛ 130line 129 didn't jump to line 130, because the condition on line 129 was never true
130 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
131 connectionsClass = kwargs["pipelineConnections"]
132 if not issubclass(connectionsClass, PipelineTaskConnections): 132 ↛ 133line 132 didn't jump to line 133, because the condition on line 132 was never true
133 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
135 # Create all the fields that will be used in the newly created sub
136 # config (under the attribute name "connections")
137 configConnectionsNamespace: dict[str, pexConfig.Field] = {}
138 for fieldName, obj in connectionsClass.allConnections.items():
139 configConnectionsNamespace[fieldName] = pexConfig.Field[str](
140 doc=f"name for connection {fieldName}", default=obj.name
141 )
142 # If there are default templates also add them as fields to
143 # configure the template values
144 if hasattr(connectionsClass, "defaultTemplates"): 144 ↛ 152line 144 didn't jump to line 152, because the condition on line 144 was never false
145 docString = "Template parameter used to format corresponding field template parameter"
146 for templateName, default in connectionsClass.defaultTemplates.items():
147 configConnectionsNamespace[templateName] = TemplateField(
148 dtype=str, doc=docString, default=default
149 )
150 # add a reference to the connection class used to create this sub
151 # config
152 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
154 # Create a new config class with the fields defined above
155 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace)
156 # add it to the Config class that is currently being declared
157 dct["connections"] = pexConfig.ConfigField(
158 dtype=Connections,
159 doc="Configurations describing the connections of the PipelineTask to datatypes",
160 )
161 dct["ConnectionsConfigClass"] = Connections
162 dct["ConnectionsClass"] = connectionsClass
163 inst = super().__new__(cls, name, bases, dct)
164 return inst
166 def __init__(
167 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any
168 ):
169 # This overrides the default init to drop the kwargs argument. Python
170 # metaclasses will have this argument set if any kwargs are passes at
171 # class construction time, but should be consumed before calling
172 # __init__ on the type metaclass. This is in accordance with python
173 # documentation on metaclasses
174 super().__init__(name, bases, dct)
176 ConnectionsClass: type[PipelineTaskConnections]
177 ConnectionsConfigClass: type[pexConfig.Config]
180class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
181 """Configuration class for `PipelineTask`
183 This Configuration class functions in largely the same manner as any other
184 derived from `lsst.pex.config.Config`. The only difference is in how it is
185 declared. `PipelineTaskConfig` children need to be declared with a
186 pipelineConnections argument. This argument should specify a child class of
187 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
188 a config class is created with information from the supplied connections
189 class to allow configuration of the connections class. This dynamically
190 created config class is then attached to the `PipelineTaskConfig` via a
191 `~lsst.pex.config.ConfigField` with the attribute name `connections`.
192 """
194 connections: pexConfig.ConfigField
195 """Field which refers to a dynamically added configuration class which is
196 based on a PipelineTaskConnections class.
197 """
199 saveMetadata = pexConfig.Field[bool](
200 default=True,
201 optional=False,
202 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
203 )
204 saveLogOutput = pexConfig.Field[bool](
205 default=True,
206 optional=False,
207 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
208 )
210 def applyConfigOverrides(
211 self,
212 instrument: Instrument | None,
213 taskDefaultName: str,
214 pipelineConfigs: Iterable[ConfigIR] | None,
215 parameters: ParametersIR,
216 label: str,
217 ) -> None:
218 r"""Apply config overrides to this config instance.
220 Parameters
221 ---------
222 instrument : `Instrument` or `None`
223 An instance of the `Instrument` specified in a pipeline.
224 If `None` then the pipeline did not specify and instrument.
225 taskDefaultName : `str`
226 The default name associated with the `Task` class. This
227 may be used with instrumental overrides.
228 pipelineConfigs : `Iterable` of `ConfigIR`
229 An iterable of `ConfigIR` objects that contain overrides
230 to apply to this config instance.
231 parameters : `ParametersIR`
232 Parameters defined in a Pipeline which are used in formatting
233 of config values across multiple `Task`\ s in a pipeline.
234 label : `str`
235 The label associated with this class's Task in a pipeline.
236 """
237 overrides = ConfigOverrides()
238 if instrument is not None:
239 overrides.addInstrumentOverride(instrument, taskDefaultName)
240 if pipelineConfigs is not None:
241 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs):
242 if subConfig.dataId is not None:
243 raise NotImplementedError(
244 "Specializing a config on a partial data id is not yet "
245 "supported in Pipeline definition"
246 )
247 # only apply override if it applies to everything
248 if subConfig.dataId is None:
249 if subConfig.file:
250 for configFile in subConfig.file:
251 overrides.addFileOverride(os.path.expandvars(configFile))
252 if subConfig.python is not None:
253 overrides.addPythonOverride(subConfig.python)
254 for key, value in subConfig.rest.items():
255 overrides.addValueOverride(key, value)
256 overrides.applyTo(self)
259class ResourceConfig(pexConfig.Config):
260 """Configuration for resource requirements.
262 This configuration class will be used by some activators to estimate
263 resource use by pipeline. Additionally some tasks could use it to adjust
264 their resource use (e.g. reduce the number of threads).
266 For some resources their limit can be estimated by corresponding task,
267 in that case task could set the field value. For many fields defined in
268 this class their associated resource used by a task will depend on the
269 size of the data and is not known in advance. For these resources their
270 value will be configured through overrides based on some external
271 estimates.
272 """
274 minMemoryMB = pexConfig.Field[int](
275 default=None,
276 optional=True,
277 doc="Minimal memory needed by task, can be None if estimate is unknown.",
278 )
279 minNumCores = pexConfig.Field[int](default=1, doc="Minimal number of cores needed by task.")