Coverage for python / lsst / pipe / base / config.py: 47%
76 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-28 08:44 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Module defining config classes for PipelineTask."""
30from __future__ import annotations
32__all__ = ["PipelineTaskConfig"]
34# -------------------------------
35# Imports of standard modules --
36# -------------------------------
37import os
38from collections.abc import Iterable
39from numbers import Number
40from typing import TYPE_CHECKING, Any, TypeVar
42# -----------------------------
43# Imports for other modules --
44# -----------------------------
45import lsst.pex.config as pexConfig
47from ._instrument import Instrument
48from .configOverrides import ConfigOverrides
49from .connections import PipelineTaskConnections
50from .pipelineIR import ConfigIR, ParametersIR
52if TYPE_CHECKING:
53 from lsst.pex.config.callStack import StackFrame
55# ----------------------------------
56# Local non-exported definitions --
57# ----------------------------------
59_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
61# ------------------------
62# Exported definitions --
63# ------------------------
66class TemplateField(pexConfig.Field):
67 """Field specialized for use with connection templates.
69 Specifically it treats strings or numbers as valid input, as occasionally
70 numbers are used as a cycle counter in templates.
72 The reason for the specialized field, is that when numbers are involved
73 with the config override system through pipelines or from the command line,
74 sometimes the quoting to get appropriate values as strings gets
75 complicated. This will simplify the process greatly.
76 """
78 def _validateValue(self, value: Any) -> None:
79 if value is None:
80 return
82 if not (isinstance(value, str | Number)):
83 raise TypeError(
84 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
85 " Expected type str or a number"
86 )
87 if self.check is not None and not self.check(value):
88 ValueError("Value {value} is not a valid value")
90 def __set__(
91 self,
92 instance: pexConfig.Config,
93 value: Any,
94 at: StackFrame | None = None,
95 label: str = "assignment",
96 ) -> None:
97 # validate first, even though validate will be called in super
98 self._validateValue(value)
99 # now, explicitly make it into a string
100 value = str(value)
101 super().__set__(instance, value, at, label)
104class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
105 """Metaclass used in the creation of PipelineTaskConfig classes.
107 This metaclass ensures a `PipelineTaskConnections` class is specified in
108 the class construction parameters with a parameter name of
109 pipelineConnections. Using the supplied connection class, this metaclass
110 constructs a `lsst.pex.config.Config` instance which can be used to
111 configure the connections class. This config is added to the config class
112 under declaration with the name "connections" used as an identifier. The
113 connections config also has a reference to the connections class used in
114 its construction associated with an atttribute named ``ConnectionsClass``.
115 Finally the newly constructed config class (not an instance of it) is
116 assigned to the Config class under construction with the attribute name
117 ``ConnectionsConfigClass``.
119 Parameters
120 ----------
121 name : `str`
122 Name of config.
123 bases : `~collections.abc.Collection`
124 Base classes.
125 dct : `~collections.abc.Mapping`
126 Parameter dict.
127 **kwargs : `~typing.Any`
128 Additional parameters.
129 """
131 def __new__(
132 cls: type[_S],
133 name: str,
134 bases: tuple[type[PipelineTaskConfig], ...],
135 dct: dict[str, Any],
136 **kwargs: Any,
137 ) -> _S:
138 if name != "PipelineTaskConfig":
139 # Verify that a connection class was specified and the argument is
140 # an instance of PipelineTaskConfig
141 if "pipelineConnections" not in kwargs: 141 ↛ 142line 141 didn't jump to line 142 because the condition on line 141 was never true
142 for base in bases:
143 if hasattr(base, "connections"):
144 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
145 break
146 if "pipelineConnections" not in kwargs: 146 ↛ 147line 146 didn't jump to line 147 because the condition on line 146 was never true
147 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
148 connectionsClass = kwargs["pipelineConnections"]
149 if not issubclass(connectionsClass, PipelineTaskConnections): 149 ↛ 150line 149 didn't jump to line 150 because the condition on line 149 was never true
150 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
152 # Create all the fields that will be used in the newly created sub
153 # config (under the attribute name "connections")
154 configConnectionsNamespace: dict[str, pexConfig.Field] = {}
155 for fieldName, obj in connectionsClass.allConnections.items():
156 configConnectionsNamespace[fieldName] = pexConfig.Field[str](
157 doc=f"name for connection {fieldName}", default=obj.name, deprecated=obj.deprecated
158 )
159 # If there are default templates also add them as fields to
160 # configure the template values
161 if hasattr(connectionsClass, "defaultTemplates"): 161 ↛ 172line 161 didn't jump to line 172 because the condition on line 161 was always true
162 docString = "Template parameter used to format corresponding field template parameter"
163 for templateName, default in connectionsClass.defaultTemplates.items():
164 configConnectionsNamespace[templateName] = TemplateField(
165 dtype=str,
166 doc=docString,
167 default=default,
168 deprecated=connectionsClass.deprecatedTemplates.get(templateName),
169 )
170 # add a reference to the connection class used to create this sub
171 # config
172 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
174 # Create a new config class with the fields defined above
175 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace)
176 # add it to the Config class that is currently being declared
177 dct["connections"] = pexConfig.ConfigField(
178 dtype=Connections,
179 doc="Configurations describing the connections of the PipelineTask to datatypes",
180 )
181 dct["ConnectionsConfigClass"] = Connections
182 dct["ConnectionsClass"] = connectionsClass
183 inst = super().__new__(cls, name, bases, dct)
184 return inst
186 def __init__(
187 self, name: str, bases: tuple[type[PipelineTaskConfig], ...], dct: dict[str, Any], **kwargs: Any
188 ):
189 # This overrides the default init to drop the kwargs argument. Python
190 # metaclasses will have this argument set if any kwargs are passes at
191 # class construction time, but should be consumed before calling
192 # __init__ on the type metaclass. This is in accordance with python
193 # documentation on metaclasses
194 super().__init__(name, bases, dct)
196 ConnectionsClass: type[PipelineTaskConnections]
197 ConnectionsConfigClass: type[pexConfig.Config]
200class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
201 """Configuration class for `PipelineTask`.
203 This Configuration class functions in largely the same manner as any other
204 derived from `lsst.pex.config.Config`. The only difference is in how it is
205 declared. `PipelineTaskConfig` children need to be declared with a
206 pipelineConnections argument. This argument should specify a child class of
207 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
208 a config class is created with information from the supplied connections
209 class to allow configuration of the connections class. This dynamically
210 created config class is then attached to the `PipelineTaskConfig` via a
211 `~lsst.pex.config.ConfigField` with the attribute name ``connections``.
212 """
214 connections: pexConfig.ConfigField
215 """Field which refers to a dynamically added configuration class which is
216 based on a PipelineTaskConnections class.
217 """
219 saveLogOutput = pexConfig.Field[bool](
220 default=True,
221 optional=False,
222 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
223 )
225 def applyConfigOverrides(
226 self,
227 instrument: Instrument | None,
228 taskDefaultName: str,
229 pipelineConfigs: Iterable[ConfigIR] | None,
230 parameters: ParametersIR,
231 label: str,
232 ) -> None:
233 """Apply config overrides to this config instance.
235 Parameters
236 ----------
237 instrument : `Instrument` or `None`
238 An instance of the `Instrument` specified in a pipeline.
239 If `None` then the pipeline did not specify and instrument.
240 taskDefaultName : `str`
241 The default name associated with the `Task` class. This
242 may be used with instrumental overrides.
243 pipelineConfigs : `~collections.abc.Iterable` \
244 of `~.pipelineIR.ConfigIR`
245 An iterable of `~.pipelineIR.ConfigIR` objects that contain
246 overrides to apply to this config instance.
247 parameters : `~.pipelineIR.ParametersIR`
248 Parameters defined in a Pipeline which are used in formatting
249 of config values across multiple `Task` in a pipeline.
250 label : `str`
251 The label associated with this class's Task in a pipeline.
252 """
253 overrides = ConfigOverrides()
254 overrides.addParameters(parameters)
255 if instrument is not None:
256 overrides.addInstrumentOverride(instrument, taskDefaultName)
257 if pipelineConfigs is not None:
258 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs):
259 if subConfig.dataId is not None:
260 raise NotImplementedError(
261 "Specializing a config on a partial data id is not yet "
262 "supported in Pipeline definition"
263 )
264 # only apply override if it applies to everything
265 if subConfig.dataId is None:
266 if subConfig.file:
267 for configFile in subConfig.file:
268 overrides.addFileOverride(os.path.expandvars(configFile))
269 if subConfig.python is not None:
270 overrides.addPythonOverride(subConfig.python)
271 for key, value in subConfig.rest.items():
272 overrides.addValueOverride(key, value)
273 overrides.applyTo(self)