Coverage for python/lsst/pipe/base/config.py: 48%
78 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:09 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 12:09 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28"""Module defining config classes for PipelineTask.
29"""
31from __future__ import annotations
33__all__ = ["PipelineTaskConfig"]
35# -------------------------------
36# Imports of standard modules --
37# -------------------------------
38import os
39from collections.abc import Iterable
40from numbers import Number
41from typing import TYPE_CHECKING, Any, TypeVar
43# -----------------------------
44# Imports for other modules --
45# -----------------------------
46import lsst.pex.config as pexConfig
48from ._instrument import Instrument
49from .configOverrides import ConfigOverrides
50from .connections import PipelineTaskConnections
51from .pipelineIR import ConfigIR, ParametersIR
53if TYPE_CHECKING:
54 from lsst.pex.config.callStack import StackFrame
56# ----------------------------------
57# Local non-exported definitions --
58# ----------------------------------
60_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
62# ------------------------
63# Exported definitions --
64# ------------------------
67class TemplateField(pexConfig.Field):
68 """Field specialized for use with connection templates.
70 Specifically it treats strings or numbers as valid input, as occasionally
71 numbers are used as a cycle counter in templates.
73 The reason for the specialized field, is that when numbers are involved
74 with the config override system through pipelines or from the command line,
75 sometimes the quoting to get appropriate values as strings gets
76 complicated. This will simplify the process greatly.
77 """
79 def _validateValue(self, value: Any) -> None:
80 if value is None:
81 return
83 if not (isinstance(value, str | Number)):
84 raise TypeError(
85 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
86 " Expected type str or a number"
87 )
88 if self.check is not None and not self.check(value):
89 ValueError("Value {value} is not a valid value")
91 def __set__(
92 self,
93 instance: pexConfig.Config,
94 value: Any,
95 at: StackFrame | None = None,
96 label: str = "assignment",
97 ) -> None:
98 # validate first, even though validate will be called in super
99 self._validateValue(value)
100 # now, explicitly make it into a string
101 value = str(value)
102 super().__set__(instance, value, at, label)
105class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
106 """Metaclass used in the creation of PipelineTaskConfig classes
108 This metaclass ensures a `PipelineTaskConnections` class is specified in
109 the class construction parameters with a parameter name of
110 pipelineConnections. Using the supplied connection class, this metaclass
111 constructs a `lsst.pex.config.Config` instance which can be used to
112 configure the connections class. This config is added to the config class
113 under declaration with the name "connections" used as an identifier. The
114 connections config also has a reference to the connections class used in
115 its construction associated with an atttribute named ``ConnectionsClass``.
116 Finally the newly constructed config class (not an instance of it) is
117 assigned to the Config class under construction with the attribute name
118 ``ConnectionsConfigClass``.
119 """
121 def __new__(
122 cls: type[_S],
123 name: str,
124 bases: tuple[type[PipelineTaskConfig], ...],
125 dct: dict[str, Any],
126 **kwargs: Any,
127 ) -> _S:
128 if name != "PipelineTaskConfig":
129 # Verify that a connection class was specified and the argument is
130 # an instance of PipelineTaskConfig
131 if "pipelineConnections" not in kwargs: 131 ↛ 132line 131 didn't jump to line 132, because the condition on line 131 was never true
132 for base in bases:
133 if hasattr(base, "connections"):
134 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
135 break
136 if "pipelineConnections" not in kwargs: 136 ↛ 137line 136 didn't jump to line 137, because the condition on line 136 was never true
137 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
138 connectionsClass = kwargs["pipelineConnections"]
139 if not issubclass(connectionsClass, PipelineTaskConnections): 139 ↛ 140line 139 didn't jump to line 140, because the condition on line 139 was never true
140 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
142 # Create all the fields that will be used in the newly created sub
143 # config (under the attribute name "connections")
144 configConnectionsNamespace: dict[str, pexConfig.Field] = {}
145 for fieldName, obj in connectionsClass.allConnections.items():
146 configConnectionsNamespace[fieldName] = pexConfig.Field[str](
147 doc=f"name for connection {fieldName}", default=obj.name, deprecated=obj.deprecated
148 )
149 # If there are default templates also add them as fields to
150 # configure the template values
151 if hasattr(connectionsClass, "defaultTemplates"): 151 ↛ 162line 151 didn't jump to line 162, because the condition on line 151 was never false
152 docString = "Template parameter used to format corresponding field template parameter"
153 for templateName, default in connectionsClass.defaultTemplates.items():
154 configConnectionsNamespace[templateName] = TemplateField(
155 dtype=str,
156 doc=docString,
157 default=default,
158 deprecated=connectionsClass.deprecatedTemplates.get(templateName),
159 )
160 # add a reference to the connection class used to create this sub
161 # config
162 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
164 # Create a new config class with the fields defined above
165 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace)
166 # add it to the Config class that is currently being declared
167 dct["connections"] = pexConfig.ConfigField(
168 dtype=Connections,
169 doc="Configurations describing the connections of the PipelineTask to datatypes",
170 )
171 dct["ConnectionsConfigClass"] = Connections
172 dct["ConnectionsClass"] = connectionsClass
173 inst = super().__new__(cls, name, bases, dct)
174 return inst
176 def __init__(
177 self, name: str, bases: tuple[type[PipelineTaskConfig], ...], dct: dict[str, Any], **kwargs: Any
178 ):
179 # This overrides the default init to drop the kwargs argument. Python
180 # metaclasses will have this argument set if any kwargs are passes at
181 # class construction time, but should be consumed before calling
182 # __init__ on the type metaclass. This is in accordance with python
183 # documentation on metaclasses
184 super().__init__(name, bases, dct)
186 ConnectionsClass: type[PipelineTaskConnections]
187 ConnectionsConfigClass: type[pexConfig.Config]
190class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
191 """Configuration class for `PipelineTask`
193 This Configuration class functions in largely the same manner as any other
194 derived from `lsst.pex.config.Config`. The only difference is in how it is
195 declared. `PipelineTaskConfig` children need to be declared with a
196 pipelineConnections argument. This argument should specify a child class of
197 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
198 a config class is created with information from the supplied connections
199 class to allow configuration of the connections class. This dynamically
200 created config class is then attached to the `PipelineTaskConfig` via a
201 `~lsst.pex.config.ConfigField` with the attribute name ``connections``.
202 """
204 connections: pexConfig.ConfigField
205 """Field which refers to a dynamically added configuration class which is
206 based on a PipelineTaskConnections class.
207 """
209 saveMetadata = pexConfig.Field[bool](
210 default=True,
211 optional=False,
212 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
213 deprecated="This field is deprecated and will be removed after v26.",
214 )
215 saveLogOutput = pexConfig.Field[bool](
216 default=True,
217 optional=False,
218 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
219 )
221 def applyConfigOverrides(
222 self,
223 instrument: Instrument | None,
224 taskDefaultName: str,
225 pipelineConfigs: Iterable[ConfigIR] | None,
226 parameters: ParametersIR,
227 label: str,
228 ) -> None:
229 """Apply config overrides to this config instance.
231 Parameters
232 ----------
233 instrument : `Instrument` or `None`
234 An instance of the `Instrument` specified in a pipeline.
235 If `None` then the pipeline did not specify and instrument.
236 taskDefaultName : `str`
237 The default name associated with the `Task` class. This
238 may be used with instrumental overrides.
239 pipelineConfigs : `~collections.abc.Iterable` \
240 of `~.pipelineIR.ConfigIR`
241 An iterable of `~.pipelineIR.ConfigIR` objects that contain
242 overrides to apply to this config instance.
243 parameters : `~.pipelineIR.ParametersIR`
244 Parameters defined in a Pipeline which are used in formatting
245 of config values across multiple `Task` in a pipeline.
246 label : `str`
247 The label associated with this class's Task in a pipeline.
248 """
249 overrides = ConfigOverrides()
250 overrides.addParameters(parameters)
251 if instrument is not None:
252 overrides.addInstrumentOverride(instrument, taskDefaultName)
253 if pipelineConfigs is not None:
254 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs):
255 if subConfig.dataId is not None:
256 raise NotImplementedError(
257 "Specializing a config on a partial data id is not yet "
258 "supported in Pipeline definition"
259 )
260 # only apply override if it applies to everything
261 if subConfig.dataId is None:
262 if subConfig.file:
263 for configFile in subConfig.file:
264 overrides.addFileOverride(os.path.expandvars(configFile))
265 if subConfig.python is not None:
266 overrides.addPythonOverride(subConfig.python)
267 for key, value in subConfig.rest.items():
268 overrides.addValueOverride(key, value)
269 overrides.applyTo(self)