Coverage for python/lsst/pipe/base/config.py: 48%
77 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:10 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:10 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining config classes for PipelineTask.
23"""
25from __future__ import annotations
27__all__ = ["PipelineTaskConfig"]
29# -------------------------------
30# Imports of standard modules --
31# -------------------------------
32import os
33from collections.abc import Iterable
34from numbers import Number
35from typing import TYPE_CHECKING, Any, TypeVar
37# -----------------------------
38# Imports for other modules --
39# -----------------------------
40import lsst.pex.config as pexConfig
42from ._instrument import Instrument
43from .configOverrides import ConfigOverrides
44from .connections import PipelineTaskConnections
45from .pipelineIR import ConfigIR, ParametersIR
47if TYPE_CHECKING:
48 from lsst.pex.config.callStack import StackFrame
50# ----------------------------------
51# Local non-exported definitions --
52# ----------------------------------
54_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
56# ------------------------
57# Exported definitions --
58# ------------------------
61class TemplateField(pexConfig.Field):
62 """Field specialized for use with connection templates.
64 Specifically it treats strings or numbers as valid input, as occasionally
65 numbers are used as a cycle counter in templates.
67 The reason for the specialized field, is that when numbers are involved
68 with the config override system through pipelines or from the command line,
69 sometimes the quoting to get appropriate values as strings gets
70 complicated. This will simplify the process greatly.
71 """
73 def _validateValue(self, value: Any) -> None:
74 if value is None:
75 return
77 if not (isinstance(value, str) or isinstance(value, Number)):
78 raise TypeError(
79 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
80 " Expected type str or a number"
81 )
82 if self.check is not None and not self.check(value):
83 ValueError("Value {value} is not a valid value")
85 def __set__(
86 self,
87 instance: pexConfig.Config,
88 value: Any,
89 at: StackFrame | None = None,
90 label: str = "assignment",
91 ) -> None:
92 # validate first, even though validate will be called in super
93 self._validateValue(value)
94 # now, explicitly make it into a string
95 value = str(value)
96 super().__set__(instance, value, at, label)
99class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
100 """Metaclass used in the creation of PipelineTaskConfig classes
102 This metaclass ensures a `PipelineTaskConnections` class is specified in
103 the class construction parameters with a parameter name of
104 pipelineConnections. Using the supplied connection class, this metaclass
105 constructs a `lsst.pex.config.Config` instance which can be used to
106 configure the connections class. This config is added to the config class
107 under declaration with the name "connections" used as an identifier. The
108 connections config also has a reference to the connections class used in
109 its construction associated with an atttribute named `ConnectionsClass`.
110 Finally the newly constructed config class (not an instance of it) is
111 assigned to the Config class under construction with the attribute name
112 `ConnectionsConfigClass`.
113 """
115 def __new__(
116 cls: type[_S],
117 name: str,
118 bases: tuple[type[PipelineTaskConfig], ...],
119 dct: dict[str, Any],
120 **kwargs: Any,
121 ) -> _S:
122 if name != "PipelineTaskConfig":
123 # Verify that a connection class was specified and the argument is
124 # an instance of PipelineTaskConfig
125 if "pipelineConnections" not in kwargs: 125 ↛ 126line 125 didn't jump to line 126, because the condition on line 125 was never true
126 for base in bases:
127 if hasattr(base, "connections"):
128 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
129 break
130 if "pipelineConnections" not in kwargs: 130 ↛ 131line 130 didn't jump to line 131, because the condition on line 130 was never true
131 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
132 connectionsClass = kwargs["pipelineConnections"]
133 if not issubclass(connectionsClass, PipelineTaskConnections): 133 ↛ 134line 133 didn't jump to line 134, because the condition on line 133 was never true
134 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
136 # Create all the fields that will be used in the newly created sub
137 # config (under the attribute name "connections")
138 configConnectionsNamespace: dict[str, pexConfig.Field] = {}
139 for fieldName, obj in connectionsClass.allConnections.items():
140 configConnectionsNamespace[fieldName] = pexConfig.Field[str](
141 doc=f"name for connection {fieldName}", default=obj.name
142 )
143 # If there are default templates also add them as fields to
144 # configure the template values
145 if hasattr(connectionsClass, "defaultTemplates"): 145 ↛ 153line 145 didn't jump to line 153, because the condition on line 145 was never false
146 docString = "Template parameter used to format corresponding field template parameter"
147 for templateName, default in connectionsClass.defaultTemplates.items():
148 configConnectionsNamespace[templateName] = TemplateField(
149 dtype=str, doc=docString, default=default
150 )
151 # add a reference to the connection class used to create this sub
152 # config
153 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
155 # Create a new config class with the fields defined above
156 Connections = type(f"{name}Connections", (pexConfig.Config,), configConnectionsNamespace)
157 # add it to the Config class that is currently being declared
158 dct["connections"] = pexConfig.ConfigField(
159 dtype=Connections,
160 doc="Configurations describing the connections of the PipelineTask to datatypes",
161 )
162 dct["ConnectionsConfigClass"] = Connections
163 dct["ConnectionsClass"] = connectionsClass
164 inst = super().__new__(cls, name, bases, dct)
165 return inst
167 def __init__(
168 self, name: str, bases: tuple[type[PipelineTaskConfig], ...], dct: dict[str, Any], **kwargs: Any
169 ):
170 # This overrides the default init to drop the kwargs argument. Python
171 # metaclasses will have this argument set if any kwargs are passes at
172 # class construction time, but should be consumed before calling
173 # __init__ on the type metaclass. This is in accordance with python
174 # documentation on metaclasses
175 super().__init__(name, bases, dct)
177 ConnectionsClass: type[PipelineTaskConnections]
178 ConnectionsConfigClass: type[pexConfig.Config]
181class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
182 """Configuration class for `PipelineTask`
184 This Configuration class functions in largely the same manner as any other
185 derived from `lsst.pex.config.Config`. The only difference is in how it is
186 declared. `PipelineTaskConfig` children need to be declared with a
187 pipelineConnections argument. This argument should specify a child class of
188 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
189 a config class is created with information from the supplied connections
190 class to allow configuration of the connections class. This dynamically
191 created config class is then attached to the `PipelineTaskConfig` via a
192 `~lsst.pex.config.ConfigField` with the attribute name `connections`.
193 """
195 connections: pexConfig.ConfigField
196 """Field which refers to a dynamically added configuration class which is
197 based on a PipelineTaskConnections class.
198 """
200 saveMetadata = pexConfig.Field[bool](
201 default=True,
202 optional=False,
203 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
204 deprecated="This field is deprecated and will be removed after v26.",
205 )
206 saveLogOutput = pexConfig.Field[bool](
207 default=True,
208 optional=False,
209 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
210 )
212 def applyConfigOverrides(
213 self,
214 instrument: Instrument | None,
215 taskDefaultName: str,
216 pipelineConfigs: Iterable[ConfigIR] | None,
217 parameters: ParametersIR,
218 label: str,
219 ) -> None:
220 r"""Apply config overrides to this config instance.
222 Parameters
223 ----------
224 instrument : `Instrument` or `None`
225 An instance of the `Instrument` specified in a pipeline.
226 If `None` then the pipeline did not specify and instrument.
227 taskDefaultName : `str`
228 The default name associated with the `Task` class. This
229 may be used with instrumental overrides.
230 pipelineConfigs : `~collections.abc.Iterable` of `ConfigIR`
231 An iterable of `ConfigIR` objects that contain overrides
232 to apply to this config instance.
233 parameters : `ParametersIR`
234 Parameters defined in a Pipeline which are used in formatting
235 of config values across multiple `Task`\ s in a pipeline.
236 label : `str`
237 The label associated with this class's Task in a pipeline.
238 """
239 overrides = ConfigOverrides()
240 if instrument is not None:
241 overrides.addInstrumentOverride(instrument, taskDefaultName)
242 if pipelineConfigs is not None:
243 for subConfig in (configIr.formatted(parameters) for configIr in pipelineConfigs):
244 if subConfig.dataId is not None:
245 raise NotImplementedError(
246 "Specializing a config on a partial data id is not yet "
247 "supported in Pipeline definition"
248 )
249 # only apply override if it applies to everything
250 if subConfig.dataId is None:
251 if subConfig.file:
252 for configFile in subConfig.file:
253 overrides.addFileOverride(os.path.expandvars(configFile))
254 if subConfig.python is not None:
255 overrides.addPythonOverride(subConfig.python)
256 for key, value in subConfig.rest.items():
257 overrides.addValueOverride(key, value)
258 overrides.applyTo(self)