Coverage for python/lsst/pipe/base/config.py: 66%
57 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-18 02:36 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-18 02:36 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining config classes for PipelineTask.
23"""
25from __future__ import annotations
27__all__ = ["ResourceConfig", "PipelineTaskConfig"]
29# -------------------------------
30# Imports of standard modules --
31# -------------------------------
32from numbers import Number
33from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Type, TypeVar
35# -----------------------------
36# Imports for other modules --
37# -----------------------------
38import lsst.pex.config as pexConfig
40from .connections import PipelineTaskConnections
42if TYPE_CHECKING: 42 ↛ 43line 42 didn't jump to line 43, because the condition on line 42 was never true
43 from lsst.pex.config.callStack import StackFrame
45# ----------------------------------
46# Local non-exported definitions --
47# ----------------------------------
49_S = TypeVar("_S", bound="PipelineTaskConfigMeta")
51# ------------------------
52# Exported definitions --
53# ------------------------
56class TemplateField(pexConfig.Field):
57 """This Field is specialized for use with connection templates.
58 Specifically it treats strings or numbers as valid input, as occasionally
59 numbers are used as a cycle counter in templates.
61 The reason for the specialized field, is that when numbers are involved
62 with the config override system through pipelines or from the command line,
63 sometimes the quoting to get appropriate values as strings gets
64 complicated. This will simplify the process greatly.
65 """
67 def _validateValue(self, value: Any) -> None:
68 if value is None:
69 return
71 if not (isinstance(value, str) or isinstance(value, Number)):
72 raise TypeError(
73 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
74 f" Expected type str or a number"
75 )
76 if self.check is not None and not self.check(value):
77 ValueError("Value {value} is not a valid value")
79 def __set__(
80 self,
81 instance: pexConfig.Field,
82 value: Any,
83 at: Optional[StackFrame] = None,
84 label: str = "assignment",
85 ) -> None:
86 # validate first, even though validate will be called in super
87 self._validateValue(value)
88 # now, explicitly make it into a string
89 value = str(value)
90 super().__set__(instance, value, at, label)
93class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
94 """Metaclass used in the creation of PipelineTaskConfig classes
96 This metaclass ensures a `PipelineTaskConnections` class is specified in
97 the class construction parameters with a parameter name of
98 pipelineConnections. Using the supplied connection class, this metaclass
99 constructs a `lsst.pex.config.Config` instance which can be used to
100 configure the connections class. This config is added to the config class
101 under declaration with the name "connections" used as an identifier. The
102 connections config also has a reference to the connections class used in
103 its construction associated with an atttribute named `ConnectionsClass`.
104 Finally the newly constructed config class (not an instance of it) is
105 assigned to the Config class under construction with the attribute name
106 `ConnectionsConfigClass`.
107 """
109 def __new__(
110 cls: Type[_S], name: str, bases: Tuple[PipelineTaskConfig, ...], dct: Dict[str, Any], **kwargs: Any
111 ) -> _S:
112 if name != "PipelineTaskConfig":
113 # Verify that a connection class was specified and the argument is
114 # an instance of PipelineTaskConfig
115 if "pipelineConnections" not in kwargs: 115 ↛ 116line 115 didn't jump to line 116, because the condition on line 115 was never true
116 for base in bases:
117 if hasattr(base, "connections"):
118 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
119 break
120 if "pipelineConnections" not in kwargs: 120 ↛ 121line 120 didn't jump to line 121, because the condition on line 120 was never true
121 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
122 connectionsClass = kwargs["pipelineConnections"]
123 if not issubclass(connectionsClass, PipelineTaskConnections): 123 ↛ 124line 123 didn't jump to line 124, because the condition on line 123 was never true
124 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
126 # Create all the fields that will be used in the newly created sub
127 # config (under the attribute name "connections")
128 configConnectionsNamespace = {}
129 for fieldName, obj in connectionsClass.allConnections.items():
130 configConnectionsNamespace[fieldName] = pexConfig.Field(
131 dtype=str, doc=f"name for connection {fieldName}", default=obj.name
132 )
133 # If there are default templates also add them as fields to
134 # configure the template values
135 if hasattr(connectionsClass, "defaultTemplates"): 135 ↛ 143line 135 didn't jump to line 143, because the condition on line 135 was never false
136 docString = "Template parameter used to format corresponding field template parameter"
137 for templateName, default in connectionsClass.defaultTemplates.items():
138 configConnectionsNamespace[templateName] = TemplateField(
139 dtype=str, doc=docString, default=default
140 )
141 # add a reference to the connection class used to create this sub
142 # config
143 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
145 # Create a new config class with the fields defined above
146 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace)
147 # add it to the Config class that is currently being declared
148 dct["connections"] = pexConfig.ConfigField(
149 dtype=Connections,
150 doc="Configurations describing the connections of the PipelineTask to datatypes",
151 )
152 dct["ConnectionsConfigClass"] = Connections
153 dct["ConnectionsClass"] = connectionsClass
154 inst = super().__new__(cls, name, bases, dct)
155 return inst
157 def __init__(
158 self, name: str, bases: Tuple[Type[PipelineTaskConfig], ...], dct: Dict[str, Any], **kwargs: Any
159 ):
160 # This overrides the default init to drop the kwargs argument. Python
161 # metaclasses will have this argument set if any kwargs are passes at
162 # class construction time, but should be consumed before calling
163 # __init__ on the type metaclass. This is in accordance with python
164 # documentation on metaclasses
165 super().__init__(name, bases, dct)
168class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
169 """Configuration class for `PipelineTask`
171 This Configuration class functions in largely the same manner as any other
172 derived from `lsst.pex.config.Config`. The only difference is in how it is
173 declared. `PipelineTaskConfig` children need to be declared with a
174 pipelineConnections argument. This argument should specify a child class of
175 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
176 a config class is created with information from the supplied connections
177 class to allow configuration of the connections class. This dynamically
178 created config class is then attached to the `PipelineTaskConfig` via a
179 `~lsst.pex.config.ConfigField` with the attribute name `connections`.
180 """
182 saveMetadata = pexConfig.Field(
183 dtype=bool,
184 default=True,
185 optional=False,
186 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
187 )
188 saveLogOutput = pexConfig.Field(
189 dtype=bool,
190 default=True,
191 optional=False,
192 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
193 )
196class ResourceConfig(pexConfig.Config):
197 """Configuration for resource requirements.
199 This configuration class will be used by some activators to estimate
200 resource use by pipeline. Additionally some tasks could use it to adjust
201 their resource use (e.g. reduce the number of threads).
203 For some resources their limit can be estimated by corresponding task,
204 in that case task could set the field value. For many fields defined in
205 this class their associated resource used by a task will depend on the
206 size of the data and is not known in advance. For these resources their
207 value will be configured through overrides based on some external
208 estimates.
209 """
211 minMemoryMB = pexConfig.Field(
212 dtype=int,
213 default=None,
214 optional=True,
215 doc="Minimal memory needed by task, can be None if estimate is unknown.",
216 )
217 minNumCores = pexConfig.Field(dtype=int, default=1, doc="Minimal number of cores needed by task.")