Coverage for python/lsst/pipe/base/config.py: 65%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module defining config classes for PipelineTask.
23"""
24__all__ = ["ResourceConfig", "PipelineTaskConfig"]
26# -------------------------------
27# Imports of standard modules --
28# -------------------------------
29from numbers import Number
31# -----------------------------
32# Imports for other modules --
33# -----------------------------
34import lsst.pex.config as pexConfig
36from .connections import PipelineTaskConnections
38# ----------------------------------
39# Local non-exported definitions --
40# ----------------------------------
42# ------------------------
43# Exported definitions --
44# ------------------------
47class TemplateField(pexConfig.Field):
48 """This Field is specialized for use with connection templates.
49 Specifically it treats strings or numbers as valid input, as occasionally
50 numbers are used as a cycle counter in templates.
52 The reason for the specialized field, is that when numbers are involved
53 with the config override system through pipelines or from the command line,
54 sometimes the quoting to get appropriate values as strings gets
55 complicated. This will simplify the process greatly.
56 """
58 def _validateValue(self, value):
59 if value is None:
60 return
62 if not (isinstance(value, str) or isinstance(value, Number)):
63 raise TypeError(
64 f"Value {value} is of incorrect type {pexConfig.config._typeStr(value)}."
65 f" Expected type str or a number"
66 )
67 if self.check is not None and not self.check(value):
68 ValueError("Value {value} is not a valid value")
70 def __set__(self, instance, value, at=None, label="assignment"):
71 # validate first, even though validate will be called in super
72 self._validateValue(value)
73 # now, explicitly make it into a string
74 value = str(value)
75 super().__set__(instance, value, at, label)
78class PipelineTaskConfigMeta(pexConfig.ConfigMeta):
79 """Metaclass used in the creation of PipelineTaskConfig classes
81 This metaclass ensures a `PipelineTaskConnections` class is specified in
82 the class construction parameters with a parameter name of
83 pipelineConnections. Using the supplied connection class, this metaclass
84 constructs a `lsst.pex.config.Config` instance which can be used to
85 configure the connections class. This config is added to the config class
86 under declaration with the name "connections" used as an identifier. The
87 connections config also has a reference to the connections class used in
88 its construction associated with an atttribute named `ConnectionsClass`.
89 Finally the newly constructed config class (not an instance of it) is
90 assigned to the Config class under construction with the attribute name
91 `ConnectionsConfigClass`.
92 """
94 def __new__(cls, name, bases, dct, **kwargs):
95 if name != "PipelineTaskConfig":
96 # Verify that a connection class was specified and the argument is
97 # an instance of PipelineTaskConfig
98 if "pipelineConnections" not in kwargs: 98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true
99 for base in bases:
100 if hasattr(base, "connections"):
101 kwargs["pipelineConnections"] = base.connections.dtype.ConnectionsClass
102 break
103 if "pipelineConnections" not in kwargs: 103 ↛ 104line 103 didn't jump to line 104, because the condition on line 103 was never true
104 raise NameError("PipelineTaskConfig or a base class must be defined with connections class")
105 connectionsClass = kwargs["pipelineConnections"]
106 if not issubclass(connectionsClass, PipelineTaskConnections): 106 ↛ 107line 106 didn't jump to line 107, because the condition on line 106 was never true
107 raise ValueError("Can only assign a PipelineTaskConnections Class to pipelineConnections")
109 # Create all the fields that will be used in the newly created sub
110 # config (under the attribute name "connections")
111 configConnectionsNamespace = {}
112 for fieldName, obj in connectionsClass.allConnections.items():
113 configConnectionsNamespace[fieldName] = pexConfig.Field(
114 dtype=str, doc=f"name for connection {fieldName}", default=obj.name
115 )
116 # If there are default templates also add them as fields to
117 # configure the template values
118 if hasattr(connectionsClass, "defaultTemplates"): 118 ↛ 126line 118 didn't jump to line 126, because the condition on line 118 was never false
119 docString = "Template parameter used to format corresponding field template parameter"
120 for templateName, default in connectionsClass.defaultTemplates.items():
121 configConnectionsNamespace[templateName] = TemplateField(
122 dtype=str, doc=docString, default=default
123 )
124 # add a reference to the connection class used to create this sub
125 # config
126 configConnectionsNamespace["ConnectionsClass"] = connectionsClass
128 # Create a new config class with the fields defined above
129 Connections = type("Connections", (pexConfig.Config,), configConnectionsNamespace)
130 # add it to the Config class that is currently being declared
131 dct["connections"] = pexConfig.ConfigField(
132 dtype=Connections,
133 doc="Configurations describing the connections of the PipelineTask to datatypes",
134 )
135 dct["ConnectionsConfigClass"] = Connections
136 dct["ConnectionsClass"] = connectionsClass
137 inst = super().__new__(cls, name, bases, dct)
138 return inst
140 def __init__(self, name, bases, dct, **kwargs):
141 # This overrides the default init to drop the kwargs argument. Python
142 # metaclasses will have this argument set if any kwargs are passes at
143 # class construction time, but should be consumed before calling
144 # __init__ on the type metaclass. This is in accordance with python
145 # documentation on metaclasses
146 super().__init__(name, bases, dct)
149class PipelineTaskConfig(pexConfig.Config, metaclass=PipelineTaskConfigMeta):
150 """Configuration class for `PipelineTask`
152 This Configuration class functions in largely the same manner as any other
153 derived from `lsst.pex.config.Config`. The only difference is in how it is
154 declared. `PipelineTaskConfig` children need to be declared with a
155 pipelineConnections argument. This argument should specify a child class of
156 `PipelineTaskConnections`. During the declaration of a `PipelineTaskConfig`
157 a config class is created with information from the supplied connections
158 class to allow configuration of the connections class. This dynamically
159 created config class is then attached to the `PipelineTaskConfig` via a
160 `~lsst.pex.config.ConfigField` with the attribute name `connections`.
161 """
163 saveMetadata = pexConfig.Field(
164 dtype=bool,
165 default=True,
166 optional=False,
167 doc="Flag to enable/disable metadata saving for a task, enabled by default.",
168 )
169 saveLogOutput = pexConfig.Field(
170 dtype=bool,
171 default=True,
172 optional=False,
173 doc="Flag to enable/disable saving of log output for a task, enabled by default.",
174 )
177class ResourceConfig(pexConfig.Config):
178 """Configuration for resource requirements.
180 This configuration class will be used by some activators to estimate
181 resource use by pipeline. Additionally some tasks could use it to adjust
182 their resource use (e.g. reduce the number of threads).
184 For some resources their limit can be estimated by corresponding task,
185 in that case task could set the field value. For many fields defined in
186 this class their associated resource used by a task will depend on the
187 size of the data and is not known in advance. For these resources their
188 value will be configured through overrides based on some external
189 estimates.
190 """
192 minMemoryMB = pexConfig.Field(
193 dtype=int,
194 default=None,
195 optional=True,
196 doc="Minimal memory needed by task, can be None if estimate is unknown.",
197 )
198 minNumCores = pexConfig.Field(dtype=int, default=1, doc="Minimal number of cores needed by task.")