lsst.pipe.base  20.0.0-21-gb65c2a3+a8acb598c9
connectionTypes.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining connection types to be used within a
23 `PipelineTaskConnections` class.
24 """
25 
26 __all__ = ["InitInput", "InitOutput", "Input", "PrerequisiteInput",
27  "Output", "BaseConnection"]
28 
29 import dataclasses
30 import typing
31 from typing import Callable, Iterable, Optional
32 
33 from lsst.daf.butler import (
34  CollectionSearch,
35  DataCoordinate,
36  DatasetRef,
37  DatasetType,
38  DimensionUniverse,
39  Registry,
40  StorageClass,
41 )
42 
43 
44 @dataclasses.dataclass(frozen=True)
46  """Base class used for declaring PipelineTask connections
47 
48  Parameters
49  ----------
50  name : `str`
51  The name used to identify the dataset type
52  storageClass : `str`
53  The storage class used when (un)/persisting the dataset type
54  multiple : `bool`
55  Indicates if this connection should expect to contain multiple objects
56  of the given dataset type
57  """
58  name: str
59  storageClass: str
60  doc: str = ""
61  multiple: bool = False
62 
63  def __get__(self, inst, klass):
64  """Descriptor method
65 
66  This is a method used to turn a connection into a descriptor.
67  When a connection is added to a connection class, it is a class level
68  variable. This method makes accessing this connection, on the
69  instance of the connection class owning this connection, return a
70  result specialized for that instance. In the case of connections
71  this specifically means names specified in a config instance will
72  be visible instead of the default names for the connection.
73  """
74  # If inst is None, this is being accessed by the class and not an
75  # instance, return this connection itself
76  if inst is None:
77  return self
78  # If no object cache exists, create one to track the instances this
79  # connection has been accessed by
80  if not hasattr(inst, '_connectionCache'):
81  object.__setattr__(inst, '_connectionCache', {})
82  # Look up an existing cached instance
83  idSelf = id(self)
84  if idSelf in inst._connectionCache:
85  return inst._connectionCache[idSelf]
86  # Accumulate the parameters that define this connection
87  params = {}
88  for field in dataclasses.fields(self):
89  params[field.name] = getattr(self, field.name)
90  # Get the name override defined by the instance of the connection class
91  params['name'] = inst._nameOverrides[self.varName]
92  # Return a new instance of this connection specialized with the
93  # information provided by the connection class instance
94  return inst._connectionCache.setdefault(idSelf, self.__class__(**params))
95 
96  def makeDatasetType(self, universe: DimensionUniverse,
97  parentStorageClass: Optional[StorageClass] = None):
98  """Construct a true `DatasetType` instance with normalized dimensions.
99  Parameters
100  ----------
101  universe : `lsst.daf.butler.DimensionUniverse`
102  Set of all known dimensions to be used to normalize the dimension
103  names specified in config.
104  parentStorageClass : `lsst.daf.butler.StorageClass`, optional
105  Parent storage class for component datasets; `None` otherwise.
106 
107  Returns
108  -------
109  datasetType : `DatasetType`
110  The `DatasetType` defined by this connection.
111  """
112  return DatasetType(self.name,
113  universe.empty,
114  self.storageClass,
115  parentStorageClass=parentStorageClass)
116 
117 
118 @dataclasses.dataclass(frozen=True)
120  """Class used for declaring PipelineTask connections that includes
121  dimensions
122 
123  Parameters
124  ----------
125  name : `str`
126  The name used to identify the dataset type
127  storageClass : `str`
128  The storage class used when (un)/persisting the dataset type
129  multiple : `bool`
130  Indicates if this connection should expect to contain multiple objects
131  of the given dataset type
132  dimensions : iterable of `str`
133  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
134  to identify the dataset type identified by the specified name
135  isCalibration: `bool`, optional
136  `True` if this dataset type may be included in CALIBRATION-type
137  collections to associate it with a validity range, `False` (default)
138  otherwise.
139  """
140  dimensions: typing.Iterable[str] = ()
141  isCalibration: bool = False
142 
143  def __post_init__(self):
144  if isinstance(self.dimensions, str):
145  raise TypeError("Dimensions must be iterable of dimensions, got str,"
146  "possibly omitted trailing comma")
147  if not isinstance(self.dimensions, typing.Iterable):
148  raise TypeError("Dimensions must be iterable of dimensions")
149 
150  def makeDatasetType(self, universe: DimensionUniverse,
151  parentStorageClass: Optional[StorageClass] = None):
152  """Construct a true `DatasetType` instance with normalized dimensions.
153  Parameters
154  ----------
155  universe : `lsst.daf.butler.DimensionUniverse`
156  Set of all known dimensions to be used to normalize the dimension
157  names specified in config.
158  parentStorageClass : `lsst.daf.butler.StorageClass`, optional
159  Parent storage class for component datasets; `None` otherwise.
160 
161  Returns
162  -------
163  datasetType : `DatasetType`
164  The `DatasetType` defined by this connection.
165  """
166  return DatasetType(self.name,
167  universe.extract(self.dimensions),
168  self.storageClass, isCalibration=self.isCalibration,
169  parentStorageClass=parentStorageClass)
170 
171 
172 @dataclasses.dataclass(frozen=True)
174  """Class used for declaring PipelineTask input connections
175 
176  Parameters
177  ----------
178  name : `str`
179  The default name used to identify the dataset type
180  storageClass : `str`
181  The storage class used when (un)/persisting the dataset type
182  multiple : `bool`
183  Indicates if this connection should expect to contain multiple objects
184  of the given dataset type
185  dimensions : iterable of `str`
186  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
187  to identify the dataset type identified by the specified name
188  deferLoad : `bool`
189  Indicates that this dataset type will be loaded as a
190  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
191  object to load the object at a later time.
192  """
193  deferLoad: bool = False
194 
195 
196 @dataclasses.dataclass(frozen=True)
198  pass
199 
200 
201 @dataclasses.dataclass(frozen=True)
203  """Class used for declaring PipelineTask prerequisite connections
204 
205  Parameters
206  ----------
207  name : `str`
208  The default name used to identify the dataset type
209  storageClass : `str`
210  The storage class used when (un)/persisting the dataset type
211  multiple : `bool`
212  Indicates if this connection should expect to contain multiple objects
213  of the given dataset type
214  dimensions : iterable of `str`
215  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
216  to identify the dataset type identified by the specified name
217  deferLoad : `bool`
218  Indicates that this dataset type will be loaded as a
219  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
220  object to load the object at a later time.
221  lookupFunction: `typing.Callable`, optional
222  An optional callable function that will look up PrerequisiteInputs
223  using the DatasetType, registry, quantum dataId, and input collections
224  passed to it. If no function is specified, the default temporal spatial
225  lookup will be used.
226  """
227  lookupFunction: Optional[Callable[[DatasetType, Registry, DataCoordinate, CollectionSearch],
228  Iterable[DatasetRef]]] = None
229 
230 
231 @dataclasses.dataclass(frozen=True)
233  pass
234 
235 
236 @dataclasses.dataclass(frozen=True)
238  pass
239 
240 
241 @dataclasses.dataclass(frozen=True)
243  pass
lsst::pipe::base.connectionTypes.PrerequisiteInput
Definition: connectionTypes.py:202
lsst::pipe::base.connectionTypes.BaseConnection
Definition: connectionTypes.py:45
lsst::pipe::base.connectionTypes.DimensionedConnection.makeDatasetType
def makeDatasetType(self, DimensionUniverse universe, Optional[StorageClass] parentStorageClass=None)
Definition: connectionTypes.py:150
lsst::pipe::base.connectionTypes.InitOutput
Definition: connectionTypes.py:242
lsst::pipe::base.connectionTypes.Output
Definition: connectionTypes.py:232
lsst::pipe::base.connectionTypes.BaseConnection.__get__
def __get__(self, inst, klass)
Definition: connectionTypes.py:63
lsst::pipe::base.connectionTypes.DimensionedConnection.__post_init__
def __post_init__(self)
Definition: connectionTypes.py:143
lsst::pipe::base.connectionTypes.BaseConnection.makeDatasetType
def makeDatasetType(self, DimensionUniverse universe, Optional[StorageClass] parentStorageClass=None)
Definition: connectionTypes.py:96
lsst::pipe::base.connectionTypes.DimensionedConnection
Definition: connectionTypes.py:119
lsst::pipe::base.connectionTypes.InitInput
Definition: connectionTypes.py:237
lsst::pipe::base.connectionTypes.Input
Definition: connectionTypes.py:197
lsst::pipe::base.connectionTypes.BaseInput
Definition: connectionTypes.py:173