lsst.pipe.base  19.0.0-11-ge4fa8f8+1
connectionTypes.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining connection types to be used within a
23 `PipelineTaskConnections` class.
24 """
25 
26 __all__ = ["InitInput", "InitOutput", "Input", "PrerequisiteInput",
27  "Output", "BaseConnection"]
28 
29 import dataclasses
30 import typing
31 from typing import Callable, Iterable, Optional
32 from collections.abc import Mapping
33 
34 from lsst.daf.butler import DatasetType, DimensionUniverse, Registry, ExpandedDataCoordinate, DatasetRef
35 
36 
37 @dataclasses.dataclass(frozen=True)
39  """Base class used for declaring PipelineTask connections
40 
41  Parameters
42  ----------
43  name : `str`
44  The name used to identify the dataset type
45  storageClass : `str`
46  The storage class used when (un)/persisting the dataset type
47  multiple : `bool`
48  Indicates if this connection should expect to contain multiple objects
49  of the given dataset type
50  """
51  name: str
52  storageClass: str
53  doc: str = ""
54  multiple: bool = False
55 
56  def __get__(self, inst, klass):
57  """Descriptor method
58 
59  This is a method used to turn a connection into a descriptor.
60  When a connection is added to a connection class, it is a class level
61  variable. This method makes accessing this connection, on the
62  instance of the connection class owning this connection, return a
63  result specialized for that instance. In the case of connections
64  this specifically means names specified in a config instance will
65  be visible instead of the default names for the connection.
66  """
67  # If inst is None, this is being accessed by the class and not an
68  # instance, return this connection itself
69  if inst is None:
70  return self
71  # If no object cache exists, create one to track the instances this
72  # connection has been accessed by
73  if not hasattr(inst, '_connectionCache'):
74  object.__setattr__(inst, '_connectionCache', {})
75  # Look up an existing cached instance
76  idSelf = id(self)
77  if idSelf in inst._connectionCache:
78  return inst._connectionCache[idSelf]
79  # Accumulate the parameters that define this connection
80  params = {}
81  for field in dataclasses.fields(self):
82  params[field.name] = getattr(self, field.name)
83  # Get the name override defined by the instance of the connection class
84  params['name'] = inst._nameOverrides[self.varName]
85  # Return a new instance of this connection specialized with the
86  # information provided by the connection class instance
87  return inst._connectionCache.setdefault(idSelf, self.__class__(**params))
88 
89 
90 @dataclasses.dataclass(frozen=True)
92  """Class used for declaring PipelineTask connections that includes
93  dimensions
94 
95  Parameters
96  ----------
97  name : `str`
98  The name used to identify the dataset type
99  storageClass : `str`
100  The storage class used when (un)/persisting the dataset type
101  multiple : `bool`
102  Indicates if this connection should expect to contain multiple objects
103  of the given dataset type
104  dimensions : iterable of `str`
105  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
106  to identify the dataset type identified by the specified name
107  """
108  dimensions: typing.Iterable[str] = ()
109 
110  def makeDatasetType(self, universe: DimensionUniverse):
111  """Construct a true `DatasetType` instance with normalized dimensions.
112  Parameters
113  ----------
114  universe : `lsst.daf.butler.DimensionUniverse`
115  Set of all known dimensions to be used to normalize the dimension
116  names specified in config.
117  Returns
118  -------
119  datasetType : `DatasetType`
120  The `DatasetType` defined by this connection.
121  """
122  return DatasetType(self.name,
123  universe.extract(self.dimensions),
124  self.storageClass)
125 
126 
127 @dataclasses.dataclass(frozen=True)
129  """Class used for declaring PipelineTask input connections
130 
131  Parameters
132  ----------
133  name : `str`
134  The default name used to identify the dataset type
135  storageClass : `str`
136  The storage class used when (un)/persisting the dataset type
137  multiple : `bool`
138  Indicates if this connection should expect to contain multiple objects
139  of the given dataset type
140  dimensions : iterable of `str`
141  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
142  to identify the dataset type identified by the specified name
143  deferLoad : `bool`
144  Indicates that this dataset type will be loaded as a
145  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
146  object to load the object at a later time.
147  """
148  deferLoad: bool = False
149 
150 
151 @dataclasses.dataclass(frozen=True)
153  pass
154 
155 
156 @dataclasses.dataclass(frozen=True)
158  """Class used for declaring PipelineTask prerequisite connections
159 
160  Parameters
161  ----------
162  name : `str`
163  The default name used to identify the dataset type
164  storageClass : `str`
165  The storage class used when (un)/persisting the dataset type
166  multiple : `bool`
167  Indicates if this connection should expect to contain multiple objects
168  of the given dataset type
169  dimensions : iterable of `str`
170  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
171  to identify the dataset type identified by the specified name
172  deferLoad : `bool`
173  Indicates that this dataset type will be loaded as a
174  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
175  object to load the object at a later time.
176  lookupFunction: `typing.Callable`
177  An optional callable function that will look up PrerequisiteInputs
178  using the DatasetType, registry, quantum dataId, and input collections
179  passed to it. If no function is specified, the default temporal spatial
180  lookup will be used.
181  """
182  lookupFunction: Optional[Callable[[DatasetType, Registry, ExpandedDataCoordinate, Mapping],
183  Iterable[DatasetRef]]] = None
184 
185 
186 @dataclasses.dataclass(frozen=True)
188  pass
189 
190 
191 @dataclasses.dataclass(frozen=True)
193  pass
194 
195 
196 @dataclasses.dataclass(frozen=True)
198  pass