lsst.pipe.base  20.0.0-9-gabd0d4c+10c7230d1e
connectionTypes.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module defining connection types to be used within a
23 `PipelineTaskConnections` class.
24 """
25 
26 __all__ = ["InitInput", "InitOutput", "Input", "PrerequisiteInput",
27  "Output", "BaseConnection"]
28 
29 import dataclasses
30 import typing
31 from typing import Callable, Iterable, Optional
32 
33 from lsst.daf.butler import (
34  CollectionSearch,
35  DataCoordinate,
36  DatasetRef,
37  DatasetType,
38  DimensionUniverse,
39  Registry,
40 )
41 
42 
43 @dataclasses.dataclass(frozen=True)
45  """Base class used for declaring PipelineTask connections
46 
47  Parameters
48  ----------
49  name : `str`
50  The name used to identify the dataset type
51  storageClass : `str`
52  The storage class used when (un)/persisting the dataset type
53  multiple : `bool`
54  Indicates if this connection should expect to contain multiple objects
55  of the given dataset type
56  """
57  name: str
58  storageClass: str
59  doc: str = ""
60  multiple: bool = False
61 
62  def __get__(self, inst, klass):
63  """Descriptor method
64 
65  This is a method used to turn a connection into a descriptor.
66  When a connection is added to a connection class, it is a class level
67  variable. This method makes accessing this connection, on the
68  instance of the connection class owning this connection, return a
69  result specialized for that instance. In the case of connections
70  this specifically means names specified in a config instance will
71  be visible instead of the default names for the connection.
72  """
73  # If inst is None, this is being accessed by the class and not an
74  # instance, return this connection itself
75  if inst is None:
76  return self
77  # If no object cache exists, create one to track the instances this
78  # connection has been accessed by
79  if not hasattr(inst, '_connectionCache'):
80  object.__setattr__(inst, '_connectionCache', {})
81  # Look up an existing cached instance
82  idSelf = id(self)
83  if idSelf in inst._connectionCache:
84  return inst._connectionCache[idSelf]
85  # Accumulate the parameters that define this connection
86  params = {}
87  for field in dataclasses.fields(self):
88  params[field.name] = getattr(self, field.name)
89  # Get the name override defined by the instance of the connection class
90  params['name'] = inst._nameOverrides[self.varName]
91  # Return a new instance of this connection specialized with the
92  # information provided by the connection class instance
93  return inst._connectionCache.setdefault(idSelf, self.__class__(**params))
94 
95 
96 @dataclasses.dataclass(frozen=True)
98  """Class used for declaring PipelineTask connections that includes
99  dimensions
100 
101  Parameters
102  ----------
103  name : `str`
104  The name used to identify the dataset type
105  storageClass : `str`
106  The storage class used when (un)/persisting the dataset type
107  multiple : `bool`
108  Indicates if this connection should expect to contain multiple objects
109  of the given dataset type
110  dimensions : iterable of `str`
111  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
112  to identify the dataset type identified by the specified name
113  """
114  dimensions: typing.Iterable[str] = ()
115 
116  def __post_init__(self):
117  if isinstance(self.dimensions, str):
118  raise TypeError("Dimensions must be iterable of dimensions, got str,"
119  "possibly omitted trailing comma")
120  if not isinstance(self.dimensions, typing.Iterable):
121  raise TypeError("Dimensions must be iterable of dimensions")
122 
123  def makeDatasetType(self, universe: DimensionUniverse):
124  """Construct a true `DatasetType` instance with normalized dimensions.
125  Parameters
126  ----------
127  universe : `lsst.daf.butler.DimensionUniverse`
128  Set of all known dimensions to be used to normalize the dimension
129  names specified in config.
130  Returns
131  -------
132  datasetType : `DatasetType`
133  The `DatasetType` defined by this connection.
134  """
135  return DatasetType(self.name,
136  universe.extract(self.dimensions),
137  self.storageClass)
138 
139 
140 @dataclasses.dataclass(frozen=True)
142  """Class used for declaring PipelineTask input connections
143 
144  Parameters
145  ----------
146  name : `str`
147  The default name used to identify the dataset type
148  storageClass : `str`
149  The storage class used when (un)/persisting the dataset type
150  multiple : `bool`
151  Indicates if this connection should expect to contain multiple objects
152  of the given dataset type
153  dimensions : iterable of `str`
154  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
155  to identify the dataset type identified by the specified name
156  deferLoad : `bool`
157  Indicates that this dataset type will be loaded as a
158  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
159  object to load the object at a later time.
160  """
161  deferLoad: bool = False
162 
163 
164 @dataclasses.dataclass(frozen=True)
166  pass
167 
168 
169 @dataclasses.dataclass(frozen=True)
171  """Class used for declaring PipelineTask prerequisite connections
172 
173  Parameters
174  ----------
175  name : `str`
176  The default name used to identify the dataset type
177  storageClass : `str`
178  The storage class used when (un)/persisting the dataset type
179  multiple : `bool`
180  Indicates if this connection should expect to contain multiple objects
181  of the given dataset type
182  dimensions : iterable of `str`
183  The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used
184  to identify the dataset type identified by the specified name
185  deferLoad : `bool`
186  Indicates that this dataset type will be loaded as a
187  `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this
188  object to load the object at a later time.
189  lookupFunction: `typing.Callable`, optional
190  An optional callable function that will look up PrerequisiteInputs
191  using the DatasetType, registry, quantum dataId, and input collections
192  passed to it. If no function is specified, the default temporal spatial
193  lookup will be used.
194  """
195  lookupFunction: Optional[Callable[[DatasetType, Registry, DataCoordinate, CollectionSearch],
196  Iterable[DatasetRef]]] = None
197 
198 
199 @dataclasses.dataclass(frozen=True)
201  pass
202 
203 
204 @dataclasses.dataclass(frozen=True)
206  pass
207 
208 
209 @dataclasses.dataclass(frozen=True)
211  pass
lsst::pipe::base.connectionTypes.PrerequisiteInput
Definition: connectionTypes.py:170
lsst::pipe::base.connectionTypes.BaseConnection
Definition: connectionTypes.py:44
lsst::pipe::base.connectionTypes.InitOutput
Definition: connectionTypes.py:210
lsst::pipe::base.connectionTypes.Output
Definition: connectionTypes.py:200
lsst::pipe::base.connectionTypes.BaseConnection.__get__
def __get__(self, inst, klass)
Definition: connectionTypes.py:62
lsst::pipe::base.connectionTypes.DimensionedConnection.makeDatasetType
def makeDatasetType(self, DimensionUniverse universe)
Definition: connectionTypes.py:123
lsst::pipe::base.connectionTypes.DimensionedConnection.__post_init__
def __post_init__(self)
Definition: connectionTypes.py:116
lsst::pipe::base.connectionTypes.DimensionedConnection
Definition: connectionTypes.py:97
lsst::pipe::base.connectionTypes.InitInput
Definition: connectionTypes.py:205
lsst::pipe::base.connectionTypes.Input
Definition: connectionTypes.py:165
lsst::pipe::base.connectionTypes.BaseInput
Definition: connectionTypes.py:141