Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module defining connection types to be used within a 

23`PipelineTaskConnections` class. 

24""" 

25 

26__all__ = ["InitInput", "InitOutput", "Input", "PrerequisiteInput", 

27 "Output", "BaseConnection"] 

28 

29import dataclasses 

30import typing 

31from typing import Callable, Iterable, Optional 

32 

33from lsst.daf.butler import ( 

34 CollectionSearch, 

35 DataCoordinate, 

36 DatasetRef, 

37 DatasetType, 

38 DimensionUniverse, 

39 Registry, 

40 StorageClass, 

41) 

42 

43 

44@dataclasses.dataclass(frozen=True) 

45class BaseConnection: 

46 """Base class used for declaring PipelineTask connections 

47 

48 Parameters 

49 ---------- 

50 name : `str` 

51 The name used to identify the dataset type 

52 storageClass : `str` 

53 The storage class used when (un)/persisting the dataset type 

54 multiple : `bool` 

55 Indicates if this connection should expect to contain multiple objects 

56 of the given dataset type 

57 """ 

58 name: str 

59 storageClass: str 

60 doc: str = "" 

61 multiple: bool = False 

62 

63 def __get__(self, inst, klass): 

64 """Descriptor method 

65 

66 This is a method used to turn a connection into a descriptor. 

67 When a connection is added to a connection class, it is a class level 

68 variable. This method makes accessing this connection, on the 

69 instance of the connection class owning this connection, return a 

70 result specialized for that instance. In the case of connections 

71 this specifically means names specified in a config instance will 

72 be visible instead of the default names for the connection. 

73 """ 

74 # If inst is None, this is being accessed by the class and not an 

75 # instance, return this connection itself 

76 if inst is None: 

77 return self 

78 # If no object cache exists, create one to track the instances this 

79 # connection has been accessed by 

80 if not hasattr(inst, '_connectionCache'): 

81 object.__setattr__(inst, '_connectionCache', {}) 

82 # Look up an existing cached instance 

83 idSelf = id(self) 

84 if idSelf in inst._connectionCache: 

85 return inst._connectionCache[idSelf] 

86 # Accumulate the parameters that define this connection 

87 params = {} 

88 for field in dataclasses.fields(self): 

89 params[field.name] = getattr(self, field.name) 

90 # Get the name override defined by the instance of the connection class 

91 params['name'] = inst._nameOverrides[self.varName] 

92 # Return a new instance of this connection specialized with the 

93 # information provided by the connection class instance 

94 return inst._connectionCache.setdefault(idSelf, self.__class__(**params)) 

95 

96 def makeDatasetType(self, universe: DimensionUniverse, 

97 parentStorageClass: Optional[StorageClass] = None): 

98 """Construct a true `DatasetType` instance with normalized dimensions. 

99 Parameters 

100 ---------- 

101 universe : `lsst.daf.butler.DimensionUniverse` 

102 Set of all known dimensions to be used to normalize the dimension 

103 names specified in config. 

104 parentStorageClass : `lsst.daf.butler.StorageClass`, optional 

105 Parent storage class for component datasets; `None` otherwise. 

106 

107 Returns 

108 ------- 

109 datasetType : `DatasetType` 

110 The `DatasetType` defined by this connection. 

111 """ 

112 return DatasetType(self.name, 

113 universe.empty, 

114 self.storageClass, 

115 parentStorageClass=parentStorageClass) 

116 

117 

118@dataclasses.dataclass(frozen=True) 

119class DimensionedConnection(BaseConnection): 

120 """Class used for declaring PipelineTask connections that includes 

121 dimensions 

122 

123 Parameters 

124 ---------- 

125 name : `str` 

126 The name used to identify the dataset type 

127 storageClass : `str` 

128 The storage class used when (un)/persisting the dataset type 

129 multiple : `bool` 

130 Indicates if this connection should expect to contain multiple objects 

131 of the given dataset type 

132 dimensions : iterable of `str` 

133 The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used 

134 to identify the dataset type identified by the specified name 

135 isCalibration: `bool`, optional 

136 `True` if this dataset type may be included in CALIBRATION-type 

137 collections to associate it with a validity range, `False` (default) 

138 otherwise. 

139 """ 

140 dimensions: typing.Iterable[str] = () 

141 isCalibration: bool = False 

142 

143 def __post_init__(self): 

144 if isinstance(self.dimensions, str): 144 ↛ 145line 144 didn't jump to line 145, because the condition on line 144 was never true

145 raise TypeError("Dimensions must be iterable of dimensions, got str," 

146 "possibly omitted trailing comma") 

147 if not isinstance(self.dimensions, typing.Iterable): 147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true

148 raise TypeError("Dimensions must be iterable of dimensions") 

149 

150 def makeDatasetType(self, universe: DimensionUniverse, 

151 parentStorageClass: Optional[StorageClass] = None): 

152 """Construct a true `DatasetType` instance with normalized dimensions. 

153 Parameters 

154 ---------- 

155 universe : `lsst.daf.butler.DimensionUniverse` 

156 Set of all known dimensions to be used to normalize the dimension 

157 names specified in config. 

158 parentStorageClass : `lsst.daf.butler.StorageClass`, optional 

159 Parent storage class for component datasets; `None` otherwise. 

160 

161 Returns 

162 ------- 

163 datasetType : `DatasetType` 

164 The `DatasetType` defined by this connection. 

165 """ 

166 return DatasetType(self.name, 

167 universe.extract(self.dimensions), 

168 self.storageClass, isCalibration=self.isCalibration, 

169 parentStorageClass=parentStorageClass) 

170 

171 

172@dataclasses.dataclass(frozen=True) 

173class BaseInput(DimensionedConnection): 

174 """Class used for declaring PipelineTask input connections 

175 

176 Parameters 

177 ---------- 

178 name : `str` 

179 The default name used to identify the dataset type 

180 storageClass : `str` 

181 The storage class used when (un)/persisting the dataset type 

182 multiple : `bool` 

183 Indicates if this connection should expect to contain multiple objects 

184 of the given dataset type 

185 dimensions : iterable of `str` 

186 The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used 

187 to identify the dataset type identified by the specified name 

188 deferLoad : `bool` 

189 Indicates that this dataset type will be loaded as a 

190 `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this 

191 object to load the object at a later time. 

192 """ 

193 deferLoad: bool = False 

194 

195 

196@dataclasses.dataclass(frozen=True) 

197class Input(BaseInput): 

198 pass 

199 

200 

201@dataclasses.dataclass(frozen=True) 

202class PrerequisiteInput(BaseInput): 

203 """Class used for declaring PipelineTask prerequisite connections 

204 

205 Parameters 

206 ---------- 

207 name : `str` 

208 The default name used to identify the dataset type 

209 storageClass : `str` 

210 The storage class used when (un)/persisting the dataset type 

211 multiple : `bool` 

212 Indicates if this connection should expect to contain multiple objects 

213 of the given dataset type 

214 dimensions : iterable of `str` 

215 The `lsst.daf.butler.Butler` `lsst.daf.butler.Registry` dimensions used 

216 to identify the dataset type identified by the specified name 

217 deferLoad : `bool` 

218 Indicates that this dataset type will be loaded as a 

219 `lsst.daf.butler.DeferredDatasetHandle`. PipelineTasks can use this 

220 object to load the object at a later time. 

221 lookupFunction: `typing.Callable`, optional 

222 An optional callable function that will look up PrerequisiteInputs 

223 using the DatasetType, registry, quantum dataId, and input collections 

224 passed to it. If no function is specified, the default temporal spatial 

225 lookup will be used. 

226 """ 

227 lookupFunction: Optional[Callable[[DatasetType, Registry, DataCoordinate, CollectionSearch], 

228 Iterable[DatasetRef]]] = None 

229 

230 

231@dataclasses.dataclass(frozen=True) 

232class Output(DimensionedConnection): 

233 pass 

234 

235 

236@dataclasses.dataclass(frozen=True) 

237class InitInput(BaseConnection): 

238 pass 

239 

240 

241@dataclasses.dataclass(frozen=True) 

242class InitOutput(BaseConnection): 

243 pass