Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "makeVisitRecordFromObsInfo", 

23 "addUnboundedCalibrationLabel") 

24 

25import os.path 

26from abc import ABCMeta, abstractmethod 

27import astropy.time 

28 

29from lsst.daf.butler import TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate 

30from lsst.utils import getPackageDir 

31 

32# To be a standard text curated calibration means that we use a 

33# standard definition for the corresponding DatasetType. 

34StandardCuratedCalibrationDatasetTypes = { 

35 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

36 "storageClass": "Defects"}, 

37 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

38 "storageClass": "QECurve"}, 

39} 

40 

41 

42class Instrument(metaclass=ABCMeta): 

43 """Base class for instrument-specific logic for the Gen3 Butler. 

44 

45 Concrete instrument subclasses should be directly constructable with no 

46 arguments. 

47 """ 

48 

49 configPaths = () 

50 """Paths to config files to read for specific Tasks. 

51 

52 The paths in this list should contain files of the form `task.py`, for 

53 each of the Tasks that requires special configuration. 

54 """ 

55 

56 policyName = None 

57 """Instrument specific name to use when locating a policy or configuration 

58 file in the file system.""" 

59 

60 obsDataPackage = None 

61 """Name of the package containing the text curated calibration files. 

62 Usually a obs _data package. If `None` no curated calibration files 

63 will be read. (`str`)""" 

64 

65 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

66 """The dataset types expected to be obtained from the obsDataPackage. 

67 These dataset types are all required to have standard definitions and 

68 must be known to the base class. Clearing this list will prevent 

69 any of these calibrations from being stored. If a dataset type is not 

70 known to a specific instrument it can still be included in this list 

71 since the data package is the source of truth. 

72 """ 

73 

74 @property 

75 @abstractmethod 

76 def filterDefinitions(self): 

77 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

78 for this instrument. 

79 """ 

80 return None 

81 

82 def __init__(self, *args, **kwargs): 

83 self.filterDefinitions.reset() 

84 self.filterDefinitions.defineFilters() 

85 self._obsDataPackageDir = None 

86 

87 @classmethod 

88 @abstractmethod 

89 def getName(cls): 

90 raise NotImplementedError() 

91 

92 @abstractmethod 

93 def getCamera(self): 

94 """Retrieve the cameraGeom representation of this instrument. 

95 

96 This is a temporary API that should go away once obs_ packages have 

97 a standardized approach to writing versioned cameras to a Gen3 repo. 

98 """ 

99 raise NotImplementedError() 

100 

101 @abstractmethod 

102 def register(self, registry): 

103 """Insert instrument, physical_filter, and detector entries into a 

104 `Registry`. 

105 """ 

106 raise NotImplementedError() 

107 

108 @property 

109 def obsDataPackageDir(self): 

110 if self.obsDataPackage is None: 

111 return None 

112 if self._obsDataPackageDir is None: 

113 # Defer any problems with locating the package until 

114 # we need to find it. 

115 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

116 return self._obsDataPackageDir 

117 

118 def _registerFilters(self, registry): 

119 """Register the physical and abstract filter Dimension relationships. 

120 This should be called in the ``register`` implementation. 

121 

122 Parameters 

123 ---------- 

124 registry : `lsst.daf.butler.core.Registry` 

125 The registry to add dimensions to. 

126 """ 

127 for filter in self.filterDefinitions: 

128 # fix for undefined abstract filters causing trouble in the registry: 

129 if filter.abstract_filter is None: 

130 abstract_filter = filter.physical_filter 

131 else: 

132 abstract_filter = filter.abstract_filter 

133 

134 registry.insertDimensionData("physical_filter", 

135 {"instrument": self.getName(), 

136 "name": filter.physical_filter, 

137 "abstract_filter": abstract_filter 

138 }) 

139 

140 @abstractmethod 

141 def getRawFormatter(self, dataId): 

142 """Return the Formatter class that should be used to read a particular 

143 raw file. 

144 

145 Parameters 

146 ---------- 

147 dataId : `DataCoordinate` 

148 Dimension-based ID for the raw file or files being ingested. 

149 

150 Returns 

151 ------- 

152 formatter : `Formatter` class 

153 Class to be used that reads the file into an 

154 `lsst.afw.image.Exposure` instance. 

155 """ 

156 raise NotImplementedError() 

157 

158 def writeCuratedCalibrations(self, butler): 

159 """Write human-curated calibration Datasets to the given Butler with 

160 the appropriate validity ranges. 

161 

162 Parameters 

163 ---------- 

164 butler : `lsst.daf.butler.Butler` 

165 Butler to use to store these calibrations. 

166 

167 Notes 

168 ----- 

169 Expected to be called from subclasses. The base method calls 

170 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

171 """ 

172 self.writeCameraGeom(butler) 

173 self.writeStandardTextCuratedCalibrations(butler) 

174 

175 def applyConfigOverrides(self, name, config): 

176 """Apply instrument-specific overrides for a task config. 

177 

178 Parameters 

179 ---------- 

180 name : `str` 

181 Name of the object being configured; typically the _DefaultName 

182 of a Task. 

183 config : `lsst.pex.config.Config` 

184 Config instance to which overrides should be applied. 

185 """ 

186 for root in self.configPaths: 

187 path = os.path.join(root, f"{name}.py") 

188 if os.path.exists(path): 

189 config.load(path) 

190 

191 def writeCameraGeom(self, butler): 

192 """Write the default camera geometry to the butler repository 

193 with an infinite validity range. 

194 

195 Parameters 

196 ---------- 

197 butler : `lsst.daf.butler.Butler` 

198 Butler to receive these calibration datasets. 

199 """ 

200 

201 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

202 universe=butler.registry.dimensions) 

203 butler.registry.registerDatasetType(datasetType) 

204 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

205 camera = self.getCamera() 

206 butler.put(camera, datasetType, unboundedDataId) 

207 

208 def writeStandardTextCuratedCalibrations(self, butler): 

209 """Write the set of standardized curated text calibrations to 

210 the repository. 

211 

212 Parameters 

213 ---------- 

214 butler : `lsst.daf.butler.Butler` 

215 Butler to receive these calibration datasets. 

216 """ 

217 

218 for datasetTypeName in self.standardCuratedDatasetTypes: 

219 # We need to define the dataset types. 

220 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

221 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

222 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

223 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

224 datasetType = DatasetType(datasetTypeName, 

225 universe=butler.registry.dimensions, 

226 **definition) 

227 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType) 

228 

229 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType): 

230 """Write standardized curated calibration datasets for this specific 

231 dataset type from an obs data package. 

232 

233 Parameters 

234 ---------- 

235 butler : `lsst.daf.butler.Butler` 

236 Gen3 butler in which to put the calibrations. 

237 datasetType : `lsst.daf.butler.DatasetType` 

238 Dataset type to be put. 

239 

240 Notes 

241 ----- 

242 This method scans the location defined in the ``obsDataPackageDir`` 

243 class attribute for curated calibrations corresponding to the 

244 supplied dataset type. The directory name in the data package must 

245 match the name of the dataset type. They are assumed to use the 

246 standard layout and can be read by 

247 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

248 metadata. 

249 """ 

250 if self.obsDataPackageDir is None: 

251 # if there is no data package then there can't be datasets 

252 return 

253 

254 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

255 datasetType.name) 

256 

257 if not os.path.exists(calibPath): 

258 return 

259 

260 # Register the dataset type 

261 butler.registry.registerDatasetType(datasetType) 

262 

263 # obs_base can't depend on pipe_tasks but concrete obs packages 

264 # can -- we therefore have to defer import 

265 from lsst.pipe.tasks.read_curated_calibs import read_all 

266 

267 camera = self.getCamera() 

268 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

269 endOfTime = TIMESPAN_MAX 

270 dimensionRecords = [] 

271 datasetRecords = [] 

272 for det in calibsDict: 

273 times = sorted([k for k in calibsDict[det]]) 

274 calibs = [calibsDict[det][time] for time in times] 

275 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

276 times += [endOfTime] 

277 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

278 md = calib.getMetadata() 

279 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

280 dataId = DataCoordinate.standardize( 

281 universe=butler.registry.dimensions, 

282 instrument=self.getName(), 

283 calibration_label=calibrationLabel, 

284 detector=md["DETECTOR"], 

285 ) 

286 datasetRecords.append((calib, dataId)) 

287 dimensionRecords.append({ 

288 "instrument": self.getName(), 

289 "name": calibrationLabel, 

290 "datetime_begin": beginTime, 

291 "datetime_end": endTime, 

292 }) 

293 

294 # Second loop actually does the inserts and filesystem writes. 

295 with butler.transaction(): 

296 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

297 # TODO: vectorize these puts, once butler APIs for that become 

298 # available. 

299 for calib, dataId in datasetRecords: 

300 butler.put(calib, datasetType, dataId) 

301 

302 

303def makeExposureRecordFromObsInfo(obsInfo, universe): 

304 """Construct an exposure DimensionRecord from 

305 `astro_metadata_translator.ObservationInfo`. 

306 

307 Parameters 

308 ---------- 

309 obsInfo : `astro_metadata_translator.ObservationInfo` 

310 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

311 the exposure. 

312 universe : `DimensionUniverse` 

313 Set of all known dimensions. 

314 

315 Returns 

316 ------- 

317 record : `DimensionRecord` 

318 A record containing exposure metadata, suitable for insertion into 

319 a `Registry`. 

320 """ 

321 dimension = universe["exposure"] 

322 return dimension.RecordClass.fromDict({ 

323 "instrument": obsInfo.instrument, 

324 "id": obsInfo.exposure_id, 

325 "name": obsInfo.observation_id, 

326 "group": obsInfo.exposure_group, 

327 "datetime_begin": obsInfo.datetime_begin, 

328 "datetime_end": obsInfo.datetime_end, 

329 "exposure_time": obsInfo.exposure_time.to_value("s"), 

330 "dark_time": obsInfo.dark_time.to_value("s"), 

331 "observation_type": obsInfo.observation_type, 

332 "physical_filter": obsInfo.physical_filter, 

333 "visit": obsInfo.visit_id, 

334 }) 

335 

336 

337def makeVisitRecordFromObsInfo(obsInfo, universe, *, region=None): 

338 """Construct a visit `DimensionRecord` from 

339 `astro_metadata_translator.ObservationInfo`. 

340 

341 Parameters 

342 ---------- 

343 obsInfo : `astro_metadata_translator.ObservationInfo` 

344 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

345 the exposure. 

346 universe : `DimensionUniverse` 

347 Set of all known dimensions. 

348 region : `lsst.sphgeom.Region`, optional 

349 Spatial region for the visit. 

350 

351 Returns 

352 ------- 

353 record : `DimensionRecord` 

354 A record containing visit metadata, suitable for insertion into a 

355 `Registry`. 

356 """ 

357 dimension = universe["visit"] 

358 return dimension.RecordClass.fromDict({ 

359 "instrument": obsInfo.instrument, 

360 "id": obsInfo.visit_id, 

361 "name": obsInfo.observation_id, 

362 "datetime_begin": obsInfo.datetime_begin, 

363 "datetime_end": obsInfo.datetime_end, 

364 "exposure_time": obsInfo.exposure_time.to_value("s"), 

365 "physical_filter": obsInfo.physical_filter, 

366 "region": region, 

367 }) 

368 

369 

370def addUnboundedCalibrationLabel(registry, instrumentName): 

371 """Add a special 'unbounded' calibration_label dimension entry for the 

372 given camera that is valid for any exposure. 

373 

374 If such an entry already exists, this function just returns a `DataId` 

375 for the existing entry. 

376 

377 Parameters 

378 ---------- 

379 registry : `Registry` 

380 Registry object in which to insert the dimension entry. 

381 instrumentName : `str` 

382 Name of the instrument this calibration label is associated with. 

383 

384 Returns 

385 ------- 

386 dataId : `DataId` 

387 New or existing data ID for the unbounded calibration. 

388 """ 

389 d = dict(instrument=instrumentName, calibration_label="unbounded") 

390 try: 

391 return registry.expandDataId(d) 

392 except LookupError: 

393 pass 

394 entry = d.copy() 

395 entry["datetime_begin"] = TIMESPAN_MIN 

396 entry["datetime_end"] = TIMESPAN_MAX 

397 registry.insertDimensionData("calibration_label", entry) 

398 return registry.expandDataId(d)