Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Tuple, TYPE_CHECKING 

29import astropy.time 

30 

31from lsst.afw.cameraGeom import Camera 

32from lsst.daf.butler import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate 

33from lsst.utils import getPackageDir, doImport 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from .gen2to3 import TranslatorFactory 

37 

38# To be a standard text curated calibration means that we use a 

39# standard definition for the corresponding DatasetType. 

40StandardCuratedCalibrationDatasetTypes = { 

41 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

42 "storageClass": "Defects"}, 

43 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

44 "storageClass": "QECurve"}, 

45} 

46 

47 

48class Instrument(metaclass=ABCMeta): 

49 """Base class for instrument-specific logic for the Gen3 Butler. 

50 

51 Concrete instrument subclasses should be directly constructable with no 

52 arguments. 

53 """ 

54 

55 configPaths = () 

56 """Paths to config files to read for specific Tasks. 

57 

58 The paths in this list should contain files of the form `task.py`, for 

59 each of the Tasks that requires special configuration. 

60 """ 

61 

62 policyName = None 

63 """Instrument specific name to use when locating a policy or configuration 

64 file in the file system.""" 

65 

66 obsDataPackage = None 

67 """Name of the package containing the text curated calibration files. 

68 Usually a obs _data package. If `None` no curated calibration files 

69 will be read. (`str`)""" 

70 

71 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

72 """The dataset types expected to be obtained from the obsDataPackage. 

73 These dataset types are all required to have standard definitions and 

74 must be known to the base class. Clearing this list will prevent 

75 any of these calibrations from being stored. If a dataset type is not 

76 known to a specific instrument it can still be included in this list 

77 since the data package is the source of truth. 

78 """ 

79 

80 @property 

81 @abstractmethod 

82 def filterDefinitions(self): 

83 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

84 for this instrument. 

85 """ 

86 return None 

87 

88 def __init__(self, *args, **kwargs): 

89 self.filterDefinitions.reset() 

90 self.filterDefinitions.defineFilters() 

91 self._obsDataPackageDir = None 

92 

93 @classmethod 

94 @abstractmethod 

95 def getName(cls): 

96 """Return the short (dimension) name for this instrument. 

97 

98 This is not (in general) the same as the class name - it's what is used 

99 as the value of the "instrument" field in data IDs, and is usually an 

100 abbreviation of the full name. 

101 """ 

102 raise NotImplementedError() 

103 

104 @abstractmethod 

105 def getCamera(self): 

106 """Retrieve the cameraGeom representation of this instrument. 

107 

108 This is a temporary API that should go away once obs_ packages have 

109 a standardized approach to writing versioned cameras to a Gen3 repo. 

110 """ 

111 raise NotImplementedError() 

112 

113 @abstractmethod 

114 def register(self, registry): 

115 """Insert instrument, physical_filter, and detector entries into a 

116 `Registry`. 

117 """ 

118 raise NotImplementedError() 

119 

120 @property 

121 def obsDataPackageDir(self): 

122 """The root of the obs package that provides specializations for 

123 this instrument (`str`). 

124 """ 

125 if self.obsDataPackage is None: 

126 return None 

127 if self._obsDataPackageDir is None: 

128 # Defer any problems with locating the package until 

129 # we need to find it. 

130 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

131 return self._obsDataPackageDir 

132 

133 @classmethod 

134 def fromName(cls, name, registry): 

135 """Given an instrument name and a butler, retrieve a corresponding 

136 instantiated instrument object. 

137 

138 Parameters 

139 ---------- 

140 name : `str` 

141 Name of the instrument (must match the return value of `getName`). 

142 registry : `lsst.daf.butler.Registry` 

143 Butler registry to query to find the information. 

144 

145 Returns 

146 ------- 

147 instrument : `Instrument` 

148 An instance of the relevant `Instrument`. 

149 

150 Notes 

151 ----- 

152 The instrument must be registered in the corresponding butler. 

153 

154 Raises 

155 ------ 

156 LookupError 

157 Raised if the instrument is not known to the supplied registry. 

158 ModuleNotFoundError 

159 Raised if the class could not be imported. This could mean 

160 that the relevant obs package has not been setup. 

161 TypeError 

162 Raised if the class name retrieved is not a string. 

163 """ 

164 dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name})) 

165 cls = dimensions[0].records["instrument"].class_name 

166 if not isinstance(cls, str): 

167 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

168 instrument = doImport(cls) 

169 return instrument() 

170 

171 def _registerFilters(self, registry): 

172 """Register the physical and abstract filter Dimension relationships. 

173 This should be called in the ``register`` implementation. 

174 

175 Parameters 

176 ---------- 

177 registry : `lsst.daf.butler.core.Registry` 

178 The registry to add dimensions to. 

179 """ 

180 for filter in self.filterDefinitions: 

181 # fix for undefined abstract filters causing trouble in the registry: 

182 if filter.abstract_filter is None: 

183 abstract_filter = filter.physical_filter 

184 else: 

185 abstract_filter = filter.abstract_filter 

186 

187 registry.insertDimensionData("physical_filter", 

188 {"instrument": self.getName(), 

189 "name": filter.physical_filter, 

190 "abstract_filter": abstract_filter 

191 }) 

192 

193 @abstractmethod 

194 def getRawFormatter(self, dataId): 

195 """Return the Formatter class that should be used to read a particular 

196 raw file. 

197 

198 Parameters 

199 ---------- 

200 dataId : `DataCoordinate` 

201 Dimension-based ID for the raw file or files being ingested. 

202 

203 Returns 

204 ------- 

205 formatter : `Formatter` class 

206 Class to be used that reads the file into an 

207 `lsst.afw.image.Exposure` instance. 

208 """ 

209 raise NotImplementedError() 

210 

211 def writeCuratedCalibrations(self, butler): 

212 """Write human-curated calibration Datasets to the given Butler with 

213 the appropriate validity ranges. 

214 

215 Parameters 

216 ---------- 

217 butler : `lsst.daf.butler.Butler` 

218 Butler to use to store these calibrations. 

219 

220 Notes 

221 ----- 

222 Expected to be called from subclasses. The base method calls 

223 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

224 """ 

225 self.writeCameraGeom(butler) 

226 self.writeStandardTextCuratedCalibrations(butler) 

227 

228 def applyConfigOverrides(self, name, config): 

229 """Apply instrument-specific overrides for a task config. 

230 

231 Parameters 

232 ---------- 

233 name : `str` 

234 Name of the object being configured; typically the _DefaultName 

235 of a Task. 

236 config : `lsst.pex.config.Config` 

237 Config instance to which overrides should be applied. 

238 """ 

239 for root in self.configPaths: 

240 path = os.path.join(root, f"{name}.py") 

241 if os.path.exists(path): 

242 config.load(path) 

243 

244 def writeCameraGeom(self, butler): 

245 """Write the default camera geometry to the butler repository 

246 with an infinite validity range. 

247 

248 Parameters 

249 ---------- 

250 butler : `lsst.daf.butler.Butler` 

251 Butler to receive these calibration datasets. 

252 """ 

253 

254 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

255 universe=butler.registry.dimensions) 

256 butler.registry.registerDatasetType(datasetType) 

257 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

258 camera = self.getCamera() 

259 butler.put(camera, datasetType, unboundedDataId) 

260 

261 def writeStandardTextCuratedCalibrations(self, butler): 

262 """Write the set of standardized curated text calibrations to 

263 the repository. 

264 

265 Parameters 

266 ---------- 

267 butler : `lsst.daf.butler.Butler` 

268 Butler to receive these calibration datasets. 

269 """ 

270 

271 for datasetTypeName in self.standardCuratedDatasetTypes: 

272 # We need to define the dataset types. 

273 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

274 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

275 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

276 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

277 datasetType = DatasetType(datasetTypeName, 

278 universe=butler.registry.dimensions, 

279 **definition) 

280 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType) 

281 

282 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType): 

283 """Write standardized curated calibration datasets for this specific 

284 dataset type from an obs data package. 

285 

286 Parameters 

287 ---------- 

288 butler : `lsst.daf.butler.Butler` 

289 Gen3 butler in which to put the calibrations. 

290 datasetType : `lsst.daf.butler.DatasetType` 

291 Dataset type to be put. 

292 

293 Notes 

294 ----- 

295 This method scans the location defined in the ``obsDataPackageDir`` 

296 class attribute for curated calibrations corresponding to the 

297 supplied dataset type. The directory name in the data package must 

298 match the name of the dataset type. They are assumed to use the 

299 standard layout and can be read by 

300 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

301 metadata. 

302 """ 

303 if self.obsDataPackageDir is None: 

304 # if there is no data package then there can't be datasets 

305 return 

306 

307 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

308 datasetType.name) 

309 

310 if not os.path.exists(calibPath): 

311 return 

312 

313 # Register the dataset type 

314 butler.registry.registerDatasetType(datasetType) 

315 

316 # obs_base can't depend on pipe_tasks but concrete obs packages 

317 # can -- we therefore have to defer import 

318 from lsst.pipe.tasks.read_curated_calibs import read_all 

319 

320 camera = self.getCamera() 

321 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

322 endOfTime = TIMESPAN_MAX 

323 dimensionRecords = [] 

324 datasetRecords = [] 

325 for det in calibsDict: 

326 times = sorted([k for k in calibsDict[det]]) 

327 calibs = [calibsDict[det][time] for time in times] 

328 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

329 times += [endOfTime] 

330 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

331 md = calib.getMetadata() 

332 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

333 dataId = DataCoordinate.standardize( 

334 universe=butler.registry.dimensions, 

335 instrument=self.getName(), 

336 calibration_label=calibrationLabel, 

337 detector=md["DETECTOR"], 

338 ) 

339 datasetRecords.append((calib, dataId)) 

340 dimensionRecords.append({ 

341 "instrument": self.getName(), 

342 "name": calibrationLabel, 

343 "datetime_begin": beginTime, 

344 "datetime_end": endTime, 

345 }) 

346 

347 # Second loop actually does the inserts and filesystem writes. 

348 with butler.transaction(): 

349 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

350 # TODO: vectorize these puts, once butler APIs for that become 

351 # available. 

352 for calib, dataId in datasetRecords: 

353 butler.put(calib, datasetType, dataId) 

354 

355 @abstractmethod 

356 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

357 """Return a factory for creating Gen2->Gen3 data ID translators, 

358 specialized for this instrument. 

359 

360 Derived class implementations should generally call 

361 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

362 arguments, but are not required to (and may not be able to if their 

363 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

364 norm). 

365 

366 Returns 

367 ------- 

368 factory : `TranslatorFactory`. 

369 Factory for `Translator` objects. 

370 """ 

371 raise NotImplementedError("Must be implemented by derived classes.") 

372 

373 

374def makeExposureRecordFromObsInfo(obsInfo, universe): 

375 """Construct an exposure DimensionRecord from 

376 `astro_metadata_translator.ObservationInfo`. 

377 

378 Parameters 

379 ---------- 

380 obsInfo : `astro_metadata_translator.ObservationInfo` 

381 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

382 the exposure. 

383 universe : `DimensionUniverse` 

384 Set of all known dimensions. 

385 

386 Returns 

387 ------- 

388 record : `DimensionRecord` 

389 A record containing exposure metadata, suitable for insertion into 

390 a `Registry`. 

391 """ 

392 dimension = universe["exposure"] 

393 return dimension.RecordClass.fromDict({ 

394 "instrument": obsInfo.instrument, 

395 "id": obsInfo.exposure_id, 

396 "name": obsInfo.observation_id, 

397 "group_name": obsInfo.exposure_group, 

398 "group_id": obsInfo.visit_id, 

399 "datetime_begin": obsInfo.datetime_begin, 

400 "datetime_end": obsInfo.datetime_end, 

401 "exposure_time": obsInfo.exposure_time.to_value("s"), 

402 "dark_time": obsInfo.dark_time.to_value("s"), 

403 "observation_type": obsInfo.observation_type, 

404 "physical_filter": obsInfo.physical_filter, 

405 }) 

406 

407 

408def addUnboundedCalibrationLabel(registry, instrumentName): 

409 """Add a special 'unbounded' calibration_label dimension entry for the 

410 given camera that is valid for any exposure. 

411 

412 If such an entry already exists, this function just returns a `DataId` 

413 for the existing entry. 

414 

415 Parameters 

416 ---------- 

417 registry : `Registry` 

418 Registry object in which to insert the dimension entry. 

419 instrumentName : `str` 

420 Name of the instrument this calibration label is associated with. 

421 

422 Returns 

423 ------- 

424 dataId : `DataId` 

425 New or existing data ID for the unbounded calibration. 

426 """ 

427 d = dict(instrument=instrumentName, calibration_label="unbounded") 

428 try: 

429 return registry.expandDataId(d) 

430 except LookupError: 

431 pass 

432 entry = d.copy() 

433 entry["datetime_begin"] = TIMESPAN_MIN 

434 entry["datetime_end"] = TIMESPAN_MAX 

435 registry.insertDimensionData("calibration_label", entry) 

436 return registry.expandDataId(d) 

437 

438 

439def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

440 """Attempt to load versioned camera geometry from a butler, but fall back 

441 to obtaining a nominal camera from the `Instrument` class if that fails. 

442 

443 Parameters 

444 ---------- 

445 butler : `lsst.daf.butler.Butler` 

446 Butler instance to attempt to query for and load a ``camera`` dataset 

447 from. 

448 dataId : `dict` or `DataCoordinate` 

449 Data ID that identifies at least the ``instrument`` and ``exposure`` 

450 dimensions. 

451 collections : Any, optional 

452 Collections to be searched, overriding ``self.butler.collections``. 

453 Can be any of the types supported by the ``collections`` argument 

454 to butler construction. 

455 

456 Returns 

457 ------- 

458 camera : `lsst.afw.cameraGeom.Camera` 

459 Camera object. 

460 versioned : `bool` 

461 If `True`, the camera was obtained from the butler and should represent 

462 a versioned camera from a calibration repository. If `False`, no 

463 camera datasets were found, and the returned camera was produced by 

464 instantiating the appropriate `Instrument` class and calling 

465 `Instrument.getCamera`. 

466 """ 

467 if collections is None: 

468 collections = butler.collections 

469 # Registry would do data ID expansion internally if we didn't do it first, 

470 # but we might want an expanded data ID ourselves later, so we do it here 

471 # to ensure it only happens once. 

472 # This will also catch problems with the data ID not having keys we need. 

473 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

474 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

475 deduplicate=True)) 

476 if cameraRefs: 

477 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

478 return butler.getDirect(cameraRefs[0]), True 

479 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

480 return instrument.getCamera(), False