Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Tuple, TYPE_CHECKING 

29import astropy.time 

30 

31from lsst.afw.cameraGeom import Camera 

32from lsst.daf.butler import Butler, DataId, TIMESPAN_MIN, TIMESPAN_MAX, DatasetType, DataCoordinate 

33from lsst.utils import getPackageDir, doImport 

34 

35if TYPE_CHECKING: 35 ↛ 36line 35 didn't jump to line 36, because the condition on line 35 was never true

36 from .gen2to3 import TranslatorFactory 

37 from lsst.daf.butler import Registry 

38 

39# To be a standard text curated calibration means that we use a 

40# standard definition for the corresponding DatasetType. 

41StandardCuratedCalibrationDatasetTypes = { 

42 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

43 "storageClass": "Defects"}, 

44 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

45 "storageClass": "QECurve"}, 

46 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"), 

47 "storageClass": "CrosstalkCalib"}, 

48} 

49 

50 

51class Instrument(metaclass=ABCMeta): 

52 """Base class for instrument-specific logic for the Gen3 Butler. 

53 

54 Concrete instrument subclasses should be directly constructable with no 

55 arguments. 

56 """ 

57 

58 configPaths = () 

59 """Paths to config files to read for specific Tasks. 

60 

61 The paths in this list should contain files of the form `task.py`, for 

62 each of the Tasks that requires special configuration. 

63 """ 

64 

65 policyName = None 

66 """Instrument specific name to use when locating a policy or configuration 

67 file in the file system.""" 

68 

69 obsDataPackage = None 

70 """Name of the package containing the text curated calibration files. 

71 Usually a obs _data package. If `None` no curated calibration files 

72 will be read. (`str`)""" 

73 

74 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

75 """The dataset types expected to be obtained from the obsDataPackage. 

76 These dataset types are all required to have standard definitions and 

77 must be known to the base class. Clearing this list will prevent 

78 any of these calibrations from being stored. If a dataset type is not 

79 known to a specific instrument it can still be included in this list 

80 since the data package is the source of truth. 

81 """ 

82 

83 @property 

84 @abstractmethod 

85 def filterDefinitions(self): 

86 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

87 for this instrument. 

88 """ 

89 return None 

90 

91 def __init__(self): 

92 self.filterDefinitions.reset() 

93 self.filterDefinitions.defineFilters() 

94 self._obsDataPackageDir = None 

95 

96 @classmethod 

97 @abstractmethod 

98 def getName(cls): 

99 """Return the short (dimension) name for this instrument. 

100 

101 This is not (in general) the same as the class name - it's what is used 

102 as the value of the "instrument" field in data IDs, and is usually an 

103 abbreviation of the full name. 

104 """ 

105 raise NotImplementedError() 

106 

107 @abstractmethod 

108 def getCamera(self): 

109 """Retrieve the cameraGeom representation of this instrument. 

110 

111 This is a temporary API that should go away once ``obs_`` packages have 

112 a standardized approach to writing versioned cameras to a Gen3 repo. 

113 """ 

114 raise NotImplementedError() 

115 

116 @abstractmethod 

117 def register(self, registry): 

118 """Insert instrument, physical_filter, and detector entries into a 

119 `Registry`. 

120 """ 

121 raise NotImplementedError() 

122 

123 @property 

124 def obsDataPackageDir(self): 

125 """The root of the obs package that provides specializations for 

126 this instrument (`str`). 

127 """ 

128 if self.obsDataPackage is None: 

129 return None 

130 if self._obsDataPackageDir is None: 

131 # Defer any problems with locating the package until 

132 # we need to find it. 

133 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

134 return self._obsDataPackageDir 

135 

136 @staticmethod 

137 def fromName(name: str, registry: Registry) -> Instrument: 

138 """Given an instrument name and a butler, retrieve a corresponding 

139 instantiated instrument object. 

140 

141 Parameters 

142 ---------- 

143 name : `str` 

144 Name of the instrument (must match the return value of `getName`). 

145 registry : `lsst.daf.butler.Registry` 

146 Butler registry to query to find the information. 

147 

148 Returns 

149 ------- 

150 instrument : `Instrument` 

151 An instance of the relevant `Instrument`. 

152 

153 Notes 

154 ----- 

155 The instrument must be registered in the corresponding butler. 

156 

157 Raises 

158 ------ 

159 LookupError 

160 Raised if the instrument is not known to the supplied registry. 

161 ModuleNotFoundError 

162 Raised if the class could not be imported. This could mean 

163 that the relevant obs package has not been setup. 

164 TypeError 

165 Raised if the class name retrieved is not a string. 

166 """ 

167 dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name})) 

168 cls = dimensions[0].records["instrument"].class_name 

169 if not isinstance(cls, str): 

170 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

171 instrument = doImport(cls) 

172 return instrument() 

173 

174 @staticmethod 

175 def importAll(registry: Registry) -> None: 

176 """Import all the instruments known to this registry. 

177 

178 This will ensure that all metadata translators have been registered. 

179 

180 Parameters 

181 ---------- 

182 registry : `lsst.daf.butler.Registry` 

183 Butler registry to query to find the information. 

184 

185 Notes 

186 ----- 

187 It is allowed for a particular instrument class to fail on import. 

188 This might simply indicate that a particular obs package has 

189 not been setup. 

190 """ 

191 dimensions = list(registry.queryDimensions("instrument")) 

192 for dim in dimensions: 

193 cls = dim.records["instrument"].class_name 

194 try: 

195 doImport(cls) 

196 except Exception: 

197 pass 

198 

199 def _registerFilters(self, registry): 

200 """Register the physical and abstract filter Dimension relationships. 

201 This should be called in the ``register`` implementation. 

202 

203 Parameters 

204 ---------- 

205 registry : `lsst.daf.butler.core.Registry` 

206 The registry to add dimensions to. 

207 """ 

208 for filter in self.filterDefinitions: 

209 # fix for undefined abstract filters causing trouble in the registry: 

210 if filter.abstract_filter is None: 

211 abstract_filter = filter.physical_filter 

212 else: 

213 abstract_filter = filter.abstract_filter 

214 

215 registry.insertDimensionData("physical_filter", 

216 {"instrument": self.getName(), 

217 "name": filter.physical_filter, 

218 "abstract_filter": abstract_filter 

219 }) 

220 

221 @abstractmethod 

222 def getRawFormatter(self, dataId): 

223 """Return the Formatter class that should be used to read a particular 

224 raw file. 

225 

226 Parameters 

227 ---------- 

228 dataId : `DataCoordinate` 

229 Dimension-based ID for the raw file or files being ingested. 

230 

231 Returns 

232 ------- 

233 formatter : `Formatter` class 

234 Class to be used that reads the file into an 

235 `lsst.afw.image.Exposure` instance. 

236 """ 

237 raise NotImplementedError() 

238 

239 def writeCuratedCalibrations(self, butler): 

240 """Write human-curated calibration Datasets to the given Butler with 

241 the appropriate validity ranges. 

242 

243 Parameters 

244 ---------- 

245 butler : `lsst.daf.butler.Butler` 

246 Butler to use to store these calibrations. 

247 

248 Notes 

249 ----- 

250 Expected to be called from subclasses. The base method calls 

251 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

252 """ 

253 self.writeCameraGeom(butler) 

254 self.writeStandardTextCuratedCalibrations(butler) 

255 

256 def applyConfigOverrides(self, name, config): 

257 """Apply instrument-specific overrides for a task config. 

258 

259 Parameters 

260 ---------- 

261 name : `str` 

262 Name of the object being configured; typically the _DefaultName 

263 of a Task. 

264 config : `lsst.pex.config.Config` 

265 Config instance to which overrides should be applied. 

266 """ 

267 for root in self.configPaths: 

268 path = os.path.join(root, f"{name}.py") 

269 if os.path.exists(path): 

270 config.load(path) 

271 

272 def writeCameraGeom(self, butler): 

273 """Write the default camera geometry to the butler repository 

274 with an infinite validity range. 

275 

276 Parameters 

277 ---------- 

278 butler : `lsst.daf.butler.Butler` 

279 Butler to receive these calibration datasets. 

280 """ 

281 

282 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

283 universe=butler.registry.dimensions) 

284 butler.registry.registerDatasetType(datasetType) 

285 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

286 camera = self.getCamera() 

287 butler.put(camera, datasetType, unboundedDataId) 

288 

289 def writeStandardTextCuratedCalibrations(self, butler): 

290 """Write the set of standardized curated text calibrations to 

291 the repository. 

292 

293 Parameters 

294 ---------- 

295 butler : `lsst.daf.butler.Butler` 

296 Butler to receive these calibration datasets. 

297 """ 

298 

299 for datasetTypeName in self.standardCuratedDatasetTypes: 

300 # We need to define the dataset types. 

301 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

302 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

303 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

304 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

305 datasetType = DatasetType(datasetTypeName, 

306 universe=butler.registry.dimensions, 

307 **definition) 

308 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType) 

309 

310 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType): 

311 """Write standardized curated calibration datasets for this specific 

312 dataset type from an obs data package. 

313 

314 Parameters 

315 ---------- 

316 butler : `lsst.daf.butler.Butler` 

317 Gen3 butler in which to put the calibrations. 

318 datasetType : `lsst.daf.butler.DatasetType` 

319 Dataset type to be put. 

320 

321 Notes 

322 ----- 

323 This method scans the location defined in the ``obsDataPackageDir`` 

324 class attribute for curated calibrations corresponding to the 

325 supplied dataset type. The directory name in the data package must 

326 match the name of the dataset type. They are assumed to use the 

327 standard layout and can be read by 

328 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

329 metadata. 

330 """ 

331 if self.obsDataPackageDir is None: 

332 # if there is no data package then there can't be datasets 

333 return 

334 

335 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

336 datasetType.name) 

337 

338 if not os.path.exists(calibPath): 

339 return 

340 

341 # Register the dataset type 

342 butler.registry.registerDatasetType(datasetType) 

343 

344 # obs_base can't depend on pipe_tasks but concrete obs packages 

345 # can -- we therefore have to defer import 

346 from lsst.pipe.tasks.read_curated_calibs import read_all 

347 

348 camera = self.getCamera() 

349 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

350 endOfTime = TIMESPAN_MAX 

351 dimensionRecords = [] 

352 datasetRecords = [] 

353 for det in calibsDict: 

354 times = sorted([k for k in calibsDict[det]]) 

355 calibs = [calibsDict[det][time] for time in times] 

356 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

357 times += [endOfTime] 

358 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

359 md = calib.getMetadata() 

360 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

361 dataId = DataCoordinate.standardize( 

362 universe=butler.registry.dimensions, 

363 instrument=self.getName(), 

364 calibration_label=calibrationLabel, 

365 detector=md["DETECTOR"], 

366 ) 

367 datasetRecords.append((calib, dataId)) 

368 dimensionRecords.append({ 

369 "instrument": self.getName(), 

370 "name": calibrationLabel, 

371 "datetime_begin": beginTime, 

372 "datetime_end": endTime, 

373 }) 

374 

375 # Second loop actually does the inserts and filesystem writes. 

376 with butler.transaction(): 

377 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

378 # TODO: vectorize these puts, once butler APIs for that become 

379 # available. 

380 for calib, dataId in datasetRecords: 

381 butler.put(calib, datasetType, dataId) 

382 

383 @abstractmethod 

384 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

385 """Return a factory for creating Gen2->Gen3 data ID translators, 

386 specialized for this instrument. 

387 

388 Derived class implementations should generally call 

389 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

390 arguments, but are not required to (and may not be able to if their 

391 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

392 norm). 

393 

394 Returns 

395 ------- 

396 factory : `TranslatorFactory`. 

397 Factory for `Translator` objects. 

398 """ 

399 raise NotImplementedError("Must be implemented by derived classes.") 

400 

401 

402def makeExposureRecordFromObsInfo(obsInfo, universe): 

403 """Construct an exposure DimensionRecord from 

404 `astro_metadata_translator.ObservationInfo`. 

405 

406 Parameters 

407 ---------- 

408 obsInfo : `astro_metadata_translator.ObservationInfo` 

409 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

410 the exposure. 

411 universe : `DimensionUniverse` 

412 Set of all known dimensions. 

413 

414 Returns 

415 ------- 

416 record : `DimensionRecord` 

417 A record containing exposure metadata, suitable for insertion into 

418 a `Registry`. 

419 """ 

420 dimension = universe["exposure"] 

421 return dimension.RecordClass.fromDict({ 

422 "instrument": obsInfo.instrument, 

423 "id": obsInfo.exposure_id, 

424 "name": obsInfo.observation_id, 

425 "group_name": obsInfo.exposure_group, 

426 "group_id": obsInfo.visit_id, 

427 "datetime_begin": obsInfo.datetime_begin, 

428 "datetime_end": obsInfo.datetime_end, 

429 "exposure_time": obsInfo.exposure_time.to_value("s"), 

430 "dark_time": obsInfo.dark_time.to_value("s"), 

431 "observation_type": obsInfo.observation_type, 

432 "physical_filter": obsInfo.physical_filter, 

433 }) 

434 

435 

436def addUnboundedCalibrationLabel(registry, instrumentName): 

437 """Add a special 'unbounded' calibration_label dimension entry for the 

438 given camera that is valid for any exposure. 

439 

440 If such an entry already exists, this function just returns a `DataId` 

441 for the existing entry. 

442 

443 Parameters 

444 ---------- 

445 registry : `Registry` 

446 Registry object in which to insert the dimension entry. 

447 instrumentName : `str` 

448 Name of the instrument this calibration label is associated with. 

449 

450 Returns 

451 ------- 

452 dataId : `DataId` 

453 New or existing data ID for the unbounded calibration. 

454 """ 

455 d = dict(instrument=instrumentName, calibration_label="unbounded") 

456 try: 

457 return registry.expandDataId(d) 

458 except LookupError: 

459 pass 

460 entry = d.copy() 

461 entry["datetime_begin"] = TIMESPAN_MIN 

462 entry["datetime_end"] = TIMESPAN_MAX 

463 registry.insertDimensionData("calibration_label", entry) 

464 return registry.expandDataId(d) 

465 

466 

467def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

468 """Attempt to load versioned camera geometry from a butler, but fall back 

469 to obtaining a nominal camera from the `Instrument` class if that fails. 

470 

471 Parameters 

472 ---------- 

473 butler : `lsst.daf.butler.Butler` 

474 Butler instance to attempt to query for and load a ``camera`` dataset 

475 from. 

476 dataId : `dict` or `DataCoordinate` 

477 Data ID that identifies at least the ``instrument`` and ``exposure`` 

478 dimensions. 

479 collections : Any, optional 

480 Collections to be searched, overriding ``self.butler.collections``. 

481 Can be any of the types supported by the ``collections`` argument 

482 to butler construction. 

483 

484 Returns 

485 ------- 

486 camera : `lsst.afw.cameraGeom.Camera` 

487 Camera object. 

488 versioned : `bool` 

489 If `True`, the camera was obtained from the butler and should represent 

490 a versioned camera from a calibration repository. If `False`, no 

491 camera datasets were found, and the returned camera was produced by 

492 instantiating the appropriate `Instrument` class and calling 

493 `Instrument.getCamera`. 

494 """ 

495 if collections is None: 

496 collections = butler.collections 

497 # Registry would do data ID expansion internally if we didn't do it first, 

498 # but we might want an expanded data ID ourselves later, so we do it here 

499 # to ensure it only happens once. 

500 # This will also catch problems with the data ID not having keys we need. 

501 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

502 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

503 deduplicate=True)) 

504 if cameraRefs: 

505 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

506 return butler.getDirect(cameraRefs[0]), True 

507 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

508 return instrument.getCamera(), False