Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Tuple, TYPE_CHECKING 

29import astropy.time 

30 

31from lsst.afw.cameraGeom import Camera 

32from lsst.daf.butler import ( 

33 Butler, 

34 CollectionType, 

35 DataCoordinate, 

36 DataId, 

37 DatasetType, 

38 TIMESPAN_MIN, 

39 TIMESPAN_MAX, 

40) 

41from lsst.utils import getPackageDir, doImport 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .gen2to3 import TranslatorFactory 

45 from lsst.daf.butler import Registry 

46 

47# To be a standard text curated calibration means that we use a 

48# standard definition for the corresponding DatasetType. 

49StandardCuratedCalibrationDatasetTypes = { 

50 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

51 "storageClass": "Defects"}, 

52 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

53 "storageClass": "QECurve"}, 

54 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"), 

55 "storageClass": "CrosstalkCalib"}, 

56} 

57 

58 

59class Instrument(metaclass=ABCMeta): 

60 """Base class for instrument-specific logic for the Gen3 Butler. 

61 

62 Concrete instrument subclasses should be directly constructable with no 

63 arguments. 

64 """ 

65 

66 configPaths = () 

67 """Paths to config files to read for specific Tasks. 

68 

69 The paths in this list should contain files of the form `task.py`, for 

70 each of the Tasks that requires special configuration. 

71 """ 

72 

73 policyName = None 

74 """Instrument specific name to use when locating a policy or configuration 

75 file in the file system.""" 

76 

77 obsDataPackage = None 

78 """Name of the package containing the text curated calibration files. 

79 Usually a obs _data package. If `None` no curated calibration files 

80 will be read. (`str`)""" 

81 

82 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

83 """The dataset types expected to be obtained from the obsDataPackage. 

84 These dataset types are all required to have standard definitions and 

85 must be known to the base class. Clearing this list will prevent 

86 any of these calibrations from being stored. If a dataset type is not 

87 known to a specific instrument it can still be included in this list 

88 since the data package is the source of truth. 

89 """ 

90 

91 @property 

92 @abstractmethod 

93 def filterDefinitions(self): 

94 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

95 for this instrument. 

96 """ 

97 return None 

98 

99 def __init__(self): 

100 self.filterDefinitions.reset() 

101 self.filterDefinitions.defineFilters() 

102 self._obsDataPackageDir = None 

103 

104 @classmethod 

105 @abstractmethod 

106 def getName(cls): 

107 """Return the short (dimension) name for this instrument. 

108 

109 This is not (in general) the same as the class name - it's what is used 

110 as the value of the "instrument" field in data IDs, and is usually an 

111 abbreviation of the full name. 

112 """ 

113 raise NotImplementedError() 

114 

115 @abstractmethod 

116 def getCamera(self): 

117 """Retrieve the cameraGeom representation of this instrument. 

118 

119 This is a temporary API that should go away once ``obs_`` packages have 

120 a standardized approach to writing versioned cameras to a Gen3 repo. 

121 """ 

122 raise NotImplementedError() 

123 

124 @abstractmethod 

125 def register(self, registry): 

126 """Insert instrument, physical_filter, and detector entries into a 

127 `Registry`. 

128 """ 

129 raise NotImplementedError() 

130 

131 @property 

132 def obsDataPackageDir(self): 

133 """The root of the obs package that provides specializations for 

134 this instrument (`str`). 

135 """ 

136 if self.obsDataPackage is None: 

137 return None 

138 if self._obsDataPackageDir is None: 

139 # Defer any problems with locating the package until 

140 # we need to find it. 

141 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

142 return self._obsDataPackageDir 

143 

144 @staticmethod 

145 def fromName(name: str, registry: Registry) -> Instrument: 

146 """Given an instrument name and a butler, retrieve a corresponding 

147 instantiated instrument object. 

148 

149 Parameters 

150 ---------- 

151 name : `str` 

152 Name of the instrument (must match the return value of `getName`). 

153 registry : `lsst.daf.butler.Registry` 

154 Butler registry to query to find the information. 

155 

156 Returns 

157 ------- 

158 instrument : `Instrument` 

159 An instance of the relevant `Instrument`. 

160 

161 Notes 

162 ----- 

163 The instrument must be registered in the corresponding butler. 

164 

165 Raises 

166 ------ 

167 LookupError 

168 Raised if the instrument is not known to the supplied registry. 

169 ModuleNotFoundError 

170 Raised if the class could not be imported. This could mean 

171 that the relevant obs package has not been setup. 

172 TypeError 

173 Raised if the class name retrieved is not a string. 

174 """ 

175 records = list(registry.queryDimensionRecords("instrument", instrument=name)) 

176 if not records: 

177 raise LookupError(f"No registered instrument with name '{name}'.") 

178 cls = records[0].class_name 

179 if not isinstance(cls, str): 

180 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

181 instrument = doImport(cls) 

182 return instrument() 

183 

184 @staticmethod 

185 def importAll(registry: Registry) -> None: 

186 """Import all the instruments known to this registry. 

187 

188 This will ensure that all metadata translators have been registered. 

189 

190 Parameters 

191 ---------- 

192 registry : `lsst.daf.butler.Registry` 

193 Butler registry to query to find the information. 

194 

195 Notes 

196 ----- 

197 It is allowed for a particular instrument class to fail on import. 

198 This might simply indicate that a particular obs package has 

199 not been setup. 

200 """ 

201 records = list(registry.queryDimensionRecords("instrument")) 

202 for record in records: 

203 cls = record.class_name 

204 try: 

205 doImport(cls) 

206 except Exception: 

207 pass 

208 

209 def _registerFilters(self, registry): 

210 """Register the physical and abstract filter Dimension relationships. 

211 This should be called in the ``register`` implementation. 

212 

213 Parameters 

214 ---------- 

215 registry : `lsst.daf.butler.core.Registry` 

216 The registry to add dimensions to. 

217 """ 

218 for filter in self.filterDefinitions: 

219 # fix for undefined abstract filters causing trouble in the registry: 

220 if filter.abstract_filter is None: 

221 abstract_filter = filter.physical_filter 

222 else: 

223 abstract_filter = filter.abstract_filter 

224 

225 registry.insertDimensionData("physical_filter", 

226 {"instrument": self.getName(), 

227 "name": filter.physical_filter, 

228 "abstract_filter": abstract_filter 

229 }) 

230 

231 @abstractmethod 

232 def getRawFormatter(self, dataId): 

233 """Return the Formatter class that should be used to read a particular 

234 raw file. 

235 

236 Parameters 

237 ---------- 

238 dataId : `DataCoordinate` 

239 Dimension-based ID for the raw file or files being ingested. 

240 

241 Returns 

242 ------- 

243 formatter : `Formatter` class 

244 Class to be used that reads the file into an 

245 `lsst.afw.image.Exposure` instance. 

246 """ 

247 raise NotImplementedError() 

248 

249 def writeCuratedCalibrations(self, butler, run=None): 

250 """Write human-curated calibration Datasets to the given Butler with 

251 the appropriate validity ranges. 

252 

253 Parameters 

254 ---------- 

255 butler : `lsst.daf.butler.Butler` 

256 Butler to use to store these calibrations. 

257 run : `str` 

258 Run to use for this collection of calibrations. If `None` the 

259 collection name is worked out automatically from the instrument 

260 name and other metadata. 

261 

262 Notes 

263 ----- 

264 Expected to be called from subclasses. The base method calls 

265 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

266 """ 

267 # Need to determine the run for ingestion based on the instrument 

268 # name and eventually the data package version. The camera geom 

269 # is currently special in that it is not in the _data package. 

270 if run is None: 

271 run = self.makeCollectionName("calib") 

272 butler.registry.registerCollection(run, type=CollectionType.RUN) 

273 self.writeCameraGeom(butler, run=run) 

274 self.writeStandardTextCuratedCalibrations(butler, run=run) 

275 self.writeAdditionalCuratedCalibrations(butler, run=run) 

276 

277 def writeAdditionalCuratedCalibrations(self, butler, run=None): 

278 """Write additional curated calibrations that might be instrument 

279 specific and are not part of the standard set. 

280 

281 Default implementation does nothing. 

282 

283 Parameters 

284 ---------- 

285 butler : `lsst.daf.butler.Butler` 

286 Butler to use to store these calibrations. 

287 run : `str`, optional 

288 Name of the run to use to override the default run associated 

289 with this Butler. 

290 """ 

291 return 

292 

293 def applyConfigOverrides(self, name, config): 

294 """Apply instrument-specific overrides for a task config. 

295 

296 Parameters 

297 ---------- 

298 name : `str` 

299 Name of the object being configured; typically the _DefaultName 

300 of a Task. 

301 config : `lsst.pex.config.Config` 

302 Config instance to which overrides should be applied. 

303 """ 

304 for root in self.configPaths: 

305 path = os.path.join(root, f"{name}.py") 

306 if os.path.exists(path): 

307 config.load(path) 

308 

309 def writeCameraGeom(self, butler, run=None): 

310 """Write the default camera geometry to the butler repository 

311 with an infinite validity range. 

312 

313 Parameters 

314 ---------- 

315 butler : `lsst.daf.butler.Butler` 

316 Butler to receive these calibration datasets. 

317 run : `str`, optional 

318 Name of the run to use to override the default run associated 

319 with this Butler. 

320 """ 

321 

322 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

323 universe=butler.registry.dimensions) 

324 butler.registry.registerDatasetType(datasetType) 

325 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

326 camera = self.getCamera() 

327 butler.put(camera, datasetType, unboundedDataId, run=run) 

328 

329 def writeStandardTextCuratedCalibrations(self, butler, run=None): 

330 """Write the set of standardized curated text calibrations to 

331 the repository. 

332 

333 Parameters 

334 ---------- 

335 butler : `lsst.daf.butler.Butler` 

336 Butler to receive these calibration datasets. 

337 run : `str`, optional 

338 Name of the run to use to override the default run associated 

339 with this Butler. 

340 """ 

341 

342 for datasetTypeName in self.standardCuratedDatasetTypes: 

343 # We need to define the dataset types. 

344 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

345 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

346 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

347 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

348 datasetType = DatasetType(datasetTypeName, 

349 universe=butler.registry.dimensions, 

350 **definition) 

351 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run) 

352 

353 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None): 

354 """Write standardized curated calibration datasets for this specific 

355 dataset type from an obs data package. 

356 

357 Parameters 

358 ---------- 

359 butler : `lsst.daf.butler.Butler` 

360 Gen3 butler in which to put the calibrations. 

361 datasetType : `lsst.daf.butler.DatasetType` 

362 Dataset type to be put. 

363 run : `str`, optional 

364 Name of the run to use to override the default run associated 

365 with this Butler. 

366 

367 Notes 

368 ----- 

369 This method scans the location defined in the ``obsDataPackageDir`` 

370 class attribute for curated calibrations corresponding to the 

371 supplied dataset type. The directory name in the data package must 

372 match the name of the dataset type. They are assumed to use the 

373 standard layout and can be read by 

374 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

375 metadata. 

376 """ 

377 if self.obsDataPackageDir is None: 

378 # if there is no data package then there can't be datasets 

379 return 

380 

381 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

382 datasetType.name) 

383 

384 if not os.path.exists(calibPath): 

385 return 

386 

387 # Register the dataset type 

388 butler.registry.registerDatasetType(datasetType) 

389 

390 # obs_base can't depend on pipe_tasks but concrete obs packages 

391 # can -- we therefore have to defer import 

392 from lsst.pipe.tasks.read_curated_calibs import read_all 

393 

394 camera = self.getCamera() 

395 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

396 endOfTime = TIMESPAN_MAX 

397 dimensionRecords = [] 

398 datasetRecords = [] 

399 for det in calibsDict: 

400 times = sorted([k for k in calibsDict[det]]) 

401 calibs = [calibsDict[det][time] for time in times] 

402 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

403 times += [endOfTime] 

404 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

405 md = calib.getMetadata() 

406 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

407 dataId = DataCoordinate.standardize( 

408 universe=butler.registry.dimensions, 

409 instrument=self.getName(), 

410 calibration_label=calibrationLabel, 

411 detector=md["DETECTOR"], 

412 ) 

413 datasetRecords.append((calib, dataId)) 

414 dimensionRecords.append({ 

415 "instrument": self.getName(), 

416 "name": calibrationLabel, 

417 "datetime_begin": beginTime, 

418 "datetime_end": endTime, 

419 }) 

420 

421 # Second loop actually does the inserts and filesystem writes. 

422 with butler.transaction(): 

423 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

424 # TODO: vectorize these puts, once butler APIs for that become 

425 # available. 

426 for calib, dataId in datasetRecords: 

427 butler.put(calib, datasetType, dataId, run=run) 

428 

429 @abstractmethod 

430 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

431 """Return a factory for creating Gen2->Gen3 data ID translators, 

432 specialized for this instrument. 

433 

434 Derived class implementations should generally call 

435 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

436 arguments, but are not required to (and may not be able to if their 

437 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

438 norm). 

439 

440 Returns 

441 ------- 

442 factory : `TranslatorFactory`. 

443 Factory for `Translator` objects. 

444 """ 

445 raise NotImplementedError("Must be implemented by derived classes.") 

446 

447 @classmethod 

448 def makeDefaultRawIngestRunName(cls) -> str: 

449 """Make the default instrument-specific run collection string for raw 

450 data ingest. 

451 

452 Returns 

453 ------- 

454 coll : `str` 

455 Run collection name to be used as the default for ingestion of 

456 raws. 

457 """ 

458 return cls.makeCollectionName("raw/all") 

459 

460 @classmethod 

461 def makeCollectionName(cls, label: str) -> str: 

462 """Get the instrument-specific collection string to use as derived 

463 from the supplied label. 

464 

465 Parameters 

466 ---------- 

467 label : `str` 

468 String to be combined with the instrument name to form a 

469 collection name. 

470 

471 Returns 

472 ------- 

473 name : `str` 

474 Collection name to use that includes the instrument name. 

475 """ 

476 return f"{cls.getName()}/{label}" 

477 

478 

479def makeExposureRecordFromObsInfo(obsInfo, universe): 

480 """Construct an exposure DimensionRecord from 

481 `astro_metadata_translator.ObservationInfo`. 

482 

483 Parameters 

484 ---------- 

485 obsInfo : `astro_metadata_translator.ObservationInfo` 

486 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

487 the exposure. 

488 universe : `DimensionUniverse` 

489 Set of all known dimensions. 

490 

491 Returns 

492 ------- 

493 record : `DimensionRecord` 

494 A record containing exposure metadata, suitable for insertion into 

495 a `Registry`. 

496 """ 

497 dimension = universe["exposure"] 

498 

499 ra, dec, sky_angle, zenith_angle = (None, None, None, None) 

500 if obsInfo.tracking_radec is not None: 

501 icrs = obsInfo.tracking_radec.icrs 

502 ra = icrs.ra.degree 

503 dec = icrs.dec.degree 

504 if obsInfo.boresight_rotation_coord == "sky": 

505 sky_angle = obsInfo.boresight_rotation_angle.degree 

506 if obsInfo.altaz_begin is not None: 

507 zenith_angle = obsInfo.altaz_begin.zen.degree 

508 

509 return dimension.RecordClass.fromDict({ 

510 "instrument": obsInfo.instrument, 

511 "id": obsInfo.exposure_id, 

512 "name": obsInfo.observation_id, 

513 "group_name": obsInfo.exposure_group, 

514 "group_id": obsInfo.visit_id, 

515 "datetime_begin": obsInfo.datetime_begin, 

516 "datetime_end": obsInfo.datetime_end, 

517 "exposure_time": obsInfo.exposure_time.to_value("s"), 

518 "dark_time": obsInfo.dark_time.to_value("s"), 

519 "observation_type": obsInfo.observation_type, 

520 "physical_filter": obsInfo.physical_filter, 

521 "science_program": obsInfo.science_program, 

522 "target_name": obsInfo.object, 

523 "tracking_ra": ra, 

524 "tracking_dec": dec, 

525 "sky_angle": sky_angle, 

526 "zenith_angle": zenith_angle, 

527 }) 

528 

529 

530def addUnboundedCalibrationLabel(registry, instrumentName): 

531 """Add a special 'unbounded' calibration_label dimension entry for the 

532 given camera that is valid for any exposure. 

533 

534 If such an entry already exists, this function just returns a `DataId` 

535 for the existing entry. 

536 

537 Parameters 

538 ---------- 

539 registry : `Registry` 

540 Registry object in which to insert the dimension entry. 

541 instrumentName : `str` 

542 Name of the instrument this calibration label is associated with. 

543 

544 Returns 

545 ------- 

546 dataId : `DataId` 

547 New or existing data ID for the unbounded calibration. 

548 """ 

549 d = dict(instrument=instrumentName, calibration_label="unbounded") 

550 try: 

551 return registry.expandDataId(d) 

552 except LookupError: 

553 pass 

554 entry = d.copy() 

555 entry["datetime_begin"] = TIMESPAN_MIN 

556 entry["datetime_end"] = TIMESPAN_MAX 

557 registry.insertDimensionData("calibration_label", entry) 

558 return registry.expandDataId(d) 

559 

560 

561def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

562 """Attempt to load versioned camera geometry from a butler, but fall back 

563 to obtaining a nominal camera from the `Instrument` class if that fails. 

564 

565 Parameters 

566 ---------- 

567 butler : `lsst.daf.butler.Butler` 

568 Butler instance to attempt to query for and load a ``camera`` dataset 

569 from. 

570 dataId : `dict` or `DataCoordinate` 

571 Data ID that identifies at least the ``instrument`` and ``exposure`` 

572 dimensions. 

573 collections : Any, optional 

574 Collections to be searched, overriding ``self.butler.collections``. 

575 Can be any of the types supported by the ``collections`` argument 

576 to butler construction. 

577 

578 Returns 

579 ------- 

580 camera : `lsst.afw.cameraGeom.Camera` 

581 Camera object. 

582 versioned : `bool` 

583 If `True`, the camera was obtained from the butler and should represent 

584 a versioned camera from a calibration repository. If `False`, no 

585 camera datasets were found, and the returned camera was produced by 

586 instantiating the appropriate `Instrument` class and calling 

587 `Instrument.getCamera`. 

588 """ 

589 if collections is None: 

590 collections = butler.collections 

591 # Registry would do data ID expansion internally if we didn't do it first, 

592 # but we might want an expanded data ID ourselves later, so we do it here 

593 # to ensure it only happens once. 

594 # This will also catch problems with the data ID not having keys we need. 

595 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

596 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

597 deduplicate=True)) 

598 if cameraRefs: 

599 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

600 return butler.getDirect(cameraRefs[0]), True 

601 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

602 return instrument.getCamera(), False