Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("Instrument", "makeExposureRecordFromObsInfo", "addUnboundedCalibrationLabel", "loadCamera") 

25 

26import os.path 

27from abc import ABCMeta, abstractmethod 

28from typing import Any, Tuple, TYPE_CHECKING 

29import astropy.time 

30 

31from lsst.afw.cameraGeom import Camera 

32from lsst.daf.butler import ( 

33 Butler, 

34 CollectionType, 

35 DataCoordinate, 

36 DataId, 

37 DatasetType, 

38 TIMESPAN_MIN, 

39 TIMESPAN_MAX, 

40) 

41from lsst.utils import getPackageDir, doImport 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .gen2to3 import TranslatorFactory 

45 from lsst.daf.butler import Registry 

46 

47# To be a standard text curated calibration means that we use a 

48# standard definition for the corresponding DatasetType. 

49StandardCuratedCalibrationDatasetTypes = { 

50 "defects": {"dimensions": ("instrument", "detector", "calibration_label"), 

51 "storageClass": "Defects"}, 

52 "qe_curve": {"dimensions": ("instrument", "detector", "calibration_label"), 

53 "storageClass": "QECurve"}, 

54 "crosstalk": {"dimensions": ("instrument", "detector", "calibration_label"), 

55 "storageClass": "CrosstalkCalib"}, 

56} 

57 

58 

59class Instrument(metaclass=ABCMeta): 

60 """Base class for instrument-specific logic for the Gen3 Butler. 

61 

62 Concrete instrument subclasses should be directly constructable with no 

63 arguments. 

64 """ 

65 

66 configPaths = () 

67 """Paths to config files to read for specific Tasks. 

68 

69 The paths in this list should contain files of the form `task.py`, for 

70 each of the Tasks that requires special configuration. 

71 """ 

72 

73 policyName = None 

74 """Instrument specific name to use when locating a policy or configuration 

75 file in the file system.""" 

76 

77 obsDataPackage = None 

78 """Name of the package containing the text curated calibration files. 

79 Usually a obs _data package. If `None` no curated calibration files 

80 will be read. (`str`)""" 

81 

82 standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes) 

83 """The dataset types expected to be obtained from the obsDataPackage. 

84 These dataset types are all required to have standard definitions and 

85 must be known to the base class. Clearing this list will prevent 

86 any of these calibrations from being stored. If a dataset type is not 

87 known to a specific instrument it can still be included in this list 

88 since the data package is the source of truth. 

89 """ 

90 

91 @property 

92 @abstractmethod 

93 def filterDefinitions(self): 

94 """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters 

95 for this instrument. 

96 """ 

97 return None 

98 

99 def __init__(self): 

100 self.filterDefinitions.reset() 

101 self.filterDefinitions.defineFilters() 

102 self._obsDataPackageDir = None 

103 

104 @classmethod 

105 @abstractmethod 

106 def getName(cls): 

107 """Return the short (dimension) name for this instrument. 

108 

109 This is not (in general) the same as the class name - it's what is used 

110 as the value of the "instrument" field in data IDs, and is usually an 

111 abbreviation of the full name. 

112 """ 

113 raise NotImplementedError() 

114 

115 @abstractmethod 

116 def getCamera(self): 

117 """Retrieve the cameraGeom representation of this instrument. 

118 

119 This is a temporary API that should go away once ``obs_`` packages have 

120 a standardized approach to writing versioned cameras to a Gen3 repo. 

121 """ 

122 raise NotImplementedError() 

123 

124 @abstractmethod 

125 def register(self, registry): 

126 """Insert instrument, physical_filter, and detector entries into a 

127 `Registry`. 

128 """ 

129 raise NotImplementedError() 

130 

131 @property 

132 def obsDataPackageDir(self): 

133 """The root of the obs package that provides specializations for 

134 this instrument (`str`). 

135 """ 

136 if self.obsDataPackage is None: 

137 return None 

138 if self._obsDataPackageDir is None: 

139 # Defer any problems with locating the package until 

140 # we need to find it. 

141 self._obsDataPackageDir = getPackageDir(self.obsDataPackage) 

142 return self._obsDataPackageDir 

143 

144 @staticmethod 

145 def fromName(name: str, registry: Registry) -> Instrument: 

146 """Given an instrument name and a butler, retrieve a corresponding 

147 instantiated instrument object. 

148 

149 Parameters 

150 ---------- 

151 name : `str` 

152 Name of the instrument (must match the return value of `getName`). 

153 registry : `lsst.daf.butler.Registry` 

154 Butler registry to query to find the information. 

155 

156 Returns 

157 ------- 

158 instrument : `Instrument` 

159 An instance of the relevant `Instrument`. 

160 

161 Notes 

162 ----- 

163 The instrument must be registered in the corresponding butler. 

164 

165 Raises 

166 ------ 

167 LookupError 

168 Raised if the instrument is not known to the supplied registry. 

169 ModuleNotFoundError 

170 Raised if the class could not be imported. This could mean 

171 that the relevant obs package has not been setup. 

172 TypeError 

173 Raised if the class name retrieved is not a string. 

174 """ 

175 dimensions = list(registry.queryDimensions("instrument", dataId={"instrument": name})) 

176 cls = dimensions[0].records["instrument"].class_name 

177 if not isinstance(cls, str): 

178 raise TypeError(f"Unexpected class name retrieved from {name} instrument dimension (got {cls})") 

179 instrument = doImport(cls) 

180 return instrument() 

181 

182 @staticmethod 

183 def importAll(registry: Registry) -> None: 

184 """Import all the instruments known to this registry. 

185 

186 This will ensure that all metadata translators have been registered. 

187 

188 Parameters 

189 ---------- 

190 registry : `lsst.daf.butler.Registry` 

191 Butler registry to query to find the information. 

192 

193 Notes 

194 ----- 

195 It is allowed for a particular instrument class to fail on import. 

196 This might simply indicate that a particular obs package has 

197 not been setup. 

198 """ 

199 dimensions = list(registry.queryDimensions("instrument")) 

200 for dim in dimensions: 

201 cls = dim.records["instrument"].class_name 

202 try: 

203 doImport(cls) 

204 except Exception: 

205 pass 

206 

207 def _registerFilters(self, registry): 

208 """Register the physical and abstract filter Dimension relationships. 

209 This should be called in the ``register`` implementation. 

210 

211 Parameters 

212 ---------- 

213 registry : `lsst.daf.butler.core.Registry` 

214 The registry to add dimensions to. 

215 """ 

216 for filter in self.filterDefinitions: 

217 # fix for undefined abstract filters causing trouble in the registry: 

218 if filter.abstract_filter is None: 

219 abstract_filter = filter.physical_filter 

220 else: 

221 abstract_filter = filter.abstract_filter 

222 

223 registry.insertDimensionData("physical_filter", 

224 {"instrument": self.getName(), 

225 "name": filter.physical_filter, 

226 "abstract_filter": abstract_filter 

227 }) 

228 

229 @abstractmethod 

230 def getRawFormatter(self, dataId): 

231 """Return the Formatter class that should be used to read a particular 

232 raw file. 

233 

234 Parameters 

235 ---------- 

236 dataId : `DataCoordinate` 

237 Dimension-based ID for the raw file or files being ingested. 

238 

239 Returns 

240 ------- 

241 formatter : `Formatter` class 

242 Class to be used that reads the file into an 

243 `lsst.afw.image.Exposure` instance. 

244 """ 

245 raise NotImplementedError() 

246 

247 def writeCuratedCalibrations(self, butler, run=None): 

248 """Write human-curated calibration Datasets to the given Butler with 

249 the appropriate validity ranges. 

250 

251 Parameters 

252 ---------- 

253 butler : `lsst.daf.butler.Butler` 

254 Butler to use to store these calibrations. 

255 run : `str` 

256 Run to use for this collection of calibrations. If `None` the 

257 collection name is worked out automatically from the instrument 

258 name and other metadata. 

259 

260 Notes 

261 ----- 

262 Expected to be called from subclasses. The base method calls 

263 ``writeCameraGeom`` and ``writeStandardTextCuratedCalibrations``. 

264 """ 

265 # Need to determine the run for ingestion based on the instrument 

266 # name and eventually the data package version. The camera geom 

267 # is currently special in that it is not in the _data package. 

268 if run is None: 

269 run = self.makeCollectionName("calib") 

270 butler.registry.registerCollection(run, type=CollectionType.RUN) 

271 self.writeCameraGeom(butler, run=run) 

272 self.writeStandardTextCuratedCalibrations(butler, run=run) 

273 self.writeAdditionalCuratedCalibrations(butler, run=run) 

274 

275 def writeAdditionalCuratedCalibrations(self, butler, run=None): 

276 """Write additional curated calibrations that might be instrument 

277 specific and are not part of the standard set. 

278 

279 Default implementation does nothing. 

280 

281 Parameters 

282 ---------- 

283 butler : `lsst.daf.butler.Butler` 

284 Butler to use to store these calibrations. 

285 run : `str`, optional 

286 Name of the run to use to override the default run associated 

287 with this Butler. 

288 """ 

289 return 

290 

291 def applyConfigOverrides(self, name, config): 

292 """Apply instrument-specific overrides for a task config. 

293 

294 Parameters 

295 ---------- 

296 name : `str` 

297 Name of the object being configured; typically the _DefaultName 

298 of a Task. 

299 config : `lsst.pex.config.Config` 

300 Config instance to which overrides should be applied. 

301 """ 

302 for root in self.configPaths: 

303 path = os.path.join(root, f"{name}.py") 

304 if os.path.exists(path): 

305 config.load(path) 

306 

307 def writeCameraGeom(self, butler, run=None): 

308 """Write the default camera geometry to the butler repository 

309 with an infinite validity range. 

310 

311 Parameters 

312 ---------- 

313 butler : `lsst.daf.butler.Butler` 

314 Butler to receive these calibration datasets. 

315 run : `str`, optional 

316 Name of the run to use to override the default run associated 

317 with this Butler. 

318 """ 

319 

320 datasetType = DatasetType("camera", ("instrument", "calibration_label"), "Camera", 

321 universe=butler.registry.dimensions) 

322 butler.registry.registerDatasetType(datasetType) 

323 unboundedDataId = addUnboundedCalibrationLabel(butler.registry, self.getName()) 

324 camera = self.getCamera() 

325 butler.put(camera, datasetType, unboundedDataId, run=run) 

326 

327 def writeStandardTextCuratedCalibrations(self, butler, run=None): 

328 """Write the set of standardized curated text calibrations to 

329 the repository. 

330 

331 Parameters 

332 ---------- 

333 butler : `lsst.daf.butler.Butler` 

334 Butler to receive these calibration datasets. 

335 run : `str`, optional 

336 Name of the run to use to override the default run associated 

337 with this Butler. 

338 """ 

339 

340 for datasetTypeName in self.standardCuratedDatasetTypes: 

341 # We need to define the dataset types. 

342 if datasetTypeName not in StandardCuratedCalibrationDatasetTypes: 

343 raise ValueError(f"DatasetType {datasetTypeName} not in understood list" 

344 f" [{'.'.join(StandardCuratedCalibrationDatasetTypes)}]") 

345 definition = StandardCuratedCalibrationDatasetTypes[datasetTypeName] 

346 datasetType = DatasetType(datasetTypeName, 

347 universe=butler.registry.dimensions, 

348 **definition) 

349 self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run) 

350 

351 def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None): 

352 """Write standardized curated calibration datasets for this specific 

353 dataset type from an obs data package. 

354 

355 Parameters 

356 ---------- 

357 butler : `lsst.daf.butler.Butler` 

358 Gen3 butler in which to put the calibrations. 

359 datasetType : `lsst.daf.butler.DatasetType` 

360 Dataset type to be put. 

361 run : `str`, optional 

362 Name of the run to use to override the default run associated 

363 with this Butler. 

364 

365 Notes 

366 ----- 

367 This method scans the location defined in the ``obsDataPackageDir`` 

368 class attribute for curated calibrations corresponding to the 

369 supplied dataset type. The directory name in the data package must 

370 match the name of the dataset type. They are assumed to use the 

371 standard layout and can be read by 

372 `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard 

373 metadata. 

374 """ 

375 if self.obsDataPackageDir is None: 

376 # if there is no data package then there can't be datasets 

377 return 

378 

379 calibPath = os.path.join(self.obsDataPackageDir, self.policyName, 

380 datasetType.name) 

381 

382 if not os.path.exists(calibPath): 

383 return 

384 

385 # Register the dataset type 

386 butler.registry.registerDatasetType(datasetType) 

387 

388 # obs_base can't depend on pipe_tasks but concrete obs packages 

389 # can -- we therefore have to defer import 

390 from lsst.pipe.tasks.read_curated_calibs import read_all 

391 

392 camera = self.getCamera() 

393 calibsDict = read_all(calibPath, camera)[0] # second return is calib type 

394 endOfTime = TIMESPAN_MAX 

395 dimensionRecords = [] 

396 datasetRecords = [] 

397 for det in calibsDict: 

398 times = sorted([k for k in calibsDict[det]]) 

399 calibs = [calibsDict[det][time] for time in times] 

400 times = [astropy.time.Time(t, format="datetime", scale="utc") for t in times] 

401 times += [endOfTime] 

402 for calib, beginTime, endTime in zip(calibs, times[:-1], times[1:]): 

403 md = calib.getMetadata() 

404 calibrationLabel = f"{datasetType.name}/{md['CALIBDATE']}/{md['DETECTOR']}" 

405 dataId = DataCoordinate.standardize( 

406 universe=butler.registry.dimensions, 

407 instrument=self.getName(), 

408 calibration_label=calibrationLabel, 

409 detector=md["DETECTOR"], 

410 ) 

411 datasetRecords.append((calib, dataId)) 

412 dimensionRecords.append({ 

413 "instrument": self.getName(), 

414 "name": calibrationLabel, 

415 "datetime_begin": beginTime, 

416 "datetime_end": endTime, 

417 }) 

418 

419 # Second loop actually does the inserts and filesystem writes. 

420 with butler.transaction(): 

421 butler.registry.insertDimensionData("calibration_label", *dimensionRecords) 

422 # TODO: vectorize these puts, once butler APIs for that become 

423 # available. 

424 for calib, dataId in datasetRecords: 

425 butler.put(calib, datasetType, dataId, run=run) 

426 

427 @abstractmethod 

428 def makeDataIdTranslatorFactory(self) -> TranslatorFactory: 

429 """Return a factory for creating Gen2->Gen3 data ID translators, 

430 specialized for this instrument. 

431 

432 Derived class implementations should generally call 

433 `TranslatorFactory.addGenericInstrumentRules` with appropriate 

434 arguments, but are not required to (and may not be able to if their 

435 Gen2 raw data IDs are sufficiently different from the HSC/DECam/CFHT 

436 norm). 

437 

438 Returns 

439 ------- 

440 factory : `TranslatorFactory`. 

441 Factory for `Translator` objects. 

442 """ 

443 raise NotImplementedError("Must be implemented by derived classes.") 

444 

445 @classmethod 

446 def makeDefaultRawIngestRunName(cls) -> str: 

447 """Make the default instrument-specific run collection string for raw 

448 data ingest. 

449 

450 Returns 

451 ------- 

452 coll : `str` 

453 Run collection name to be used as the default for ingestion of 

454 raws. 

455 """ 

456 return cls.makeCollectionName("raw/all") 

457 

458 @classmethod 

459 def makeCollectionName(cls, label: str) -> str: 

460 """Get the instrument-specific collection string to use as derived 

461 from the supplied label. 

462 

463 Parameters 

464 ---------- 

465 label : `str` 

466 String to be combined with the instrument name to form a 

467 collection name. 

468 

469 Returns 

470 ------- 

471 name : `str` 

472 Collection name to use that includes the instrument name. 

473 """ 

474 return f"{cls.getName()}/{label}" 

475 

476 

477def makeExposureRecordFromObsInfo(obsInfo, universe): 

478 """Construct an exposure DimensionRecord from 

479 `astro_metadata_translator.ObservationInfo`. 

480 

481 Parameters 

482 ---------- 

483 obsInfo : `astro_metadata_translator.ObservationInfo` 

484 A `~astro_metadata_translator.ObservationInfo` object corresponding to 

485 the exposure. 

486 universe : `DimensionUniverse` 

487 Set of all known dimensions. 

488 

489 Returns 

490 ------- 

491 record : `DimensionRecord` 

492 A record containing exposure metadata, suitable for insertion into 

493 a `Registry`. 

494 """ 

495 dimension = universe["exposure"] 

496 return dimension.RecordClass.fromDict({ 

497 "instrument": obsInfo.instrument, 

498 "id": obsInfo.exposure_id, 

499 "name": obsInfo.observation_id, 

500 "group_name": obsInfo.exposure_group, 

501 "group_id": obsInfo.visit_id, 

502 "datetime_begin": obsInfo.datetime_begin, 

503 "datetime_end": obsInfo.datetime_end, 

504 "exposure_time": obsInfo.exposure_time.to_value("s"), 

505 "dark_time": obsInfo.dark_time.to_value("s"), 

506 "observation_type": obsInfo.observation_type, 

507 "physical_filter": obsInfo.physical_filter, 

508 }) 

509 

510 

511def addUnboundedCalibrationLabel(registry, instrumentName): 

512 """Add a special 'unbounded' calibration_label dimension entry for the 

513 given camera that is valid for any exposure. 

514 

515 If such an entry already exists, this function just returns a `DataId` 

516 for the existing entry. 

517 

518 Parameters 

519 ---------- 

520 registry : `Registry` 

521 Registry object in which to insert the dimension entry. 

522 instrumentName : `str` 

523 Name of the instrument this calibration label is associated with. 

524 

525 Returns 

526 ------- 

527 dataId : `DataId` 

528 New or existing data ID for the unbounded calibration. 

529 """ 

530 d = dict(instrument=instrumentName, calibration_label="unbounded") 

531 try: 

532 return registry.expandDataId(d) 

533 except LookupError: 

534 pass 

535 entry = d.copy() 

536 entry["datetime_begin"] = TIMESPAN_MIN 

537 entry["datetime_end"] = TIMESPAN_MAX 

538 registry.insertDimensionData("calibration_label", entry) 

539 return registry.expandDataId(d) 

540 

541 

542def loadCamera(butler: Butler, dataId: DataId, *, collections: Any = None) -> Tuple[Camera, bool]: 

543 """Attempt to load versioned camera geometry from a butler, but fall back 

544 to obtaining a nominal camera from the `Instrument` class if that fails. 

545 

546 Parameters 

547 ---------- 

548 butler : `lsst.daf.butler.Butler` 

549 Butler instance to attempt to query for and load a ``camera`` dataset 

550 from. 

551 dataId : `dict` or `DataCoordinate` 

552 Data ID that identifies at least the ``instrument`` and ``exposure`` 

553 dimensions. 

554 collections : Any, optional 

555 Collections to be searched, overriding ``self.butler.collections``. 

556 Can be any of the types supported by the ``collections`` argument 

557 to butler construction. 

558 

559 Returns 

560 ------- 

561 camera : `lsst.afw.cameraGeom.Camera` 

562 Camera object. 

563 versioned : `bool` 

564 If `True`, the camera was obtained from the butler and should represent 

565 a versioned camera from a calibration repository. If `False`, no 

566 camera datasets were found, and the returned camera was produced by 

567 instantiating the appropriate `Instrument` class and calling 

568 `Instrument.getCamera`. 

569 """ 

570 if collections is None: 

571 collections = butler.collections 

572 # Registry would do data ID expansion internally if we didn't do it first, 

573 # but we might want an expanded data ID ourselves later, so we do it here 

574 # to ensure it only happens once. 

575 # This will also catch problems with the data ID not having keys we need. 

576 dataId = butler.registry.expandDataId(dataId, graph=butler.registry.dimensions["exposure"].graph) 

577 cameraRefs = list(butler.registry.queryDatasets("camera", dataId=dataId, collections=collections, 

578 deduplicate=True)) 

579 if cameraRefs: 

580 assert len(cameraRefs) == 1, "Should be guaranteed by deduplicate=True above." 

581 return butler.getDirect(cameraRefs[0]), True 

582 instrument = Instrument.fromName(dataId["instrument"], butler.registry) 

583 return instrument.getCamera(), False