Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22__all__ = ("FitsExposureFormatter", "FitsImageFormatter", "FitsMaskFormatter", 

23 "FitsMaskedImageFormatter") 

24 

25from abc import abstractmethod 

26import warnings 

27 

28from lsst.daf.base import PropertySet 

29from lsst.daf.butler import Formatter 

30from lsst.daf.butler.core.utils import cached_getter 

31from lsst.afw.image import ExposureFitsReader, ImageFitsReader, MaskFitsReader, MaskedImageFitsReader 

32from lsst.afw.image import ExposureInfo, FilterLabel 

33# Needed for ApCorrMap to resolve properly 

34from lsst.afw.math import BoundedField # noqa: F401 

35 

36 

37class FitsImageFormatterBase(Formatter): 

38 """Base class formatter for image-like storage classes stored via FITS. 

39 

40 Notes 

41 ----- 

42 This class makes no assumptions about how many HDUs are used to represent 

43 the image on disk, and includes no support for writing. It's really just a 

44 collection of miscellaneous boilerplate common to all FITS image 

45 formatters. 

46 

47 Concrete subclasses must implement `readComponent`, `readFull`, and `write` 

48 (even if just to disable them by raising an exception). 

49 """ 

50 

51 extension = ".fits" 

52 supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz", ".fz", ".fit"}) 

53 

54 unsupportedParameters = {} 

55 """Support all parameters.""" 

56 

57 @property 

58 @cached_getter 

59 def checked_parameters(self): 

60 """The parameters passed by the butler user, after checking them 

61 against the storage class and transforming `None` into an empty `dict` 

62 (`dict`). 

63 

64 This is computed on first use and then cached. It should never be 

65 accessed when writing. Subclasses that need additional checking should 

66 delegate to `super` and then check the result before returning it. 

67 """ 

68 parameters = self.fileDescriptor.parameters 

69 if parameters is None: 

70 parameters = {} 

71 self.fileDescriptor.storageClass.validateParameters(parameters) 

72 return parameters 

73 

74 def read(self, component=None): 

75 # Docstring inherited. 

76 if self.fileDescriptor.readStorageClass != self.fileDescriptor.storageClass: 

77 if component is not None: 

78 return self.readComponent(component) 

79 else: 

80 raise ValueError("Storage class inconsistency ({} vs {}) but no" 

81 " component requested".format(self.fileDescriptor.readStorageClass.name, 

82 self.fileDescriptor.storageClass.name)) 

83 return self.readFull() 

84 

85 @abstractmethod 

86 def readComponent(self, component): 

87 """Read a component dataset. 

88 

89 Parameters 

90 ---------- 

91 component : `str`, optional 

92 Component to read from the file. 

93 

94 Returns 

95 ------- 

96 obj : component-dependent 

97 In-memory component object. 

98 

99 Raises 

100 ------ 

101 KeyError 

102 Raised if the requested component cannot be handled. 

103 """ 

104 raise NotImplementedError() 

105 

106 @abstractmethod 

107 def readFull(self): 

108 """Read the full dataset (while still accounting for parameters). 

109 

110 Returns 

111 ------- 

112 obj : component-dependent 

113 In-memory component object. 

114 

115 """ 

116 raise NotImplementedError() 

117 

118 

119class ReaderFitsImageFormatterBase(FitsImageFormatterBase): 

120 """Base class formatter for image-like storage classes stored via FITS 

121 backed by a "reader" object similar to `lsst.afw.image.ImageFitsReader`. 

122 

123 Notes 

124 ----- 

125 This class includes no support for writing. 

126 

127 Concrete subclasses must provide at least the `ReaderClass` attribute 

128 and a `write` implementation (even just to disable writing by raising). 

129 

130 The provided implementation of `readComponent` handles only the 'bbox', 

131 'dimensions', and 'xy0' components common to all image-like storage 

132 classes. Subclasses with additional components should handle them first, 

133 then delegate to ``super()`` for these (or, if necessary, delegate first 

134 and catch `KeyError`). 

135 

136 The provided implementation of `readFull` handles only parameters that 

137 can be forwarded directly to the reader class (usually ``bbox`` and 

138 ``origin``). Concrete subclasses that need to handle additional parameters 

139 should generally reimplement without delegating (the implementation is 

140 trivial). 

141 """ 

142 

143 

144class StandardFitsImageFormatterBase(ReaderFitsImageFormatterBase): 

145 """Base class interface for image-like storage stored via FITS, 

146 written using LSST code. 

147 

148 Notes 

149 ----- 

150 Concrete subclasses must provide at least the `ReaderClass` attribute. 

151 

152 The provided implementation of `readComponent` handles only the 'bbox', 

153 'dimensions', and 'xy0' components common to all image-like storage 

154 classes. Subclasses with additional components should handle them first, 

155 then delegate to ``super()`` for these (or, if necessary, delegate first 

156 and catch `KeyError`). 

157 

158 The provided implementation of `readFull` handles only parameters that 

159 can be forwarded directly to the reader class (usually ``bbox`` and 

160 ``origin``). Concrete subclasses that need to handle additional parameters 

161 should generally reimplement without delegating (the implementation is 

162 trivial). 

163 

164 This Formatter supports write recipes, and assumes its in-memory type has 

165 ``writeFits`` and (for write recipes) ``writeFitsWithOptions`` methods. 

166 

167 Each ``StandardFitsImageFormatterBase`` recipe for FITS compression should 

168 define ``image``, ``mask`` and ``variance`` entries, each of which may 

169 contain ``compression`` and ``scaling`` entries. Defaults will be 

170 provided for any missing elements under ``compression`` and 

171 ``scaling``. 

172 

173 The allowed entries under ``compression`` are: 

174 

175 * ``algorithm`` (`str`): compression algorithm to use 

176 * ``rows`` (`int`): number of rows per tile (0 = entire dimension) 

177 * ``columns`` (`int`): number of columns per tile (0 = entire dimension) 

178 * ``quantizeLevel`` (`float`): cfitsio quantization level 

179 

180 The allowed entries under ``scaling`` are: 

181 

182 * ``algorithm`` (`str`): scaling algorithm to use 

183 * ``bitpix`` (`int`): bits per pixel (0,8,16,32,64,-32,-64) 

184 * ``fuzz`` (`bool`): fuzz the values when quantising floating-point values? 

185 * ``seed`` (`int`): seed for random number generator when fuzzing 

186 * ``maskPlanes`` (`list` of `str`): mask planes to ignore when doing 

187 statistics 

188 * ``quantizeLevel`` (`float`): divisor of the standard deviation for 

189 ``STDEV_*`` scaling 

190 * ``quantizePad`` (`float`): number of stdev to allow on the low side (for 

191 ``STDEV_POSITIVE``/``NEGATIVE``) 

192 * ``bscale`` (`float`): manually specified ``BSCALE`` 

193 (for ``MANUAL`` scaling) 

194 * ``bzero`` (`float`): manually specified ``BSCALE`` 

195 (for ``MANUAL`` scaling) 

196 

197 A very simple example YAML recipe (for the ``Exposure`` specialization): 

198 

199 .. code-block:: yaml 

200 

201 lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter: 

202 default: 

203 image: &default 

204 compression: 

205 algorithm: GZIP_SHUFFLE 

206 mask: *default 

207 variance: *default 

208 

209 """ 

210 supportedWriteParameters = frozenset({"recipe"}) 

211 ReaderClass: type # must be set by concrete subclasses 

212 

213 @property 

214 @cached_getter 

215 def reader(self): 

216 """The reader object that backs this formatter's read operations. 

217 

218 This is computed on first use and then cached. It should never be 

219 accessed when writing. 

220 """ 

221 return self.ReaderClass(self.fileDescriptor.location.path) 

222 

223 def readComponent(self, component): 

224 # Docstring inherited. 

225 if component in ("bbox", "dimensions", "xy0"): 

226 bbox = self.reader.readBBox() 

227 if component == "dimensions": 

228 return bbox.getDimensions() 

229 elif component == "xy0": 

230 return bbox.getMin() 

231 else: 

232 return bbox 

233 else: 

234 raise KeyError(f"Unknown component requested: {component}") 

235 

236 def readFull(self): 

237 # Docstring inherited. 

238 return self.reader.read(**self.checked_parameters) 

239 

240 def write(self, inMemoryDataset): 

241 """Write a Python object to a file. 

242 

243 Parameters 

244 ---------- 

245 inMemoryDataset : `object` 

246 The Python object to store. 

247 """ 

248 # Update the location with the formatter-preferred file extension 

249 self.fileDescriptor.location.updateExtension(self.extension) 

250 outputPath = self.fileDescriptor.location.path 

251 

252 # check to see if we have a recipe requested 

253 recipeName = self.writeParameters.get("recipe") 

254 recipe = self.getImageCompressionSettings(recipeName) 

255 if recipe: 

256 # Can not construct a PropertySet from a hierarchical 

257 # dict but can update one. 

258 ps = PropertySet() 

259 ps.update(recipe) 

260 inMemoryDataset.writeFitsWithOptions(outputPath, options=ps) 

261 else: 

262 inMemoryDataset.writeFits(outputPath) 

263 

264 def getImageCompressionSettings(self, recipeName): 

265 """Retrieve the relevant compression settings for this recipe. 

266 

267 Parameters 

268 ---------- 

269 recipeName : `str` 

270 Label associated with the collection of compression parameters 

271 to select. 

272 

273 Returns 

274 ------- 

275 settings : `dict` 

276 The selected settings. 

277 """ 

278 # if no recipe has been provided and there is no default 

279 # return immediately 

280 if not recipeName: 

281 if "default" not in self.writeRecipes: 

282 return {} 

283 recipeName = "default" 

284 

285 if recipeName not in self.writeRecipes: 

286 raise RuntimeError(f"Unrecognized recipe option given for compression: {recipeName}") 

287 

288 recipe = self.writeRecipes[recipeName] 

289 

290 # Set the seed based on dataId 

291 seed = hash(tuple(self.dataId.items())) % 2**31 

292 for plane in ("image", "mask", "variance"): 

293 if plane in recipe and "scaling" in recipe[plane]: 

294 scaling = recipe[plane]["scaling"] 

295 if "seed" in scaling and scaling["seed"] == 0: 

296 scaling["seed"] = seed 

297 

298 return recipe 

299 

300 @classmethod 

301 def validateWriteRecipes(cls, recipes): 

302 """Validate supplied recipes for this formatter. 

303 

304 The recipes are supplemented with default values where appropriate. 

305 

306 TODO: replace this custom validation code with Cerberus (DM-11846) 

307 

308 Parameters 

309 ---------- 

310 recipes : `dict` 

311 Recipes to validate. Can be empty dict or `None`. 

312 

313 Returns 

314 ------- 

315 validated : `dict` 

316 Validated recipes. Returns what was given if there are no 

317 recipes listed. 

318 

319 Raises 

320 ------ 

321 RuntimeError 

322 Raised if validation fails. 

323 """ 

324 # Schemas define what should be there, and the default values (and by 

325 # the default value, the expected type). 

326 compressionSchema = { 

327 "algorithm": "NONE", 

328 "rows": 1, 

329 "columns": 0, 

330 "quantizeLevel": 0.0, 

331 } 

332 scalingSchema = { 

333 "algorithm": "NONE", 

334 "bitpix": 0, 

335 "maskPlanes": ["NO_DATA"], 

336 "seed": 0, 

337 "quantizeLevel": 4.0, 

338 "quantizePad": 5.0, 

339 "fuzz": True, 

340 "bscale": 1.0, 

341 "bzero": 0.0, 

342 } 

343 

344 if not recipes: 

345 # We can not insist on recipes being specified 

346 return recipes 

347 

348 def checkUnrecognized(entry, allowed, description): 

349 """Check to see if the entry contains unrecognised keywords""" 

350 unrecognized = set(entry) - set(allowed) 

351 if unrecognized: 

352 raise RuntimeError( 

353 f"Unrecognized entries when parsing image compression recipe {description}: " 

354 f"{unrecognized}") 

355 

356 validated = {} 

357 for name in recipes: 

358 checkUnrecognized(recipes[name], ["image", "mask", "variance"], name) 

359 validated[name] = {} 

360 for plane in ("image", "mask", "variance"): 

361 checkUnrecognized(recipes[name][plane], ["compression", "scaling"], 

362 f"{name}->{plane}") 

363 

364 np = {} 

365 validated[name][plane] = np 

366 for settings, schema in (("compression", compressionSchema), 

367 ("scaling", scalingSchema)): 

368 np[settings] = {} 

369 if settings not in recipes[name][plane]: 

370 for key in schema: 

371 np[settings][key] = schema[key] 

372 continue 

373 entry = recipes[name][plane][settings] 

374 checkUnrecognized(entry, schema.keys(), f"{name}->{plane}->{settings}") 

375 for key in schema: 

376 value = type(schema[key])(entry[key]) if key in entry else schema[key] 

377 np[settings][key] = value 

378 return validated 

379 

380 

381class FitsImageFormatter(StandardFitsImageFormatterBase): 

382 """Concrete formatter for reading/writing `~lsst.afw.image.Image` 

383 from/to FITS. 

384 """ 

385 

386 ReaderClass = ImageFitsReader 

387 

388 

389class FitsMaskFormatter(StandardFitsImageFormatterBase): 

390 """Concrete formatter for reading/writing `~lsst.afw.image.Mask` 

391 from/to FITS. 

392 """ 

393 

394 ReaderClass = MaskFitsReader 

395 

396 

397class FitsMaskedImageFormatter(StandardFitsImageFormatterBase): 

398 """Concrete formatter for reading/writing `~lsst.afw.image.MaskedImage` 

399 from/to FITS. 

400 """ 

401 

402 ReaderClass = MaskedImageFitsReader 

403 

404 def readComponent(self, component): 

405 # Docstring inherited. 

406 if component == "image": 

407 return self.reader.readImage(**self.checked_parameters) 

408 elif component == "mask": 

409 return self.reader.readMask(**self.checked_parameters) 

410 elif component == "variance": 

411 return self.reader.readVariance(**self.checked_parameters) 

412 else: 

413 # Delegate to base for bbox, dimensions, xy0. 

414 return super().readComponent(component) 

415 

416 

417class FitsExposureFormatter(FitsMaskedImageFormatter): 

418 """Concrete formatter for reading/writing `~lsst.afw.image.Exposure` 

419 from/to FITS. 

420 

421 Notes 

422 ----- 

423 This class inherits from `FitsMaskedImageFormatter` even though 

424 `lsst.afw.image.Exposure` doesn't inherit from 

425 `lsst.afw.image.MaskedImage`; this is just an easy way to be able to 

426 delegate to `FitsMaskedImageFormatter.super()` for component-handling, and 

427 should be replaced with e.g. both calling a free function if that slight 

428 type covariance violation ever becomes a practical problem. 

429 """ 

430 

431 ReaderClass = ExposureFitsReader 

432 

433 def readComponent(self, component): 

434 # Docstring inherited. 

435 # Generic components can be read via a string name; DM-27754 will make 

436 # this mapping larger at the expense of the following one. 

437 genericComponents = { 

438 "summaryStats": ExposureInfo.KEY_SUMMARY_STATS, 

439 } 

440 if (genericComponentName := genericComponents.get(component)) is not None: 

441 return self.reader.readComponent(genericComponentName) 

442 # Other components have hard-coded method names, but don't take 

443 # parameters. 

444 standardComponents = { 

445 'metadata': 'readMetadata', 

446 'wcs': 'readWcs', 

447 'coaddInputs': 'readCoaddInputs', 

448 'psf': 'readPsf', 

449 'photoCalib': 'readPhotoCalib', 

450 # TODO: deprecate in DM-27170, remove in DM-27177 

451 'filter': 'readFilter', 

452 # TODO: deprecate in DM-27177, remove in DM-27811 

453 'filterLabel': 'readFilterLabel', 

454 'validPolygon': 'readValidPolygon', 

455 'apCorrMap': 'readApCorrMap', 

456 'visitInfo': 'readVisitInfo', 

457 'transmissionCurve': 'readTransmissionCurve', 

458 'detector': 'readDetector', 

459 'exposureInfo': 'readExposureInfo', 

460 } 

461 if (methodName := standardComponents.get(component)) is not None: 

462 result = getattr(self.reader, methodName)() 

463 if component == "filterLabel": 

464 return self._fixFilterLabels(result) 

465 return result 

466 # Delegate to MaskedImage and ImageBase implementations for the rest. 

467 return super().readComponent(component) 

468 

469 def readFull(self): 

470 # Docstring inherited. 

471 full = super().readFull() 

472 full.getInfo().setFilterLabel(self._fixFilterLabels(full.getInfo().getFilterLabel())) 

473 return full 

474 

475 def _fixFilterLabels(self, file_filter_label, should_be_standardized=None): 

476 """Compare the filter label read from the file with the one in the 

477 data ID. 

478 

479 Parameters 

480 ---------- 

481 file_filter_label : `lsst.afw.image.FilterLabel` or `None` 

482 Filter label read from the file, if there was one. 

483 should_be_standardized : `bool`, optional 

484 If `True`, expect ``file_filter_label`` to be consistent with the 

485 data ID and warn only if it is not. If `False`, expect it to be 

486 inconsistent and warn only if the data ID is incomplete and hence 

487 the `FilterLabel` cannot be fixed. If `None` (default) guess 

488 whether the file should be standardized by looking at the 

489 serialization version number in file, which requires this method to 

490 have been run after `readFull` or `readComponent`. 

491 

492 Returns 

493 ------- 

494 filter_label : `lsst.afw.image.FilterLabel` or `None` 

495 The preferred filter label; may be the given one or one built from 

496 the data ID. `None` is returned if there should never be any 

497 filters associated with this dataset type. 

498 

499 Notes 

500 ----- 

501 Most test coverage for this method is in ci_hsc_gen3, where we have 

502 much easier access to test data that exhibits the problems it attempts 

503 to solve. 

504 """ 

505 # Remember filter data ID keys that weren't in this particular data ID, 

506 # so we can warn about them later. 

507 missing = [] 

508 band = None 

509 physical_filter = None 

510 if "band" in self.dataId.graph.dimensions.names: 

511 band = self.dataId.get("band") 

512 # band isn't in the data ID; is that just because this data ID 

513 # hasn't been filled in with everything the Registry knows, or 

514 # because this dataset is never associated with a band? 

515 if band is None and not self.dataId.hasFull() and "band" in self.dataId.graph.implied.names: 

516 missing.append("band") 

517 if "physical_filter" in self.dataId.graph.dimensions.names: 

518 physical_filter = self.dataId.get("physical_filter") 

519 # Same check as above for band, but for physical_filter. 

520 if (physical_filter is None and not self.dataId.hasFull() 

521 and "physical_filter" in self.dataId.graph.implied.names): 

522 missing.append("physical_filter") 

523 if should_be_standardized is None: 

524 version = self.reader.readSerializationVersion() 

525 should_be_standardized = (version >= 2) 

526 if missing: 

527 # Data ID identifies a filter but the actual filter label values 

528 # haven't been fetched from the database; we have no choice but 

529 # to use the one in the file. 

530 # Warn if that's more likely than not to be bad, because the file 

531 # predates filter standardization. 

532 if not should_be_standardized: 

533 warnings.warn(f"Data ID {self.dataId} is missing (implied) value(s) for {missing}; " 

534 "the correctness of this Exposure's FilterLabel cannot be guaranteed. " 

535 "Call Registry.expandDataId before Butler.get to avoid this.") 

536 return file_filter_label 

537 if band is None and physical_filter is None: 

538 data_id_filter_label = None 

539 else: 

540 data_id_filter_label = FilterLabel(band=band, physical=physical_filter) 

541 if data_id_filter_label != file_filter_label and should_be_standardized: 

542 # File was written after FilterLabel and standardization, but its 

543 # FilterLabel doesn't agree with the data ID: this indicates a bug 

544 # in whatever code produced the Exposure (though it may be one that 

545 # has been fixed since the file was written). 

546 warnings.warn(f"Reading {self.fileDescriptor.location} with data ID {self.dataId}: " 

547 f"filter label mismatch (file is {file_filter_label}, data ID is " 

548 f"{data_id_filter_label}). This is probably a bug in the code that produced it.") 

549 return data_id_filter_label