Coverage for tests/test_butlerFits.py: 16%

223 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-02 10:52 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import shutil 

26import tempfile 

27import unittest 

28from typing import TYPE_CHECKING 

29 

30import astropy.table 

31import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

32import lsst.afw.image 

33import lsst.pex.config 

34import lsst.utils.tests 

35from lsst.afw.fits import readMetadata 

36from lsst.afw.image import LOCAL, ExposureFitsReader 

37from lsst.afw.math import flipImage 

38from lsst.daf.base import PropertyList, PropertySet 

39from lsst.daf.butler import Config, DatasetType, StorageClassFactory 

40from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo 

41from lsst.geom import Box2I, Extent2I, Point2I 

42from lsst.obs.base.exposureAssembler import ExposureAssembler 

43from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

44 

45if TYPE_CHECKING: 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true

46 from lsst.daf.butler import DatasetRef 

47 

48TESTDIR = os.path.dirname(__file__) 

49 

50BUTLER_CONFIG = """ 

51storageClasses: 

52 ExposureCompositeF: 

53 inheritsFrom: ExposureF 

54datastore: 

55 # Want to check disassembly so can't use InMemory 

56 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

57 formatters: 

58 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

59 lossless: 

60 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

61 parameters: 

62 recipe: lossless 

63 uncompressed: 

64 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

65 parameters: 

66 recipe: noCompression 

67 lossy: 

68 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

69 parameters: 

70 recipe: lossyBasic 

71 composites: 

72 disassembled: 

73 ExposureCompositeF: True 

74""" 

75 

76# Components present in the test file 

77COMPONENTS = { 

78 "wcs", 

79 "image", 

80 "mask", 

81 "coaddInputs", 

82 "psf", 

83 "visitInfo", 

84 "variance", 

85 "metadata", 

86 "photoCalib", 

87 "filter", 

88 "validPolygon", 

89 "transmissionCurve", 

90 "detector", 

91 "apCorrMap", 

92 "summaryStats", 

93 "id", 

94} 

95READ_COMPONENTS = { 

96 "bbox", 

97 "xy0", 

98 "dimensions", 

99} 

100 

101 

102class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

103 @classmethod 

104 def setUpClass(cls): 

105 """Create a new butler once only.""" 

106 

107 cls.storageClassFactory = StorageClassFactory() 

108 

109 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

110 

111 dataIds = { 

112 "instrument": ["DummyCam"], 

113 "physical_filter": ["d-r"], 

114 "visit": [42, 43, 44], 

115 } 

116 

117 # Ensure that we test in a directory that will include some 

118 # metacharacters 

119 subdir = "sub?#dir" 

120 butlerRoot = os.path.join(cls.root, subdir) 

121 

122 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

123 

124 # Create dataset types used by the tests 

125 for datasetTypeName, storageClassName in ( 

126 ("calexp", "ExposureF"), 

127 ("unknown", "ExposureCompositeF"), 

128 ("testCatalog", "SourceCatalog"), 

129 ("lossless", "ExposureF"), 

130 ("uncompressed", "ExposureF"), 

131 ("lossy", "ExposureF"), 

132 ): 

133 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

134 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

135 

136 # And some dataset types that have no dimensions for easy testing 

137 for datasetTypeName, storageClassName in ( 

138 ("ps", "PropertySet"), 

139 ("pl", "PropertyList"), 

140 ("int_exp_trimmed", "ExposureI"), 

141 ("int_exp_untrimmed", "ExposureI"), 

142 ): 

143 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

144 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

145 

146 @classmethod 

147 def tearDownClass(cls): 

148 if cls.root is not None: 

149 shutil.rmtree(cls.root, ignore_errors=True) 

150 

151 def setUp(self): 

152 self.butler = makeTestCollection(self.creatorButler) 

153 

154 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

155 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

156 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

157 

158 def assertCatalogEqual( 

159 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog 

160 ) -> None: 

161 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

162 inputTable = inputCatalog.getTable() 

163 inputRecord = inputCatalog[0] 

164 outputTable = outputCatalog.getTable() 

165 outputRecord = outputCatalog[0] 

166 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

167 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

168 self.assertEqual( 

169 inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

170 outputTable.getSchema().getAliasMap().get("slot_Centroid"), 

171 ) 

172 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

173 self.assertFloatsAlmostEqual( 

174 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6 

175 ) 

176 self.assertFloatsAlmostEqual( 

177 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6 

178 ) 

179 self.assertEqual( 

180 inputTable.getSchema().getAliasMap().get("slot_Shape"), 

181 outputTable.getSchema().getAliasMap().get("slot_Shape"), 

182 ) 

183 self.assertFloatsAlmostEqual( 

184 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6 

185 ) 

186 self.assertFloatsAlmostEqual( 

187 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6 

188 ) 

189 self.assertFloatsAlmostEqual( 

190 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6 

191 ) 

192 

193 def runFundamentalTypeTest(self, datasetTypeName, entity): 

194 """Put and get the supplied entity and compare.""" 

195 ref = self.butler.put(entity, datasetTypeName) 

196 butler_ps = self.butler.get(ref) 

197 self.assertEqual(butler_ps, entity) 

198 

199 # Break the contact by ensuring that we are writing YAML 

200 uri = self.butler.getURI(ref) 

201 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}") 

202 

203 def testFundamentalTypes(self) -> None: 

204 """Ensure that some fundamental stack types round trip.""" 

205 ps = PropertySet() 

206 ps["a.b"] = 5 

207 ps["c.d.e"] = "string" 

208 self.runFundamentalTypeTest("ps", ps) 

209 

210 pl = PropertyList() 

211 pl["A"] = 1 

212 pl.setComment("A", "An int comment") 

213 pl["B"] = "string" 

214 pl.setComment("B", "A string comment") 

215 self.runFundamentalTypeTest("pl", pl) 

216 

217 def testFitsCatalog(self) -> None: 

218 """Test reading of a FITS catalog""" 

219 catalog = self.makeExampleCatalog() 

220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

221 ref = self.butler.put(catalog, "testCatalog", dataId) 

222 stored = self.butler.get(ref) 

223 self.assertCatalogEqual(catalog, stored) 

224 

225 # Override the storage class. 

226 astropy_table = self.butler.get(ref, storageClass="AstropyTable") 

227 self.assertIsInstance(astropy_table, astropy.table.Table) 

228 self.assertEqual(len(astropy_table), len(stored)) 

229 

230 def testExposureCompositePutGetConcrete(self) -> None: 

231 """Test composite with no disassembly""" 

232 ref = self.runExposureCompositePutGetTest("calexp") 

233 

234 uri = self.butler.getURI(ref) 

235 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

236 

237 def testExposureCompositePutGetVirtual(self) -> None: 

238 """Testing composite disassembly""" 

239 ref = self.runExposureCompositePutGetTest("unknown") 

240 

241 primary, components = self.butler.getURIs(ref) 

242 self.assertIsNone(primary) 

243 self.assertEqual(set(components), COMPONENTS) 

244 for compName, uri in components.items(): 

245 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}") 

246 

247 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

248 example = os.path.join(TESTDIR, "data", "calexp.fits") 

249 exposure = lsst.afw.image.ExposureF(example) 

250 

251 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

252 ref = self.butler.put(exposure, datasetTypeName, dataId) 

253 

254 # Get the full thing 

255 composite = self.butler.get(datasetTypeName, dataId) 

256 

257 # There is no assert for Exposure so just look at maskedImage 

258 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

259 

260 # Helper for extracting components 

261 assembler = ExposureAssembler(ref.datasetType.storageClass) 

262 

263 # Check all possible components that can be read 

264 allComponents = set() 

265 allComponents.update(COMPONENTS, READ_COMPONENTS) 

266 

267 # Get each component from butler independently 

268 for compName in allComponents: 

269 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

270 component = self.butler.get(compTypeName, dataId) 

271 

272 reference = assembler.getComponent(exposure, compName) 

273 

274 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

275 

276 if compName in ("image", "variance"): 

277 self.assertImagesEqual(component, reference) 

278 elif compName == "mask": 

279 self.assertMasksEqual(component, reference) 

280 elif compName == "wcs": 

281 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

282 elif compName == "coaddInputs": 

283 self.assertEqual( 

284 len(component.visits), len(reference.visits), f"cf visits {component.visits}" 

285 ) 

286 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}") 

287 elif compName == "psf": 

288 # Equality for PSF does not work 

289 pass 

290 elif compName == "filter": 

291 self.assertEqual(component, reference) 

292 elif compName == "id": 

293 self.assertEqual(component, reference) 

294 elif compName == "visitInfo": 

295 self.assertEqual(component, reference, "VisitInfo comparison") 

296 elif compName == "metadata": 

297 # The component metadata has extra fields in it so cannot 

298 # compare directly. 

299 for k, v in reference.items(): 

300 self.assertEqual(component[k], v) 

301 elif compName == "photoCalib": 

302 # This example has a 

303 # "spatially constant with mean: inf error: nan" entry 

304 # which does not compare directly. 

305 self.assertEqual(str(component), str(reference)) 

306 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib") 

307 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

308 self.assertEqual(component, reference) 

309 elif compName == "apCorrMap": 

310 self.assertEqual(set(component.keys()), set(reference.keys())) 

311 elif compName == "transmissionCurve": 

312 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds()) 

313 elif compName == "detector": 

314 c_amps = {a.getName() for a in component.getAmplifiers()} 

315 r_amps = {a.getName() for a in reference.getAmplifiers()} 

316 self.assertEqual(c_amps, r_amps) 

317 elif compName == "summaryStats": 

318 self.assertEqual(component.psfSigma, reference.psfSigma) 

319 else: 

320 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

321 

322 # Full Exposure with parameters 

323 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

324 parameters = dict(bbox=inBBox, origin=LOCAL) 

325 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

326 outBBox = subset.getBBox() 

327 self.assertEqual(inBBox, outBBox) 

328 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

329 

330 return ref 

331 

332 def putFits(self, exposure, datasetTypeName, visit): 

333 """Put different datasetTypes and return information.""" 

334 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

335 refC = self.butler.put(exposure, datasetTypeName, dataId) 

336 uriC = self.butler.getURI(refC) 

337 stat = os.stat(uriC.ospath) 

338 size = stat.st_size 

339 # We can't use butler's Exposure storage class metadata component, 

340 # because that intentionally strips keywords that science code 

341 # shouldn't ever read (to at least weaken our assumptions that we write 

342 # to FITS). Instead use a lower-level method on the URI for this test. 

343 meta = readMetadata(uriC.ospath) 

344 return meta, size 

345 

346 def testCompression(self): 

347 """Test that we can write compressed and uncompressed FITS.""" 

348 example = os.path.join(TESTDIR, "data", "small.fits") 

349 exposure = lsst.afw.image.ExposureF(example) 

350 

351 # Write a lossless compressed 

352 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

353 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

354 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

355 

356 # Write an uncompressed FITS file 

357 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

358 self.assertNotIn("ZCMPTYPE", metaN) 

359 

360 # Write an uncompressed FITS file 

361 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

362 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

363 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

364 

365 self.assertNotEqual(sizeC, sizeN) 

366 # Data file is so small that Lossy and Compressed are dominated 

367 # by the extra compression tables 

368 self.assertEqual(sizeL, sizeC) 

369 

370 def testExposureFormatterAmpParameter(self): 

371 """Test the FitsExposureFormatter implementation of the Exposure 

372 StorageClass's 'amp' and 'detector' parameters. 

373 """ 

374 # Our example exposure file has a realistic detector (looks like HSC), 

375 # but the image itself doesn't match it. So we just load the detector, 

376 # and use it to make our own more useful images and put them. 

377 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

378 trimmed_full = make_ramp_exposure_trimmed(detector) 

379 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

380 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

381 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

382 for n, amp in enumerate(detector): 

383 # Try to read each amp as it is on disk, with a variety of 

384 # parameters that should all do the same thing. 

385 for amp_parameter in [amp, amp.getName(), n]: 

386 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

387 with self.subTest(parameters=parameters): 

388 test_trimmed = self.butler.get(trimmed_ref, parameters=parameters) 

389 test_untrimmed = self.butler.get(untrimmed_ref, parameters=parameters) 

390 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

391 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

392 self.assertEqual(len(test_trimmed.getDetector()), 1) 

393 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

394 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

395 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

396 # Try to read various transformed versions of the original amp, 

397 # to make sure flips and offsets are applied correctly. 

398 # First flip X only. 

399 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

400 test_t1_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t1}) 

401 self.assertImagesEqual( 

402 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False) 

403 ) 

404 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

405 test_t1_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t1}) 

406 self.assertImagesEqual( 

407 test_t1_untrimmed.image, 

408 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False), 

409 ) 

410 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

411 # Flip Y only. 

412 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

413 test_t2_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t2}) 

414 self.assertImagesEqual( 

415 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True) 

416 ) 

417 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

418 test_t2_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t2}) 

419 self.assertImagesEqual( 

420 test_t2_untrimmed.image, 

421 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True), 

422 ) 

423 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

424 # Add an XY offset only. 

425 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

426 test_t3_trimmed = self.butler.get(trimmed_ref, parameters={"amp": amp_t3}) 

427 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image) 

428 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

429 test_t3_untrimmed = self.butler.get(untrimmed_ref, parameters={"amp": amp_t3}) 

430 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

431 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

432 

433 

434if __name__ == "__main__": 434 ↛ 435line 434 didn't jump to line 435, because the condition on line 434 was never true

435 unittest.main()