Coverage for tests/test_butlerFits.py: 16%

221 statements  

« prev     ^ index     » next       coverage.py v6.4, created at 2022-05-27 11:22 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import shutil 

26import tempfile 

27import unittest 

28from typing import TYPE_CHECKING 

29 

30import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

31import lsst.afw.image 

32import lsst.pex.config 

33import lsst.utils.tests 

34from lsst.afw.fits import readMetadata 

35from lsst.afw.image import LOCAL, ExposureFitsReader 

36from lsst.afw.math import flipImage 

37from lsst.daf.base import PropertyList, PropertySet 

38from lsst.daf.butler import Config, DatasetType, StorageClassFactory 

39from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo 

40from lsst.geom import Box2I, Extent2I, Point2I 

41from lsst.obs.base.exposureAssembler import ExposureAssembler 

42from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

43 

44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from lsst.daf.butler import DatasetRef 

46 

47TESTDIR = os.path.dirname(__file__) 

48 

49BUTLER_CONFIG = """ 

50storageClasses: 

51 ExposureCompositeF: 

52 inheritsFrom: ExposureF 

53datastore: 

54 # Want to check disassembly so can't use InMemory 

55 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

56 formatters: 

57 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

58 lossless: 

59 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

60 parameters: 

61 recipe: lossless 

62 uncompressed: 

63 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

64 parameters: 

65 recipe: noCompression 

66 lossy: 

67 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

68 parameters: 

69 recipe: lossyBasic 

70 composites: 

71 disassembled: 

72 ExposureCompositeF: True 

73""" 

74 

75# Components present in the test file 

76COMPONENTS = { 

77 "wcs", 

78 "image", 

79 "mask", 

80 "coaddInputs", 

81 "psf", 

82 "visitInfo", 

83 "variance", 

84 "metadata", 

85 "photoCalib", 

86 "filterLabel", 

87 "validPolygon", 

88 "transmissionCurve", 

89 "detector", 

90 "apCorrMap", 

91 "summaryStats", 

92 "id", 

93} 

94READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"} 

95 

96 

97class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

98 @classmethod 

99 def setUpClass(cls): 

100 """Create a new butler once only.""" 

101 

102 cls.storageClassFactory = StorageClassFactory() 

103 

104 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

105 

106 dataIds = { 

107 "instrument": ["DummyCam"], 

108 "physical_filter": ["d-r"], 

109 "visit": [42, 43, 44], 

110 } 

111 

112 # Ensure that we test in a directory that will include some 

113 # metacharacters 

114 subdir = "sub?#dir" 

115 butlerRoot = os.path.join(cls.root, subdir) 

116 

117 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

118 

119 # Create dataset types used by the tests 

120 for datasetTypeName, storageClassName in ( 

121 ("calexp", "ExposureF"), 

122 ("unknown", "ExposureCompositeF"), 

123 ("testCatalog", "SourceCatalog"), 

124 ("lossless", "ExposureF"), 

125 ("uncompressed", "ExposureF"), 

126 ("lossy", "ExposureF"), 

127 ): 

128 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

129 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

130 

131 # And some dataset types that have no dimensions for easy testing 

132 for datasetTypeName, storageClassName in ( 

133 ("ps", "PropertySet"), 

134 ("pl", "PropertyList"), 

135 ("int_exp_trimmed", "ExposureI"), 

136 ("int_exp_untrimmed", "ExposureI"), 

137 ): 

138 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

139 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

140 

141 @classmethod 

142 def tearDownClass(cls): 

143 if cls.root is not None: 

144 shutil.rmtree(cls.root, ignore_errors=True) 

145 

146 def setUp(self): 

147 self.butler = makeTestCollection(self.creatorButler) 

148 

149 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

150 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

151 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

152 

153 def assertCatalogEqual( 

154 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog 

155 ) -> None: 

156 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

157 inputTable = inputCatalog.getTable() 

158 inputRecord = inputCatalog[0] 

159 outputTable = outputCatalog.getTable() 

160 outputRecord = outputCatalog[0] 

161 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

162 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

163 self.assertEqual( 

164 inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

165 outputTable.getSchema().getAliasMap().get("slot_Centroid"), 

166 ) 

167 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

168 self.assertFloatsAlmostEqual( 

169 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6 

170 ) 

171 self.assertFloatsAlmostEqual( 

172 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6 

173 ) 

174 self.assertEqual( 

175 inputTable.getSchema().getAliasMap().get("slot_Shape"), 

176 outputTable.getSchema().getAliasMap().get("slot_Shape"), 

177 ) 

178 self.assertFloatsAlmostEqual( 

179 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6 

180 ) 

181 self.assertFloatsAlmostEqual( 

182 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6 

183 ) 

184 self.assertFloatsAlmostEqual( 

185 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6 

186 ) 

187 

188 def runFundamentalTypeTest(self, datasetTypeName, entity): 

189 """Put and get the supplied entity and compare.""" 

190 ref = self.butler.put(entity, datasetTypeName) 

191 butler_ps = self.butler.get(ref) 

192 self.assertEqual(butler_ps, entity) 

193 

194 # Break the contact by ensuring that we are writing YAML 

195 uri = self.butler.getURI(ref) 

196 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}") 

197 

198 def testFundamentalTypes(self) -> None: 

199 """Ensure that some fundamental stack types round trip.""" 

200 ps = PropertySet() 

201 ps["a.b"] = 5 

202 ps["c.d.e"] = "string" 

203 self.runFundamentalTypeTest("ps", ps) 

204 

205 pl = PropertyList() 

206 pl["A"] = 1 

207 pl.setComment("A", "An int comment") 

208 pl["B"] = "string" 

209 pl.setComment("B", "A string comment") 

210 self.runFundamentalTypeTest("pl", pl) 

211 

212 def testFitsCatalog(self) -> None: 

213 """Test reading of a FITS catalog""" 

214 catalog = self.makeExampleCatalog() 

215 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

216 ref = self.butler.put(catalog, "testCatalog", dataId) 

217 stored = self.butler.get(ref) 

218 self.assertCatalogEqual(catalog, stored) 

219 

220 def testExposureCompositePutGetConcrete(self) -> None: 

221 """Test composite with no disassembly""" 

222 ref = self.runExposureCompositePutGetTest("calexp") 

223 

224 uri = self.butler.getURI(ref) 

225 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

226 

227 def testExposureCompositePutGetVirtual(self) -> None: 

228 """Testing composite disassembly""" 

229 ref = self.runExposureCompositePutGetTest("unknown") 

230 

231 primary, components = self.butler.getURIs(ref) 

232 self.assertIsNone(primary) 

233 self.assertEqual(set(components), COMPONENTS) 

234 for compName, uri in components.items(): 

235 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}") 

236 

237 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

238 example = os.path.join(TESTDIR, "data", "calexp.fits") 

239 exposure = lsst.afw.image.ExposureF(example) 

240 

241 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

242 ref = self.butler.put(exposure, datasetTypeName, dataId) 

243 

244 # Get the full thing 

245 composite = self.butler.get(datasetTypeName, dataId) 

246 

247 # There is no assert for Exposure so just look at maskedImage 

248 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

249 

250 # Helper for extracting components 

251 assembler = ExposureAssembler(ref.datasetType.storageClass) 

252 

253 # Check all possible components that can be read 

254 allComponents = set() 

255 allComponents.update(COMPONENTS, READ_COMPONENTS) 

256 

257 # Get each component from butler independently 

258 for compName in allComponents: 

259 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

260 component = self.butler.get(compTypeName, dataId) 

261 

262 reference = assembler.getComponent(exposure, compName) 

263 

264 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

265 

266 if compName in ("image", "variance"): 

267 self.assertImagesEqual(component, reference) 

268 elif compName == "mask": 

269 self.assertMasksEqual(component, reference) 

270 elif compName == "wcs": 

271 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

272 elif compName == "coaddInputs": 

273 self.assertEqual( 

274 len(component.visits), len(reference.visits), f"cf visits {component.visits}" 

275 ) 

276 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}") 

277 elif compName == "psf": 

278 # Equality for PSF does not work 

279 pass 

280 elif compName == "filter": 

281 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

282 elif compName == "filterLabel": 

283 self.assertEqual(component, reference) 

284 elif compName == "id": 

285 self.assertEqual(component, reference) 

286 elif compName == "visitInfo": 

287 self.assertEqual(component, reference, "VisitInfo comparison") 

288 elif compName == "metadata": 

289 # The component metadata has extra fields in it so cannot 

290 # compare directly. 

291 for k, v in reference.items(): 

292 self.assertEqual(component[k], v) 

293 elif compName == "photoCalib": 

294 # This example has a 

295 # "spatially constant with mean: inf error: nan" entry 

296 # which does not compare directly. 

297 self.assertEqual(str(component), str(reference)) 

298 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib") 

299 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

300 self.assertEqual(component, reference) 

301 elif compName == "apCorrMap": 

302 self.assertEqual(set(component.keys()), set(reference.keys())) 

303 elif compName == "transmissionCurve": 

304 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds()) 

305 elif compName == "detector": 

306 c_amps = {a.getName() for a in component.getAmplifiers()} 

307 r_amps = {a.getName() for a in reference.getAmplifiers()} 

308 self.assertEqual(c_amps, r_amps) 

309 elif compName == "summaryStats": 

310 self.assertEqual(component.psfSigma, reference.psfSigma) 

311 else: 

312 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

313 

314 # Full Exposure with parameters 

315 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

316 parameters = dict(bbox=inBBox, origin=LOCAL) 

317 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

318 outBBox = subset.getBBox() 

319 self.assertEqual(inBBox, outBBox) 

320 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

321 

322 return ref 

323 

324 def putFits(self, exposure, datasetTypeName, visit): 

325 """Put different datasetTypes and return information.""" 

326 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

327 refC = self.butler.put(exposure, datasetTypeName, dataId) 

328 uriC = self.butler.getURI(refC) 

329 stat = os.stat(uriC.ospath) 

330 size = stat.st_size 

331 # We can't use butler's Exposure storage class metadata component, 

332 # because that intentionally strips keywords that science code 

333 # shouldn't ever read (to at least weaken our assumptions that we write 

334 # to FITS). Instead use a lower-level method on the URI for this test. 

335 meta = readMetadata(uriC.ospath) 

336 return meta, size 

337 

338 def testCompression(self): 

339 """Test that we can write compressed and uncompressed FITS.""" 

340 example = os.path.join(TESTDIR, "data", "small.fits") 

341 exposure = lsst.afw.image.ExposureF(example) 

342 

343 # Write a lossless compressed 

344 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

345 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

346 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

347 

348 # Write an uncompressed FITS file 

349 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

350 self.assertNotIn("ZCMPTYPE", metaN) 

351 

352 # Write an uncompressed FITS file 

353 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

354 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

355 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

356 

357 self.assertNotEqual(sizeC, sizeN) 

358 # Data file is so small that Lossy and Compressed are dominated 

359 # by the extra compression tables 

360 self.assertEqual(sizeL, sizeC) 

361 

362 def testExposureFormatterAmpParameter(self): 

363 """Test the FitsExposureFormatter implementation of the Exposure 

364 StorageClass's 'amp' and 'detector' parameters. 

365 """ 

366 # Our example exposure file has a realistic detector (looks like HSC), 

367 # but the image itself doesn't match it. So we just load the detector, 

368 # and use it to make our own more useful images and put them. 

369 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

370 trimmed_full = make_ramp_exposure_trimmed(detector) 

371 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

372 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

373 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

374 for n, amp in enumerate(detector): 

375 # Try to read each amp as it is on disk, with a variety of 

376 # parameters that should all do the same thing. 

377 for amp_parameter in [amp, amp.getName(), n]: 

378 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

379 with self.subTest(parameters=parameters): 

380 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters) 

381 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters) 

382 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

383 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

384 self.assertEqual(len(test_trimmed.getDetector()), 1) 

385 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

386 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

387 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

388 # Try to read various transformed versions of the original amp, 

389 # to make sure flips and offsets are applied correctly. 

390 # First flip X only. 

391 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

392 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1}) 

393 self.assertImagesEqual( 

394 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False) 

395 ) 

396 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

397 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1}) 

398 self.assertImagesEqual( 

399 test_t1_untrimmed.image, 

400 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False), 

401 ) 

402 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

403 # Flip Y only. 

404 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

405 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2}) 

406 self.assertImagesEqual( 

407 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True) 

408 ) 

409 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

410 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2}) 

411 self.assertImagesEqual( 

412 test_t2_untrimmed.image, 

413 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True), 

414 ) 

415 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

416 # Add an XY offset only. 

417 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

418 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3}) 

419 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image) 

420 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

421 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3}) 

422 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

423 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

424 

425 

426if __name__ == "__main__": 426 ↛ 427line 426 didn't jump to line 427, because the condition on line 426 was never true

427 unittest.main()