Coverage for tests/test_butlerFits.py: 16%

221 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-24 02:02 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import shutil 

26import tempfile 

27import unittest 

28from typing import TYPE_CHECKING 

29 

30import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

31import lsst.afw.image 

32import lsst.pex.config 

33import lsst.utils.tests 

34from lsst.afw.fits import readMetadata 

35from lsst.afw.image import LOCAL, ExposureFitsReader 

36from lsst.afw.math import flipImage 

37from lsst.daf.base import PropertyList, PropertySet 

38from lsst.daf.butler import Config, DatasetType, StorageClassFactory 

39from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo 

40from lsst.geom import Box2I, Extent2I, Point2I 

41from lsst.obs.base.exposureAssembler import ExposureAssembler 

42from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

43 

44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from lsst.daf.butler import DatasetRef 

46 

47TESTDIR = os.path.dirname(__file__) 

48 

49BUTLER_CONFIG = """ 

50storageClasses: 

51 ExposureCompositeF: 

52 inheritsFrom: ExposureF 

53datastore: 

54 # Want to check disassembly so can't use InMemory 

55 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

56 formatters: 

57 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

58 lossless: 

59 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

60 parameters: 

61 recipe: lossless 

62 uncompressed: 

63 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

64 parameters: 

65 recipe: noCompression 

66 lossy: 

67 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

68 parameters: 

69 recipe: lossyBasic 

70 composites: 

71 disassembled: 

72 ExposureCompositeF: True 

73""" 

74 

75# Components present in the test file 

76COMPONENTS = { 

77 "wcs", 

78 "image", 

79 "mask", 

80 "coaddInputs", 

81 "psf", 

82 "visitInfo", 

83 "variance", 

84 "metadata", 

85 "photoCalib", 

86 "filter", 

87 "validPolygon", 

88 "transmissionCurve", 

89 "detector", 

90 "apCorrMap", 

91 "summaryStats", 

92 "id", 

93} 

94READ_COMPONENTS = { 

95 "bbox", 

96 "xy0", 

97 "dimensions", 

98 "filterLabel", 

99} 

100 

101 

102class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

103 @classmethod 

104 def setUpClass(cls): 

105 """Create a new butler once only.""" 

106 

107 cls.storageClassFactory = StorageClassFactory() 

108 

109 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

110 

111 dataIds = { 

112 "instrument": ["DummyCam"], 

113 "physical_filter": ["d-r"], 

114 "visit": [42, 43, 44], 

115 } 

116 

117 # Ensure that we test in a directory that will include some 

118 # metacharacters 

119 subdir = "sub?#dir" 

120 butlerRoot = os.path.join(cls.root, subdir) 

121 

122 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

123 

124 # Create dataset types used by the tests 

125 for datasetTypeName, storageClassName in ( 

126 ("calexp", "ExposureF"), 

127 ("unknown", "ExposureCompositeF"), 

128 ("testCatalog", "SourceCatalog"), 

129 ("lossless", "ExposureF"), 

130 ("uncompressed", "ExposureF"), 

131 ("lossy", "ExposureF"), 

132 ): 

133 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

134 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

135 

136 # And some dataset types that have no dimensions for easy testing 

137 for datasetTypeName, storageClassName in ( 

138 ("ps", "PropertySet"), 

139 ("pl", "PropertyList"), 

140 ("int_exp_trimmed", "ExposureI"), 

141 ("int_exp_untrimmed", "ExposureI"), 

142 ): 

143 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

144 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

145 

146 @classmethod 

147 def tearDownClass(cls): 

148 if cls.root is not None: 

149 shutil.rmtree(cls.root, ignore_errors=True) 

150 

151 def setUp(self): 

152 self.butler = makeTestCollection(self.creatorButler) 

153 

154 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

155 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

156 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

157 

158 def assertCatalogEqual( 

159 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog 

160 ) -> None: 

161 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

162 inputTable = inputCatalog.getTable() 

163 inputRecord = inputCatalog[0] 

164 outputTable = outputCatalog.getTable() 

165 outputRecord = outputCatalog[0] 

166 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

167 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

168 self.assertEqual( 

169 inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

170 outputTable.getSchema().getAliasMap().get("slot_Centroid"), 

171 ) 

172 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

173 self.assertFloatsAlmostEqual( 

174 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6 

175 ) 

176 self.assertFloatsAlmostEqual( 

177 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6 

178 ) 

179 self.assertEqual( 

180 inputTable.getSchema().getAliasMap().get("slot_Shape"), 

181 outputTable.getSchema().getAliasMap().get("slot_Shape"), 

182 ) 

183 self.assertFloatsAlmostEqual( 

184 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6 

185 ) 

186 self.assertFloatsAlmostEqual( 

187 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6 

188 ) 

189 self.assertFloatsAlmostEqual( 

190 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6 

191 ) 

192 

193 def runFundamentalTypeTest(self, datasetTypeName, entity): 

194 """Put and get the supplied entity and compare.""" 

195 ref = self.butler.put(entity, datasetTypeName) 

196 butler_ps = self.butler.get(ref) 

197 self.assertEqual(butler_ps, entity) 

198 

199 # Break the contact by ensuring that we are writing YAML 

200 uri = self.butler.getURI(ref) 

201 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}") 

202 

203 def testFundamentalTypes(self) -> None: 

204 """Ensure that some fundamental stack types round trip.""" 

205 ps = PropertySet() 

206 ps["a.b"] = 5 

207 ps["c.d.e"] = "string" 

208 self.runFundamentalTypeTest("ps", ps) 

209 

210 pl = PropertyList() 

211 pl["A"] = 1 

212 pl.setComment("A", "An int comment") 

213 pl["B"] = "string" 

214 pl.setComment("B", "A string comment") 

215 self.runFundamentalTypeTest("pl", pl) 

216 

217 def testFitsCatalog(self) -> None: 

218 """Test reading of a FITS catalog""" 

219 catalog = self.makeExampleCatalog() 

220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

221 ref = self.butler.put(catalog, "testCatalog", dataId) 

222 stored = self.butler.get(ref) 

223 self.assertCatalogEqual(catalog, stored) 

224 

225 def testExposureCompositePutGetConcrete(self) -> None: 

226 """Test composite with no disassembly""" 

227 ref = self.runExposureCompositePutGetTest("calexp") 

228 

229 uri = self.butler.getURI(ref) 

230 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

231 

232 def testExposureCompositePutGetVirtual(self) -> None: 

233 """Testing composite disassembly""" 

234 ref = self.runExposureCompositePutGetTest("unknown") 

235 

236 primary, components = self.butler.getURIs(ref) 

237 self.assertIsNone(primary) 

238 self.assertEqual(set(components), COMPONENTS) 

239 for compName, uri in components.items(): 

240 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}") 

241 

242 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

243 example = os.path.join(TESTDIR, "data", "calexp.fits") 

244 exposure = lsst.afw.image.ExposureF(example) 

245 

246 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

247 ref = self.butler.put(exposure, datasetTypeName, dataId) 

248 

249 # Get the full thing 

250 composite = self.butler.get(datasetTypeName, dataId) 

251 

252 # There is no assert for Exposure so just look at maskedImage 

253 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

254 

255 # Helper for extracting components 

256 assembler = ExposureAssembler(ref.datasetType.storageClass) 

257 

258 # Check all possible components that can be read 

259 allComponents = set() 

260 allComponents.update(COMPONENTS, READ_COMPONENTS) 

261 

262 # Get each component from butler independently 

263 for compName in allComponents: 

264 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

265 component = self.butler.get(compTypeName, dataId) 

266 

267 reference = assembler.getComponent(exposure, compName) 

268 

269 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

270 

271 if compName in ("image", "variance"): 

272 self.assertImagesEqual(component, reference) 

273 elif compName == "mask": 

274 self.assertMasksEqual(component, reference) 

275 elif compName == "wcs": 

276 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

277 elif compName == "coaddInputs": 

278 self.assertEqual( 

279 len(component.visits), len(reference.visits), f"cf visits {component.visits}" 

280 ) 

281 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}") 

282 elif compName == "psf": 

283 # Equality for PSF does not work 

284 pass 

285 elif compName == "filter": 

286 self.assertEqual(component, reference) 

287 elif compName == "filterLabel": 

288 self.assertEqual(component, reference) 

289 elif compName == "id": 

290 self.assertEqual(component, reference) 

291 elif compName == "visitInfo": 

292 self.assertEqual(component, reference, "VisitInfo comparison") 

293 elif compName == "metadata": 

294 # The component metadata has extra fields in it so cannot 

295 # compare directly. 

296 for k, v in reference.items(): 

297 self.assertEqual(component[k], v) 

298 elif compName == "photoCalib": 

299 # This example has a 

300 # "spatially constant with mean: inf error: nan" entry 

301 # which does not compare directly. 

302 self.assertEqual(str(component), str(reference)) 

303 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib") 

304 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

305 self.assertEqual(component, reference) 

306 elif compName == "apCorrMap": 

307 self.assertEqual(set(component.keys()), set(reference.keys())) 

308 elif compName == "transmissionCurve": 

309 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds()) 

310 elif compName == "detector": 

311 c_amps = {a.getName() for a in component.getAmplifiers()} 

312 r_amps = {a.getName() for a in reference.getAmplifiers()} 

313 self.assertEqual(c_amps, r_amps) 

314 elif compName == "summaryStats": 

315 self.assertEqual(component.psfSigma, reference.psfSigma) 

316 else: 

317 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

318 

319 # Full Exposure with parameters 

320 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

321 parameters = dict(bbox=inBBox, origin=LOCAL) 

322 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

323 outBBox = subset.getBBox() 

324 self.assertEqual(inBBox, outBBox) 

325 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

326 

327 return ref 

328 

329 def putFits(self, exposure, datasetTypeName, visit): 

330 """Put different datasetTypes and return information.""" 

331 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

332 refC = self.butler.put(exposure, datasetTypeName, dataId) 

333 uriC = self.butler.getURI(refC) 

334 stat = os.stat(uriC.ospath) 

335 size = stat.st_size 

336 # We can't use butler's Exposure storage class metadata component, 

337 # because that intentionally strips keywords that science code 

338 # shouldn't ever read (to at least weaken our assumptions that we write 

339 # to FITS). Instead use a lower-level method on the URI for this test. 

340 meta = readMetadata(uriC.ospath) 

341 return meta, size 

342 

343 def testCompression(self): 

344 """Test that we can write compressed and uncompressed FITS.""" 

345 example = os.path.join(TESTDIR, "data", "small.fits") 

346 exposure = lsst.afw.image.ExposureF(example) 

347 

348 # Write a lossless compressed 

349 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

350 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

351 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

352 

353 # Write an uncompressed FITS file 

354 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

355 self.assertNotIn("ZCMPTYPE", metaN) 

356 

357 # Write an uncompressed FITS file 

358 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

359 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

360 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

361 

362 self.assertNotEqual(sizeC, sizeN) 

363 # Data file is so small that Lossy and Compressed are dominated 

364 # by the extra compression tables 

365 self.assertEqual(sizeL, sizeC) 

366 

367 def testExposureFormatterAmpParameter(self): 

368 """Test the FitsExposureFormatter implementation of the Exposure 

369 StorageClass's 'amp' and 'detector' parameters. 

370 """ 

371 # Our example exposure file has a realistic detector (looks like HSC), 

372 # but the image itself doesn't match it. So we just load the detector, 

373 # and use it to make our own more useful images and put them. 

374 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

375 trimmed_full = make_ramp_exposure_trimmed(detector) 

376 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

377 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

378 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

379 for n, amp in enumerate(detector): 

380 # Try to read each amp as it is on disk, with a variety of 

381 # parameters that should all do the same thing. 

382 for amp_parameter in [amp, amp.getName(), n]: 

383 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

384 with self.subTest(parameters=parameters): 

385 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters) 

386 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters) 

387 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

388 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

389 self.assertEqual(len(test_trimmed.getDetector()), 1) 

390 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

391 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

392 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

393 # Try to read various transformed versions of the original amp, 

394 # to make sure flips and offsets are applied correctly. 

395 # First flip X only. 

396 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

397 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1}) 

398 self.assertImagesEqual( 

399 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False) 

400 ) 

401 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

402 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1}) 

403 self.assertImagesEqual( 

404 test_t1_untrimmed.image, 

405 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False), 

406 ) 

407 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

408 # Flip Y only. 

409 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

410 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2}) 

411 self.assertImagesEqual( 

412 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True) 

413 ) 

414 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

415 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2}) 

416 self.assertImagesEqual( 

417 test_t2_untrimmed.image, 

418 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True), 

419 ) 

420 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

421 # Add an XY offset only. 

422 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

423 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3}) 

424 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image) 

425 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

426 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3}) 

427 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

428 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

429 

430 

431if __name__ == "__main__": 431 ↛ 432line 431 didn't jump to line 432, because the condition on line 431 was never true

432 unittest.main()