Coverage for tests/test_butlerFits.py: 18%

234 statements  

« prev     ^ index     » next       coverage.py v7.2.1, created at 2023-03-12 01:53 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.pex.config 

34import lsst.afw.image 

35from lsst.afw.image import ExposureFitsReader, LOCAL 

36from lsst.afw.fits import readMetadata 

37from lsst.afw.math import flipImage 

38import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

39from lsst.geom import Box2I, Extent2I, Point2I 

40from lsst.base import Packages 

41from lsst.daf.base import PropertyList, PropertySet 

42 

43from lsst.daf.butler import Config 

44from lsst.daf.butler import StorageClassFactory 

45from lsst.daf.butler import DatasetType 

46from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

47 

48from lsst.obs.base.exposureAssembler import ExposureAssembler 

49from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

50 

51if TYPE_CHECKING: 51 ↛ 52line 51 didn't jump to line 52, because the condition on line 51 was never true

52 from lsst.daf.butler import DatasetRef 

53 

54TESTDIR = os.path.dirname(__file__) 

55 

56BUTLER_CONFIG = """ 

57storageClasses: 

58 ExposureCompositeF: 

59 inheritsFrom: ExposureF 

60datastore: 

61 # Want to check disassembly so can't use InMemory 

62 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

63 formatters: 

64 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

65 lossless: 

66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

67 parameters: 

68 recipe: lossless 

69 uncompressed: 

70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

71 parameters: 

72 recipe: noCompression 

73 lossy: 

74 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

75 parameters: 

76 recipe: lossyBasic 

77 composites: 

78 disassembled: 

79 ExposureCompositeF: True 

80""" 

81 

82# Components present in the test file 

83COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib", 

84 "filterLabel", "validPolygon", "transmissionCurve", "detector", "apCorrMap", "summaryStats"} 

85READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"} 

86 

87 

88class SimpleConfig(lsst.pex.config.Config): 

89 """Config to use in tests for butler put/get""" 

90 i = lsst.pex.config.Field("integer test", int) 

91 c = lsst.pex.config.Field("string", str) 

92 

93 

94class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

95 

96 @classmethod 

97 def setUpClass(cls): 

98 """Create a new butler once only.""" 

99 

100 cls.storageClassFactory = StorageClassFactory() 

101 

102 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

103 

104 dataIds = { 

105 "instrument": ["DummyCam"], 

106 "physical_filter": ["d-r"], 

107 "visit": [42, 43, 44], 

108 } 

109 

110 # Ensure that we test in a directory that will include some 

111 # metacharacters 

112 subdir = "sub?#dir" 

113 butlerRoot = os.path.join(cls.root, subdir) 

114 

115 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

116 

117 # Create dataset types used by the tests 

118 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

119 ("unknown", "ExposureCompositeF"), 

120 ("testCatalog", "SourceCatalog"), 

121 ("lossless", "ExposureF"), 

122 ("uncompressed", "ExposureF"), 

123 ("lossy", "ExposureF"), 

124 ): 

125 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

126 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

127 

128 # And some dataset types that have no dimensions for easy testing 

129 for datasetTypeName, storageClassName in (("ps", "PropertySet"), 

130 ("pl", "PropertyList"), 

131 ("pkg", "Packages"), 

132 ("config", "Config"), 

133 ("int_exp_trimmed", "ExposureI"), 

134 ("int_exp_untrimmed", "ExposureI"), 

135 ): 

136 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

137 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

138 

139 @classmethod 

140 def tearDownClass(cls): 

141 if cls.root is not None: 

142 shutil.rmtree(cls.root, ignore_errors=True) 

143 

144 def setUp(self): 

145 self.butler = makeTestCollection(self.creatorButler) 

146 

147 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

148 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

149 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

150 

151 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

152 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

153 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

154 inputTable = inputCatalog.getTable() 

155 inputRecord = inputCatalog[0] 

156 outputTable = outputCatalog.getTable() 

157 outputRecord = outputCatalog[0] 

158 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

159 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

160 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

161 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

162 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

163 self.assertFloatsAlmostEqual( 

164 inputRecord.getCentroidErr()[0, 0], 

165 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

166 self.assertFloatsAlmostEqual( 

167 inputRecord.getCentroidErr()[1, 1], 

168 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

169 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

170 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

171 self.assertFloatsAlmostEqual( 

172 inputRecord.getShapeErr()[0, 0], 

173 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

174 self.assertFloatsAlmostEqual( 

175 inputRecord.getShapeErr()[1, 1], 

176 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

177 self.assertFloatsAlmostEqual( 

178 inputRecord.getShapeErr()[2, 2], 

179 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

180 

181 def runFundamentalTypeTest(self, datasetTypeName, entity): 

182 """Put and get the supplied entity and compare.""" 

183 ref = self.butler.put(entity, datasetTypeName) 

184 butler_ps = self.butler.get(ref) 

185 self.assertEqual(butler_ps, entity) 

186 

187 # Break the contact by ensuring that we are writing YAML 

188 uri = self.butler.getURI(ref) 

189 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}") 

190 

191 def testFundamentalTypes(self) -> None: 

192 """Ensure that some fundamental stack types round trip.""" 

193 ps = PropertySet() 

194 ps["a.b"] = 5 

195 ps["c.d.e"] = "string" 

196 self.runFundamentalTypeTest("ps", ps) 

197 

198 pl = PropertyList() 

199 pl["A"] = 1 

200 pl.setComment("A", "An int comment") 

201 pl["B"] = "string" 

202 pl.setComment("B", "A string comment") 

203 self.runFundamentalTypeTest("pl", pl) 

204 

205 pkg = Packages.fromSystem() 

206 self.runFundamentalTypeTest("pkg", pkg) 

207 

208 def testPexConfig(self) -> None: 

209 """Test that we can put and get pex_config Configs""" 

210 c = SimpleConfig(i=10, c="hello") 

211 self.assertEqual(c.i, 10) 

212 ref = self.butler.put(c, "config") 

213 butler_c = self.butler.get(ref) 

214 self.assertEqual(c, butler_c) 

215 self.assertIsInstance(butler_c, SimpleConfig) 

216 

217 def testFitsCatalog(self) -> None: 

218 """Test reading of a FITS catalog""" 

219 catalog = self.makeExampleCatalog() 

220 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

221 ref = self.butler.put(catalog, "testCatalog", dataId) 

222 stored = self.butler.get(ref) 

223 self.assertCatalogEqual(catalog, stored) 

224 

225 def testExposureCompositePutGetConcrete(self) -> None: 

226 """Test composite with no disassembly""" 

227 ref = self.runExposureCompositePutGetTest("calexp") 

228 

229 uri = self.butler.getURI(ref) 

230 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

231 

232 def testExposureCompositePutGetVirtual(self) -> None: 

233 """Testing composite disassembly""" 

234 ref = self.runExposureCompositePutGetTest("unknown") 

235 

236 primary, components = self.butler.getURIs(ref) 

237 self.assertIsNone(primary) 

238 self.assertEqual(set(components), COMPONENTS) 

239 for compName, uri in components.items(): 

240 self.assertTrue(uri.exists(), 

241 f"Checking URI {uri} existence for component {compName}") 

242 

243 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

244 example = os.path.join(TESTDIR, "data", "calexp.fits") 

245 exposure = lsst.afw.image.ExposureF(example) 

246 

247 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

248 ref = self.butler.put(exposure, datasetTypeName, dataId) 

249 

250 # Get the full thing 

251 composite = self.butler.get(datasetTypeName, dataId) 

252 

253 # There is no assert for Exposure so just look at maskedImage 

254 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

255 

256 # Helper for extracting components 

257 assembler = ExposureAssembler(ref.datasetType.storageClass) 

258 

259 # Check all possible components that can be read 

260 allComponents = set() 

261 allComponents.update(COMPONENTS, READ_COMPONENTS) 

262 

263 # Get each component from butler independently 

264 for compName in allComponents: 

265 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

266 component = self.butler.get(compTypeName, dataId) 

267 

268 reference = assembler.getComponent(exposure, compName) 

269 

270 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

271 

272 if compName in ("image", "variance"): 

273 self.assertImagesEqual(component, reference) 

274 elif compName == "mask": 

275 self.assertMasksEqual(component, reference) 

276 elif compName == "wcs": 

277 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

278 elif compName == "coaddInputs": 

279 self.assertEqual(len(component.visits), len(reference.visits), 

280 f"cf visits {component.visits}") 

281 self.assertEqual(len(component.ccds), len(reference.ccds), 

282 f"cf CCDs {component.ccds}") 

283 elif compName == "psf": 

284 # Equality for PSF does not work 

285 pass 

286 elif compName == "filter": 

287 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

288 elif compName == "filterLabel": 

289 self.assertEqual(component, reference) 

290 elif compName == "visitInfo": 

291 self.assertEqual(component.getExposureId(), reference.getExposureId(), 

292 "VisitInfo comparison") 

293 elif compName == "metadata": 

294 # The component metadata has extra fields in it so cannot 

295 # compare directly. 

296 for k, v in reference.items(): 

297 self.assertEqual(component[k], v) 

298 elif compName == "photoCalib": 

299 # This example has a 

300 # "spatially constant with mean: inf error: nan" entry 

301 # which does not compare directly. 

302 self.assertEqual(str(component), str(reference)) 

303 self.assertIn("spatially constant with mean: 1.99409", str(component), 

304 "Checking photoCalib") 

305 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

306 self.assertEqual(component, reference) 

307 elif compName == "apCorrMap": 

308 self.assertEqual(set(component.keys()), set(reference.keys())) 

309 elif compName == "transmissionCurve": 

310 self.assertEqual(component.getThroughputAtBounds(), 

311 reference.getThroughputAtBounds()) 

312 elif compName == "detector": 

313 c_amps = {a.getName() for a in component.getAmplifiers()} 

314 r_amps = {a.getName() for a in reference.getAmplifiers()} 

315 self.assertEqual(c_amps, r_amps) 

316 elif compName == 'summaryStats': 

317 self.assertEqual(component.psfSigma, reference.psfSigma) 

318 else: 

319 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

320 

321 # Full Exposure with parameters 

322 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

323 parameters = dict(bbox=inBBox, origin=LOCAL) 

324 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

325 outBBox = subset.getBBox() 

326 self.assertEqual(inBBox, outBBox) 

327 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

328 

329 return ref 

330 

331 def putFits(self, exposure, datasetTypeName, visit): 

332 """Put different datasetTypes and return information.""" 

333 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

334 refC = self.butler.put(exposure, datasetTypeName, dataId) 

335 uriC = self.butler.getURI(refC) 

336 stat = os.stat(uriC.ospath) 

337 size = stat.st_size 

338 # We can't use butler's Exposure storage class metadata component, 

339 # because that intentionally strips keywords that science code 

340 # shouldn't ever read (to at least weaken our assumptions that we write 

341 # to FITS). Instead use a lower-level method on the URI for this test. 

342 meta = readMetadata(uriC.ospath) 

343 return meta, size 

344 

345 def testCompression(self): 

346 """Test that we can write compressed and uncompressed FITS.""" 

347 example = os.path.join(TESTDIR, "data", "small.fits") 

348 exposure = lsst.afw.image.ExposureF(example) 

349 

350 # Write a lossless compressed 

351 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

352 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

353 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

354 

355 # Write an uncompressed FITS file 

356 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

357 self.assertNotIn("ZCMPTYPE", metaN) 

358 

359 # Write an uncompressed FITS file 

360 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

361 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

362 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

363 

364 self.assertNotEqual(sizeC, sizeN) 

365 # Data file is so small that Lossy and Compressed are dominated 

366 # by the extra compression tables 

367 self.assertEqual(sizeL, sizeC) 

368 

369 def testExposureFormatterAmpParameter(self): 

370 """Test the FitsExposureFormatter implementation of the Exposure 

371 StorageClass's 'amp' and 'detector' parameters. 

372 """ 

373 # Our example exposure file has a realistic detector (looks like HSC), 

374 # but the image itself doesn't match it. So we just load the detector, 

375 # and use it to make our own more useful images and put them. 

376 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

377 trimmed_full = make_ramp_exposure_trimmed(detector) 

378 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

379 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

380 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

381 for n, amp in enumerate(detector): 

382 # Try to read each amp as it is on disk, with a variety of 

383 # parameters that should all do the same thing. 

384 for amp_parameter in [amp, amp.getName(), n]: 

385 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

386 with self.subTest(parameters=parameters): 

387 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters) 

388 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters) 

389 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

390 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

391 self.assertEqual(len(test_trimmed.getDetector()), 1) 

392 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

393 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

394 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

395 # Try to read various transformed versions of the original amp, 

396 # to make sure flips and offsets are applied correctly. 

397 # First flip X only. 

398 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

399 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1}) 

400 self.assertImagesEqual(test_t1_trimmed.image, 

401 flipImage(trimmed_full[amp.getBBox()].image, 

402 flipLR=True, flipTB=False)) 

403 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

404 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1}) 

405 self.assertImagesEqual(test_t1_untrimmed.image, 

406 flipImage(untrimmed_full[amp.getRawBBox()].image, 

407 flipLR=True, flipTB=False)) 

408 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

409 # Flip Y only. 

410 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

411 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2}) 

412 self.assertImagesEqual(test_t2_trimmed.image, 

413 flipImage(trimmed_full[amp.getBBox()].image, 

414 flipLR=False, flipTB=True)) 

415 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

416 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2}) 

417 self.assertImagesEqual(test_t2_untrimmed.image, 

418 flipImage(untrimmed_full[amp.getRawBBox()].image, 

419 flipLR=False, flipTB=True)) 

420 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

421 # Add an XY offset only. 

422 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

423 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3}) 

424 self.assertImagesEqual(test_t3_trimmed.image, 

425 trimmed_full[amp.getBBox()].image) 

426 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

427 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3}) 

428 self.assertImagesEqual(test_t3_untrimmed.image, 

429 untrimmed_full[amp.getRawBBox()].image) 

430 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

431 

432 

433if __name__ == "__main__": 433 ↛ 434line 433 didn't jump to line 434, because the condition on line 433 was never true

434 unittest.main()