Coverage for tests/test_butlerFits.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

236 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.pex.config 

34import lsst.afw.image 

35from lsst.afw.image import ExposureFitsReader, LOCAL 

36from lsst.afw.fits import readMetadata 

37from lsst.afw.math import flipImage 

38import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

39from lsst.geom import Box2I, Extent2I, Point2I 

40from lsst.base import Packages 

41from lsst.daf.base import PropertyList, PropertySet 

42 

43from lsst.daf.butler import Config 

44from lsst.daf.butler import StorageClassFactory 

45from lsst.daf.butler import DatasetType 

46from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

47 

48from lsst.obs.base.exposureAssembler import ExposureAssembler 

49from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

50 

51if TYPE_CHECKING: 51 ↛ 52line 51 didn't jump to line 52, because the condition on line 51 was never true

52 from lsst.daf.butler import DatasetRef 

53 

54TESTDIR = os.path.dirname(__file__) 

55 

56BUTLER_CONFIG = """ 

57storageClasses: 

58 ExposureCompositeF: 

59 inheritsFrom: ExposureF 

60datastore: 

61 # Want to check disassembly so can't use InMemory 

62 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

63 formatters: 

64 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

65 lossless: 

66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

67 parameters: 

68 recipe: lossless 

69 uncompressed: 

70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

71 parameters: 

72 recipe: noCompression 

73 lossy: 

74 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

75 parameters: 

76 recipe: lossyBasic 

77 composites: 

78 disassembled: 

79 ExposureCompositeF: True 

80""" 

81 

82# Components present in the test file 

83COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib", 

84 "filterLabel", "validPolygon", "transmissionCurve", "detector", "apCorrMap", "summaryStats", 

85 "id", 

86 } 

87READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"} 

88 

89 

90class SimpleConfig(lsst.pex.config.Config): 

91 """Config to use in tests for butler put/get""" 

92 i = lsst.pex.config.Field("integer test", int) 

93 c = lsst.pex.config.Field("string", str) 

94 

95 

96class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

97 

98 @classmethod 

99 def setUpClass(cls): 

100 """Create a new butler once only.""" 

101 

102 cls.storageClassFactory = StorageClassFactory() 

103 

104 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

105 

106 dataIds = { 

107 "instrument": ["DummyCam"], 

108 "physical_filter": ["d-r"], 

109 "visit": [42, 43, 44], 

110 } 

111 

112 # Ensure that we test in a directory that will include some 

113 # metacharacters 

114 subdir = "sub?#dir" 

115 butlerRoot = os.path.join(cls.root, subdir) 

116 

117 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

118 

119 # Create dataset types used by the tests 

120 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

121 ("unknown", "ExposureCompositeF"), 

122 ("testCatalog", "SourceCatalog"), 

123 ("lossless", "ExposureF"), 

124 ("uncompressed", "ExposureF"), 

125 ("lossy", "ExposureF"), 

126 ): 

127 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

128 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

129 

130 # And some dataset types that have no dimensions for easy testing 

131 for datasetTypeName, storageClassName in (("ps", "PropertySet"), 

132 ("pl", "PropertyList"), 

133 ("pkg", "Packages"), 

134 ("config", "Config"), 

135 ("int_exp_trimmed", "ExposureI"), 

136 ("int_exp_untrimmed", "ExposureI"), 

137 ): 

138 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

139 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

140 

141 @classmethod 

142 def tearDownClass(cls): 

143 if cls.root is not None: 

144 shutil.rmtree(cls.root, ignore_errors=True) 

145 

146 def setUp(self): 

147 self.butler = makeTestCollection(self.creatorButler) 

148 

149 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

150 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

151 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

152 

153 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

154 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

155 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

156 inputTable = inputCatalog.getTable() 

157 inputRecord = inputCatalog[0] 

158 outputTable = outputCatalog.getTable() 

159 outputRecord = outputCatalog[0] 

160 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

161 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

162 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

163 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

164 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

165 self.assertFloatsAlmostEqual( 

166 inputRecord.getCentroidErr()[0, 0], 

167 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

168 self.assertFloatsAlmostEqual( 

169 inputRecord.getCentroidErr()[1, 1], 

170 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

171 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

172 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

173 self.assertFloatsAlmostEqual( 

174 inputRecord.getShapeErr()[0, 0], 

175 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

176 self.assertFloatsAlmostEqual( 

177 inputRecord.getShapeErr()[1, 1], 

178 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

179 self.assertFloatsAlmostEqual( 

180 inputRecord.getShapeErr()[2, 2], 

181 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

182 

183 def runFundamentalTypeTest(self, datasetTypeName, entity): 

184 """Put and get the supplied entity and compare.""" 

185 ref = self.butler.put(entity, datasetTypeName) 

186 butler_ps = self.butler.get(ref) 

187 self.assertEqual(butler_ps, entity) 

188 

189 # Break the contact by ensuring that we are writing YAML 

190 uri = self.butler.getURI(ref) 

191 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}") 

192 

193 def testFundamentalTypes(self) -> None: 

194 """Ensure that some fundamental stack types round trip.""" 

195 ps = PropertySet() 

196 ps["a.b"] = 5 

197 ps["c.d.e"] = "string" 

198 self.runFundamentalTypeTest("ps", ps) 

199 

200 pl = PropertyList() 

201 pl["A"] = 1 

202 pl.setComment("A", "An int comment") 

203 pl["B"] = "string" 

204 pl.setComment("B", "A string comment") 

205 self.runFundamentalTypeTest("pl", pl) 

206 

207 pkg = Packages.fromSystem() 

208 self.runFundamentalTypeTest("pkg", pkg) 

209 

210 def testPexConfig(self) -> None: 

211 """Test that we can put and get pex_config Configs""" 

212 c = SimpleConfig(i=10, c="hello") 

213 self.assertEqual(c.i, 10) 

214 ref = self.butler.put(c, "config") 

215 butler_c = self.butler.get(ref) 

216 self.assertEqual(c, butler_c) 

217 self.assertIsInstance(butler_c, SimpleConfig) 

218 

219 def testFitsCatalog(self) -> None: 

220 """Test reading of a FITS catalog""" 

221 catalog = self.makeExampleCatalog() 

222 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

223 ref = self.butler.put(catalog, "testCatalog", dataId) 

224 stored = self.butler.get(ref) 

225 self.assertCatalogEqual(catalog, stored) 

226 

227 def testExposureCompositePutGetConcrete(self) -> None: 

228 """Test composite with no disassembly""" 

229 ref = self.runExposureCompositePutGetTest("calexp") 

230 

231 uri = self.butler.getURI(ref) 

232 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

233 

234 def testExposureCompositePutGetVirtual(self) -> None: 

235 """Testing composite disassembly""" 

236 ref = self.runExposureCompositePutGetTest("unknown") 

237 

238 primary, components = self.butler.getURIs(ref) 

239 self.assertIsNone(primary) 

240 self.assertEqual(set(components), COMPONENTS) 

241 for compName, uri in components.items(): 

242 self.assertTrue(uri.exists(), 

243 f"Checking URI {uri} existence for component {compName}") 

244 

245 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

246 example = os.path.join(TESTDIR, "data", "calexp.fits") 

247 exposure = lsst.afw.image.ExposureF(example) 

248 

249 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

250 ref = self.butler.put(exposure, datasetTypeName, dataId) 

251 

252 # Get the full thing 

253 composite = self.butler.get(datasetTypeName, dataId) 

254 

255 # There is no assert for Exposure so just look at maskedImage 

256 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

257 

258 # Helper for extracting components 

259 assembler = ExposureAssembler(ref.datasetType.storageClass) 

260 

261 # Check all possible components that can be read 

262 allComponents = set() 

263 allComponents.update(COMPONENTS, READ_COMPONENTS) 

264 

265 # Get each component from butler independently 

266 for compName in allComponents: 

267 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

268 component = self.butler.get(compTypeName, dataId) 

269 

270 reference = assembler.getComponent(exposure, compName) 

271 

272 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

273 

274 if compName in ("image", "variance"): 

275 self.assertImagesEqual(component, reference) 

276 elif compName == "mask": 

277 self.assertMasksEqual(component, reference) 

278 elif compName == "wcs": 

279 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

280 elif compName == "coaddInputs": 

281 self.assertEqual(len(component.visits), len(reference.visits), 

282 f"cf visits {component.visits}") 

283 self.assertEqual(len(component.ccds), len(reference.ccds), 

284 f"cf CCDs {component.ccds}") 

285 elif compName == "psf": 

286 # Equality for PSF does not work 

287 pass 

288 elif compName == "filter": 

289 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

290 elif compName == "filterLabel": 

291 self.assertEqual(component, reference) 

292 elif compName == "id": 

293 self.assertEqual(component, reference) 

294 elif compName == "visitInfo": 

295 self.assertEqual(component, reference, 

296 "VisitInfo comparison") 

297 elif compName == "metadata": 

298 # The component metadata has extra fields in it so cannot 

299 # compare directly. 

300 for k, v in reference.items(): 

301 self.assertEqual(component[k], v) 

302 elif compName == "photoCalib": 

303 # This example has a 

304 # "spatially constant with mean: inf error: nan" entry 

305 # which does not compare directly. 

306 self.assertEqual(str(component), str(reference)) 

307 self.assertIn("spatially constant with mean: 1.99409", str(component), 

308 "Checking photoCalib") 

309 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

310 self.assertEqual(component, reference) 

311 elif compName == "apCorrMap": 

312 self.assertEqual(set(component.keys()), set(reference.keys())) 

313 elif compName == "transmissionCurve": 

314 self.assertEqual(component.getThroughputAtBounds(), 

315 reference.getThroughputAtBounds()) 

316 elif compName == "detector": 

317 c_amps = {a.getName() for a in component.getAmplifiers()} 

318 r_amps = {a.getName() for a in reference.getAmplifiers()} 

319 self.assertEqual(c_amps, r_amps) 

320 elif compName == 'summaryStats': 

321 self.assertEqual(component.psfSigma, reference.psfSigma) 

322 else: 

323 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

324 

325 # Full Exposure with parameters 

326 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

327 parameters = dict(bbox=inBBox, origin=LOCAL) 

328 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

329 outBBox = subset.getBBox() 

330 self.assertEqual(inBBox, outBBox) 

331 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

332 

333 return ref 

334 

335 def putFits(self, exposure, datasetTypeName, visit): 

336 """Put different datasetTypes and return information.""" 

337 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

338 refC = self.butler.put(exposure, datasetTypeName, dataId) 

339 uriC = self.butler.getURI(refC) 

340 stat = os.stat(uriC.ospath) 

341 size = stat.st_size 

342 # We can't use butler's Exposure storage class metadata component, 

343 # because that intentionally strips keywords that science code 

344 # shouldn't ever read (to at least weaken our assumptions that we write 

345 # to FITS). Instead use a lower-level method on the URI for this test. 

346 meta = readMetadata(uriC.ospath) 

347 return meta, size 

348 

349 def testCompression(self): 

350 """Test that we can write compressed and uncompressed FITS.""" 

351 example = os.path.join(TESTDIR, "data", "small.fits") 

352 exposure = lsst.afw.image.ExposureF(example) 

353 

354 # Write a lossless compressed 

355 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

356 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

357 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

358 

359 # Write an uncompressed FITS file 

360 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

361 self.assertNotIn("ZCMPTYPE", metaN) 

362 

363 # Write an uncompressed FITS file 

364 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

365 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

366 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

367 

368 self.assertNotEqual(sizeC, sizeN) 

369 # Data file is so small that Lossy and Compressed are dominated 

370 # by the extra compression tables 

371 self.assertEqual(sizeL, sizeC) 

372 

373 def testExposureFormatterAmpParameter(self): 

374 """Test the FitsExposureFormatter implementation of the Exposure 

375 StorageClass's 'amp' and 'detector' parameters. 

376 """ 

377 # Our example exposure file has a realistic detector (looks like HSC), 

378 # but the image itself doesn't match it. So we just load the detector, 

379 # and use it to make our own more useful images and put them. 

380 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

381 trimmed_full = make_ramp_exposure_trimmed(detector) 

382 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

383 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

384 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

385 for n, amp in enumerate(detector): 

386 # Try to read each amp as it is on disk, with a variety of 

387 # parameters that should all do the same thing. 

388 for amp_parameter in [amp, amp.getName(), n]: 

389 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

390 with self.subTest(parameters=parameters): 

391 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters) 

392 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters) 

393 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

394 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

395 self.assertEqual(len(test_trimmed.getDetector()), 1) 

396 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

397 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

398 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

399 # Try to read various transformed versions of the original amp, 

400 # to make sure flips and offsets are applied correctly. 

401 # First flip X only. 

402 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

403 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1}) 

404 self.assertImagesEqual(test_t1_trimmed.image, 

405 flipImage(trimmed_full[amp.getBBox()].image, 

406 flipLR=True, flipTB=False)) 

407 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

408 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1}) 

409 self.assertImagesEqual(test_t1_untrimmed.image, 

410 flipImage(untrimmed_full[amp.getRawBBox()].image, 

411 flipLR=True, flipTB=False)) 

412 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

413 # Flip Y only. 

414 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

415 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2}) 

416 self.assertImagesEqual(test_t2_trimmed.image, 

417 flipImage(trimmed_full[amp.getBBox()].image, 

418 flipLR=False, flipTB=True)) 

419 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

420 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2}) 

421 self.assertImagesEqual(test_t2_untrimmed.image, 

422 flipImage(untrimmed_full[amp.getRawBBox()].image, 

423 flipLR=False, flipTB=True)) 

424 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

425 # Add an XY offset only. 

426 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

427 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3}) 

428 self.assertImagesEqual(test_t3_trimmed.image, 

429 trimmed_full[amp.getBBox()].image) 

430 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

431 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3}) 

432 self.assertImagesEqual(test_t3_untrimmed.image, 

433 untrimmed_full[amp.getRawBBox()].image) 

434 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

435 

436 

437if __name__ == "__main__": 437 ↛ 438line 437 didn't jump to line 438, because the condition on line 437 was never true

438 unittest.main()