Coverage for tests/test_butlerFits.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

234 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import shutil 

26import tempfile 

27import unittest 

28from typing import TYPE_CHECKING 

29 

30import lsst.afw.cameraGeom.testUtils # for test asserts injected into TestCase 

31import lsst.afw.image 

32import lsst.pex.config 

33import lsst.utils.tests 

34from lsst.afw.fits import readMetadata 

35from lsst.afw.image import LOCAL, ExposureFitsReader 

36from lsst.afw.math import flipImage 

37from lsst.daf.base import PropertyList, PropertySet 

38from lsst.daf.butler import Config, DatasetType, StorageClassFactory 

39from lsst.daf.butler.tests import DatasetTestHelper, addDatasetType, makeTestCollection, makeTestRepo 

40from lsst.geom import Box2I, Extent2I, Point2I 

41from lsst.obs.base.exposureAssembler import ExposureAssembler 

42from lsst.obs.base.tests import make_ramp_exposure_trimmed, make_ramp_exposure_untrimmed 

43from lsst.utils.packages import Packages 

44 

45if TYPE_CHECKING: 45 ↛ 46line 45 didn't jump to line 46, because the condition on line 45 was never true

46 from lsst.daf.butler import DatasetRef 

47 

48TESTDIR = os.path.dirname(__file__) 

49 

50BUTLER_CONFIG = """ 

51storageClasses: 

52 ExposureCompositeF: 

53 inheritsFrom: ExposureF 

54datastore: 

55 # Want to check disassembly so can't use InMemory 

56 cls: lsst.daf.butler.datastores.fileDatastore.FileDatastore 

57 formatters: 

58 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

59 lossless: 

60 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

61 parameters: 

62 recipe: lossless 

63 uncompressed: 

64 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

65 parameters: 

66 recipe: noCompression 

67 lossy: 

68 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

69 parameters: 

70 recipe: lossyBasic 

71 composites: 

72 disassembled: 

73 ExposureCompositeF: True 

74""" 

75 

76# Components present in the test file 

77COMPONENTS = { 

78 "wcs", 

79 "image", 

80 "mask", 

81 "coaddInputs", 

82 "psf", 

83 "visitInfo", 

84 "variance", 

85 "metadata", 

86 "photoCalib", 

87 "filterLabel", 

88 "validPolygon", 

89 "transmissionCurve", 

90 "detector", 

91 "apCorrMap", 

92 "summaryStats", 

93 "id", 

94} 

95READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"} 

96 

97 

98class SimpleConfig(lsst.pex.config.Config): 

99 """Config to use in tests for butler put/get""" 

100 

101 i = lsst.pex.config.Field("integer test", int) 

102 c = lsst.pex.config.Field("string", str) 

103 

104 

105class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

106 @classmethod 

107 def setUpClass(cls): 

108 """Create a new butler once only.""" 

109 

110 cls.storageClassFactory = StorageClassFactory() 

111 

112 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

113 

114 dataIds = { 

115 "instrument": ["DummyCam"], 

116 "physical_filter": ["d-r"], 

117 "visit": [42, 43, 44], 

118 } 

119 

120 # Ensure that we test in a directory that will include some 

121 # metacharacters 

122 subdir = "sub?#dir" 

123 butlerRoot = os.path.join(cls.root, subdir) 

124 

125 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

126 

127 # Create dataset types used by the tests 

128 for datasetTypeName, storageClassName in ( 

129 ("calexp", "ExposureF"), 

130 ("unknown", "ExposureCompositeF"), 

131 ("testCatalog", "SourceCatalog"), 

132 ("lossless", "ExposureF"), 

133 ("uncompressed", "ExposureF"), 

134 ("lossy", "ExposureF"), 

135 ): 

136 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

137 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

138 

139 # And some dataset types that have no dimensions for easy testing 

140 for datasetTypeName, storageClassName in ( 

141 ("ps", "PropertySet"), 

142 ("pl", "PropertyList"), 

143 ("pkg", "Packages"), 

144 ("config", "Config"), 

145 ("int_exp_trimmed", "ExposureI"), 

146 ("int_exp_untrimmed", "ExposureI"), 

147 ): 

148 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

149 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

150 

151 @classmethod 

152 def tearDownClass(cls): 

153 if cls.root is not None: 

154 shutil.rmtree(cls.root, ignore_errors=True) 

155 

156 def setUp(self): 

157 self.butler = makeTestCollection(self.creatorButler) 

158 

159 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

160 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

161 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

162 

163 def assertCatalogEqual( 

164 self, inputCatalog: lsst.afw.table.SourceCatalog, outputCatalog: lsst.afw.table.SourceCatalog 

165 ) -> None: 

166 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

167 inputTable = inputCatalog.getTable() 

168 inputRecord = inputCatalog[0] 

169 outputTable = outputCatalog.getTable() 

170 outputRecord = outputCatalog[0] 

171 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

172 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

173 self.assertEqual( 

174 inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

175 outputTable.getSchema().getAliasMap().get("slot_Centroid"), 

176 ) 

177 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

178 self.assertFloatsAlmostEqual( 

179 inputRecord.getCentroidErr()[0, 0], outputRecord.getCentroidErr()[0, 0], rtol=1e-6 

180 ) 

181 self.assertFloatsAlmostEqual( 

182 inputRecord.getCentroidErr()[1, 1], outputRecord.getCentroidErr()[1, 1], rtol=1e-6 

183 ) 

184 self.assertEqual( 

185 inputTable.getSchema().getAliasMap().get("slot_Shape"), 

186 outputTable.getSchema().getAliasMap().get("slot_Shape"), 

187 ) 

188 self.assertFloatsAlmostEqual( 

189 inputRecord.getShapeErr()[0, 0], outputRecord.getShapeErr()[0, 0], rtol=1e-6 

190 ) 

191 self.assertFloatsAlmostEqual( 

192 inputRecord.getShapeErr()[1, 1], outputRecord.getShapeErr()[1, 1], rtol=1e-6 

193 ) 

194 self.assertFloatsAlmostEqual( 

195 inputRecord.getShapeErr()[2, 2], outputRecord.getShapeErr()[2, 2], rtol=1e-6 

196 ) 

197 

198 def runFundamentalTypeTest(self, datasetTypeName, entity): 

199 """Put and get the supplied entity and compare.""" 

200 ref = self.butler.put(entity, datasetTypeName) 

201 butler_ps = self.butler.get(ref) 

202 self.assertEqual(butler_ps, entity) 

203 

204 # Break the contact by ensuring that we are writing YAML 

205 uri = self.butler.getURI(ref) 

206 self.assertEqual(uri.getExtension(), ".yaml", f"Check extension of {uri}") 

207 

208 def testFundamentalTypes(self) -> None: 

209 """Ensure that some fundamental stack types round trip.""" 

210 ps = PropertySet() 

211 ps["a.b"] = 5 

212 ps["c.d.e"] = "string" 

213 self.runFundamentalTypeTest("ps", ps) 

214 

215 pl = PropertyList() 

216 pl["A"] = 1 

217 pl.setComment("A", "An int comment") 

218 pl["B"] = "string" 

219 pl.setComment("B", "A string comment") 

220 self.runFundamentalTypeTest("pl", pl) 

221 

222 pkg = Packages.fromSystem() 

223 self.runFundamentalTypeTest("pkg", pkg) 

224 

225 def testPexConfig(self) -> None: 

226 """Test that we can put and get pex_config Configs""" 

227 c = SimpleConfig(i=10, c="hello") 

228 self.assertEqual(c.i, 10) 

229 ref = self.butler.put(c, "config") 

230 butler_c = self.butler.get(ref) 

231 self.assertEqual(c, butler_c) 

232 self.assertIsInstance(butler_c, SimpleConfig) 

233 

234 def testFitsCatalog(self) -> None: 

235 """Test reading of a FITS catalog""" 

236 catalog = self.makeExampleCatalog() 

237 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

238 ref = self.butler.put(catalog, "testCatalog", dataId) 

239 stored = self.butler.get(ref) 

240 self.assertCatalogEqual(catalog, stored) 

241 

242 def testExposureCompositePutGetConcrete(self) -> None: 

243 """Test composite with no disassembly""" 

244 ref = self.runExposureCompositePutGetTest("calexp") 

245 

246 uri = self.butler.getURI(ref) 

247 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

248 

249 def testExposureCompositePutGetVirtual(self) -> None: 

250 """Testing composite disassembly""" 

251 ref = self.runExposureCompositePutGetTest("unknown") 

252 

253 primary, components = self.butler.getURIs(ref) 

254 self.assertIsNone(primary) 

255 self.assertEqual(set(components), COMPONENTS) 

256 for compName, uri in components.items(): 

257 self.assertTrue(uri.exists(), f"Checking URI {uri} existence for component {compName}") 

258 

259 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

260 example = os.path.join(TESTDIR, "data", "calexp.fits") 

261 exposure = lsst.afw.image.ExposureF(example) 

262 

263 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

264 ref = self.butler.put(exposure, datasetTypeName, dataId) 

265 

266 # Get the full thing 

267 composite = self.butler.get(datasetTypeName, dataId) 

268 

269 # There is no assert for Exposure so just look at maskedImage 

270 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

271 

272 # Helper for extracting components 

273 assembler = ExposureAssembler(ref.datasetType.storageClass) 

274 

275 # Check all possible components that can be read 

276 allComponents = set() 

277 allComponents.update(COMPONENTS, READ_COMPONENTS) 

278 

279 # Get each component from butler independently 

280 for compName in allComponents: 

281 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

282 component = self.butler.get(compTypeName, dataId) 

283 

284 reference = assembler.getComponent(exposure, compName) 

285 

286 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

287 

288 if compName in ("image", "variance"): 

289 self.assertImagesEqual(component, reference) 

290 elif compName == "mask": 

291 self.assertMasksEqual(component, reference) 

292 elif compName == "wcs": 

293 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

294 elif compName == "coaddInputs": 

295 self.assertEqual( 

296 len(component.visits), len(reference.visits), f"cf visits {component.visits}" 

297 ) 

298 self.assertEqual(len(component.ccds), len(reference.ccds), f"cf CCDs {component.ccds}") 

299 elif compName == "psf": 

300 # Equality for PSF does not work 

301 pass 

302 elif compName == "filter": 

303 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

304 elif compName == "filterLabel": 

305 self.assertEqual(component, reference) 

306 elif compName == "id": 

307 self.assertEqual(component, reference) 

308 elif compName == "visitInfo": 

309 self.assertEqual(component, reference, "VisitInfo comparison") 

310 elif compName == "metadata": 

311 # The component metadata has extra fields in it so cannot 

312 # compare directly. 

313 for k, v in reference.items(): 

314 self.assertEqual(component[k], v) 

315 elif compName == "photoCalib": 

316 # This example has a 

317 # "spatially constant with mean: inf error: nan" entry 

318 # which does not compare directly. 

319 self.assertEqual(str(component), str(reference)) 

320 self.assertIn("spatially constant with mean: 1.99409", str(component), "Checking photoCalib") 

321 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

322 self.assertEqual(component, reference) 

323 elif compName == "apCorrMap": 

324 self.assertEqual(set(component.keys()), set(reference.keys())) 

325 elif compName == "transmissionCurve": 

326 self.assertEqual(component.getThroughputAtBounds(), reference.getThroughputAtBounds()) 

327 elif compName == "detector": 

328 c_amps = {a.getName() for a in component.getAmplifiers()} 

329 r_amps = {a.getName() for a in reference.getAmplifiers()} 

330 self.assertEqual(c_amps, r_amps) 

331 elif compName == "summaryStats": 

332 self.assertEqual(component.psfSigma, reference.psfSigma) 

333 else: 

334 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

335 

336 # Full Exposure with parameters 

337 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

338 parameters = dict(bbox=inBBox, origin=LOCAL) 

339 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

340 outBBox = subset.getBBox() 

341 self.assertEqual(inBBox, outBBox) 

342 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

343 

344 return ref 

345 

346 def putFits(self, exposure, datasetTypeName, visit): 

347 """Put different datasetTypes and return information.""" 

348 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

349 refC = self.butler.put(exposure, datasetTypeName, dataId) 

350 uriC = self.butler.getURI(refC) 

351 stat = os.stat(uriC.ospath) 

352 size = stat.st_size 

353 # We can't use butler's Exposure storage class metadata component, 

354 # because that intentionally strips keywords that science code 

355 # shouldn't ever read (to at least weaken our assumptions that we write 

356 # to FITS). Instead use a lower-level method on the URI for this test. 

357 meta = readMetadata(uriC.ospath) 

358 return meta, size 

359 

360 def testCompression(self): 

361 """Test that we can write compressed and uncompressed FITS.""" 

362 example = os.path.join(TESTDIR, "data", "small.fits") 

363 exposure = lsst.afw.image.ExposureF(example) 

364 

365 # Write a lossless compressed 

366 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

367 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

368 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

369 

370 # Write an uncompressed FITS file 

371 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

372 self.assertNotIn("ZCMPTYPE", metaN) 

373 

374 # Write an uncompressed FITS file 

375 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

376 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

377 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

378 

379 self.assertNotEqual(sizeC, sizeN) 

380 # Data file is so small that Lossy and Compressed are dominated 

381 # by the extra compression tables 

382 self.assertEqual(sizeL, sizeC) 

383 

384 def testExposureFormatterAmpParameter(self): 

385 """Test the FitsExposureFormatter implementation of the Exposure 

386 StorageClass's 'amp' and 'detector' parameters. 

387 """ 

388 # Our example exposure file has a realistic detector (looks like HSC), 

389 # but the image itself doesn't match it. So we just load the detector, 

390 # and use it to make our own more useful images and put them. 

391 detector = ExposureFitsReader(os.path.join(TESTDIR, "data", "calexp.fits")).readDetector() 

392 trimmed_full = make_ramp_exposure_trimmed(detector) 

393 untrimmed_full = make_ramp_exposure_untrimmed(detector) 

394 trimmed_ref = self.butler.put(trimmed_full, "int_exp_trimmed") 

395 untrimmed_ref = self.butler.put(untrimmed_full, "int_exp_untrimmed") 

396 for n, amp in enumerate(detector): 

397 # Try to read each amp as it is on disk, with a variety of 

398 # parameters that should all do the same thing. 

399 for amp_parameter in [amp, amp.getName(), n]: 

400 for parameters in [{"amp": amp_parameter}, {"amp": amp_parameter, "detector": detector}]: 

401 with self.subTest(parameters=parameters): 

402 test_trimmed = self.butler.getDirect(trimmed_ref, parameters=parameters) 

403 test_untrimmed = self.butler.getDirect(untrimmed_ref, parameters=parameters) 

404 self.assertImagesEqual(test_trimmed.image, trimmed_full[amp.getBBox()].image) 

405 self.assertImagesEqual(test_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

406 self.assertEqual(len(test_trimmed.getDetector()), 1) 

407 self.assertEqual(len(test_untrimmed.getDetector()), 1) 

408 self.assertAmplifiersEqual(test_trimmed.getDetector()[0], amp) 

409 self.assertAmplifiersEqual(test_untrimmed.getDetector()[0], amp) 

410 # Try to read various transformed versions of the original amp, 

411 # to make sure flips and offsets are applied correctly. 

412 # First flip X only. 

413 amp_t1 = amp.rebuild().transform(outFlipX=True).finish() 

414 test_t1_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t1}) 

415 self.assertImagesEqual( 

416 test_t1_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=True, flipTB=False) 

417 ) 

418 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

419 test_t1_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t1}) 

420 self.assertImagesEqual( 

421 test_t1_untrimmed.image, 

422 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=True, flipTB=False), 

423 ) 

424 self.assertAmplifiersEqual(test_t1_trimmed.getDetector()[0], amp_t1) 

425 # Flip Y only. 

426 amp_t2 = amp.rebuild().transform(outFlipY=True).finish() 

427 test_t2_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t2}) 

428 self.assertImagesEqual( 

429 test_t2_trimmed.image, flipImage(trimmed_full[amp.getBBox()].image, flipLR=False, flipTB=True) 

430 ) 

431 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

432 test_t2_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t2}) 

433 self.assertImagesEqual( 

434 test_t2_untrimmed.image, 

435 flipImage(untrimmed_full[amp.getRawBBox()].image, flipLR=False, flipTB=True), 

436 ) 

437 self.assertAmplifiersEqual(test_t2_trimmed.getDetector()[0], amp_t2) 

438 # Add an XY offset only. 

439 amp_t3 = amp.rebuild().transform(outOffset=Extent2I(5, 4)).finish() 

440 test_t3_trimmed = self.butler.getDirect(trimmed_ref, parameters={"amp": amp_t3}) 

441 self.assertImagesEqual(test_t3_trimmed.image, trimmed_full[amp.getBBox()].image) 

442 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

443 test_t3_untrimmed = self.butler.getDirect(untrimmed_ref, parameters={"amp": amp_t3}) 

444 self.assertImagesEqual(test_t3_untrimmed.image, untrimmed_full[amp.getRawBBox()].image) 

445 self.assertAmplifiersEqual(test_t3_trimmed.getDetector()[0], amp_t3) 

446 

447 

448if __name__ == "__main__": 448 ↛ 449line 448 didn't jump to line 449, because the condition on line 448 was never true

449 unittest.main()