Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.pex.config 

34import lsst.afw.image 

35from lsst.afw.image import LOCAL 

36from lsst.geom import Box2I, Point2I 

37from lsst.base import Packages 

38from lsst.daf.base import PropertyList, PropertySet 

39 

40from lsst.daf.butler import Config 

41from lsst.daf.butler import StorageClassFactory 

42from lsst.daf.butler import DatasetType 

43from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

44 

45from lsst.obs.base.exposureAssembler import ExposureAssembler 

46 

47if TYPE_CHECKING: 47 ↛ 48line 47 didn't jump to line 48, because the condition on line 47 was never true

48 from lsst.daf.butler import DatasetRef 

49 

50TESTDIR = os.path.dirname(__file__) 

51 

52BUTLER_CONFIG = """ 

53storageClasses: 

54 ExposureCompositeF: 

55 inheritsFrom: ExposureF 

56datastore: 

57 # Want to check disassembly so can't use InMemory 

58 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore 

59 formatters: 

60 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

61 lossless: 

62 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

63 parameters: 

64 recipe: lossless 

65 uncompressed: 

66 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

67 parameters: 

68 recipe: noCompression 

69 lossy: 

70 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

71 parameters: 

72 recipe: lossyBasic 

73 composites: 

74 disassembled: 

75 ExposureCompositeF: True 

76""" 

77 

78# Components present in the test file 

79COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib", 

80 "filterLabel", "validPolygon", "transmissionCurve", "detector", "apCorrMap", "summaryStats"} 

81READ_COMPONENTS = {"bbox", "xy0", "dimensions", "filter"} 

82 

83 

84class SimpleConfig(lsst.pex.config.Config): 

85 """Config to use in tests for butler put/get""" 

86 i = lsst.pex.config.Field("integer test", int) 

87 c = lsst.pex.config.Field("string", str) 

88 

89 

90class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

91 

92 @classmethod 

93 def setUpClass(cls): 

94 """Create a new butler once only.""" 

95 

96 cls.storageClassFactory = StorageClassFactory() 

97 

98 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

99 

100 dataIds = { 

101 "instrument": ["DummyCam"], 

102 "physical_filter": ["d-r"], 

103 "visit": [42, 43, 44], 

104 } 

105 

106 # Ensure that we test in a directory that will include some 

107 # metacharacters 

108 subdir = "sub?#dir" 

109 butlerRoot = os.path.join(cls.root, subdir) 

110 

111 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

112 

113 # Create dataset types used by the tests 

114 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

115 ("unknown", "ExposureCompositeF"), 

116 ("testCatalog", "SourceCatalog"), 

117 ("lossless", "ExposureF"), 

118 ("uncompressed", "ExposureF"), 

119 ("lossy", "ExposureF"), 

120 ): 

121 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

122 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

123 

124 # And some dataset types that have no dimensions for easy testing 

125 for datasetTypeName, storageClassName in (("ps", "PropertySet"), 

126 ("pl", "PropertyList"), 

127 ("pkg", "Packages"), 

128 ("config", "Config"), 

129 ): 

130 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

131 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

132 

133 @classmethod 

134 def tearDownClass(cls): 

135 if cls.root is not None: 

136 shutil.rmtree(cls.root, ignore_errors=True) 

137 

138 def setUp(self): 

139 self.butler = makeTestCollection(self.creatorButler) 

140 

141 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

142 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

143 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

144 

145 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

146 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

147 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

148 inputTable = inputCatalog.getTable() 

149 inputRecord = inputCatalog[0] 

150 outputTable = outputCatalog.getTable() 

151 outputRecord = outputCatalog[0] 

152 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

153 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

154 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

155 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

156 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

157 self.assertFloatsAlmostEqual( 

158 inputRecord.getCentroidErr()[0, 0], 

159 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

160 self.assertFloatsAlmostEqual( 

161 inputRecord.getCentroidErr()[1, 1], 

162 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

163 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

164 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

165 self.assertFloatsAlmostEqual( 

166 inputRecord.getShapeErr()[0, 0], 

167 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

168 self.assertFloatsAlmostEqual( 

169 inputRecord.getShapeErr()[1, 1], 

170 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

171 self.assertFloatsAlmostEqual( 

172 inputRecord.getShapeErr()[2, 2], 

173 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

174 

175 def runFundamentalTypeTest(self, datasetTypeName, entity): 

176 """Put and get the supplied entity and compare.""" 

177 ref = self.butler.put(entity, datasetTypeName) 

178 butler_ps = self.butler.get(ref) 

179 self.assertEqual(butler_ps, entity) 

180 

181 # Break the contact by ensuring that we are writing YAML 

182 uri = self.butler.getURI(ref) 

183 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}") 

184 

185 def testFundamentalTypes(self) -> None: 

186 """Ensure that some fundamental stack types round trip.""" 

187 ps = PropertySet() 

188 ps["a.b"] = 5 

189 ps["c.d.e"] = "string" 

190 self.runFundamentalTypeTest("ps", ps) 

191 

192 pl = PropertyList() 

193 pl["A"] = 1 

194 pl.setComment("A", "An int comment") 

195 pl["B"] = "string" 

196 pl.setComment("B", "A string comment") 

197 self.runFundamentalTypeTest("pl", pl) 

198 

199 pkg = Packages.fromSystem() 

200 self.runFundamentalTypeTest("pkg", pkg) 

201 

202 def testPexConfig(self) -> None: 

203 """Test that we can put and get pex_config Configs""" 

204 c = SimpleConfig(i=10, c="hello") 

205 self.assertEqual(c.i, 10) 

206 ref = self.butler.put(c, "config") 

207 butler_c = self.butler.get(ref) 

208 self.assertEqual(c, butler_c) 

209 self.assertIsInstance(butler_c, SimpleConfig) 

210 

211 def testFitsCatalog(self) -> None: 

212 """Test reading of a FITS catalog""" 

213 catalog = self.makeExampleCatalog() 

214 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

215 ref = self.butler.put(catalog, "testCatalog", dataId) 

216 stored = self.butler.get(ref) 

217 self.assertCatalogEqual(catalog, stored) 

218 

219 def testExposureCompositePutGetConcrete(self) -> None: 

220 """Test composite with no disassembly""" 

221 ref = self.runExposureCompositePutGetTest("calexp") 

222 

223 uri = self.butler.getURI(ref) 

224 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

225 

226 def testExposureCompositePutGetVirtual(self) -> None: 

227 """Testing composite disassembly""" 

228 ref = self.runExposureCompositePutGetTest("unknown") 

229 

230 primary, components = self.butler.getURIs(ref) 

231 self.assertIsNone(primary) 

232 self.assertEqual(set(components), COMPONENTS) 

233 for compName, uri in components.items(): 

234 self.assertTrue(uri.exists(), 

235 f"Checking URI {uri} existence for component {compName}") 

236 

237 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

238 example = os.path.join(TESTDIR, "data", "calexp.fits") 

239 exposure = lsst.afw.image.ExposureF(example) 

240 

241 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

242 ref = self.butler.put(exposure, datasetTypeName, dataId) 

243 

244 # Get the full thing 

245 composite = self.butler.get(datasetTypeName, dataId) 

246 

247 # There is no assert for Exposure so just look at maskedImage 

248 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

249 

250 # Helper for extracting components 

251 assembler = ExposureAssembler(ref.datasetType.storageClass) 

252 

253 # Check all possible components that can be read 

254 allComponents = set() 

255 allComponents.update(COMPONENTS, READ_COMPONENTS) 

256 

257 # Get each component from butler independently 

258 for compName in allComponents: 

259 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

260 component = self.butler.get(compTypeName, dataId) 

261 

262 reference = assembler.getComponent(exposure, compName) 

263 

264 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

265 

266 if compName in ("image", "variance"): 

267 self.assertImagesEqual(component, reference) 

268 elif compName == "mask": 

269 self.assertMasksEqual(component, reference) 

270 elif compName == "wcs": 

271 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

272 elif compName == "coaddInputs": 

273 self.assertEqual(len(component.visits), len(reference.visits), 

274 f"cf visits {component.visits}") 

275 self.assertEqual(len(component.ccds), len(reference.ccds), 

276 f"cf CCDs {component.ccds}") 

277 elif compName == "psf": 

278 # Equality for PSF does not work 

279 pass 

280 elif compName == "filter": 

281 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

282 elif compName == "filterLabel": 

283 self.assertEqual(component, reference) 

284 elif compName == "visitInfo": 

285 self.assertEqual(component.getExposureId(), reference.getExposureId(), 

286 "VisitInfo comparison") 

287 elif compName == "metadata": 

288 # The component metadata has extra fields in it so cannot 

289 # compare directly. 

290 for k, v in reference.items(): 

291 self.assertEqual(component[k], v) 

292 elif compName == "photoCalib": 

293 # This example has a 

294 # "spatially constant with mean: inf error: nan" entry 

295 # which does not compare directly. 

296 self.assertEqual(str(component), str(reference)) 

297 self.assertIn("spatially constant with mean: 1.99409", str(component), 

298 "Checking photoCalib") 

299 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

300 self.assertEqual(component, reference) 

301 elif compName == "apCorrMap": 

302 self.assertEqual(set(component.keys()), set(reference.keys())) 

303 elif compName == "transmissionCurve": 

304 self.assertEqual(component.getThroughputAtBounds(), 

305 reference.getThroughputAtBounds()) 

306 elif compName == "detector": 

307 c_amps = {a.getName() for a in component.getAmplifiers()} 

308 r_amps = {a.getName() for a in reference.getAmplifiers()} 

309 self.assertEqual(c_amps, r_amps) 

310 elif compName == 'summaryStats': 

311 self.assertEqual(component.psfSigma, reference.psfSigma) 

312 else: 

313 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

314 

315 # Full Exposure with parameters 

316 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

317 parameters = dict(bbox=inBBox, origin=LOCAL) 

318 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

319 outBBox = subset.getBBox() 

320 self.assertEqual(inBBox, outBBox) 

321 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

322 

323 return ref 

324 

325 def putFits(self, exposure, datasetTypeName, visit): 

326 """Put different datasetTypes and return information.""" 

327 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

328 refC = self.butler.put(exposure, datasetTypeName, dataId) 

329 uriC = self.butler.getURI(refC) 

330 stat = os.stat(uriC.ospath) 

331 size = stat.st_size 

332 metaDatasetTypeName = DatasetType.nameWithComponent(datasetTypeName, "metadata") 

333 meta = self.butler.get(metaDatasetTypeName, dataId) 

334 return meta, size 

335 

336 def testCompression(self): 

337 """Test that we can write compressed and uncompressed FITS.""" 

338 example = os.path.join(TESTDIR, "data", "small.fits") 

339 exposure = lsst.afw.image.ExposureF(example) 

340 

341 # Write a lossless compressed 

342 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

343 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

344 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

345 

346 # Write an uncompressed FITS file 

347 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

348 self.assertNotIn("ZCMPTYPE", metaN) 

349 

350 # Write an uncompressed FITS file 

351 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

352 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

353 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

354 

355 self.assertNotEqual(sizeC, sizeN) 

356 # Data file is so small that Lossy and Compressed are dominated 

357 # by the extra compression tables 

358 self.assertEqual(sizeL, sizeC) 

359 

360 

361if __name__ == "__main__": 361 ↛ 362line 361 didn't jump to line 362, because the condition on line 361 was never true

362 unittest.main()