Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.afw.image 

34from lsst.afw.image import LOCAL 

35from lsst.geom import Box2I, Point2I 

36from lsst.base import Packages 

37from lsst.daf.base import PropertyList, PropertySet 

38 

39from lsst.daf.butler import Config 

40from lsst.daf.butler import StorageClassFactory 

41from lsst.daf.butler import DatasetType 

42from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

43 

44from lsst.obs.base.exposureAssembler import ExposureAssembler 

45 

46if TYPE_CHECKING: 46 ↛ 47line 46 didn't jump to line 47, because the condition on line 46 was never true

47 from lsst.daf.butler import DatasetRef 

48 

49TESTDIR = os.path.dirname(__file__) 

50 

51BUTLER_CONFIG = """ 

52storageClasses: 

53 ExposureCompositeF: 

54 inheritsFrom: ExposureF 

55datastore: 

56 # Want to check disassembly so can't use InMemory 

57 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore 

58 formatters: 

59 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

60 lossless: 

61 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

62 parameters: 

63 recipe: lossless 

64 uncompressed: 

65 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

66 parameters: 

67 recipe: noCompression 

68 lossy: 

69 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

70 parameters: 

71 recipe: lossyBasic 

72 composites: 

73 disassembled: 

74 ExposureCompositeF: True 

75""" 

76 

77# Components present in the test file 

78COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib", 

79 "filter", "validPolygon", "transmissionCurve", "detector", "apCorrMap"} 

80READ_COMPONENTS = {"bbox", "xy0", "dimensions"} 

81 

82 

83class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

84 

85 @classmethod 

86 def setUpClass(cls): 

87 """Create a new butler once only.""" 

88 

89 cls.storageClassFactory = StorageClassFactory() 

90 

91 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

92 

93 dataIds = { 

94 "instrument": ["DummyCam"], 

95 "physical_filter": ["d-r"], 

96 "visit": [42, 43, 44], 

97 } 

98 

99 # Ensure that we test in a directory that will include some 

100 # metacharacters 

101 subdir = "sub?#dir" 

102 butlerRoot = os.path.join(cls.root, subdir) 

103 

104 cls.creatorButler = makeTestRepo(butlerRoot, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

105 

106 # Create dataset types used by the tests 

107 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

108 ("unknown", "ExposureCompositeF"), 

109 ("testCatalog", "SourceCatalog"), 

110 ("lossless", "ExposureF"), 

111 ("uncompressed", "ExposureF"), 

112 ("lossy", "ExposureF"), 

113 ): 

114 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

115 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

116 

117 # And some dataset types that have no dimensions for easy testing 

118 for datasetTypeName, storageClassName in (("ps", "PropertySet"), 

119 ("pl", "PropertyList"), 

120 ("pkg", "Packages") 

121 ): 

122 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

123 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

124 

125 @classmethod 

126 def tearDownClass(cls): 

127 if cls.root is not None: 

128 shutil.rmtree(cls.root, ignore_errors=True) 

129 

130 def setUp(self): 

131 self.butler = makeTestCollection(self.creatorButler) 

132 

133 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

134 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

135 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

136 

137 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

138 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

139 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

140 inputTable = inputCatalog.getTable() 

141 inputRecord = inputCatalog[0] 

142 outputTable = outputCatalog.getTable() 

143 outputRecord = outputCatalog[0] 

144 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

145 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

146 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

147 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

148 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

149 self.assertFloatsAlmostEqual( 

150 inputRecord.getCentroidErr()[0, 0], 

151 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

152 self.assertFloatsAlmostEqual( 

153 inputRecord.getCentroidErr()[1, 1], 

154 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

155 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

156 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

157 self.assertFloatsAlmostEqual( 

158 inputRecord.getShapeErr()[0, 0], 

159 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

160 self.assertFloatsAlmostEqual( 

161 inputRecord.getShapeErr()[1, 1], 

162 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

163 self.assertFloatsAlmostEqual( 

164 inputRecord.getShapeErr()[2, 2], 

165 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

166 

167 def runFundamentalTypeTest(self, datasetTypeName, entity): 

168 """Put and get the supplied entity and compare.""" 

169 ref = self.butler.put(entity, datasetTypeName) 

170 butler_ps = self.butler.get(ref) 

171 self.assertEqual(butler_ps, entity) 

172 

173 # Break the contact by ensuring that we are writing YAML 

174 uri = self.butler.getURI(ref) 

175 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}") 

176 

177 def testFundamentalTypes(self) -> None: 

178 """Ensure that some fundamental stack types round trip.""" 

179 ps = PropertySet() 

180 ps["a.b"] = 5 

181 ps["c.d.e"] = "string" 

182 self.runFundamentalTypeTest("ps", ps) 

183 

184 pl = PropertyList() 

185 pl["A"] = 1 

186 pl.setComment("A", "An int comment") 

187 pl["B"] = "string" 

188 pl.setComment("B", "A string comment") 

189 self.runFundamentalTypeTest("pl", pl) 

190 

191 pkg = Packages.fromSystem() 

192 self.runFundamentalTypeTest("pkg", pkg) 

193 

194 def testFitsCatalog(self) -> None: 

195 """Test reading of a FITS catalog""" 

196 catalog = self.makeExampleCatalog() 

197 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

198 ref = self.butler.put(catalog, "testCatalog", dataId) 

199 stored = self.butler.get(ref) 

200 self.assertCatalogEqual(catalog, stored) 

201 

202 def testExposureCompositePutGetConcrete(self) -> None: 

203 """Test composite with no disassembly""" 

204 ref = self.runExposureCompositePutGetTest("calexp") 

205 

206 uri = self.butler.getURI(ref) 

207 self.assertTrue(uri.exists(), f"Checking URI {uri} existence") 

208 

209 def testExposureCompositePutGetVirtual(self) -> None: 

210 """Testing composite disassembly""" 

211 ref = self.runExposureCompositePutGetTest("unknown") 

212 

213 primary, components = self.butler.getURIs(ref) 

214 self.assertIsNone(primary) 

215 self.assertEqual(set(components), COMPONENTS) 

216 for compName, uri in components.items(): 

217 self.assertTrue(uri.exists(), 

218 f"Checking URI {uri} existence for component {compName}") 

219 

220 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

221 example = os.path.join(TESTDIR, "data", "calexp.fits") 

222 exposure = lsst.afw.image.ExposureF(example) 

223 

224 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

225 ref = self.butler.put(exposure, datasetTypeName, dataId) 

226 

227 # Get the full thing 

228 composite = self.butler.get(datasetTypeName, dataId) 

229 

230 # There is no assert for Exposure so just look at maskedImage 

231 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

232 

233 # Helper for extracting components 

234 assembler = ExposureAssembler(ref.datasetType.storageClass) 

235 

236 # Check all possible components that can be read 

237 allComponents = set() 

238 allComponents.update(COMPONENTS, READ_COMPONENTS) 

239 

240 # Get each component from butler independently 

241 for compName in allComponents: 

242 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

243 component = self.butler.get(compTypeName, dataId) 

244 

245 reference = assembler.getComponent(exposure, compName) 

246 

247 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

248 

249 if compName in ("image", "variance"): 

250 self.assertImagesEqual(component, reference) 

251 elif compName == "mask": 

252 self.assertMasksEqual(component, reference) 

253 elif compName == "wcs": 

254 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

255 elif compName == "coaddInputs": 

256 self.assertEqual(len(component.visits), len(reference.visits), 

257 f"cf visits {component.visits}") 

258 self.assertEqual(len(component.ccds), len(reference.ccds), 

259 f"cf CCDs {component.ccds}") 

260 elif compName == "psf": 

261 # Equality for PSF does not work 

262 pass 

263 elif compName == "filter": 

264 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

265 elif compName == "visitInfo": 

266 self.assertEqual(component.getExposureId(), reference.getExposureId(), 

267 "VisitInfo comparison") 

268 elif compName == "metadata": 

269 # The component metadata has extra fields in it so cannot 

270 # compare directly. 

271 for k, v in reference.items(): 

272 self.assertEqual(component[k], v) 

273 elif compName == "photoCalib": 

274 # This example has a 

275 # "spatially constant with mean: inf error: nan" entry 

276 # which does not compare directly. 

277 self.assertEqual(str(component), str(reference)) 

278 self.assertIn("spatially constant with mean: 1.99409", str(component), 

279 "Checking photoCalib") 

280 elif compName in ("bbox", "xy0", "dimensions", "validPolygon"): 

281 self.assertEqual(component, reference) 

282 elif compName == "apCorrMap": 

283 self.assertEqual(set(component.keys()), set(reference.keys())) 

284 elif compName == "transmissionCurve": 

285 self.assertEqual(component.getThroughputAtBounds(), 

286 reference.getThroughputAtBounds()) 

287 elif compName == "detector": 

288 c_amps = {a.getName() for a in component.getAmplifiers()} 

289 r_amps = {a.getName() for a in reference.getAmplifiers()} 

290 self.assertEqual(c_amps, r_amps) 

291 else: 

292 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

293 

294 # Full Exposure with parameters 

295 inBBox = Box2I(minimum=Point2I(3, 3), maximum=Point2I(21, 16)) 

296 parameters = dict(bbox=inBBox, origin=LOCAL) 

297 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

298 outBBox = subset.getBBox() 

299 self.assertEqual(inBBox, outBBox) 

300 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

301 

302 return ref 

303 

304 def putFits(self, exposure, datasetTypeName, visit): 

305 """Put different datasetTypes and return information.""" 

306 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

307 refC = self.butler.put(exposure, datasetTypeName, dataId) 

308 uriC = self.butler.getURI(refC) 

309 stat = os.stat(uriC.ospath) 

310 size = stat.st_size 

311 metaDatasetTypeName = DatasetType.nameWithComponent(datasetTypeName, "metadata") 

312 meta = self.butler.get(metaDatasetTypeName, dataId) 

313 return meta, size 

314 

315 def testCompression(self): 

316 """Test that we can write compressed and uncompressed FITS.""" 

317 example = os.path.join(TESTDIR, "data", "small.fits") 

318 exposure = lsst.afw.image.ExposureF(example) 

319 

320 # Write a lossless compressed 

321 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

322 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

323 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

324 

325 # Write an uncompressed FITS file 

326 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

327 self.assertNotIn("ZCMPTYPE", metaN) 

328 

329 # Write an uncompressed FITS file 

330 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

331 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

332 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

333 

334 self.assertNotEqual(sizeC, sizeN) 

335 # Data file is so small that Lossy and Compressed are dominated 

336 # by the extra compression tables 

337 self.assertEqual(sizeL, sizeC) 

338 

339 

340if __name__ == "__main__": 340 ↛ 341line 340 didn't jump to line 341, because the condition on line 340 was never true

341 unittest.main()