Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.afw.image 

34from lsst.afw.image import LOCAL 

35from lsst.geom import Box2I, Point2I 

36from lsst.base import Packages 

37from lsst.daf.base import PropertyList, PropertySet 

38 

39from lsst.daf.butler import Config 

40from lsst.daf.butler import StorageClassFactory 

41from lsst.daf.butler import DatasetType 

42from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

43 

44from lsst.obs.base.exposureAssembler import ExposureAssembler 

45 

46if TYPE_CHECKING: 46 ↛ 47line 46 didn't jump to line 47, because the condition on line 46 was never true

47 from lsst.daf.butler import DatasetRef 

48 

49TESTDIR = os.path.dirname(__file__) 

50 

51BUTLER_CONFIG = """ 

52storageClasses: 

53 ExposureCompositeF: 

54 inheritsFrom: ExposureF 

55datastore: 

56 # Want to check disassembly so can't use InMemory 

57 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore 

58 formatters: 

59 ExposureCompositeF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

60 lossless: 

61 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

62 parameters: 

63 recipe: lossless 

64 uncompressed: 

65 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

66 parameters: 

67 recipe: noCompression 

68 lossy: 

69 formatter: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter 

70 parameters: 

71 recipe: lossyBasic 

72 composites: 

73 disassembled: 

74 ExposureCompositeF: True 

75""" 

76 

77# Components present in the test file 

78COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib", 

79 "filter"} 

80READ_COMPONENTS = {"bbox", "xy0", "dimensions"} 

81 

82 

83class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

84 

85 @classmethod 

86 def setUpClass(cls): 

87 """Create a new butler once only.""" 

88 

89 cls.storageClassFactory = StorageClassFactory() 

90 

91 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

92 

93 dataIds = { 

94 "instrument": ["DummyCam"], 

95 "physical_filter": ["d-r"], 

96 "visit": [42, 43, 44], 

97 } 

98 

99 cls.creatorButler = makeTestRepo(cls.root, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

100 

101 # Create dataset types used by the tests 

102 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

103 ("unknown", "ExposureCompositeF"), 

104 ("testCatalog", "SourceCatalog"), 

105 ("lossless", "ExposureF"), 

106 ("uncompressed", "ExposureF"), 

107 ("lossy", "ExposureF"), 

108 ): 

109 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

110 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

111 

112 # And some dataset types that have no dimensions for easy testing 

113 for datasetTypeName, storageClassName in (("ps", "PropertySet"), 

114 ("pl", "PropertyList"), 

115 ("pkg", "Packages") 

116 ): 

117 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

118 addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass) 

119 

120 @classmethod 

121 def tearDownClass(cls): 

122 if cls.root is not None: 

123 shutil.rmtree(cls.root, ignore_errors=True) 

124 

125 def setUp(self): 

126 self.butler = makeTestCollection(self.creatorButler) 

127 

128 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

129 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

130 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

131 

132 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

133 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

134 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

135 inputTable = inputCatalog.getTable() 

136 inputRecord = inputCatalog[0] 

137 outputTable = outputCatalog.getTable() 

138 outputRecord = outputCatalog[0] 

139 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

140 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

141 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

142 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

143 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

144 self.assertFloatsAlmostEqual( 

145 inputRecord.getCentroidErr()[0, 0], 

146 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

147 self.assertFloatsAlmostEqual( 

148 inputRecord.getCentroidErr()[1, 1], 

149 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

150 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

151 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

152 self.assertFloatsAlmostEqual( 

153 inputRecord.getShapeErr()[0, 0], 

154 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

155 self.assertFloatsAlmostEqual( 

156 inputRecord.getShapeErr()[1, 1], 

157 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

158 self.assertFloatsAlmostEqual( 

159 inputRecord.getShapeErr()[2, 2], 

160 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

161 

162 def runFundamentalTypeTest(self, datasetTypeName, entity): 

163 """Put and get the supplied entity and compare.""" 

164 ref = self.butler.put(entity, datasetTypeName) 

165 butler_ps = self.butler.get(ref) 

166 self.assertEqual(butler_ps, entity) 

167 

168 # Break the contact by ensuring that we are writing YAML 

169 uri = self.butler.getURI(ref) 

170 self.assertTrue(uri.path.endswith(".yaml"), f"Check extension of {uri}") 

171 

172 def testFundamentalTypes(self) -> None: 

173 """Ensure that some fundamental stack types round trip.""" 

174 ps = PropertySet() 

175 ps["a.b"] = 5 

176 ps["c.d.e"] = "string" 

177 self.runFundamentalTypeTest("ps", ps) 

178 

179 pl = PropertyList() 

180 pl["A"] = 1 

181 pl.setComment("A", "An int comment") 

182 pl["B"] = "string" 

183 pl.setComment("B", "A string comment") 

184 self.runFundamentalTypeTest("pl", pl) 

185 

186 pkg = Packages.fromSystem() 

187 self.runFundamentalTypeTest("pkg", pkg) 

188 

189 def testFitsCatalog(self) -> None: 

190 """Test reading of a FITS catalog""" 

191 catalog = self.makeExampleCatalog() 

192 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

193 ref = self.butler.put(catalog, "testCatalog", dataId) 

194 stored = self.butler.get(ref) 

195 self.assertCatalogEqual(catalog, stored) 

196 

197 def testExposureCompositePutGetConcrete(self) -> None: 

198 """Test composite with no disassembly""" 

199 ref = self.runExposureCompositePutGetTest("calexp") 

200 

201 uri = self.butler.getURI(ref) 

202 self.assertTrue(os.path.exists(uri.path), f"Checking URI {uri} existence") 

203 

204 def testExposureCompositePutGetVirtual(self) -> None: 

205 """Testing composite disassembly""" 

206 ref = self.runExposureCompositePutGetTest("unknown") 

207 

208 primary, components = self.butler.getURIs(ref) 

209 self.assertIsNone(primary) 

210 self.assertEqual(set(components), COMPONENTS) 

211 for compName, uri in components.items(): 

212 self.assertTrue(os.path.exists(uri.path), 

213 f"Checking URI {uri} existence for component {compName}") 

214 

215 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

216 example = os.path.join(TESTDIR, "data", "small.fits") 

217 exposure = lsst.afw.image.ExposureF(example) 

218 

219 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

220 ref = self.butler.put(exposure, datasetTypeName, dataId) 

221 

222 # Get the full thing 

223 composite = self.butler.get(datasetTypeName, dataId) 

224 

225 # There is no assert for Exposure so just look at maskedImage 

226 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

227 

228 # Helper for extracting components 

229 assembler = ExposureAssembler(ref.datasetType.storageClass) 

230 

231 # Check all possible components that can be read 

232 allComponents = set() 

233 allComponents.update(COMPONENTS, READ_COMPONENTS) 

234 

235 # Get each component from butler independently 

236 for compName in allComponents: 

237 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

238 component = self.butler.get(compTypeName, dataId) 

239 

240 reference = assembler.getComponent(exposure, compName) 

241 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

242 

243 if compName in ("image", "variance"): 

244 self.assertImagesEqual(component, reference) 

245 elif compName == "mask": 

246 self.assertMasksEqual(component, reference) 

247 elif compName == "wcs": 

248 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

249 elif compName == "coaddInputs": 

250 self.assertEqual(len(component.visits), len(reference.visits), 

251 f"cf visits {component.visits}") 

252 self.assertEqual(len(component.ccds), len(reference.ccds), 

253 f"cf CCDs {component.ccds}") 

254 elif compName == "psf": 

255 # Equality for PSF does not work 

256 pass 

257 elif compName == "filter": 

258 self.assertEqual(component.getCanonicalName(), reference.getCanonicalName()) 

259 elif compName == "visitInfo": 

260 self.assertEqual(component.getExposureId(), reference.getExposureId(), 

261 f"VisitInfo comparison") 

262 elif compName == "metadata": 

263 # The component metadata has extra fields in it so cannot 

264 # compare directly. 

265 for k, v in reference.items(): 

266 self.assertEqual(component[k], v) 

267 elif compName == "photoCalib": 

268 # This example has a 

269 # "spatially constant with mean: inf error: nan" entry 

270 # which does not compare directly. 

271 self.assertEqual(str(component), str(reference)) 

272 self.assertIn("spatially constant with mean: inf", str(component), "Checking photoCalib") 

273 elif compName in ("bbox", "xy0", "dimensions"): 

274 self.assertEqual(component, reference) 

275 else: 

276 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

277 

278 # With parameters 

279 inBBox = Box2I(minimum=Point2I(0, 0), maximum=Point2I(3, 3)) 

280 parameters = dict(bbox=inBBox, origin=LOCAL) 

281 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

282 outBBox = subset.getBBox() 

283 self.assertEqual(inBBox, outBBox) 

284 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

285 

286 return ref 

287 

288 def putFits(self, exposure, datasetTypeName, visit): 

289 """Put different datasetTypes and return information.""" 

290 dataId = {"visit": visit, "instrument": "DummyCam", "physical_filter": "d-r"} 

291 refC = self.butler.put(exposure, datasetTypeName, dataId) 

292 uriC = self.butler.getURI(refC) 

293 stat = os.stat(uriC.path) 

294 size = stat.st_size 

295 metaDatasetTypeName = DatasetType.nameWithComponent(datasetTypeName, "metadata") 

296 meta = self.butler.get(metaDatasetTypeName, dataId) 

297 return meta, size 

298 

299 def testCompression(self): 

300 """Test that we can write compressed and uncompressed FITS.""" 

301 example = os.path.join(TESTDIR, "data", "small.fits") 

302 exposure = lsst.afw.image.ExposureF(example) 

303 

304 # Write a lossless compressed 

305 metaC, sizeC = self.putFits(exposure, "lossless", 42) 

306 self.assertEqual(metaC["TTYPE1"], "COMPRESSED_DATA") 

307 self.assertEqual(metaC["ZCMPTYPE"], "GZIP_2") 

308 

309 # Write an uncompressed FITS file 

310 metaN, sizeN = self.putFits(exposure, "uncompressed", 43) 

311 self.assertNotIn("ZCMPTYPE", metaN) 

312 

313 # Write an uncompressed FITS file 

314 metaL, sizeL = self.putFits(exposure, "lossy", 44) 

315 self.assertEqual(metaL["TTYPE1"], "COMPRESSED_DATA") 

316 self.assertEqual(metaL["ZCMPTYPE"], "RICE_1") 

317 

318 self.assertNotEqual(sizeC, sizeN) 

319 # Data file is so small that Lossy and Compressed are dominated 

320 # by the extra compression tables 

321 self.assertEqual(sizeL, sizeC) 

322 

323 

324if __name__ == "__main__": 324 ↛ 325line 324 didn't jump to line 325, because the condition on line 324 was never true

325 unittest.main()