Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24import os 

25import unittest 

26import tempfile 

27import shutil 

28 

29from typing import TYPE_CHECKING 

30 

31import lsst.utils.tests 

32 

33import lsst.afw.image 

34from lsst.afw.image import LOCAL 

35from lsst.geom import Box2I, Point2I 

36 

37from lsst.daf.butler import Config 

38from lsst.daf.butler import StorageClassFactory 

39from lsst.daf.butler import DatasetType 

40from lsst.daf.butler.tests import DatasetTestHelper, makeTestRepo, addDatasetType, makeTestCollection 

41 

42from lsst.obs.base.exposureAssembler import ExposureAssembler 

43 

44if TYPE_CHECKING: 44 ↛ 45line 44 didn't jump to line 45, because the condition on line 44 was never true

45 from lsst.daf.butler import DatasetRef 

46 

47TESTDIR = os.path.dirname(__file__) 

48 

49BUTLER_CONFIG = """ 

50storageClasses: 

51 ExposureCompositeF: 

52 inheritsFrom: ExposureF 

53datastore: 

54 # Want to check disassembly so can't use InMemory 

55 cls: lsst.daf.butler.datastores.posixDatastore.PosixDatastore 

56 formatters: 

57 ExposureCompositeF: lsst.obs.base.fitsExposureFormatter.FitsExposureFormatter 

58 composites: 

59 disassembled: 

60 ExposureCompositeF: True 

61""" 

62 

63# Components present in the test file 

64COMPONENTS = {"wcs", "image", "mask", "coaddInputs", "psf", "visitInfo", "variance", "metadata", "photoCalib"} 

65 

66 

67class ButlerFitsTests(DatasetTestHelper, lsst.utils.tests.TestCase): 

68 

69 @classmethod 

70 def setUpClass(cls): 

71 """Create a new butler once only.""" 

72 

73 cls.storageClassFactory = StorageClassFactory() 

74 

75 cls.root = tempfile.mkdtemp(dir=TESTDIR) 

76 

77 dataIds = { 

78 "instrument": ["DummyCam"], 

79 "physical_filter": ["d-r"], 

80 "visit": [42], 

81 } 

82 

83 cls.creatorButler = makeTestRepo(cls.root, dataIds, config=Config.fromYaml(BUTLER_CONFIG)) 

84 

85 # Create dataset types used by the tests 

86 for datasetTypeName, storageClassName in (("calexp", "ExposureF"), 

87 ("unknown", "ExposureCompositeF"), 

88 ("testCatalog", "SourceCatalog"), 

89 ): 

90 storageClass = cls.storageClassFactory.getStorageClass(storageClassName) 

91 addDatasetType(cls.creatorButler, datasetTypeName, set(dataIds), storageClass) 

92 

93 @classmethod 

94 def tearDownClass(cls): 

95 if cls.root is not None: 

96 shutil.rmtree(cls.root, ignore_errors=True) 

97 

98 def setUp(self): 

99 self.butler = makeTestCollection(self.creatorButler) 

100 

101 def makeExampleCatalog(self) -> lsst.afw.table.SourceCatalog: 

102 catalogPath = os.path.join(TESTDIR, "data", "source_catalog.fits") 

103 return lsst.afw.table.SourceCatalog.readFits(catalogPath) 

104 

105 def assertCatalogEqual(self, inputCatalog: lsst.afw.table.SourceCatalog, 

106 outputCatalog: lsst.afw.table.SourceCatalog) -> None: 

107 self.assertIsInstance(outputCatalog, lsst.afw.table.SourceCatalog) 

108 inputTable = inputCatalog.getTable() 

109 inputRecord = inputCatalog[0] 

110 outputTable = outputCatalog.getTable() 

111 outputRecord = outputCatalog[0] 

112 self.assertEqual(inputRecord.getPsfInstFlux(), outputRecord.getPsfInstFlux()) 

113 self.assertEqual(inputRecord.getPsfFluxFlag(), outputRecord.getPsfFluxFlag()) 

114 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Centroid"), 

115 outputTable.getSchema().getAliasMap().get("slot_Centroid")) 

116 self.assertEqual(inputRecord.getCentroid(), outputRecord.getCentroid()) 

117 self.assertFloatsAlmostEqual( 

118 inputRecord.getCentroidErr()[0, 0], 

119 outputRecord.getCentroidErr()[0, 0], rtol=1e-6) 

120 self.assertFloatsAlmostEqual( 

121 inputRecord.getCentroidErr()[1, 1], 

122 outputRecord.getCentroidErr()[1, 1], rtol=1e-6) 

123 self.assertEqual(inputTable.getSchema().getAliasMap().get("slot_Shape"), 

124 outputTable.getSchema().getAliasMap().get("slot_Shape")) 

125 self.assertFloatsAlmostEqual( 

126 inputRecord.getShapeErr()[0, 0], 

127 outputRecord.getShapeErr()[0, 0], rtol=1e-6) 

128 self.assertFloatsAlmostEqual( 

129 inputRecord.getShapeErr()[1, 1], 

130 outputRecord.getShapeErr()[1, 1], rtol=1e-6) 

131 self.assertFloatsAlmostEqual( 

132 inputRecord.getShapeErr()[2, 2], 

133 outputRecord.getShapeErr()[2, 2], rtol=1e-6) 

134 

135 def testFitsCatalog(self) -> None: 

136 catalog = self.makeExampleCatalog() 

137 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

138 ref = self.butler.put(catalog, "testCatalog", dataId) 

139 stored = self.butler.get(ref) 

140 self.assertCatalogEqual(catalog, stored) 

141 

142 def testExposureCompositePutGetConcrete(self) -> None: 

143 ref = self.runExposureCompositePutGetTest("calexp") 

144 

145 uri = self.butler.getURI(ref) 

146 self.assertTrue(os.path.exists(uri.path), f"Checking URI {uri} existence") 

147 

148 def testExposureCompositePutGetVirtual(self) -> None: 

149 ref = self.runExposureCompositePutGetTest("unknown") 

150 

151 primary, components = self.butler.getURIs(ref) 

152 self.assertIsNone(primary) 

153 self.assertEqual(set(components), COMPONENTS) 

154 for compName, uri in components.items(): 

155 self.assertTrue(os.path.exists(uri.path), 

156 f"Checking URI {uri} existence for component {compName}") 

157 

158 def runExposureCompositePutGetTest(self, datasetTypeName: str) -> DatasetRef: 

159 example = os.path.join(TESTDIR, "data", "small.fits") 

160 exposure = lsst.afw.image.ExposureF(example) 

161 

162 dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"} 

163 ref = self.butler.put(exposure, datasetTypeName, dataId) 

164 

165 # Get the full thing 

166 composite = self.butler.get(datasetTypeName, dataId) 

167 

168 # There is no assert for Exposure so just look at maskedImage 

169 self.assertMaskedImagesEqual(composite.maskedImage, exposure.maskedImage) 

170 

171 # Helper for extracting components 

172 assembler = ExposureAssembler(ref.datasetType.storageClass) 

173 

174 # Get each component from butler independently 

175 for compName in COMPONENTS: 

176 compTypeName = DatasetType.nameWithComponent(datasetTypeName, compName) 

177 component = self.butler.get(compTypeName, dataId) 

178 

179 reference = assembler.getComponent(exposure, compName) 

180 self.assertIsInstance(component, type(reference), f"Checking type of component {compName}") 

181 

182 if compName in ("image", "variance"): 

183 self.assertImagesEqual(component, reference) 

184 elif compName == "mask": 

185 self.assertMasksEqual(component, reference) 

186 elif compName == "wcs": 

187 self.assertWcsAlmostEqualOverBBox(component, reference, exposure.getBBox()) 

188 elif compName == "coaddInputs": 

189 self.assertEqual(len(component.visits), len(reference.visits), 

190 f"cf visits {component.visits}") 

191 self.assertEqual(len(component.ccds), len(reference.ccds), 

192 f"cf CCDs {component.ccds}") 

193 elif compName == "psf": 

194 # Equality for PSF does not work 

195 pass 

196 elif compName == "visitInfo": 

197 self.assertEqual(component.getExposureId(), reference.getExposureId(), 

198 f"VisitInfo comparison") 

199 elif compName == "metadata": 

200 # The component metadata has extra fields in it so cannot 

201 # compare directly. 

202 for k, v in reference.items(): 

203 self.assertEqual(component[k], v) 

204 elif compName == "photoCalib": 

205 # This example has a 

206 # "spatially constant with mean: inf error: nan" entry 

207 # which does not compare directly. 

208 self.assertEqual(str(component), str(reference)) 

209 self.assertIn("spatially constant with mean: inf", str(component), "Checking photoCalib") 

210 else: 

211 raise RuntimeError(f"Unexpected component '{compName}' encountered in test") 

212 

213 # With parameters 

214 inBBox = Box2I(minimum=Point2I(0, 0), maximum=Point2I(3, 3)) 

215 parameters = dict(bbox=inBBox, origin=LOCAL) 

216 subset = self.butler.get(datasetTypeName, dataId, parameters=parameters) 

217 outBBox = subset.getBBox() 

218 self.assertEqual(inBBox, outBBox) 

219 self.assertImagesEqual(subset.getImage(), exposure.subset(inBBox, origin=LOCAL).getImage()) 

220 

221 return ref 

222 

223 

224if __name__ == "__main__": 224 ↛ 225line 224 didn't jump to line 225, because the condition on line 224 was never true

225 unittest.main()