Coverage for tests/test_transformDiaSourceCatalog.py: 18%

134 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 10:38 +0000

1# This file is part of ap_association 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24 

25import numpy as np 

26 

27from lsst.ap.association.transformDiaSourceCatalog import (TransformDiaSourceCatalogConfig, 

28 TransformDiaSourceCatalogTask) 

29from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

30import lsst.daf.base as dafBase 

31import lsst.afw.image as afwImage 

32import lsst.geom as geom 

33import lsst.meas.base.tests as measTests 

34from lsst.pipe.base import Struct 

35import lsst.utils.tests 

36 

37from lsst.ap.association.transformDiaSourceCatalog import UnpackApdbFlags 

38 

39TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

40 

41 

42class TestTransformDiaSourceCatalogTask(unittest.TestCase): 

43 def setUp(self): 

44 # Create an instance of random generator with fixed seed. 

45 rng = np.random.default_rng(1234) 

46 

47 # The first source will be a sky source. 

48 self.nSources = 10 

49 # Default PSF size (psfDim in makeEmptyExposure) in TestDataset results 

50 # in an 18 pixel wide source box. 

51 self.bboxSize = 18 

52 self.yLoc = 100 

53 self.bbox = geom.Box2I(geom.Point2I(0, 0), 

54 geom.Extent2I(1024, 1153)) 

55 dataset = measTests.TestDataset(self.bbox) 

56 for srcIdx in range(self.nSources-1): 

57 # Place sources at (index, yLoc), so we can distinguish them later. 

58 dataset.addSource(100000.0, geom.Point2D(srcIdx, self.yLoc)) 

59 # Ensure the last source has no peak `significance` field. 

60 dataset.addSource(100000.0, geom.Point2D(srcIdx+1, self.yLoc), setPeakSignificance=False) 

61 schema = dataset.makeMinimalSchema() 

62 schema.addField("base_PixelFlags_flag", type="Flag") 

63 schema.addField("base_PixelFlags_flag_offimage", type="Flag") 

64 schema.addField("sky_source", type="Flag", doc="Sky objects.") 

65 self.exposure, self.inputCatalog = dataset.realize(10.0, schema, randomSeed=1234) 

66 self.inputCatalog[0]['sky_source'] = True 

67 # Create schemas for use in initializing the TransformDiaSourceCatalog task. 

68 self.initInputs = {"diaSourceSchema": Struct(schema=schema)} 

69 self.initInputsBadFlags = {"diaSourceSchema": Struct(schema=dataset.makeMinimalSchema())} 

70 

71 # Separate real/bogus score table, indexed on the above catalog ids. 

72 reliabilitySchema = lsst.afw.table.Schema() 

73 reliabilitySchema.addField(self.inputCatalog.schema["id"].asField()) 

74 reliabilitySchema.addField("score", doc="real/bogus score of this source", type=float) 

75 self.reliability = lsst.afw.table.BaseCatalog(reliabilitySchema) 

76 self.reliability.resize(len(self.inputCatalog)) 

77 self.reliability["id"] = self.inputCatalog["id"] 

78 self.reliability["score"] = rng.random(len(self.inputCatalog)) 

79 

80 self.expId = 4321 

81 self.date = dafBase.DateTime(nsecs=1400000000 * 10**9) 

82 detector = DetectorWrapper(id=23, bbox=self.exposure.getBBox()).detector 

83 visit = afwImage.VisitInfo( 

84 id=self.expId, 

85 exposureTime=200., 

86 date=self.date) 

87 self.exposure.info.id = self.expId + (10000*detector.getId()) 

88 self.exposure.setDetector(detector) 

89 self.exposure.info.setVisitInfo(visit) 

90 self.band = 'g' 

91 self.exposure.setFilter(afwImage.FilterLabel(band=self.band, physical='g.MP9401')) 

92 scale = 2 

93 scaleErr = 1 

94 self.photoCalib = afwImage.PhotoCalib(scale, scaleErr) 

95 self.exposure.setPhotoCalib(self.photoCalib) 

96 

97 self.config = TransformDiaSourceCatalogConfig() 

98 self.config.flagMap = os.path.join(TESTDIR, "data", "test-flag-map.yaml") 

99 self.config.functorFile = os.path.join(TESTDIR, 

100 "data", 

101 "testDiaSource.yaml") 

102 

103 def test_run(self): 

104 """Test output dataFrame is created and values are correctly inserted 

105 from the exposure. 

106 """ 

107 transformTask = TransformDiaSourceCatalogTask(initInputs=self.initInputs, 

108 config=self.config) 

109 result = transformTask.run(self.inputCatalog, 

110 self.exposure, 

111 self.band) 

112 

113 self.assertEqual(len(result.diaSourceTable), len(self.inputCatalog)) 

114 np.testing.assert_array_equal(result.diaSourceTable["bboxSize"], [self.bboxSize]*self.nSources) 

115 np.testing.assert_array_equal(result.diaSourceTable["visit"], 

116 [self.exposure.visitInfo.id]*self.nSources) 

117 np.testing.assert_array_equal(result.diaSourceTable["detector"], 

118 [self.exposure.detector.getId()]*self.nSources) 

119 np.testing.assert_array_equal(result.diaSourceTable["band"], [self.band]*self.nSources) 

120 np.testing.assert_array_equal(result.diaSourceTable["midpointMjdTai"], 

121 [self.date.get(system=dafBase.DateTime.MJD)]*self.nSources) 

122 np.testing.assert_array_equal(result.diaSourceTable["diaObjectId"], [0]*self.nSources) 

123 np.testing.assert_array_equal(result.diaSourceTable["x"], np.arange(self.nSources)) 

124 # The final snr value should be NaN because it doesn't have a peak significance field. 

125 expect_snr = [397.887353515625]*9 

126 expect_snr.append(np.nan) 

127 # Have to use allclose because assert_array_equal doesn't support equal_nan. 

128 np.testing.assert_allclose(result.diaSourceTable["snr"], expect_snr, equal_nan=True, rtol=0) 

129 

130 def test_run_with_reliability(self): 

131 self.config.doIncludeReliability = True 

132 transformTask = TransformDiaSourceCatalogTask(initInputs=self.initInputs, 

133 config=self.config) 

134 result = transformTask.run(self.inputCatalog, 

135 self.exposure, 

136 self.band, 

137 reliability=self.reliability) 

138 self.assertEqual(len(result.diaSourceTable), len(self.inputCatalog)) 

139 np.testing.assert_array_equal(result.diaSourceTable["reliability"], self.reliability["score"]) 

140 

141 def test_run_doSkySources(self): 

142 """Test that we get the correct output with doSkySources=True; the one 

143 sky source should be missing, but the other records should be the same. 

144 

145 We only test the fields here that could be different, not the ones that 

146 are the same for all sources. 

147 """ 

148 # Make the sky source have a different significance value, to distinguish it. 

149 self.inputCatalog[0].getFootprint().updatePeakSignificance(5.0) 

150 

151 self.config.doRemoveSkySources = True 

152 task = TransformDiaSourceCatalogTask(initInputs=self.initInputs, config=self.config) 

153 result = task.run(self.inputCatalog, self.exposure, self.band) 

154 

155 self.assertEqual(len(result.diaSourceTable), self.nSources-1) 

156 # 0th source was removed, so x positions of the remaining sources are at x=1,2,3... 

157 np.testing.assert_array_equal(result.diaSourceTable["x"], np.arange(self.nSources-1)+1) 

158 # The final snr value should be NaN because it doesn't have a peak significance field. 

159 expect_snr = [397.887353515625]*8 

160 expect_snr.append(np.nan) 

161 # Have to use allclose because assert_array_equal doesn't support equal_nan. 

162 np.testing.assert_allclose(result.diaSourceTable["snr"], expect_snr, equal_nan=True, rtol=0) 

163 

164 def test_run_dia_source_wrong_flags(self): 

165 """Test that the proper errors are thrown when requesting flag columns 

166 that are not in the input schema. 

167 """ 

168 with self.assertRaises(KeyError): 

169 TransformDiaSourceCatalogTask(initInputs=self.initInputsBadFlags) 

170 

171 def test_computeBBoxSize(self): 

172 transform = TransformDiaSourceCatalogTask(initInputs=self.initInputs, 

173 config=self.config) 

174 boxSizes = transform.computeBBoxSizes(self.inputCatalog) 

175 

176 for size in boxSizes: 

177 self.assertEqual(size, self.bboxSize) 

178 self.assertEqual(len(boxSizes), self.nSources) 

179 

180 # TODO: remove in DM-41532 

181 def test_bit_unpacker(self): 

182 """Test that the integer bit packer is functioning correctly. 

183 """ 

184 self.config.doPackFlags = True 

185 transform = TransformDiaSourceCatalogTask(initInputs=self.initInputs, 

186 config=self.config) 

187 for idx, obj in enumerate(self.inputCatalog): 

188 if idx in [1, 3, 5]: 

189 obj.set("base_PixelFlags_flag", 1) 

190 if idx in [1, 4, 6]: 

191 obj.set("base_PixelFlags_flag_offimage", 1) 

192 outputCatalog = transform.run(self.inputCatalog, 

193 self.exposure, 

194 self.band).diaSourceTable 

195 

196 unpacker = UnpackApdbFlags(self.config.flagMap, "DiaSource") 

197 flag_values = unpacker.unpack(outputCatalog["flags"], "flags") 

198 

199 for idx, flag in enumerate(flag_values): 

200 if idx in [1, 3, 5]: 

201 self.assertTrue(flag['base_PixelFlags_flag']) 

202 else: 

203 self.assertFalse(flag['base_PixelFlags_flag']) 

204 

205 if idx in [1, 4, 6]: 

206 self.assertTrue(flag['base_PixelFlags_flag_offimage']) 

207 else: 

208 self.assertFalse(flag['base_PixelFlags_flag_offimage']) 

209 

210 def test_flag_existence_check(self): 

211 unpacker = UnpackApdbFlags(self.config.flagMap, "DiaSource") 

212 

213 self.assertTrue(unpacker.flagExists('base_PixelFlags_flag')) 

214 self.assertFalse(unpacker.flagExists('')) 

215 with self.assertRaisesRegex(ValueError, 'column doesNotExist not in flag map'): 

216 unpacker.flagExists('base_PixelFlags_flag', columnName='doesNotExist') 

217 

218 def test_flag_bitmask(self): 

219 """Test that we get the expected bitmask back from supplied flag names. 

220 """ 

221 unpacker = UnpackApdbFlags(self.config.flagMap, "DiaSource") 

222 

223 with self.assertRaisesRegex(ValueError, "flag '' not included"): 

224 unpacker.makeFlagBitMask(['']) 

225 with self.assertRaisesRegex(ValueError, 'column doesNotExist not in flag map'): 

226 unpacker.makeFlagBitMask(['base_PixelFlags_flag'], columnName='doesNotExist') 

227 self.assertEqual(unpacker.makeFlagBitMask(['base_PixelFlags_flag']), np.uint64(1)) 

228 self.assertEqual(unpacker.makeFlagBitMask(['base_PixelFlags_flag_offimage']), np.uint64(4)) 

229 self.assertEqual(unpacker.makeFlagBitMask(['base_PixelFlags_flag', 

230 'base_PixelFlags_flag_offimage']), 

231 np.uint64(5)) 

232 

233 

234class MemoryTester(lsst.utils.tests.MemoryTestCase): 

235 pass 

236 

237 

238def setup_module(module): 

239 lsst.utils.tests.init() 

240 

241 

242if __name__ == "__main__": 242 ↛ 243line 242 didn't jump to line 243, because the condition on line 242 was never true

243 lsst.utils.tests.init() 

244 unittest.main()