Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ap_association. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24 

25from lsst.ap.association import ( 

26 MapApDataConfig, 

27 MapApDataTask, 

28 MapDiaSourceConfig, 

29 MapDiaSourceTask, 

30 UnpackApdbFlags) 

31from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

32import lsst.afw.table as afwTable 

33import lsst.daf.base as dafBase 

34import lsst.afw.image as afwImage 

35import lsst.afw.image.utils as afwImageUtils 

36import lsst.geom as geom 

37import lsst.meas.base.tests as measTests 

38from lsst.utils import getPackageDir 

39import lsst.utils.tests 

40 

41 

42def make_input_source_catalog(dataset, add_flags=False): 

43 """Create tests objects to map into apData products. 

44 

45 Parameters 

46 ---------- 

47 dataset : `lsst.meas.base.tests.TestDataset` 

48 Test dataset to create random exposure realizations with inserted 

49 sources and detected sources in a catalog. 

50 """ 

51 schema = dataset.makeMinimalSchema() 

52 schema.addField("base_NaiveCentroid_x", type="D") 

53 schema.addField("base_NaiveCentroid_y", type="D") 

54 schema.addField("base_PsfFlux_instFlux", type="D") 

55 schema.addField("base_PsfFlux_instFluxErr", type="D") 

56 schema.addField("ip_diffim_DipoleFit_separation", type="D") 

57 schema.addField("ip_diffim_DipoleFit_orientation", type="D") 

58 schema.addField("ip_diffim_DipoleFit_neg_instFlux", type="D") 

59 schema.addField("ip_diffim_DipoleFit_neg_instFluxErr", type="D") 

60 schema.addField("ip_diffim_DipoleFit_pos_instFlux", type="D") 

61 schema.addField("ip_diffim_DipoleFit_pos_instFluxErr", type="D") 

62 schema.addField("ip_diffim_forced_PsfFlux_instFlux", type="D") 

63 schema.addField("ip_diffim_forced_PsfFlux_instFluxErr", type="D") 

64 if add_flags: 

65 schema.addField("base_PixelFlags_flag", type="Flag") 

66 schema.addField("base_PixelFlags_flag_offimage", type="Flag") 

67 

68 exposure, catalog = dataset.realize(10.0, schema, randomSeed=1234) 

69 catalog.definePsfFlux("base_PsfFlux") 

70 catalog.defineCentroid("base_NaiveCentroid") 

71 

72 # Fill in simple values for the test DiaSources. Create specific values 

73 # where required such as negative values for the negative lobe of the 

74 # dipole flux. 

75 for src in catalog: 

76 for subSchema in schema: 

77 if subSchema.getField().getName() == "ip_diffim_DipoleFit_neg_instFlux": 

78 src.set(subSchema.getKey(), -1) 

79 elif (subSchema.getField().getName().startswith("ip") 

80 or subSchema.getField().getName().startswith("base")): 

81 src.set(subSchema.getKey(), 1) 

82 src.set("base_NaiveCentroid_x", src["truth_x"]) 

83 src.set("base_NaiveCentroid_y", src["truth_y"]) 

84 src.setCoord(exposure.getWcs().pixelToSky(src.getCentroid())) 

85 return catalog, exposure 

86 

87 

88class TestAPDataMapperTask(unittest.TestCase): 

89 

90 def setUp(self): 

91 

92 nSources = 10 

93 # CFHT Filters from the camera mapper. 

94 afwImageUtils.resetFilters() 

95 afwImageUtils.defineFilter('u', lambdaEff=374, alias="u.MP9301") 

96 afwImageUtils.defineFilter('g', lambdaEff=487, alias="g.MP9401") 

97 afwImageUtils.defineFilter('r', lambdaEff=628, alias="r.MP9601") 

98 afwImageUtils.defineFilter('i', lambdaEff=778, alias="i.MP9701") 

99 afwImageUtils.defineFilter('z', lambdaEff=1170, alias="z.MP9801") 

100 

101 self.bbox = geom.Box2I(geom.Point2I(0, 0), 

102 geom.Extent2I(1024, 1153)) 

103 dataset = measTests.TestDataset(self.bbox) 

104 for srcIdx in range(nSources): 

105 dataset.addSource(100000.0, geom.Point2D(100, 100)) 

106 self.inputCatalogNoFlags, _ = make_input_source_catalog(dataset, False) 

107 self.inputCatalog, self.exposure = \ 

108 make_input_source_catalog(dataset, True) 

109 

110 detector = DetectorWrapper(id=23, bbox=self.exposure.getBBox()).detector 

111 visit = afwImage.VisitInfo( 

112 exposureId=4321, 

113 exposureTime=200., 

114 date=dafBase.DateTime(nsecs=1400000000 * 10**9)) 

115 self.exposure.setDetector(detector) 

116 self.exposure.getInfo().setVisitInfo(visit) 

117 self.exposure.setFilter(afwImage.Filter('g.MP9401')) 

118 scale = 2 

119 scaleErr = 1 

120 self.photoCalib = afwImage.PhotoCalib(scale, scaleErr) 

121 self.exposure.setPhotoCalib(self.photoCalib) 

122 

123 def test_run(self): 

124 """Test the generic data product mapper. 

125 """ 

126 outSchema = afwTable.SourceTable.makeMinimalSchema() 

127 outSchema.addField("psFlux", type="D") 

128 outSchema.addField("psFluxErr", type="D") 

129 

130 mapApDConfig = MapApDataConfig() 

131 mapApDConfig.copyColumns = { 

132 "id": "id", 

133 "parent": "parent", 

134 "coord_ra": "coord_ra", 

135 "coord_dec": "coord_dec", 

136 "slot_PsfFlux_instFlux": "psFlux", 

137 "slot_PsfFlux_instFluxErr": "psFluxErr" 

138 } 

139 

140 mapApD = MapApDataTask(inputSchema=self.inputCatalog.schema, 

141 outputSchema=outSchema, 

142 config=mapApDConfig) 

143 outputCatalog = mapApD.run(self.inputCatalog) 

144 

145 for inObj, outObj in zip(self.inputCatalog, outputCatalog): 

146 for inputName, outputName in mapApDConfig.copyColumns.items(): 

147 self.assertEqual(inObj[inputName], outObj[outputName]) 

148 

149 def test_run_dia_source(self): 

150 """Test the DiaSource specific data product mapper/calibrator. 

151 """ 

152 mapApDConfig = self._create_map_dia_source_config() 

153 mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, 

154 config=mapApDConfig) 

155 outputCatalog = mapApD.run(self.inputCatalog, self.exposure) 

156 

157 expectedMeanDip = 2. 

158 expectedDiffFlux = 0. 

159 expectedLength = 0.19999994425496748 

160 expectedBBox = 18 

161 

162 for inObj, outObj in zip(self.inputCatalog, outputCatalog): 

163 self.assertEqual( 

164 outObj["ccdVisitId"], 

165 self.exposure.getInfo().getVisitInfo().getExposureId()) 

166 self.assertEqual( 

167 outObj["midPointTai"], 

168 self.exposure.getInfo().getVisitInfo().getDate().get( 

169 system=dafBase.DateTime.MJD)) 

170 self.assertEqual( 

171 outObj["flags"], 

172 1 * 2 ** 0 + 1 * 2 ** 1) 

173 for inputName, outputName in mapApDConfig.copyColumns.items(): 

174 if inputName.startswith("slot_PsfFlux"): 

175 self._test_calibrated_flux(inObj, outObj) 

176 else: 

177 self.assertEqual(inObj[inputName], outObj[outputName]) 

178 self.assertEqual(outObj["dipMeanFlux"], expectedMeanDip) 

179 self.assertEqual(outObj["dipFluxDiff"], expectedDiffFlux) 

180 self.assertAlmostEqual(outObj["dipLength"], expectedLength) 

181 self.assertEqual(outObj["bboxSize"], expectedBBox) 

182 # Mapper should always emit standardized filters 

183 self.assertEqual(outObj["filterName"], 'g') 

184 

185 def _create_map_dia_source_config(self): 

186 """Create a test config for use in MapDiaSourceTask. 

187 

188 Returns 

189 ------- 

190 configurable : `lsst.pex.config.Config` 

191 Configurable for use in MapDiaSourceTask. 

192 """ 

193 configurable = MapDiaSourceConfig() 

194 configurable.copyColumns = { 

195 "id": "id", 

196 "parent": "parent", 

197 "coord_ra": "coord_ra", 

198 "coord_dec": "coord_dec", 

199 "slot_PsfFlux_instFlux": "psFlux", 

200 "slot_PsfFlux_instFluxErr": "psFluxErr" 

201 } 

202 configurable.calibrateColumns = ["slot_PsfFlux"] 

203 configurable.flagMap = os.path.join( 

204 getPackageDir("ap_association"), 

205 "tests", 

206 "test-flag-map.yaml") 

207 

208 return configurable 

209 

210 def test_run_dia_source_wrong_flags(self): 

211 """Test that the proper errors are thrown when requesting flag columns 

212 that are not in the input schema. 

213 """ 

214 mapApDConfig = self._create_map_dia_source_config() 

215 with self.assertRaises(KeyError): 

216 MapDiaSourceTask(inputSchema=self.inputCatalogNoFlags.schema, 

217 config=mapApDConfig) 

218 

219 def test_calibrateFluxes(self): 

220 """Test that flux calibration works as expected. 

221 """ 

222 outSchema = afwTable.SourceTable.makeMinimalSchema() 

223 outSchema.addField("psFlux", type="D") 

224 outSchema.addField("psFluxErr", type="D") 

225 

226 outputCatalog = afwTable.SourceCatalog(outSchema) 

227 outRecord = outputCatalog.addNew() 

228 

229 mapApDConfig = self._create_map_dia_source_config() 

230 mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, 

231 config=mapApDConfig) 

232 

233 mapApD.calibrateFluxes(self.inputCatalog[0], 

234 outRecord, 

235 self.photoCalib) 

236 self._test_calibrated_flux(self.inputCatalog[0], outRecord) 

237 

238 def _test_calibrated_flux(self, inputRecord, outputRecord): 

239 """Compare calibrated fluxes to expectation from zero point. 

240 

241 Parameters 

242 ---------- 

243 inputRecord: `lsst.afw.table.SourceRecord` 

244 Input source record with uncalibrated flux values. 

245 outputRecord: `lsst.afw.table.SourceRecord` 

246 Source record with calibrated fluxes. 

247 """ 

248 expected = self.photoCalib.instFluxToNanojansky(inputRecord["slot_PsfFlux_instFlux"], 

249 inputRecord["slot_PsfFlux_instFluxErr"]) 

250 self.assertAlmostEqual(outputRecord["psFlux"], expected.value) 

251 self.assertAlmostEqual(outputRecord["psFluxErr"], expected.error) 

252 

253 def test_computeBBoxSize(self): 

254 """Test the values created for diaSourceBBox. 

255 """ 

256 outSchema = afwTable.SourceTable.makeMinimalSchema() 

257 outSchema.addField("bboxSize", type="I") 

258 outputCatalog = afwTable.SourceCatalog(outSchema) 

259 outRecord = outputCatalog.addNew() 

260 mapApDConfig = self._create_map_dia_source_config() 

261 mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, 

262 config=mapApDConfig) 

263 mapApD.computeBBoxSize(self.inputCatalog[0], outRecord) 

264 

265 self.assertEqual(outRecord["bboxSize"], 18) 

266 

267 def test_bit_unpacker(self): 

268 """Test that the integer bit packer is functioning correctly. 

269 """ 

270 mapApDConfig = self._create_map_dia_source_config() 

271 mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, 

272 config=mapApDConfig) 

273 for idx, obj in enumerate(self.inputCatalog): 

274 if idx in [1, 3, 5]: 

275 obj.set("base_PixelFlags_flag", 0) 

276 if idx in [1, 4, 6]: 

277 obj.set("base_PixelFlags_flag_offimage", 0) 

278 outputCatalog = mapApD.run(self.inputCatalog, self.exposure) 

279 

280 unpacker = UnpackApdbFlags(mapApDConfig.flagMap, "DiaSource") 

281 flag_values = unpacker.unpack(outputCatalog.get("flags"), "flags") 

282 

283 for idx, flag in enumerate(flag_values): 

284 if idx in [1, 3, 5]: 

285 self.assertFalse(flag['base_PixelFlags_flag']) 

286 else: 

287 self.assertTrue(flag['base_PixelFlags_flag']) 

288 

289 if idx in [1, 4, 6]: 

290 self.assertFalse(flag['base_PixelFlags_flag_offimage']) 

291 else: 

292 self.assertTrue(flag['base_PixelFlags_flag_offimage']) 

293 

294 

295class MemoryTester(lsst.utils.tests.MemoryTestCase): 

296 pass 

297 

298 

299def setup_module(module): 

300 lsst.utils.tests.init() 

301 

302 

303if __name__ == "__main__": 303 ↛ 304line 303 didn't jump to line 304, because the condition on line 303 was never true

304 lsst.utils.tests.init() 

305 unittest.main()