Coverage for tests/test_loadReferenceObjects.py: 10%

216 statements  

« prev     ^ index     » next       coverage.py v7.2.2, created at 2023-03-23 16:58 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2016 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import itertools 

25import unittest 

26 

27import lsst.afw.table as afwTable 

28import lsst.log 

29from lsst.meas.algorithms import ReferenceObjectLoaderBase, getRefFluxField, getRefFluxKeys 

30from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits, convertToNanojansky 

31import lsst.pex.config 

32import lsst.utils.tests 

33 

34from ingestIndexTestBase import makeConvertConfig 

35 

36 

37class TrivialLoader(ReferenceObjectLoaderBase): 

38 """Minimal subclass of ReferenceObjectLoaderBase to allow instantiation 

39 """ 

40 

41 def loadSkyCircle(self, ctrCoord, radius, filterName): 

42 pass 

43 

44 

45class ReferenceObjectLoaderBaseTestCase(lsst.utils.tests.TestCase): 

46 """Test case for ReferenceObjectLoaderBase abstract base class. 

47 

48 Only methods with concrete implementations are tested (hence not loadSkyCircle) 

49 """ 

50 

51 def testFilterMapVsAnyFilterMapsToThis(self): 

52 config = TrivialLoader.ConfigClass() 

53 # check that a filterMap-only config passes validation 

54 config.filterMap = {"b": "a"} 

55 try: 

56 config.validate() 

57 except lsst.pex.config.FieldValidationError: 

58 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.") 

59 

60 # anyFilterMapsToThis and filterMap are mutually exclusive 

61 config.anyFilterMapsToThis = "c" 

62 with self.assertRaises(lsst.pex.config.FieldValidationError): 

63 config.validate() 

64 

65 # check that a anyFilterMapsToThis-only config passes validation 

66 config.filterMap = {} 

67 try: 

68 config.validate() 

69 except lsst.pex.config.FieldValidationError: 

70 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.") 

71 

72 def testMakeMinimalSchema(self): 

73 """Make a schema and check it.""" 

74 for filterNameList in (["r"], ["foo", "_bar"]): 

75 for (addIsPhotometric, addIsResolved, addIsVariable, 

76 coordErrDim, addProperMotion, properMotionErrDim, 

77 addParallax) in itertools.product( 

78 (False, True), (False, True), (False, True), 

79 (-1, 0, 1, 2, 3, 4), (False, True), (-1, 0, 1, 2, 3, 4), 

80 (False, True)): 

81 argDict = dict( 

82 filterNameList=filterNameList, 

83 addIsPhotometric=addIsPhotometric, 

84 addIsResolved=addIsResolved, 

85 addIsVariable=addIsVariable, 

86 coordErrDim=coordErrDim, 

87 addProperMotion=addProperMotion, 

88 properMotionErrDim=properMotionErrDim, 

89 addParallax=addParallax, 

90 ) 

91 if coordErrDim not in (0, 2, 3) or \ 

92 (addProperMotion and properMotionErrDim not in (0, 2, 3)): 

93 with self.assertRaises(ValueError): 

94 ReferenceObjectLoaderBase.makeMinimalSchema(**argDict) 

95 else: 

96 refSchema = ReferenceObjectLoaderBase.makeMinimalSchema(**argDict) 

97 self.assertTrue("coord_ra" in refSchema) 

98 self.assertTrue("coord_dec" in refSchema) 

99 for filterName in filterNameList: 

100 fluxField = filterName + "_flux" 

101 self.assertIn(fluxField, refSchema) 

102 self.assertNotIn("x" + fluxField, refSchema) 

103 fluxErrField = fluxField + "Err" 

104 self.assertIn(fluxErrField, refSchema) 

105 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux") 

106 self.assertEqual("resolved" in refSchema, addIsResolved) 

107 self.assertEqual("variable" in refSchema, addIsVariable) 

108 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

109 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

110 self.assertEqual("epoch" in refSchema, addProperMotion or addParallax) 

111 self.assertEqual("coord_raErr" in refSchema, coordErrDim > 0) 

112 self.assertEqual("coord_decErr" in refSchema, coordErrDim > 0) 

113 self.assertEqual("coord_ra_dec_Cov" in refSchema, coordErrDim == 3) 

114 self.assertEqual("pm_ra" in refSchema, addProperMotion) 

115 self.assertEqual("pm_dec" in refSchema, addProperMotion) 

116 self.assertEqual("pm_raErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

117 self.assertEqual("pm_decErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

118 self.assertEqual("pm_flag" in refSchema, addProperMotion) 

119 self.assertEqual("pm_ra_dec_Cov" in refSchema, 

120 addProperMotion and properMotionErrDim == 3) 

121 self.assertEqual("parallax" in refSchema, addParallax) 

122 self.assertEqual("parallaxErr" in refSchema, addParallax) 

123 self.assertEqual("parallax_flag" in refSchema, addParallax) 

124 

125 def testFilterAliasMap(self): 

126 """Make a schema with filter aliases.""" 

127 for filterMap in ({}, {"camr": "r"}): 

128 config = TrivialLoader.ConfigClass() 

129 config.filterMap = filterMap 

130 loader = TrivialLoader(config=config) 

131 refSchema = TrivialLoader.makeMinimalSchema(filterNameList="r") 

132 loader._addFluxAliases(refSchema, 

133 anyFilterMapsToThis=config.anyFilterMapsToThis, 

134 filterMap=config.filterMap) 

135 

136 self.assertIn("r_flux", refSchema) 

137 self.assertIn("r_fluxErr", refSchema) 

138 

139 # camera filters aliases are named <filter>_camFlux 

140 if "camr" in filterMap: 

141 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux") 

142 else: 

143 with self.assertRaisesRegex(RuntimeError, 

144 r"Could not find flux field\(s\) camr_camFlux, camr_flux"): 

145 getRefFluxField(refSchema, "camr") 

146 

147 refCat = afwTable.SimpleCatalog(refSchema) 

148 refObj = refCat.addNew() 

149 refObj["r_flux"] = 1.23 

150 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23) 

151 if "camr" in filterMap: 

152 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23) 

153 refObj["r_fluxErr"] = 0.111 

154 if "camr" in filterMap: 

155 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111) 

156 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r") 

157 self.assertEqual(refCat[0].get(fluxKey), 1.23) 

158 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

159 if "camr" in filterMap: 

160 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr") 

161 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

162 else: 

163 with self.assertRaises(RuntimeError): 

164 getRefFluxKeys(refSchema, "camr") 

165 

166 def testAnyFilterMapsToThisAlias(self): 

167 # test anyFilterMapsToThis 

168 config = TrivialLoader.ConfigClass() 

169 config.anyFilterMapsToThis = "gg" 

170 loader = TrivialLoader(config=config) 

171 refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["gg"]) 

172 loader._addFluxAliases(refSchema, 

173 anyFilterMapsToThis=config.anyFilterMapsToThis, 

174 filterMap=config.filterMap) 

175 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux") 

176 # raise if "gg" is not in the refcat filter list 

177 with self.assertRaises(RuntimeError): 

178 refSchema = TrivialLoader.makeMinimalSchema(filterNameList=["rr"]) 

179 refSchema = loader._addFluxAliases(refSchema, 

180 anyFilterMapsToThis=config.anyFilterMapsToThis, 

181 filterMap=config.filterMap) 

182 

183 def testCheckFluxUnits(self): 

184 """Test that we can identify old style fluxes in a schema.""" 

185 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

186 # the default schema should pass 

187 self.assertTrue(hasNanojanskyFluxUnits(schema)) 

188 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

189 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

190 

191 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

192 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

193 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

194 

195 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

196 schema.addField('bad_flux', doc='old flux units', type=float, units='Jy') 

197 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

198 

199 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

200 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='') 

201 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

202 

203 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

204 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

205 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

206 

207 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

208 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

209 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

210 

211 def testConvertOldFluxes(self): 

212 """Check that we can convert old style fluxes in a catalog.""" 

213 flux = 1.234 

214 fluxErr = 5.678 

215 log = lsst.log.Log() 

216 

217 def make_catalog(): 

218 schema = ReferenceObjectLoaderBase.makeMinimalSchema(['r', 'z']) 

219 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

220 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

221 refCat = afwTable.SimpleCatalog(schema) 

222 refObj = refCat.addNew() 

223 refObj["bad_flux"] = flux 

224 refObj["bad_fluxErr"] = fluxErr 

225 return refCat 

226 

227 oldRefCat = make_catalog() 

228 newRefCat = convertToNanojansky(oldRefCat, log) 

229 self.assertEqual(newRefCat['bad_flux'], [flux*1e9, ]) 

230 self.assertEqual(newRefCat['bad_fluxErr'], [fluxErr*1e9, ]) 

231 self.assertEqual(newRefCat.schema['bad_flux'].asField().getUnits(), 'nJy') 

232 self.assertEqual(newRefCat.schema['bad_fluxErr'].asField().getUnits(), 'nJy') 

233 

234 # check that doConvert=False returns None (it also logs a summary) 

235 oldRefCat = make_catalog() 

236 newRefCat = convertToNanojansky(oldRefCat, log, doConvert=False) 

237 self.assertIsNone(newRefCat) 

238 

239 

240class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase): 

241 """Test validation of ConvertReferenceCatalogConfig.""" 

242 def testValidateRaDecMag(self): 

243 config = makeConvertConfig() 

244 config.validate() 

245 

246 for name in ("ra_name", "dec_name", "mag_column_list"): 

247 with self.subTest(name=name): 

248 config = makeConvertConfig() 

249 setattr(config, name, None) 

250 with self.assertRaises(ValueError): 

251 config.validate() 

252 

253 def testValidateRaDecErr(self): 

254 # check that a basic config validates 

255 config = makeConvertConfig(withRaDecErr=True) 

256 config.validate() 

257 

258 # check that a config with any of these fields missing does not validate 

259 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

260 with self.subTest(name=name): 

261 config = makeConvertConfig(withRaDecErr=True) 

262 setattr(config, name, None) 

263 with self.assertRaises(ValueError): 

264 config.validate() 

265 

266 # check that coord_err_unit must be an astropy unit 

267 config = makeConvertConfig(withRaDecErr=True) 

268 config.coord_err_unit = "nonsense unit" 

269 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

270 config.validate() 

271 

272 def testValidateMagErr(self): 

273 config = makeConvertConfig(withMagErr=True) 

274 config.validate() 

275 

276 # test for missing names 

277 for name in config.mag_column_list: 

278 with self.subTest(name=name): 

279 config = makeConvertConfig(withMagErr=True) 

280 del config.mag_err_column_map[name] 

281 with self.assertRaises(ValueError): 

282 config.validate() 

283 

284 # test for incorrect names 

285 for name in config.mag_column_list: 

286 with self.subTest(name=name): 

287 config = makeConvertConfig(withMagErr=True) 

288 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

289 del config.mag_err_column_map[name] 

290 with self.assertRaises(ValueError): 

291 config.validate() 

292 

293 def testValidatePm(self): 

294 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"] 

295 

296 for withPmErr in (False, True): 

297 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

298 config.validate() 

299 del config 

300 

301 if withPmErr: 

302 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"] 

303 else: 

304 names = basicNames 

305 for name in names: 

306 with self.subTest(name=name, withPmErr=withPmErr): 

307 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

308 setattr(config, name, None) 

309 with self.assertRaises(ValueError): 

310 config.validate() 

311 

312 def testValidateParallax(self): 

313 """Validation should fail if any parallax-related fields are missing. 

314 """ 

315 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

316 

317 config = makeConvertConfig(withParallax=True) 

318 config.validate() 

319 del config 

320 

321 for name in names: 

322 with self.subTest(name=name): 

323 config = makeConvertConfig(withParallax=True) 

324 setattr(config, name, None) 

325 with self.assertRaises(ValueError, msg=name): 

326 config.validate() 

327 

328 

329class TestMemory(lsst.utils.tests.MemoryTestCase): 

330 pass 

331 

332 

333def setup_module(module): 

334 lsst.utils.tests.init() 

335 

336 

337if __name__ == "__main__": 337 ↛ 338line 337 didn't jump to line 338, because the condition on line 337 was never true

338 lsst.utils.tests.init() 

339 unittest.main()