Coverage for tests/test_loadReferenceObjects.py: 11%

213 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-23 02:49 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2016 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import itertools 

25import unittest 

26 

27import lsst.afw.table as afwTable 

28import lsst.log 

29from lsst.meas.algorithms import ReferenceObjectLoader, getRefFluxField, getRefFluxKeys 

30from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits, convertToNanojansky 

31import lsst.pex.config 

32import lsst.utils.tests 

33 

34from ingestIndexTestBase import makeConvertConfig 

35 

36 

37class ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase): 

38 """Test generic parts of loader, but not the actual catalog loading.""" 

39 def testFilterMapVsAnyFilterMapsToThis(self): 

40 config = ReferenceObjectLoader.ConfigClass() 

41 # check that a filterMap-only config passes validation 

42 config.filterMap = {"b": "a"} 

43 try: 

44 config.validate() 

45 except lsst.pex.config.FieldValidationError: 

46 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.") 

47 

48 # anyFilterMapsToThis and filterMap are mutually exclusive 

49 config.anyFilterMapsToThis = "c" 

50 with self.assertRaises(lsst.pex.config.FieldValidationError): 

51 config.validate() 

52 

53 # check that a anyFilterMapsToThis-only config passes validation 

54 config.filterMap = {} 

55 try: 

56 config.validate() 

57 except lsst.pex.config.FieldValidationError: 

58 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.") 

59 

60 def testMakeMinimalSchema(self): 

61 """Make a schema and check it.""" 

62 for filterNameList in (["r"], ["foo", "_bar"]): 

63 for (addIsPhotometric, addIsResolved, addIsVariable, 

64 coordErrDim, addProperMotion, properMotionErrDim, 

65 addParallax) in itertools.product( 

66 (False, True), (False, True), (False, True), 

67 (-1, 0, 1, 2, 3, 4), (False, True), (-1, 0, 1, 2, 3, 4), 

68 (False, True)): 

69 argDict = dict( 

70 filterNameList=filterNameList, 

71 addIsPhotometric=addIsPhotometric, 

72 addIsResolved=addIsResolved, 

73 addIsVariable=addIsVariable, 

74 coordErrDim=coordErrDim, 

75 addProperMotion=addProperMotion, 

76 properMotionErrDim=properMotionErrDim, 

77 addParallax=addParallax, 

78 ) 

79 if coordErrDim not in (0, 2, 3) or \ 

80 (addProperMotion and properMotionErrDim not in (0, 2, 3)): 

81 with self.assertRaises(ValueError): 

82 ReferenceObjectLoader.makeMinimalSchema(**argDict) 

83 else: 

84 refSchema = ReferenceObjectLoader.makeMinimalSchema(**argDict) 

85 self.assertTrue("coord_ra" in refSchema) 

86 self.assertTrue("coord_dec" in refSchema) 

87 for filterName in filterNameList: 

88 fluxField = filterName + "_flux" 

89 self.assertIn(fluxField, refSchema) 

90 self.assertNotIn("x" + fluxField, refSchema) 

91 fluxErrField = fluxField + "Err" 

92 self.assertIn(fluxErrField, refSchema) 

93 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux") 

94 self.assertEqual("resolved" in refSchema, addIsResolved) 

95 self.assertEqual("variable" in refSchema, addIsVariable) 

96 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

97 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

98 self.assertEqual("epoch" in refSchema, addProperMotion or addParallax) 

99 self.assertEqual("coord_raErr" in refSchema, coordErrDim > 0) 

100 self.assertEqual("coord_decErr" in refSchema, coordErrDim > 0) 

101 self.assertEqual("coord_ra_dec_Cov" in refSchema, coordErrDim == 3) 

102 self.assertEqual("pm_ra" in refSchema, addProperMotion) 

103 self.assertEqual("pm_dec" in refSchema, addProperMotion) 

104 self.assertEqual("pm_raErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

105 self.assertEqual("pm_decErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

106 self.assertEqual("pm_flag" in refSchema, addProperMotion) 

107 self.assertEqual("pm_ra_dec_Cov" in refSchema, 

108 addProperMotion and properMotionErrDim == 3) 

109 self.assertEqual("parallax" in refSchema, addParallax) 

110 self.assertEqual("parallaxErr" in refSchema, addParallax) 

111 self.assertEqual("parallax_flag" in refSchema, addParallax) 

112 

113 def testFilterAliasMap(self): 

114 """Make a schema with filter aliases.""" 

115 for filterMap in ({}, {"camr": "r"}): 

116 config = ReferenceObjectLoader.ConfigClass() 

117 config.filterMap = filterMap 

118 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

119 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList="r") 

120 loader._addFluxAliases(refSchema, 

121 anyFilterMapsToThis=config.anyFilterMapsToThis, 

122 filterMap=config.filterMap) 

123 

124 self.assertIn("r_flux", refSchema) 

125 self.assertIn("r_fluxErr", refSchema) 

126 

127 # camera filters aliases are named <filter>_camFlux 

128 if "camr" in filterMap: 

129 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux") 

130 else: 

131 with self.assertRaisesRegex(RuntimeError, 

132 r"Could not find flux field\(s\) camr_camFlux, camr_flux"): 

133 getRefFluxField(refSchema, "camr") 

134 

135 refCat = afwTable.SimpleCatalog(refSchema) 

136 refObj = refCat.addNew() 

137 refObj["r_flux"] = 1.23 

138 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23) 

139 if "camr" in filterMap: 

140 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23) 

141 refObj["r_fluxErr"] = 0.111 

142 if "camr" in filterMap: 

143 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111) 

144 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r") 

145 self.assertEqual(refCat[0].get(fluxKey), 1.23) 

146 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

147 if "camr" in filterMap: 

148 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr") 

149 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

150 else: 

151 with self.assertRaises(RuntimeError): 

152 getRefFluxKeys(refSchema, "camr") 

153 

154 def testAnyFilterMapsToThisAlias(self): 

155 # test anyFilterMapsToThis 

156 config = ReferenceObjectLoader.ConfigClass() 

157 config.anyFilterMapsToThis = "gg" 

158 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

159 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList=["gg"]) 

160 loader._addFluxAliases(refSchema, 

161 anyFilterMapsToThis=config.anyFilterMapsToThis, 

162 filterMap=config.filterMap) 

163 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux") 

164 # raise if "gg" is not in the refcat filter list 

165 with self.assertRaises(RuntimeError): 

166 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList=["rr"]) 

167 refSchema = loader._addFluxAliases(refSchema, 

168 anyFilterMapsToThis=config.anyFilterMapsToThis, 

169 filterMap=config.filterMap) 

170 

171 def testCheckFluxUnits(self): 

172 """Test that we can identify old style fluxes in a schema.""" 

173 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

174 # the default schema should pass 

175 self.assertTrue(hasNanojanskyFluxUnits(schema)) 

176 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

177 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

178 

179 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

180 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

181 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

182 

183 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

184 schema.addField('bad_flux', doc='old flux units', type=float, units='Jy') 

185 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

186 

187 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

188 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='') 

189 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

190 

191 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

192 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

193 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

194 

195 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

196 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

197 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

198 

199 def testConvertOldFluxes(self): 

200 """Check that we can convert old style fluxes in a catalog.""" 

201 flux = 1.234 

202 fluxErr = 5.678 

203 log = lsst.log.Log() 

204 

205 def make_catalog(): 

206 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

207 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

208 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

209 refCat = afwTable.SimpleCatalog(schema) 

210 refObj = refCat.addNew() 

211 refObj["bad_flux"] = flux 

212 refObj["bad_fluxErr"] = fluxErr 

213 return refCat 

214 

215 oldRefCat = make_catalog() 

216 newRefCat = convertToNanojansky(oldRefCat, log) 

217 self.assertEqual(newRefCat['bad_flux'], [flux*1e9, ]) 

218 self.assertEqual(newRefCat['bad_fluxErr'], [fluxErr*1e9, ]) 

219 self.assertEqual(newRefCat.schema['bad_flux'].asField().getUnits(), 'nJy') 

220 self.assertEqual(newRefCat.schema['bad_fluxErr'].asField().getUnits(), 'nJy') 

221 

222 # check that doConvert=False returns None (it also logs a summary) 

223 oldRefCat = make_catalog() 

224 newRefCat = convertToNanojansky(oldRefCat, log, doConvert=False) 

225 self.assertIsNone(newRefCat) 

226 

227 

228class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase): 

229 """Test validation of ConvertReferenceCatalogConfig.""" 

230 def testValidateRaDecMag(self): 

231 config = makeConvertConfig() 

232 config.validate() 

233 

234 for name in ("ra_name", "dec_name", "mag_column_list"): 

235 with self.subTest(name=name): 

236 config = makeConvertConfig() 

237 setattr(config, name, None) 

238 with self.assertRaises(ValueError): 

239 config.validate() 

240 

241 def testValidateRaDecErr(self): 

242 # check that a basic config validates 

243 config = makeConvertConfig(withRaDecErr=True) 

244 config.validate() 

245 

246 # check that a config with any of these fields missing does not validate 

247 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

248 with self.subTest(name=name): 

249 config = makeConvertConfig(withRaDecErr=True) 

250 setattr(config, name, None) 

251 with self.assertRaises(ValueError): 

252 config.validate() 

253 

254 # check that coord_err_unit must be an astropy unit 

255 config = makeConvertConfig(withRaDecErr=True) 

256 config.coord_err_unit = "nonsense unit" 

257 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

258 config.validate() 

259 

260 def testValidateMagErr(self): 

261 config = makeConvertConfig(withMagErr=True) 

262 config.validate() 

263 

264 # test for missing names 

265 for name in config.mag_column_list: 

266 with self.subTest(name=name): 

267 config = makeConvertConfig(withMagErr=True) 

268 del config.mag_err_column_map[name] 

269 with self.assertRaises(ValueError): 

270 config.validate() 

271 

272 # test for incorrect names 

273 for name in config.mag_column_list: 

274 with self.subTest(name=name): 

275 config = makeConvertConfig(withMagErr=True) 

276 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

277 del config.mag_err_column_map[name] 

278 with self.assertRaises(ValueError): 

279 config.validate() 

280 

281 def testValidatePm(self): 

282 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"] 

283 

284 for withPmErr in (False, True): 

285 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

286 config.validate() 

287 del config 

288 

289 if withPmErr: 

290 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"] 

291 else: 

292 names = basicNames 

293 for name in names: 

294 with self.subTest(name=name, withPmErr=withPmErr): 

295 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

296 setattr(config, name, None) 

297 with self.assertRaises(ValueError): 

298 config.validate() 

299 

300 def testValidateParallax(self): 

301 """Validation should fail if any parallax-related fields are missing. 

302 """ 

303 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

304 

305 config = makeConvertConfig(withParallax=True) 

306 config.validate() 

307 del config 

308 

309 for name in names: 

310 with self.subTest(name=name): 

311 config = makeConvertConfig(withParallax=True) 

312 setattr(config, name, None) 

313 with self.assertRaises(ValueError, msg=name): 

314 config.validate() 

315 

316 

317class TestMemory(lsst.utils.tests.MemoryTestCase): 

318 pass 

319 

320 

321def setup_module(module): 

322 lsst.utils.tests.init() 

323 

324 

325if __name__ == "__main__": 325 ↛ 326line 325 didn't jump to line 326, because the condition on line 325 was never true

326 lsst.utils.tests.init() 

327 unittest.main()