Coverage for tests/test_loadReferenceObjects.py: 12%

230 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-15 03:09 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2016 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import itertools 

25import unittest 

26 

27import astropy.time 

28 

29import lsst.afw.table as afwTable 

30import lsst.geom 

31import lsst.log 

32from lsst.meas.algorithms import ReferenceObjectLoader, getRefFluxField, getRefFluxKeys 

33from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits, convertToNanojansky 

34import lsst.pex.config 

35import lsst.utils.tests 

36 

37from ingestIndexTestBase import makeConvertConfig 

38 

39 

40class ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase): 

41 """Test generic parts of loader, but not the actual catalog loading.""" 

42 def testFilterMapVsAnyFilterMapsToThis(self): 

43 config = ReferenceObjectLoader.ConfigClass() 

44 # check that a filterMap-only config passes validation 

45 config.filterMap = {"b": "a"} 

46 try: 

47 config.validate() 

48 except lsst.pex.config.FieldValidationError: 

49 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.") 

50 

51 # anyFilterMapsToThis and filterMap are mutually exclusive 

52 config.anyFilterMapsToThis = "c" 

53 with self.assertRaises(lsst.pex.config.FieldValidationError): 

54 config.validate() 

55 

56 # check that a anyFilterMapsToThis-only config passes validation 

57 config.filterMap = {} 

58 try: 

59 config.validate() 

60 except lsst.pex.config.FieldValidationError: 

61 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.") 

62 

63 def testMakeMinimalSchema(self): 

64 """Make a schema and check it.""" 

65 for filterNameList in (["r"], ["foo", "_bar"]): 

66 for (addIsPhotometric, addIsResolved, addIsVariable, 

67 coordErrDim, addProperMotion, properMotionErrDim, 

68 addParallax) in itertools.product( 

69 (False, True), (False, True), (False, True), 

70 (-1, 0, 1, 2, 3, 4), (False, True), (-1, 0, 1, 2, 3, 4), 

71 (False, True)): 

72 argDict = dict( 

73 filterNameList=filterNameList, 

74 addIsPhotometric=addIsPhotometric, 

75 addIsResolved=addIsResolved, 

76 addIsVariable=addIsVariable, 

77 coordErrDim=coordErrDim, 

78 addProperMotion=addProperMotion, 

79 properMotionErrDim=properMotionErrDim, 

80 addParallax=addParallax, 

81 ) 

82 if coordErrDim not in (0, 2, 3) or \ 

83 (addProperMotion and properMotionErrDim not in (0, 2, 3)): 

84 with self.assertRaises(ValueError): 

85 ReferenceObjectLoader.makeMinimalSchema(**argDict) 

86 else: 

87 refSchema = ReferenceObjectLoader.makeMinimalSchema(**argDict) 

88 self.assertTrue("coord_ra" in refSchema) 

89 self.assertTrue("coord_dec" in refSchema) 

90 for filterName in filterNameList: 

91 fluxField = filterName + "_flux" 

92 self.assertIn(fluxField, refSchema) 

93 self.assertNotIn("x" + fluxField, refSchema) 

94 fluxErrField = fluxField + "Err" 

95 self.assertIn(fluxErrField, refSchema) 

96 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux") 

97 self.assertEqual("resolved" in refSchema, addIsResolved) 

98 self.assertEqual("variable" in refSchema, addIsVariable) 

99 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

100 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

101 self.assertEqual("epoch" in refSchema, addProperMotion or addParallax) 

102 self.assertEqual("coord_raErr" in refSchema, coordErrDim > 0) 

103 self.assertEqual("coord_decErr" in refSchema, coordErrDim > 0) 

104 self.assertEqual("coord_ra_dec_Cov" in refSchema, coordErrDim == 3) 

105 self.assertEqual("pm_ra" in refSchema, addProperMotion) 

106 self.assertEqual("pm_dec" in refSchema, addProperMotion) 

107 self.assertEqual("pm_raErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

108 self.assertEqual("pm_decErr" in refSchema, addProperMotion and properMotionErrDim > 0) 

109 self.assertEqual("pm_flag" in refSchema, addProperMotion) 

110 self.assertEqual("pm_ra_dec_Cov" in refSchema, 

111 addProperMotion and properMotionErrDim == 3) 

112 self.assertEqual("parallax" in refSchema, addParallax) 

113 self.assertEqual("parallaxErr" in refSchema, addParallax) 

114 self.assertEqual("parallax_flag" in refSchema, addParallax) 

115 

116 def testFilterAliasMap(self): 

117 """Make a schema with filter aliases.""" 

118 for filterMap in ({}, {"camr": "r"}): 

119 config = ReferenceObjectLoader.ConfigClass() 

120 config.filterMap = filterMap 

121 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

122 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList="r") 

123 loader._addFluxAliases(refSchema, 

124 anyFilterMapsToThis=config.anyFilterMapsToThis, 

125 filterMap=config.filterMap) 

126 

127 self.assertIn("r_flux", refSchema) 

128 self.assertIn("r_fluxErr", refSchema) 

129 

130 # camera filters aliases are named <filter>_camFlux 

131 if "camr" in filterMap: 

132 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux") 

133 else: 

134 with self.assertRaisesRegex(RuntimeError, 

135 r"Could not find flux field\(s\) camr_camFlux, camr_flux"): 

136 getRefFluxField(refSchema, "camr") 

137 

138 refCat = afwTable.SimpleCatalog(refSchema) 

139 refObj = refCat.addNew() 

140 refObj["r_flux"] = 1.23 

141 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23) 

142 if "camr" in filterMap: 

143 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23) 

144 refObj["r_fluxErr"] = 0.111 

145 if "camr" in filterMap: 

146 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111) 

147 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r") 

148 self.assertEqual(refCat[0].get(fluxKey), 1.23) 

149 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

150 if "camr" in filterMap: 

151 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr") 

152 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

153 else: 

154 with self.assertRaises(RuntimeError): 

155 getRefFluxKeys(refSchema, "camr") 

156 

157 def testAnyFilterMapsToThisAlias(self): 

158 # test anyFilterMapsToThis 

159 config = ReferenceObjectLoader.ConfigClass() 

160 config.anyFilterMapsToThis = "gg" 

161 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

162 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList=["gg"]) 

163 loader._addFluxAliases(refSchema, 

164 anyFilterMapsToThis=config.anyFilterMapsToThis, 

165 filterMap=config.filterMap) 

166 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux") 

167 # raise if "gg" is not in the refcat filter list 

168 with self.assertRaises(RuntimeError): 

169 refSchema = ReferenceObjectLoader.makeMinimalSchema(filterNameList=["rr"]) 

170 refSchema = loader._addFluxAliases(refSchema, 

171 anyFilterMapsToThis=config.anyFilterMapsToThis, 

172 filterMap=config.filterMap) 

173 

174 def testCheckFluxUnits(self): 

175 """Test that we can identify old style fluxes in a schema.""" 

176 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

177 # the default schema should pass 

178 self.assertTrue(hasNanojanskyFluxUnits(schema)) 

179 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

180 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

181 

182 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

183 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

184 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

185 

186 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

187 schema.addField('bad_flux', doc='old flux units', type=float, units='Jy') 

188 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

189 

190 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

191 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='') 

192 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

193 

194 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

195 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

196 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

197 

198 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

199 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='') 

200 self.assertFalse(hasNanojanskyFluxUnits(schema)) 

201 

202 def testConvertOldFluxes(self): 

203 """Check that we can convert old style fluxes in a catalog.""" 

204 flux = 1.234 

205 fluxErr = 5.678 

206 log = lsst.log.Log() 

207 

208 def make_catalog(): 

209 schema = ReferenceObjectLoader.makeMinimalSchema(['r', 'z']) 

210 schema.addField('bad_flux', doc='old flux units', type=float, units='') 

211 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy') 

212 refCat = afwTable.SimpleCatalog(schema) 

213 refObj = refCat.addNew() 

214 refObj["bad_flux"] = flux 

215 refObj["bad_fluxErr"] = fluxErr 

216 return refCat 

217 

218 oldRefCat = make_catalog() 

219 newRefCat = convertToNanojansky(oldRefCat, log) 

220 self.assertEqual(newRefCat['bad_flux'], [flux*1e9, ]) 

221 self.assertEqual(newRefCat['bad_fluxErr'], [fluxErr*1e9, ]) 

222 self.assertEqual(newRefCat.schema['bad_flux'].asField().getUnits(), 'nJy') 

223 self.assertEqual(newRefCat.schema['bad_fluxErr'].asField().getUnits(), 'nJy') 

224 

225 # check that doConvert=False returns None (it also logs a summary) 

226 oldRefCat = make_catalog() 

227 newRefCat = convertToNanojansky(oldRefCat, log, doConvert=False) 

228 self.assertIsNone(newRefCat) 

229 

230 def testGetMetadataCircle(self): 

231 center = lsst.geom.SpherePoint(100*lsst.geom.degrees, 45*lsst.geom.degrees) 

232 radius = lsst.geom.Angle(1*lsst.geom.degrees) 

233 loader = ReferenceObjectLoader(None, None, name=None) 

234 metadata = loader.getMetadataCircle(center, radius, "fakeR") 

235 self.assertEqual(metadata['RA'], center.getLongitude().asDegrees()) 

236 self.assertEqual(metadata['DEC'], center.getLatitude().asDegrees()) 

237 self.assertEqual(metadata['RADIUS'], radius.asDegrees()) 

238 self.assertEqual(metadata['SMATCHV'], 2) 

239 self.assertEqual(metadata['FILTER'], 'fakeR') 

240 self.assertEqual(metadata['JEPOCH'], None) 

241 self.assertEqual(metadata['TIMESYS'], 'TAI') 

242 

243 epoch = astropy.time.Time(2023.0, format="jyear", scale="tai") 

244 metadata = loader.getMetadataCircle(center, radius, "fakeR", epoch=epoch) 

245 self.assertEqual(metadata['JEPOCH'], epoch.jyear) 

246 

247 

248class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase): 

249 """Test validation of ConvertReferenceCatalogConfig.""" 

250 def testValidateRaDecMag(self): 

251 config = makeConvertConfig() 

252 config.validate() 

253 

254 for name in ("ra_name", "dec_name", "mag_column_list"): 

255 with self.subTest(name=name): 

256 config = makeConvertConfig() 

257 setattr(config, name, None) 

258 with self.assertRaises(ValueError): 

259 config.validate() 

260 

261 def testValidateRaDecErr(self): 

262 # check that a basic config validates 

263 config = makeConvertConfig(withRaDecErr=True) 

264 config.validate() 

265 

266 # check that a config with any of these fields missing does not validate 

267 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

268 with self.subTest(name=name): 

269 config = makeConvertConfig(withRaDecErr=True) 

270 setattr(config, name, None) 

271 with self.assertRaises(ValueError): 

272 config.validate() 

273 

274 # check that coord_err_unit must be an astropy unit 

275 config = makeConvertConfig(withRaDecErr=True) 

276 config.coord_err_unit = "nonsense unit" 

277 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

278 config.validate() 

279 

280 def testValidateMagErr(self): 

281 config = makeConvertConfig(withMagErr=True) 

282 config.validate() 

283 

284 # test for missing names 

285 for name in config.mag_column_list: 

286 with self.subTest(name=name): 

287 config = makeConvertConfig(withMagErr=True) 

288 del config.mag_err_column_map[name] 

289 with self.assertRaises(ValueError): 

290 config.validate() 

291 

292 # test for incorrect names 

293 for name in config.mag_column_list: 

294 with self.subTest(name=name): 

295 config = makeConvertConfig(withMagErr=True) 

296 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

297 del config.mag_err_column_map[name] 

298 with self.assertRaises(ValueError): 

299 config.validate() 

300 

301 def testValidatePm(self): 

302 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"] 

303 

304 for withPmErr in (False, True): 

305 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

306 config.validate() 

307 del config 

308 

309 if withPmErr: 

310 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"] 

311 else: 

312 names = basicNames 

313 for name in names: 

314 with self.subTest(name=name, withPmErr=withPmErr): 

315 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

316 setattr(config, name, None) 

317 with self.assertRaises(ValueError): 

318 config.validate() 

319 

320 def testValidateParallax(self): 

321 """Validation should fail if any parallax-related fields are missing. 

322 """ 

323 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

324 

325 config = makeConvertConfig(withParallax=True) 

326 config.validate() 

327 del config 

328 

329 for name in names: 

330 with self.subTest(name=name): 

331 config = makeConvertConfig(withParallax=True) 

332 setattr(config, name, None) 

333 with self.assertRaises(ValueError, msg=name): 

334 config.validate() 

335 

336 

337class TestMemory(lsst.utils.tests.MemoryTestCase): 

338 pass 

339 

340 

341def setup_module(module): 

342 lsst.utils.tests.init() 

343 

344 

345if __name__ == "__main__": 345 ↛ 346line 345 didn't jump to line 346, because the condition on line 345 was never true

346 lsst.utils.tests.init() 

347 unittest.main()