Coverage for tests/test_htmIndex.py: 16%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

237 statements  

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2016 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import os 

25import unittest 

26from collections import Counter 

27 

28import astropy.time 

29import astropy.units 

30import numpy as np 

31 

32import lsst.geom 

33import lsst.afw.table as afwTable 

34import lsst.afw.geom as afwGeom 

35import lsst.daf.persistence as dafPersist 

36from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask, 

37 LoadIndexedReferenceObjectsConfig, getRefFluxField) 

38from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits 

39import lsst.utils 

40 

41from ingestIndexTestBase import (makeConvertConfig, ConvertReferenceCatalogTestBase, 

42 make_coord) 

43 

44 

45class IngestIndexTaskValidateTestCase(lsst.utils.tests.TestCase): 

46 """Test validation of IngestIndexReferenceConfig.""" 

47 def testValidateRaDecMag(self): 

48 config = makeConvertConfig() 

49 config.validate() 

50 

51 for name in ("ra_name", "dec_name", "mag_column_list"): 

52 with self.subTest(name=name): 

53 config = makeConvertConfig() 

54 setattr(config, name, None) 

55 with self.assertRaises(ValueError): 

56 config.validate() 

57 

58 def testValidateRaDecErr(self): 

59 # check that a basic config validates 

60 config = makeConvertConfig(withRaDecErr=True) 

61 config.validate() 

62 

63 # check that a config with any of these fields missing does not validate 

64 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

65 with self.subTest(name=name): 

66 config = makeConvertConfig(withRaDecErr=True) 

67 setattr(config, name, None) 

68 with self.assertRaises(ValueError): 

69 config.validate() 

70 

71 # check that coord_err_unit must be an astropy unit 

72 config = makeConvertConfig(withRaDecErr=True) 

73 config.coord_err_unit = "nonsense unit" 

74 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

75 config.validate() 

76 

77 def testValidateMagErr(self): 

78 config = makeConvertConfig(withMagErr=True) 

79 config.validate() 

80 

81 # test for missing names 

82 for name in config.mag_column_list: 

83 with self.subTest(name=name): 

84 config = makeConvertConfig(withMagErr=True) 

85 del config.mag_err_column_map[name] 

86 with self.assertRaises(ValueError): 

87 config.validate() 

88 

89 # test for incorrect names 

90 for name in config.mag_column_list: 

91 with self.subTest(name=name): 

92 config = makeConvertConfig(withMagErr=True) 

93 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

94 del config.mag_err_column_map[name] 

95 with self.assertRaises(ValueError): 

96 config.validate() 

97 

98 def testValidatePm(self): 

99 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"] 

100 

101 for withPmErr in (False, True): 

102 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

103 config.validate() 

104 del config 

105 

106 if withPmErr: 

107 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"] 

108 else: 

109 names = basicNames 

110 for name in names: 

111 with self.subTest(name=name, withPmErr=withPmErr): 

112 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

113 setattr(config, name, None) 

114 with self.assertRaises(ValueError): 

115 config.validate() 

116 

117 def testValidateParallax(self): 

118 """Validation should fail if any parallax-related fields are missing. 

119 """ 

120 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

121 

122 config = makeConvertConfig(withParallax=True) 

123 config.validate() 

124 del config 

125 

126 for name in names: 

127 with self.subTest(name=name): 

128 config = makeConvertConfig(withParallax=True) 

129 setattr(config, name, None) 

130 with self.assertRaises(ValueError, msg=name): 

131 config.validate() 

132 

133 

134class ReferenceCatalogIngestAndLoadTestCase(ConvertReferenceCatalogTestBase, lsst.utils.tests.TestCase): 

135 """Tests of converting, ingesting, loading and validating an HTM Indexed 

136 Reference Catalog (gen2 code path). 

137 """ 

138 @classmethod 

139 def setUpClass(cls): 

140 super().setUpClass() 

141 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test') 

142 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input") 

143 

144 # Run the ingest once to create a butler repo we can compare to 

145 config = makeConvertConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True, 

146 withParallax=True) 

147 # Pregenerated gen2 test refcats have the "cal_ref_cat" name. 

148 config.dataset_config.ref_dataset_name = "cal_ref_cat" 

149 config.dataset_config.indexer.active.depth = cls.depth 

150 config.id_name = 'id' 

151 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr 

152 config.parallax_scale = 1e3 # arcsec -> milliarcsec 

153 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

154 config.file_reader.format = 'ascii.commented_header' 

155 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath, 

156 cls.skyCatalogFile], config=config) 

157 cls.testButler = dafPersist.Butler(cls.testRepoPath) 

158 

159 @classmethod 

160 def tearDownClass(cls): 

161 del cls.testButler 

162 

163 def testSanity(self): 

164 """Sanity-check that compCats contains some entries with sources.""" 

165 numWithSources = 0 

166 for idList in self.compCats.values(): 

167 if len(idList) > 0: 

168 numWithSources += 1 

169 self.assertGreater(numWithSources, 0) 

170 

171 def testIngestSetsVersion(self): 

172 """Test that newly ingested catalogs get the correct version number set. 

173 """ 

174 def runTest(withRaDecErr): 

175 outputPath = os.path.join(self.outPath, "output_setsVersion" 

176 + "_withRaDecErr" if withRaDecErr else "") 

177 # Test with multiple files and standard config 

178 config = makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

179 withPm=True, withPmErr=True) 

180 # Pregenerated gen2 test refcats have the "cal_ref_cat" name. 

181 config.dataset_config.ref_dataset_name = "cal_ref_cat" 

182 # don't use the default depth, to avoid taking the time to create thousands of file locks 

183 config.dataset_config.indexer.active.depth = self.depth 

184 IngestIndexedReferenceTask.parseAndRun( 

185 args=[self.input_dir, "--output", outputPath, self.skyCatalogFile], 

186 config=config) 

187 # A newly-ingested refcat should be marked format_version=1. 

188 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath)) 

189 self.assertEqual(loader.dataset_config.format_version, 1) 

190 

191 runTest(withRaDecErr=True) 

192 runTest(withRaDecErr=False) 

193 

194 def testLoadSkyCircle(self): 

195 """Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config.""" 

196 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

197 for tupl, idList in self.compCats.items(): 

198 cent = make_coord(*tupl) 

199 lcat = loader.loadSkyCircle(cent, self.searchRadius, 'a') 

200 self.assertTrue(lcat.refCat.isContiguous()) 

201 self.assertFalse("camFlux" in lcat.refCat.schema) 

202 self.assertEqual(Counter(lcat.refCat['id']), Counter(idList)) 

203 if len(lcat.refCat) > 0: 

204 # make sure there are no duplicate ids 

205 self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1) 

206 self.assertEqual(len(set(Counter(idList).values())), 1) 

207 # A default-loaded sky circle should not have centroids 

208 self.assertNotIn("centroid_x", lcat.refCat.schema) 

209 self.assertNotIn("centroid_y", lcat.refCat.schema) 

210 self.assertNotIn("hasCentroid", lcat.refCat.schema) 

211 else: 

212 self.assertEqual(len(idList), 0) 

213 

214 def testLoadPixelBox(self): 

215 """Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config.""" 

216 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

217 numFound = 0 

218 for tupl, idList in self.compCats.items(): 

219 cent = make_coord(*tupl) 

220 bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary 

221 ctr_pix = bbox.getCenter() 

222 # catalog is sparse, so set pixel scale such that bbox encloses region 

223 # used to generate compCats 

224 pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth()) 

225 cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale) 

226 wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix) 

227 result = loader.loadPixelBox(bbox, wcs, "a") 

228 # The following is to ensure the reference catalog coords are 

229 # getting corrected for proper motion when an epoch is provided. 

230 # Use an extreme epoch so that differences in corrected coords 

231 # will be significant. Note that this simply tests that the coords 

232 # do indeed change when the epoch is passed. It makes no attempt 

233 # at assessing the correctness of the change. This is left to the 

234 # explicit testProperMotion() test below. 

235 resultWithEpoch = loader.loadPixelBox(bbox, wcs, "a", 

236 epoch=astropy.time.Time(30000, format='mjd', scale="tai")) 

237 self.assertFloatsNotEqual(result.refCat["coord_ra"], resultWithEpoch.refCat["coord_ra"], 

238 rtol=1.0e-4) 

239 self.assertFloatsNotEqual(result.refCat["coord_dec"], resultWithEpoch.refCat["coord_dec"], 

240 rtol=1.0e-4) 

241 self.assertFalse("camFlux" in result.refCat.schema) 

242 self.assertGreaterEqual(len(result.refCat), len(idList)) 

243 numFound += len(result.refCat) 

244 self.assertGreater(numFound, 0) 

245 

246 def testFilterMap(self): 

247 """Test filterMap parameters of LoadIndexedReferenceObjectsConfig.""" 

248 config = LoadIndexedReferenceObjectsConfig() 

249 config.filterMap = {"aprime": "a"} 

250 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

251 for tupl, idList in self.compCats.items(): 

252 cent = make_coord(*tupl) 

253 lcat = loader.loadSkyCircle(cent, self.searchRadius, "a") 

254 self.assertEqual(lcat.fluxField, "a_flux") 

255 if len(idList) > 0: 

256 aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime") 

257 self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema) 

258 break # just need one test 

259 

260 def testProperMotion(self): 

261 """Test proper motion correction""" 

262 center = make_coord(93.0, -90.0) 

263 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

264 references = loader.loadSkyCircle(center, self.searchRadius, 'a').refCat 

265 original = references.copy(True) 

266 

267 # Zero epoch change --> no proper motion correction (except minor numerical effects) 

268 loader.applyProperMotions(references, self.epoch) 

269 self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14) 

270 self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14) 

271 self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"]) 

272 self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"]) 

273 

274 # One year difference 

275 loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr) 

276 self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"]) 

277 self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"]) 

278 for orig, ref in zip(original, references): 

279 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()), 

280 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds) 

281 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()), 

282 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds) 

283 predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"]) 

284 predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"]) 

285 self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr) 

286 self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr) 

287 

288 def testRequireProperMotion(self): 

289 """Tests of the requireProperMotion config field. 

290 

291 Requiring proper motion corrections for a catalog that does not 

292 contain valid PM data should result in an exception. 

293 

294 `data/testHtmIndex-ps1-bad-pm.fits` is a random shard taken from the 

295 ps1_pv3_3pi_20170110 refcat (that has the unitless PM fields), 

296 stripped to only 2 rows: we patch it in here to simplify test setup. 

297 """ 

298 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/testHtmIndex-ps1-bad-pm.fits') 

299 refcatData = lsst.afw.table.SimpleCatalog.readFits(path) 

300 center = make_coord(93.0, -90.0) 

301 epoch = self.epoch + 1.0*astropy.units.yr 

302 

303 # malformatted catalogs should warn and raise if we require proper motion corrections 

304 config = LoadIndexedReferenceObjectsConfig() 

305 config.requireProperMotion = True 

306 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

307 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

308 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData): 

309 msg = "requireProperMotion=True but refcat pm_ra field is not an Angle" 

310 with self.assertRaisesRegex(RuntimeError, msg): 

311 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=epoch) 

312 

313 # not specifying `epoch` with requireProperMotion=True should raise for any catalog 

314 config = LoadIndexedReferenceObjectsConfig() 

315 config.requireProperMotion = True 

316 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

317 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

318 msg = "requireProperMotion=True but epoch not provided to loader" 

319 with self.assertRaisesRegex(RuntimeError, msg): 

320 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=None) 

321 

322 # malformatted catalogs should just warn if we do not require proper motion corrections 

323 config = LoadIndexedReferenceObjectsConfig() 

324 config.requireProperMotion = False 

325 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

326 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

327 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData): 

328 with self.assertLogs("lsst.LoadIndexedReferenceObjectsTask", level="WARNING") as cm: 

329 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=epoch) 

330 warnLog1 = "Reference catalog pm_ra field is not an Angle; cannot apply proper motion." 

331 self.assertEqual(cm.records[0].message, warnLog1) 

332 

333 def testLoadVersion0(self): 

334 """Test reading a pre-written format_version=0 (Jy flux) catalog. 

335 It should be converted to have nJy fluxes. 

336 """ 

337 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version0') 

338 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path)) 

339 self.assertEqual(loader.dataset_config.format_version, 0) 

340 result = loader.loadSkyCircle(make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

341 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

342 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits')) 

343 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux']) 

344 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr']) 

345 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux']) 

346 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr']) 

347 

348 def testLoadVersion1(self): 

349 """Test reading a format_version=1 catalog (fluxes unchanged).""" 

350 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1') 

351 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path)) 

352 self.assertEqual(loader.dataset_config.format_version, 1) 

353 result = loader.loadSkyCircle(make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

354 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

355 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits')) 

356 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux']) 

357 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr']) 

358 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux']) 

359 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr']) 

360 

361 

362class TestMemory(lsst.utils.tests.MemoryTestCase): 

363 pass 

364 

365 

366def setup_module(module): 

367 lsst.utils.tests.init() 

368 

369 

370if __name__ == "__main__": 370 ↛ 371line 370 didn't jump to line 371, because the condition on line 370 was never true

371 lsst.utils.tests.init() 

372 unittest.main()