Coverage for tests/test_htmIndex.py: 16%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

240 statements  

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2016 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import os 

25import unittest 

26from collections import Counter 

27 

28import astropy.time 

29import astropy.units 

30import numpy as np 

31 

32import lsst.geom 

33import lsst.afw.table as afwTable 

34import lsst.afw.geom as afwGeom 

35import lsst.daf.persistence as dafPersist 

36from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask, 

37 LoadIndexedReferenceObjectsConfig, getRefFluxField) 

38from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits 

39import lsst.utils 

40 

41from ingestIndexTestBase import (makeConvertConfig, ConvertReferenceCatalogTestBase, 

42 make_coord) 

43 

44 

45class IngestIndexTaskValidateTestCase(lsst.utils.tests.TestCase): 

46 """Test validation of IngestIndexReferenceConfig.""" 

47 def testValidateRaDecMag(self): 

48 config = makeConvertConfig() 

49 config.validate() 

50 

51 for name in ("ra_name", "dec_name", "mag_column_list"): 

52 with self.subTest(name=name): 

53 config = makeConvertConfig() 

54 setattr(config, name, None) 

55 with self.assertRaises(ValueError): 

56 config.validate() 

57 

58 def testValidateRaDecErr(self): 

59 # check that a basic config validates 

60 config = makeConvertConfig(withRaDecErr=True) 

61 config.validate() 

62 

63 # check that a config with any of these fields missing does not validate 

64 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

65 with self.subTest(name=name): 

66 config = makeConvertConfig(withRaDecErr=True) 

67 setattr(config, name, None) 

68 with self.assertRaises(ValueError): 

69 config.validate() 

70 

71 # check that coord_err_unit must be an astropy unit 

72 config = makeConvertConfig(withRaDecErr=True) 

73 config.coord_err_unit = "nonsense unit" 

74 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

75 config.validate() 

76 

77 def testValidateMagErr(self): 

78 config = makeConvertConfig(withMagErr=True) 

79 config.validate() 

80 

81 # test for missing names 

82 for name in config.mag_column_list: 

83 with self.subTest(name=name): 

84 config = makeConvertConfig(withMagErr=True) 

85 del config.mag_err_column_map[name] 

86 with self.assertRaises(ValueError): 

87 config.validate() 

88 

89 # test for incorrect names 

90 for name in config.mag_column_list: 

91 with self.subTest(name=name): 

92 config = makeConvertConfig(withMagErr=True) 

93 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

94 del config.mag_err_column_map[name] 

95 with self.assertRaises(ValueError): 

96 config.validate() 

97 

98 def testValidatePm(self): 

99 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"] 

100 

101 for withPmErr in (False, True): 

102 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

103 config.validate() 

104 del config 

105 

106 if withPmErr: 

107 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"] 

108 else: 

109 names = basicNames 

110 for name in names: 

111 with self.subTest(name=name, withPmErr=withPmErr): 

112 config = makeConvertConfig(withPm=True, withPmErr=withPmErr) 

113 setattr(config, name, None) 

114 with self.assertRaises(ValueError): 

115 config.validate() 

116 

117 def testValidateParallax(self): 

118 """Validation should fail if any parallax-related fields are missing. 

119 """ 

120 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

121 

122 config = makeConvertConfig(withParallax=True) 

123 config.validate() 

124 del config 

125 

126 for name in names: 

127 with self.subTest(name=name): 

128 config = makeConvertConfig(withParallax=True) 

129 setattr(config, name, None) 

130 with self.assertRaises(ValueError, msg=name): 

131 config.validate() 

132 

133 

134class ReferenceCatalogIngestAndLoadTestCase(ConvertReferenceCatalogTestBase, lsst.utils.tests.TestCase): 

135 """Tests of converting, ingesting, loading and validating an HTM Indexed 

136 Reference Catalog (gen2 code path). 

137 """ 

138 @classmethod 

139 def setUpClass(cls): 

140 super().setUpClass() 

141 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test') 

142 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input") 

143 

144 # Run the ingest once to create a butler repo we can compare to 

145 config = makeConvertConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True, 

146 withParallax=True) 

147 # Pregenerated gen2 test refcats have the "cal_ref_cat" name. 

148 config.dataset_config.ref_dataset_name = "cal_ref_cat" 

149 config.dataset_config.indexer.active.depth = cls.depth 

150 config.id_name = 'id' 

151 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr 

152 config.parallax_scale = 1e3 # arcsec -> milliarcsec 

153 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

154 config.file_reader.format = 'ascii.commented_header' 

155 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath, 

156 cls.skyCatalogFile], config=config) 

157 cls.testButler = dafPersist.Butler(cls.testRepoPath) 

158 

159 @classmethod 

160 def tearDownClass(cls): 

161 del cls.testButler 

162 

163 def testSanity(self): 

164 """Sanity-check that compCats contains some entries with sources.""" 

165 numWithSources = 0 

166 for idList in self.compCats.values(): 

167 if len(idList) > 0: 

168 numWithSources += 1 

169 self.assertGreater(numWithSources, 0) 

170 

171 def testIngestSetsVersion(self): 

172 """Test that newly ingested catalogs get the correct version number set. 

173 """ 

174 def runTest(withRaDecErr): 

175 outputPath = os.path.join(self.outPath, "output_setsVersion" 

176 + "_withRaDecErr" if withRaDecErr else "") 

177 # Test with multiple files and standard config 

178 config = makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True, 

179 withPm=True, withPmErr=True) 

180 # Pregenerated gen2 test refcats have the "cal_ref_cat" name. 

181 config.dataset_config.ref_dataset_name = "cal_ref_cat" 

182 # don't use the default depth, to avoid taking the time to create thousands of file locks 

183 config.dataset_config.indexer.active.depth = self.depth 

184 IngestIndexedReferenceTask.parseAndRun( 

185 args=[self.input_dir, "--output", outputPath, self.skyCatalogFile], 

186 config=config) 

187 # A newly-ingested refcat should be marked format_version=1. 

188 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath)) 

189 self.assertEqual(loader.dataset_config.format_version, 1) 

190 

191 runTest(withRaDecErr=True) 

192 runTest(withRaDecErr=False) 

193 

194 def testLoadSkyCircle(self): 

195 """Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config.""" 

196 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

197 for tupl, idList in self.compCats.items(): 

198 cent = make_coord(*tupl) 

199 lcat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a') 

200 self.assertTrue(lcat.refCat.isContiguous()) 

201 self.assertFalse("camFlux" in lcat.refCat.schema) 

202 self.assertEqual(Counter(lcat.refCat['id']), Counter(idList)) 

203 if len(lcat.refCat) > 0: 

204 # make sure there are no duplicate ids 

205 self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1) 

206 self.assertEqual(len(set(Counter(idList).values())), 1) 

207 # A default-loaded sky circle should not have centroids 

208 self.assertNotIn("centroid_x", lcat.refCat.schema) 

209 self.assertNotIn("centroid_y", lcat.refCat.schema) 

210 self.assertNotIn("hasCentroid", lcat.refCat.schema) 

211 else: 

212 self.assertEqual(len(idList), 0) 

213 

214 def testLoadPixelBox(self): 

215 """Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config.""" 

216 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

217 numFound = 0 

218 for tupl, idList in self.compCats.items(): 

219 cent = make_coord(*tupl) 

220 bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary 

221 ctr_pix = bbox.getCenter() 

222 # catalog is sparse, so set pixel scale such that bbox encloses region 

223 # used to generate compCats 

224 pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth()) 

225 cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale) 

226 wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix) 

227 result = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a") 

228 # The following is to ensure the reference catalog coords are 

229 # getting corrected for proper motion when an epoch is provided. 

230 # Use an extreme epoch so that differences in corrected coords 

231 # will be significant. Note that this simply tests that the coords 

232 # do indeed change when the epoch is passed. It makes no attempt 

233 # at assessing the correctness of the change. This is left to the 

234 # explicit testProperMotion() test below. 

235 resultWithEpoch = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a", 

236 epoch=astropy.time.Time(30000, format='mjd', scale="tai")) 

237 self.assertFloatsNotEqual(result.refCat["coord_ra"], resultWithEpoch.refCat["coord_ra"], 

238 rtol=1.0e-4) 

239 self.assertFloatsNotEqual(result.refCat["coord_dec"], resultWithEpoch.refCat["coord_dec"], 

240 rtol=1.0e-4) 

241 self.assertFalse("camFlux" in result.refCat.schema) 

242 self.assertGreaterEqual(len(result.refCat), len(idList)) 

243 numFound += len(result.refCat) 

244 self.assertGreater(numFound, 0) 

245 

246 def testDefaultFilterAndFilterMap(self): 

247 """Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig.""" 

248 config = LoadIndexedReferenceObjectsConfig() 

249 config.defaultFilter = "b" 

250 config.filterMap = {"aprime": "a"} 

251 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

252 for tupl, idList in self.compCats.items(): 

253 cent = make_coord(*tupl) 

254 lcat = loader.loadSkyCircle(cent, self.searchRadius) 

255 self.assertEqual(lcat.fluxField, "camFlux") 

256 if len(idList) > 0: 

257 defFluxFieldName = getRefFluxField(lcat.refCat.schema, None) 

258 self.assertTrue(defFluxFieldName in lcat.refCat.schema) 

259 aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime") 

260 self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema) 

261 break # just need one test 

262 

263 def testProperMotion(self): 

264 """Test proper motion correction""" 

265 center = make_coord(93.0, -90.0) 

266 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler) 

267 references = loader.loadSkyCircle(center, self.searchRadius, filterName='a').refCat 

268 original = references.copy(True) 

269 

270 # Zero epoch change --> no proper motion correction (except minor numerical effects) 

271 loader.applyProperMotions(references, self.epoch) 

272 self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14) 

273 self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14) 

274 self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"]) 

275 self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"]) 

276 

277 # One year difference 

278 loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr) 

279 self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"]) 

280 self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"]) 

281 for orig, ref in zip(original, references): 

282 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()), 

283 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds) 

284 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()), 

285 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds) 

286 predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"]) 

287 predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"]) 

288 self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr) 

289 self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr) 

290 

291 def testRequireProperMotion(self): 

292 """Tests of the requireProperMotion config field. 

293 

294 Requiring proper motion corrections for a catalog that does not 

295 contain valid PM data should result in an exception. 

296 

297 `data/testHtmIndex-ps1-bad-pm.fits` is a random shard taken from the 

298 ps1_pv3_3pi_20170110 refcat (that has the unitless PM fields), 

299 stripped to only 2 rows: we patch it in here to simplify test setup. 

300 """ 

301 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/testHtmIndex-ps1-bad-pm.fits') 

302 refcatData = lsst.afw.table.SimpleCatalog.readFits(path) 

303 center = make_coord(93.0, -90.0) 

304 epoch = self.epoch + 1.0*astropy.units.yr 

305 

306 # malformatted catalogs should warn and raise if we require proper motion corrections 

307 config = LoadIndexedReferenceObjectsConfig() 

308 config.requireProperMotion = True 

309 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

310 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

311 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData): 

312 msg = "requireProperMotion=True but refcat pm_ra field is not an Angle" 

313 with self.assertRaisesRegex(RuntimeError, msg): 

314 loader.loadSkyCircle(center, self.searchRadius, epoch=epoch) 

315 

316 # not specifying `epoch` with requireProperMotion=True should raise for any catalog 

317 config = LoadIndexedReferenceObjectsConfig() 

318 config.requireProperMotion = True 

319 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

320 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

321 msg = "requireProperMotion=True but epoch not provided to loader" 

322 with self.assertRaisesRegex(RuntimeError, msg): 

323 loader.loadSkyCircle(center, self.searchRadius, epoch=None) 

324 

325 # malformatted catalogs should just warn if we do not require proper motion corrections 

326 config = LoadIndexedReferenceObjectsConfig() 

327 config.requireProperMotion = False 

328 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test 

329 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config) 

330 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData): 

331 with self.assertLogs("lsst.LoadIndexedReferenceObjectsTask", level="WARNING") as cm: 

332 loader.loadSkyCircle(center, self.searchRadius, epoch=epoch) 

333 warnLog1 = "Reference catalog pm_ra field is not an Angle; cannot apply proper motion." 

334 self.assertEqual(cm.records[0].message, warnLog1) 

335 

336 def testLoadVersion0(self): 

337 """Test reading a pre-written format_version=0 (Jy flux) catalog. 

338 It should be converted to have nJy fluxes. 

339 """ 

340 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version0') 

341 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path)) 

342 self.assertEqual(loader.dataset_config.format_version, 0) 

343 result = loader.loadSkyCircle(make_coord(10, 20), 

344 5*lsst.geom.degrees, filterName='a') 

345 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

346 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits')) 

347 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux']) 

348 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr']) 

349 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux']) 

350 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr']) 

351 

352 def testLoadVersion1(self): 

353 """Test reading a format_version=1 catalog (fluxes unchanged).""" 

354 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1') 

355 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path)) 

356 self.assertEqual(loader.dataset_config.format_version, 1) 

357 result = loader.loadSkyCircle(make_coord(10, 20), 

358 5*lsst.geom.degrees, filterName='a') 

359 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

360 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits')) 

361 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux']) 

362 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr']) 

363 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux']) 

364 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr']) 

365 

366 

367class TestMemory(lsst.utils.tests.MemoryTestCase): 

368 pass 

369 

370 

371def setup_module(module): 

372 lsst.utils.tests.init() 

373 

374 

375if __name__ == "__main__": 375 ↛ 376line 375 didn't jump to line 376, because the condition on line 375 was never true

376 lsst.utils.tests.init() 

377 unittest.main()