Coverage for tests/test_referenceObjectLoader.py: 15%

230 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-30 03:11 -0700

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os.path 

23import tempfile 

24import unittest 

25import glob 

26 

27import numpy as np 

28from smatch.matcher import sphdist 

29import astropy.time 

30 

31import lsst.daf.butler 

32import lsst.afw.geom as afwGeom 

33import lsst.afw.table as afwTable 

34from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

35from lsst.daf.butler.script import ingest_files 

36from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader, 

37 getRefFluxField, getRefFluxKeys) 

38from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles 

39from lsst.meas.algorithms.convertReferenceCatalog import _makeSchema 

40import lsst.utils 

41import lsst.geom 

42 

43import convertReferenceCatalogTestBase 

44 

45 

46class ReferenceObjectLoaderGenericTests(lsst.utils.tests.TestCase): 

47 """Test parts of the reference loader that don't depend on loading a 

48 catalog, for example schema creation, filter maps, units, and metadata. 

49 """ 

50 def testFilterMapVsAnyFilterMapsToThis(self): 

51 config = ReferenceObjectLoader.ConfigClass() 

52 # check that a filterMap-only config passes validation 

53 config.filterMap = {"b": "a"} 

54 try: 

55 config.validate() 

56 except lsst.pex.config.FieldValidationError: 

57 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.") 

58 

59 # anyFilterMapsToThis and filterMap are mutually exclusive 

60 config.anyFilterMapsToThis = "c" 

61 with self.assertRaises(lsst.pex.config.FieldValidationError): 

62 config.validate() 

63 

64 # check that a anyFilterMapsToThis-only config passes validation 

65 config.filterMap = {} 

66 try: 

67 config.validate() 

68 except lsst.pex.config.FieldValidationError: 

69 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.") 

70 

71 def testFilterAliasMap(self): 

72 """Make a schema with filter aliases.""" 

73 for filterMap in ({}, {"camr": "r"}): 

74 config = ReferenceObjectLoader.ConfigClass() 

75 config.filterMap = filterMap 

76 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

77 refSchema = _makeSchema(filterNameList="r") 

78 loader._addFluxAliases(refSchema, 

79 anyFilterMapsToThis=config.anyFilterMapsToThis, 

80 filterMap=config.filterMap) 

81 

82 self.assertIn("r_flux", refSchema) 

83 self.assertIn("r_fluxErr", refSchema) 

84 

85 # camera filters aliases are named <filter>_camFlux 

86 if "camr" in filterMap: 

87 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux") 

88 else: 

89 with self.assertRaisesRegex(RuntimeError, 

90 r"Could not find flux field\(s\) camr_camFlux, camr_flux"): 

91 getRefFluxField(refSchema, "camr") 

92 

93 refCat = afwTable.SimpleCatalog(refSchema) 

94 refObj = refCat.addNew() 

95 refObj["r_flux"] = 1.23 

96 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23) 

97 if "camr" in filterMap: 

98 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23) 

99 refObj["r_fluxErr"] = 0.111 

100 if "camr" in filterMap: 

101 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111) 

102 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r") 

103 self.assertEqual(refCat[0].get(fluxKey), 1.23) 

104 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

105 if "camr" in filterMap: 

106 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr") 

107 self.assertEqual(refCat[0].get(fluxErrKey), 0.111) 

108 else: 

109 with self.assertRaises(RuntimeError): 

110 getRefFluxKeys(refSchema, "camr") 

111 

112 def testAnyFilterMapsToThisAlias(self): 

113 # test anyFilterMapsToThis 

114 config = ReferenceObjectLoader.ConfigClass() 

115 config.anyFilterMapsToThis = "gg" 

116 loader = ReferenceObjectLoader(None, None, name=None, config=config) 

117 refSchema = _makeSchema(filterNameList=["gg"]) 

118 loader._addFluxAliases(refSchema, 

119 anyFilterMapsToThis=config.anyFilterMapsToThis, 

120 filterMap=config.filterMap) 

121 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux") 

122 # raise if "gg" is not in the refcat filter list 

123 with self.assertRaises(RuntimeError): 

124 refSchema = _makeSchema(filterNameList=["rr"]) 

125 refSchema = loader._addFluxAliases(refSchema, 

126 anyFilterMapsToThis=config.anyFilterMapsToThis, 

127 filterMap=config.filterMap) 

128 

129 def testGetMetadataCircle(self): 

130 center = lsst.geom.SpherePoint(100*lsst.geom.degrees, 45*lsst.geom.degrees) 

131 radius = lsst.geom.Angle(1*lsst.geom.degrees) 

132 loader = ReferenceObjectLoader(None, None, name=None) 

133 metadata = loader.getMetadataCircle(center, radius, "fakeR") 

134 self.assertEqual(metadata['RA'], center.getLongitude().asDegrees()) 

135 self.assertEqual(metadata['DEC'], center.getLatitude().asDegrees()) 

136 self.assertEqual(metadata['RADIUS'], radius.asDegrees()) 

137 self.assertEqual(metadata['SMATCHV'], 2) 

138 self.assertEqual(metadata['FILTER'], 'fakeR') 

139 self.assertEqual(metadata['JEPOCH'], None) 

140 self.assertEqual(metadata['TIMESYS'], 'TAI') 

141 

142 epoch = astropy.time.Time(2023.0, format="jyear", scale="tai") 

143 metadata = loader.getMetadataCircle(center, radius, "fakeR", epoch=epoch) 

144 self.assertEqual(metadata['JEPOCH'], epoch.jyear) 

145 

146 

147class ReferenceObjectLoaderLoadTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase, 

148 lsst.utils.tests.TestCase): 

149 """Tests of loading reference catalogs, using an in-memory generated fake 

150 sky catalog that is converted to an LSST refcat. 

151 

152 This effectively is a partial integration test of the refcat conversion, 

153 ingestion, and loading sequence, focusing mostly on testing the different 

154 ways to load a refcat. It significantly overlaps in coverage with 

155 ``nopytest_convertReferenceCatalog.py``, but uses a very trivial test 

156 refcat and only one core during the conversion. 

157 """ 

158 @classmethod 

159 def setUpClass(cls): 

160 super().setUpClass() 

161 

162 # Generate a catalog, with arbitrary ids 

163 inTempDir = tempfile.TemporaryDirectory() 

164 inPath = inTempDir.name 

165 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123) 

166 

167 cls.skyCatalog = skyCatalog 

168 

169 # override some field names. 

170 config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True, 

171 withPm=True, withParallax=True, 

172 withFullPositionInformation=True) 

173 # use a very small HTM pixelization depth 

174 depth = 2 

175 config.dataset_config.indexer.active.depth = depth 

176 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

177 config.file_reader.format = 'ascii.commented_header' 

178 config.n_processes = 1 

179 config.id_name = 'id' # Use the ids from the generated catalogs 

180 cls.repoTempDir = tempfile.TemporaryDirectory() 

181 repoPath = cls.repoTempDir.name 

182 

183 # Convert the input data files to our HTM indexed format. 

184 dataTempDir = tempfile.TemporaryDirectory() 

185 dataPath = dataTempDir.name 

186 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

187 converter.run([skyCatalogFile]) 

188 

189 # Make a temporary butler to ingest them into. 

190 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

191 dimensions = [f"htm{depth}"] 

192 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

193 dimensions, 

194 "SimpleCatalog", 

195 universe=butler.dimensions, 

196 isCalibration=False) 

197 butler.registry.registerDatasetType(datasetType) 

198 

199 # Ingest the files into the new butler. 

200 run = "testingRun" 

201 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

202 ingest_files(repoPath, 

203 config.dataset_config.ref_dataset_name, 

204 run, 

205 htmTableFile, 

206 transfer="auto") 

207 

208 # Test if we can get back the catalogs, with a new butler. 

209 butler = lsst.daf.butler.Butler(repoPath) 

210 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

211 collections=[run]).expanded()) 

212 handles = [] 

213 for dataRef in datasetRefs: 

214 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

215 

216 cls.datasetRefs = datasetRefs 

217 cls.handles = handles 

218 

219 inTempDir.cleanup() 

220 dataTempDir.cleanup() 

221 

222 def test_loadSkyCircle(self): 

223 """Test the loadSkyCircle routine.""" 

224 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

225 self.handles, 

226 name="testrefcat") 

227 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

228 cat = loader.loadSkyCircle( 

229 center, 

230 30.0*lsst.geom.degrees, 

231 filterName='a', 

232 ).refCat 

233 # Check that the max distance is less than the radius 

234 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec'])) 

235 self.assertLess(np.max(dist), 30.0) 

236 

237 # Check that all the objects from the two catalogs are here. 

238 dist = sphdist(180.0, 0.0, self.skyCatalog['ra'], self.skyCatalog['dec']) 

239 inside, = (dist < 30.0).nonzero() 

240 self.assertEqual(len(cat), len(inside)) 

241 

242 self.assertTrue(cat.isContiguous()) 

243 self.assertEqual(len(np.unique(cat['id'])), len(cat)) 

244 # A default-loaded sky circle should not have centroids 

245 self.assertNotIn('centroid_x', cat.schema) 

246 self.assertNotIn('centroid_y', cat.schema) 

247 self.assertNotIn('hasCentroid', cat.schema) 

248 

249 def test_loadPixelBox(self): 

250 """Test the loadPixelBox routine.""" 

251 # This will create a box 50 degrees on a side. 

252 loaderConfig = ReferenceObjectLoader.ConfigClass() 

253 loaderConfig.pixelMargin = 0 

254 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

255 self.handles, 

256 name="testrefcat", 

257 config=loaderConfig) 

258 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000)) 

259 crpix = lsst.geom.Point2D(500, 500) 

260 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

261 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees) 

262 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

263 

264 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat 

265 

266 # This is a sanity check on the ranges; the exact selection depends 

267 # on cos(dec) and the tangent-plane projection. 

268 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0) 

269 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0) 

270 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0) 

271 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0) 

272 

273 # The following is to ensure the reference catalog coords are 

274 # getting corrected for proper motion when an epoch is provided. 

275 # Use an extreme epoch so that differences in corrected coords 

276 # will be significant. Note that this simply tests that the coords 

277 # do indeed change when the epoch is passed. It makes no attempt 

278 # at assessing the correctness of the change. This is left to the 

279 # explicit testProperMotion() test below. 

280 catWithEpoch = loader.loadPixelBox( 

281 bbox, 

282 wcs, 

283 'a', 

284 bboxToSpherePadding=0, 

285 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat 

286 

287 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4) 

288 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4) 

289 

290 def test_filterMap(self): 

291 """Test filterMap parameters.""" 

292 loaderConfig = ReferenceObjectLoader.ConfigClass() 

293 loaderConfig.filterMap = {'aprime': 'a'} 

294 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

295 self.handles, 

296 name="testrefcat", 

297 config=loaderConfig) 

298 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

299 result = loader.loadSkyCircle( 

300 center, 

301 30.0*lsst.geom.degrees, 

302 filterName='aprime', 

303 ) 

304 self.assertEqual(result.fluxField, 'aprime_camFlux') 

305 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux']) 

306 

307 def test_properMotion(self): 

308 """Test proper motion correction.""" 

309 loaderConfig = ReferenceObjectLoader.ConfigClass() 

310 loaderConfig.filterMap = {'aprime': 'a'} 

311 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

312 self.handles, 

313 name="testrefcat", 

314 config=loaderConfig) 

315 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

316 cat = loader.loadSkyCircle( 

317 center, 

318 30.0*lsst.geom.degrees, 

319 filterName='a' 

320 ).refCat 

321 

322 # Zero epoch change --> no proper motion correction (except minor numerical effects) 

323 cat_pm = loader.loadSkyCircle( 

324 center, 

325 30.0*lsst.geom.degrees, 

326 filterName='a', 

327 epoch=self.epoch 

328 ).refCat 

329 

330 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14) 

331 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14) 

332 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr']) 

333 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr']) 

334 

335 # One year difference 

336 cat_pm = loader.loadSkyCircle( 

337 center, 

338 30.0*lsst.geom.degrees, 

339 filterName='a', 

340 epoch=self.epoch + 1.0*astropy.units.yr 

341 ).refCat 

342 

343 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr']) 

344 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr']) 

345 

346 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds() 

347 for i in range(len(cat))]) 

348 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds() 

349 for i in range(len(cat))]) 

350 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10) 

351 self.assertFloatsAlmostEqual(bearings, self.properMotionDir.asArcseconds(), rtol=1.0e-10) 

352 

353 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"]) 

354 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"]) 

355 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr) 

356 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr) 

357 

358 # One year negative difference. This demonstrates a fix for DM-38808, 

359 # when the refcat epoch is later in time than the data. 

360 cat_pm = loader.loadSkyCircle( 

361 center, 

362 30.0*lsst.geom.degrees, 

363 filterName='a', 

364 epoch=self.epoch - 1.0*astropy.units.yr 

365 ).refCat 

366 

367 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr']) 

368 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr']) 

369 

370 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds() 

371 for i in range(len(cat))]) 

372 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds() 

373 for i in range(len(cat))]) 

374 reverse_proper_motion_dir = self.properMotionDir + 180 * lsst.geom.degrees 

375 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10) 

376 self.assertFloatsAlmostEqual(bearings, reverse_proper_motion_dir.asArcseconds(), rtol=1.0e-10) 

377 

378 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"]) 

379 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"]) 

380 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr) 

381 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr) 

382 

383 def test_requireProperMotion(self): 

384 """Tests of the requireProperMotion config field.""" 

385 loaderConfig = ReferenceObjectLoader.ConfigClass() 

386 loaderConfig.requireProperMotion = True 

387 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

388 self.handles, 

389 name="testrefcat", 

390 config=loaderConfig) 

391 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

392 

393 # Test that we require an epoch set. 

394 msg = 'requireProperMotion=True but epoch not provided to loader' 

395 with self.assertRaisesRegex(RuntimeError, msg): 

396 loader.loadSkyCircle( 

397 center, 

398 30.0*lsst.geom.degrees, 

399 filterName='a' 

400 ) 

401 

402 

403class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase): 

404 """Test cases for reading version 0 and version 1 catalogs.""" 

405 def testLoadVersion0(self): 

406 """Attempting to read version 0 refcats should raise. 

407 """ 

408 path = os.path.join( 

409 os.path.dirname(os.path.abspath(__file__)), 

410 'data', 

411 'version0', 

412 'ref_cats', 

413 'cal_ref_cat' 

414 ) 

415 

416 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

417 

418 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

419 with self.assertRaisesRegex(ValueError, "Version 0 refcats are no longer supported"): 

420 loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20), 

421 5*lsst.geom.degrees, 

422 'a') 

423 

424 def testLoadVersion1(self): 

425 """Test reading a format_version=1 catalog (fluxes unchanged).""" 

426 path = os.path.join( 

427 os.path.dirname(os.path.abspath(__file__)), 

428 'data', 

429 'version1', 

430 'ref_cats', 

431 'cal_ref_cat' 

432 ) 

433 

434 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

435 

436 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

437 result = loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20), 

438 5*lsst.geom.degrees, 

439 'a') 

440 

441 # version>=1 should not change units on read (they're already nJy). 

442 catalog = afwTable.SimpleCatalog.readFits(filenames[0]) 

443 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux']) 

444 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr']) 

445 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux']) 

446 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr']) 

447 

448 

449class TestMemory(lsst.utils.tests.MemoryTestCase): 

450 pass 

451 

452 

453def setup_module(module): 

454 lsst.utils.tests.init() 

455 

456 

457if __name__ == "__main__": 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true

458 lsst.utils.tests.init() 

459 unittest.main()