Coverage for tests/test_referenceObjectLoader.py: 14%
266 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-23 01:51 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-12-23 01:51 -0800
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import tempfile
24import unittest
25import glob
27import numpy as np
28from smatch.matcher import sphdist
29import astropy.time
31import lsst.daf.butler
32import lsst.afw.geom as afwGeom
33import lsst.afw.table as afwTable
34from lsst.daf.butler import DatasetType, DeferredDatasetHandle
35from lsst.daf.butler.script import ingest_files
36from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader,
37 getRefFluxField, getRefFluxKeys)
38from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles
39from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits, convertToNanojansky
40from lsst.meas.algorithms.convertReferenceCatalog import _makeSchema
41import lsst.utils
42import lsst.geom
44import convertReferenceCatalogTestBase
47class ReferenceObjectLoaderGenericTests(lsst.utils.tests.TestCase):
48 """Test parts of the reference loader that don't depend on loading a
49 catalog, for example schema creation, filter maps, units, and metadata.
50 """
51 def testFilterMapVsAnyFilterMapsToThis(self):
52 config = ReferenceObjectLoader.ConfigClass()
53 # check that a filterMap-only config passes validation
54 config.filterMap = {"b": "a"}
55 try:
56 config.validate()
57 except lsst.pex.config.FieldValidationError:
58 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.")
60 # anyFilterMapsToThis and filterMap are mutually exclusive
61 config.anyFilterMapsToThis = "c"
62 with self.assertRaises(lsst.pex.config.FieldValidationError):
63 config.validate()
65 # check that a anyFilterMapsToThis-only config passes validation
66 config.filterMap = {}
67 try:
68 config.validate()
69 except lsst.pex.config.FieldValidationError:
70 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.")
72 def testFilterAliasMap(self):
73 """Make a schema with filter aliases."""
74 for filterMap in ({}, {"camr": "r"}):
75 config = ReferenceObjectLoader.ConfigClass()
76 config.filterMap = filterMap
77 loader = ReferenceObjectLoader(None, None, name=None, config=config)
78 refSchema = _makeSchema(filterNameList="r")
79 loader._addFluxAliases(refSchema,
80 anyFilterMapsToThis=config.anyFilterMapsToThis,
81 filterMap=config.filterMap)
83 self.assertIn("r_flux", refSchema)
84 self.assertIn("r_fluxErr", refSchema)
86 # camera filters aliases are named <filter>_camFlux
87 if "camr" in filterMap:
88 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux")
89 else:
90 with self.assertRaisesRegex(RuntimeError,
91 r"Could not find flux field\(s\) camr_camFlux, camr_flux"):
92 getRefFluxField(refSchema, "camr")
94 refCat = afwTable.SimpleCatalog(refSchema)
95 refObj = refCat.addNew()
96 refObj["r_flux"] = 1.23
97 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23)
98 if "camr" in filterMap:
99 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23)
100 refObj["r_fluxErr"] = 0.111
101 if "camr" in filterMap:
102 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111)
103 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r")
104 self.assertEqual(refCat[0].get(fluxKey), 1.23)
105 self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
106 if "camr" in filterMap:
107 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr")
108 self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
109 else:
110 with self.assertRaises(RuntimeError):
111 getRefFluxKeys(refSchema, "camr")
113 def testCheckFluxUnits(self):
114 """Test that we can identify old style fluxes in a schema."""
115 schema = _makeSchema(['r', 'z'])
116 # the default schema should pass
117 self.assertTrue(hasNanojanskyFluxUnits(schema))
118 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='')
119 self.assertFalse(hasNanojanskyFluxUnits(schema))
121 schema = _makeSchema(['r', 'z'])
122 schema.addField('bad_flux', doc='old flux units', type=float, units='')
123 self.assertFalse(hasNanojanskyFluxUnits(schema))
125 schema = _makeSchema(['r', 'z'])
126 schema.addField('bad_flux', doc='old flux units', type=float, units='Jy')
127 self.assertFalse(hasNanojanskyFluxUnits(schema))
129 schema = _makeSchema(['r', 'z'])
130 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='')
131 self.assertFalse(hasNanojanskyFluxUnits(schema))
133 schema = _makeSchema(['r', 'z'])
134 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy')
135 self.assertFalse(hasNanojanskyFluxUnits(schema))
137 schema = _makeSchema(['r', 'z'])
138 schema.addField('bad_fluxSigma', doc='old flux units', type=float, units='')
139 self.assertFalse(hasNanojanskyFluxUnits(schema))
141 def testAnyFilterMapsToThisAlias(self):
142 # test anyFilterMapsToThis
143 config = ReferenceObjectLoader.ConfigClass()
144 config.anyFilterMapsToThis = "gg"
145 loader = ReferenceObjectLoader(None, None, name=None, config=config)
146 refSchema = _makeSchema(filterNameList=["gg"])
147 loader._addFluxAliases(refSchema,
148 anyFilterMapsToThis=config.anyFilterMapsToThis,
149 filterMap=config.filterMap)
150 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux")
151 # raise if "gg" is not in the refcat filter list
152 with self.assertRaises(RuntimeError):
153 refSchema = _makeSchema(filterNameList=["rr"])
154 refSchema = loader._addFluxAliases(refSchema,
155 anyFilterMapsToThis=config.anyFilterMapsToThis,
156 filterMap=config.filterMap)
158 def testConvertOldFluxes(self):
159 """Check that we can convert old style fluxes in a catalog."""
160 flux = 1.234
161 fluxErr = 5.678
162 log = lsst.log.Log()
164 def make_catalog():
165 schema = _makeSchema(['r', 'z'])
166 schema.addField('bad_flux', doc='old flux units', type=float, units='')
167 schema.addField('bad_fluxErr', doc='old flux units', type=float, units='Jy')
168 refCat = afwTable.SimpleCatalog(schema)
169 refObj = refCat.addNew()
170 refObj["bad_flux"] = flux
171 refObj["bad_fluxErr"] = fluxErr
172 return refCat
174 oldRefCat = make_catalog()
175 newRefCat = convertToNanojansky(oldRefCat, log)
176 self.assertEqual(newRefCat['bad_flux'], [flux*1e9, ])
177 self.assertEqual(newRefCat['bad_fluxErr'], [fluxErr*1e9, ])
178 self.assertEqual(newRefCat.schema['bad_flux'].asField().getUnits(), 'nJy')
179 self.assertEqual(newRefCat.schema['bad_fluxErr'].asField().getUnits(), 'nJy')
181 # check that doConvert=False returns None (it also logs a summary)
182 oldRefCat = make_catalog()
183 newRefCat = convertToNanojansky(oldRefCat, log, doConvert=False)
184 self.assertIsNone(newRefCat)
186 def testGetMetadataCircle(self):
187 center = lsst.geom.SpherePoint(100*lsst.geom.degrees, 45*lsst.geom.degrees)
188 radius = lsst.geom.Angle(1*lsst.geom.degrees)
189 loader = ReferenceObjectLoader(None, None, name=None)
190 metadata = loader.getMetadataCircle(center, radius, "fakeR")
191 self.assertEqual(metadata['RA'], center.getLongitude().asDegrees())
192 self.assertEqual(metadata['DEC'], center.getLatitude().asDegrees())
193 self.assertEqual(metadata['RADIUS'], radius.asDegrees())
194 self.assertEqual(metadata['SMATCHV'], 2)
195 self.assertEqual(metadata['FILTER'], 'fakeR')
196 self.assertEqual(metadata['JEPOCH'], None)
197 self.assertEqual(metadata['TIMESYS'], 'TAI')
199 epoch = astropy.time.Time(2023.0, format="jyear", scale="tai")
200 metadata = loader.getMetadataCircle(center, radius, "fakeR", epoch=epoch)
201 self.assertEqual(metadata['JEPOCH'], epoch.jyear)
204class ReferenceObjectLoaderLoadTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase,
205 lsst.utils.tests.TestCase):
206 """Tests of loading reference catalogs, using an in-memory generated fake
207 sky catalog that is converted to an LSST refcat.
209 This effectively is a partial integration test of the refcat conversion,
210 ingestion, and loading sequence, focusing mostly on testing the different
211 ways to load a refcat. It significantly overlaps in coverage with
212 ``nopytest_convertReferenceCatalog.py``, but uses a very trivial test
213 refcat and only one core during the conversion.
214 """
215 @classmethod
216 def setUpClass(cls):
217 super().setUpClass()
219 # Generate a catalog, with arbitrary ids
220 inTempDir = tempfile.TemporaryDirectory()
221 inPath = inTempDir.name
222 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123)
224 cls.skyCatalog = skyCatalog
226 # override some field names.
227 config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True,
228 withPm=True, withParallax=True,
229 withFullPositionInformation=True)
230 # use a very small HTM pixelization depth
231 depth = 2
232 config.dataset_config.indexer.active.depth = depth
233 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
234 config.file_reader.format = 'ascii.commented_header'
235 config.n_processes = 1
236 config.id_name = 'id' # Use the ids from the generated catalogs
237 cls.repoTempDir = tempfile.TemporaryDirectory()
238 repoPath = cls.repoTempDir.name
240 # Convert the input data files to our HTM indexed format.
241 dataTempDir = tempfile.TemporaryDirectory()
242 dataPath = dataTempDir.name
243 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
244 converter.run([skyCatalogFile])
246 # Make a temporary butler to ingest them into.
247 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
248 dimensions = [f"htm{depth}"]
249 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
250 dimensions,
251 "SimpleCatalog",
252 universe=butler.registry.dimensions,
253 isCalibration=False)
254 butler.registry.registerDatasetType(datasetType)
256 # Ingest the files into the new butler.
257 run = "testingRun"
258 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
259 ingest_files(repoPath,
260 config.dataset_config.ref_dataset_name,
261 run,
262 htmTableFile,
263 transfer="auto")
265 # Test if we can get back the catalogs, with a new butler.
266 butler = lsst.daf.butler.Butler(repoPath)
267 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
268 collections=[run]).expanded())
269 handles = []
270 for dataRef in datasetRefs:
271 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
273 cls.datasetRefs = datasetRefs
274 cls.handles = handles
276 inTempDir.cleanup()
277 dataTempDir.cleanup()
279 def test_loadSkyCircle(self):
280 """Test the loadSkyCircle routine."""
281 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
282 self.handles,
283 name="testrefcat")
284 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
285 cat = loader.loadSkyCircle(
286 center,
287 30.0*lsst.geom.degrees,
288 filterName='a',
289 ).refCat
290 # Check that the max distance is less than the radius
291 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec']))
292 self.assertLess(np.max(dist), 30.0)
294 # Check that all the objects from the two catalogs are here.
295 dist = sphdist(180.0, 0.0, self.skyCatalog['ra'], self.skyCatalog['dec'])
296 inside, = (dist < 30.0).nonzero()
297 self.assertEqual(len(cat), len(inside))
299 self.assertTrue(cat.isContiguous())
300 self.assertEqual(len(np.unique(cat['id'])), len(cat))
301 # A default-loaded sky circle should not have centroids
302 self.assertNotIn('centroid_x', cat.schema)
303 self.assertNotIn('centroid_y', cat.schema)
304 self.assertNotIn('hasCentroid', cat.schema)
306 def test_loadPixelBox(self):
307 """Test the loadPixelBox routine."""
308 # This will create a box 50 degrees on a side.
309 loaderConfig = ReferenceObjectLoader.ConfigClass()
310 loaderConfig.pixelMargin = 0
311 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
312 self.handles,
313 name="testrefcat",
314 config=loaderConfig)
315 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000))
316 crpix = lsst.geom.Point2D(500, 500)
317 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
318 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees)
319 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
321 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
323 # This is a sanity check on the ranges; the exact selection depends
324 # on cos(dec) and the tangent-plane projection.
325 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
326 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
327 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
328 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
330 # The following is to ensure the reference catalog coords are
331 # getting corrected for proper motion when an epoch is provided.
332 # Use an extreme epoch so that differences in corrected coords
333 # will be significant. Note that this simply tests that the coords
334 # do indeed change when the epoch is passed. It makes no attempt
335 # at assessing the correctness of the change. This is left to the
336 # explicit testProperMotion() test below.
337 catWithEpoch = loader.loadPixelBox(
338 bbox,
339 wcs,
340 'a',
341 bboxToSpherePadding=0,
342 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat
344 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4)
345 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4)
347 def test_filterMap(self):
348 """Test filterMap parameters."""
349 loaderConfig = ReferenceObjectLoader.ConfigClass()
350 loaderConfig.filterMap = {'aprime': 'a'}
351 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
352 self.handles,
353 name="testrefcat",
354 config=loaderConfig)
355 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
356 result = loader.loadSkyCircle(
357 center,
358 30.0*lsst.geom.degrees,
359 filterName='aprime',
360 )
361 self.assertEqual(result.fluxField, 'aprime_camFlux')
362 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux'])
364 def test_properMotion(self):
365 """Test proper motion correction."""
366 loaderConfig = ReferenceObjectLoader.ConfigClass()
367 loaderConfig.filterMap = {'aprime': 'a'}
368 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
369 self.handles,
370 name="testrefcat",
371 config=loaderConfig)
372 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
373 cat = loader.loadSkyCircle(
374 center,
375 30.0*lsst.geom.degrees,
376 filterName='a'
377 ).refCat
379 # Zero epoch change --> no proper motion correction (except minor numerical effects)
380 cat_pm = loader.loadSkyCircle(
381 center,
382 30.0*lsst.geom.degrees,
383 filterName='a',
384 epoch=self.epoch
385 ).refCat
387 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14)
388 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14)
389 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
390 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
392 # One year difference
393 cat_pm = loader.loadSkyCircle(
394 center,
395 30.0*lsst.geom.degrees,
396 filterName='a',
397 epoch=self.epoch + 1.0*astropy.units.yr
398 ).refCat
400 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
401 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
402 for orig, ref in zip(cat, cat_pm):
403 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
404 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
405 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
406 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
407 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
408 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
409 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
410 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
412 def test_requireProperMotion(self):
413 """Tests of the requireProperMotion config field."""
414 loaderConfig = ReferenceObjectLoader.ConfigClass()
415 loaderConfig.requireProperMotion = True
416 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
417 self.handles,
418 name="testrefcat",
419 config=loaderConfig)
420 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
422 # Test that we require an epoch set.
423 msg = 'requireProperMotion=True but epoch not provided to loader'
424 with self.assertRaisesRegex(RuntimeError, msg):
425 loader.loadSkyCircle(
426 center,
427 30.0*lsst.geom.degrees,
428 filterName='a'
429 )
432class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase):
433 """Test cases for reading version 0 and version 1 catalogs."""
434 def testLoadVersion0(self):
435 """Test reading a pre-written format_version=0 (Jy flux) catalog.
436 It should be converted to have nJy fluxes.
437 """
438 path = os.path.join(
439 os.path.dirname(os.path.abspath(__file__)),
440 'data',
441 'version0',
442 'ref_cats',
443 'cal_ref_cat'
444 )
446 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
448 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
449 result = loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20),
450 5*lsst.geom.degrees,
451 'a')
453 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
454 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
455 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
456 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
457 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
458 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
460 def testLoadVersion1(self):
461 """Test reading a format_version=1 catalog (fluxes unchanged)."""
462 path = os.path.join(
463 os.path.dirname(os.path.abspath(__file__)),
464 'data',
465 'version1',
466 'ref_cats',
467 'cal_ref_cat'
468 )
470 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
472 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
473 result = loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20),
474 5*lsst.geom.degrees,
475 'a')
477 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
478 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
479 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
480 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
481 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
482 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
485class TestMemory(lsst.utils.tests.MemoryTestCase):
486 pass
489def setup_module(module):
490 lsst.utils.tests.init()
493if __name__ == "__main__": 493 ↛ 494line 493 didn't jump to line 494, because the condition on line 493 was never true
494 lsst.utils.tests.init()
495 unittest.main()