Coverage for tests/test_referenceObjectLoader.py: 15%
230 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-04 12:18 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-04 12:18 +0000
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import tempfile
24import unittest
25import glob
27import numpy as np
28from smatch.matcher import sphdist
29import astropy.time
31import lsst.daf.butler
32import lsst.afw.geom as afwGeom
33import lsst.afw.table as afwTable
34from lsst.daf.butler import DatasetType, DeferredDatasetHandle
35from lsst.daf.butler.script import ingest_files
36from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader,
37 getRefFluxField, getRefFluxKeys)
38from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles
39from lsst.meas.algorithms.convertReferenceCatalog import _makeSchema
40import lsst.utils
41import lsst.geom
43import convertReferenceCatalogTestBase
46class ReferenceObjectLoaderGenericTests(lsst.utils.tests.TestCase):
47 """Test parts of the reference loader that don't depend on loading a
48 catalog, for example schema creation, filter maps, units, and metadata.
49 """
50 def testFilterMapVsAnyFilterMapsToThis(self):
51 config = ReferenceObjectLoader.ConfigClass()
52 # check that a filterMap-only config passes validation
53 config.filterMap = {"b": "a"}
54 try:
55 config.validate()
56 except lsst.pex.config.FieldValidationError:
57 self.fail("`filterMap`-only LoadReferenceObjectsConfig should not fail validation.")
59 # anyFilterMapsToThis and filterMap are mutually exclusive
60 config.anyFilterMapsToThis = "c"
61 with self.assertRaises(lsst.pex.config.FieldValidationError):
62 config.validate()
64 # check that a anyFilterMapsToThis-only config passes validation
65 config.filterMap = {}
66 try:
67 config.validate()
68 except lsst.pex.config.FieldValidationError:
69 self.fail("`anyFilterMapsToThis`-only LoadReferenceObjectsConfig should not fail validation.")
71 def testFilterAliasMap(self):
72 """Make a schema with filter aliases."""
73 for filterMap in ({}, {"camr": "r"}):
74 config = ReferenceObjectLoader.ConfigClass()
75 config.filterMap = filterMap
76 loader = ReferenceObjectLoader(None, None, name=None, config=config)
77 refSchema = _makeSchema(filterNameList="r")
78 loader._addFluxAliases(refSchema,
79 anyFilterMapsToThis=config.anyFilterMapsToThis,
80 filterMap=config.filterMap)
82 self.assertIn("r_flux", refSchema)
83 self.assertIn("r_fluxErr", refSchema)
85 # camera filters aliases are named <filter>_camFlux
86 if "camr" in filterMap:
87 self.assertEqual(getRefFluxField(refSchema, "camr"), "camr_camFlux")
88 else:
89 with self.assertRaisesRegex(RuntimeError,
90 r"Could not find flux field\(s\) camr_camFlux, camr_flux"):
91 getRefFluxField(refSchema, "camr")
93 refCat = afwTable.SimpleCatalog(refSchema)
94 refObj = refCat.addNew()
95 refObj["r_flux"] = 1.23
96 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "r")), 1.23)
97 if "camr" in filterMap:
98 self.assertAlmostEqual(refCat[0].get(getRefFluxField(refSchema, "camr")), 1.23)
99 refObj["r_fluxErr"] = 0.111
100 if "camr" in filterMap:
101 self.assertEqual(refCat[0].get("camr_camFluxErr"), 0.111)
102 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "r")
103 self.assertEqual(refCat[0].get(fluxKey), 1.23)
104 self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
105 if "camr" in filterMap:
106 fluxKey, fluxErrKey = getRefFluxKeys(refSchema, "camr")
107 self.assertEqual(refCat[0].get(fluxErrKey), 0.111)
108 else:
109 with self.assertRaises(RuntimeError):
110 getRefFluxKeys(refSchema, "camr")
112 def testAnyFilterMapsToThisAlias(self):
113 # test anyFilterMapsToThis
114 config = ReferenceObjectLoader.ConfigClass()
115 config.anyFilterMapsToThis = "gg"
116 loader = ReferenceObjectLoader(None, None, name=None, config=config)
117 refSchema = _makeSchema(filterNameList=["gg"])
118 loader._addFluxAliases(refSchema,
119 anyFilterMapsToThis=config.anyFilterMapsToThis,
120 filterMap=config.filterMap)
121 self.assertEqual(getRefFluxField(refSchema, "r"), "gg_flux")
122 # raise if "gg" is not in the refcat filter list
123 with self.assertRaises(RuntimeError):
124 refSchema = _makeSchema(filterNameList=["rr"])
125 refSchema = loader._addFluxAliases(refSchema,
126 anyFilterMapsToThis=config.anyFilterMapsToThis,
127 filterMap=config.filterMap)
129 def testGetMetadataCircle(self):
130 center = lsst.geom.SpherePoint(100*lsst.geom.degrees, 45*lsst.geom.degrees)
131 radius = lsst.geom.Angle(1*lsst.geom.degrees)
132 loader = ReferenceObjectLoader(None, None, name=None)
133 metadata = loader.getMetadataCircle(center, radius, "fakeR")
134 self.assertEqual(metadata['RA'], center.getLongitude().asDegrees())
135 self.assertEqual(metadata['DEC'], center.getLatitude().asDegrees())
136 self.assertEqual(metadata['RADIUS'], radius.asDegrees())
137 self.assertEqual(metadata['SMATCHV'], 2)
138 self.assertEqual(metadata['FILTER'], 'fakeR')
139 self.assertEqual(metadata['JEPOCH'], None)
140 self.assertEqual(metadata['TIMESYS'], 'TAI')
142 epoch = astropy.time.Time(2023.0, format="jyear", scale="tai")
143 metadata = loader.getMetadataCircle(center, radius, "fakeR", epoch=epoch)
144 self.assertEqual(metadata['JEPOCH'], epoch.jyear)
147class ReferenceObjectLoaderLoadTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase,
148 lsst.utils.tests.TestCase):
149 """Tests of loading reference catalogs, using an in-memory generated fake
150 sky catalog that is converted to an LSST refcat.
152 This effectively is a partial integration test of the refcat conversion,
153 ingestion, and loading sequence, focusing mostly on testing the different
154 ways to load a refcat. It significantly overlaps in coverage with
155 ``nopytest_convertReferenceCatalog.py``, but uses a very trivial test
156 refcat and only one core during the conversion.
157 """
158 @classmethod
159 def setUpClass(cls):
160 super().setUpClass()
162 # Generate a catalog, with arbitrary ids
163 inTempDir = tempfile.TemporaryDirectory()
164 inPath = inTempDir.name
165 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123)
167 cls.skyCatalog = skyCatalog
169 # override some field names.
170 config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True,
171 withPm=True, withParallax=True,
172 withFullPositionInformation=True)
173 # use a very small HTM pixelization depth
174 depth = 2
175 config.dataset_config.indexer.active.depth = depth
176 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
177 config.file_reader.format = 'ascii.commented_header'
178 config.n_processes = 1
179 config.id_name = 'id' # Use the ids from the generated catalogs
180 cls.repoTempDir = tempfile.TemporaryDirectory()
181 repoPath = cls.repoTempDir.name
183 # Convert the input data files to our HTM indexed format.
184 dataTempDir = tempfile.TemporaryDirectory()
185 dataPath = dataTempDir.name
186 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
187 converter.run([skyCatalogFile])
189 # Make a temporary butler to ingest them into.
190 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
191 dimensions = [f"htm{depth}"]
192 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
193 dimensions,
194 "SimpleCatalog",
195 universe=butler.dimensions,
196 isCalibration=False)
197 butler.registry.registerDatasetType(datasetType)
199 # Ingest the files into the new butler.
200 run = "testingRun"
201 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
202 ingest_files(repoPath,
203 config.dataset_config.ref_dataset_name,
204 run,
205 htmTableFile,
206 transfer="auto")
208 # Test if we can get back the catalogs, with a new butler.
209 butler = lsst.daf.butler.Butler(repoPath)
210 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
211 collections=[run]).expanded())
212 handles = []
213 for dataRef in datasetRefs:
214 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
216 cls.datasetRefs = datasetRefs
217 cls.handles = handles
219 inTempDir.cleanup()
220 dataTempDir.cleanup()
222 def test_loadSkyCircle(self):
223 """Test the loadSkyCircle routine."""
224 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
225 self.handles,
226 name="testrefcat")
227 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
228 cat = loader.loadSkyCircle(
229 center,
230 30.0*lsst.geom.degrees,
231 filterName='a',
232 ).refCat
233 # Check that the max distance is less than the radius
234 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec']))
235 self.assertLess(np.max(dist), 30.0)
237 # Check that all the objects from the two catalogs are here.
238 dist = sphdist(180.0, 0.0, self.skyCatalog['ra'], self.skyCatalog['dec'])
239 inside, = (dist < 30.0).nonzero()
240 self.assertEqual(len(cat), len(inside))
242 self.assertTrue(cat.isContiguous())
243 self.assertEqual(len(np.unique(cat['id'])), len(cat))
244 # A default-loaded sky circle should not have centroids
245 self.assertNotIn('centroid_x', cat.schema)
246 self.assertNotIn('centroid_y', cat.schema)
247 self.assertNotIn('hasCentroid', cat.schema)
249 def test_loadPixelBox(self):
250 """Test the loadPixelBox routine."""
251 # This will create a box 50 degrees on a side.
252 loaderConfig = ReferenceObjectLoader.ConfigClass()
253 loaderConfig.pixelMargin = 0
254 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
255 self.handles,
256 name="testrefcat",
257 config=loaderConfig)
258 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000))
259 crpix = lsst.geom.Point2D(500, 500)
260 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
261 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees)
262 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
264 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
266 # This is a sanity check on the ranges; the exact selection depends
267 # on cos(dec) and the tangent-plane projection.
268 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
269 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
270 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
271 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
273 # The following is to ensure the reference catalog coords are
274 # getting corrected for proper motion when an epoch is provided.
275 # Use an extreme epoch so that differences in corrected coords
276 # will be significant. Note that this simply tests that the coords
277 # do indeed change when the epoch is passed. It makes no attempt
278 # at assessing the correctness of the change. This is left to the
279 # explicit testProperMotion() test below.
280 catWithEpoch = loader.loadPixelBox(
281 bbox,
282 wcs,
283 'a',
284 bboxToSpherePadding=0,
285 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat
287 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4)
288 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4)
290 def test_filterMap(self):
291 """Test filterMap parameters."""
292 loaderConfig = ReferenceObjectLoader.ConfigClass()
293 loaderConfig.filterMap = {'aprime': 'a'}
294 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
295 self.handles,
296 name="testrefcat",
297 config=loaderConfig)
298 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
299 result = loader.loadSkyCircle(
300 center,
301 30.0*lsst.geom.degrees,
302 filterName='aprime',
303 )
304 self.assertEqual(result.fluxField, 'aprime_camFlux')
305 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux'])
307 def test_properMotion(self):
308 """Test proper motion correction."""
309 loaderConfig = ReferenceObjectLoader.ConfigClass()
310 loaderConfig.filterMap = {'aprime': 'a'}
311 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
312 self.handles,
313 name="testrefcat",
314 config=loaderConfig)
315 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
316 cat = loader.loadSkyCircle(
317 center,
318 30.0*lsst.geom.degrees,
319 filterName='a'
320 ).refCat
322 # Zero epoch change --> no proper motion correction (except minor numerical effects)
323 cat_pm = loader.loadSkyCircle(
324 center,
325 30.0*lsst.geom.degrees,
326 filterName='a',
327 epoch=self.epoch
328 ).refCat
330 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14)
331 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14)
332 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
333 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
335 # One year difference
336 cat_pm = loader.loadSkyCircle(
337 center,
338 30.0*lsst.geom.degrees,
339 filterName='a',
340 epoch=self.epoch + 1.0*astropy.units.yr
341 ).refCat
343 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
344 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
346 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds()
347 for i in range(len(cat))])
348 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds()
349 for i in range(len(cat))])
350 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10)
351 self.assertFloatsAlmostEqual(bearings, self.properMotionDir.asArcseconds(), rtol=1.0e-10)
353 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
354 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
355 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
356 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
358 # One year negative difference. This demonstrates a fix for DM-38808,
359 # when the refcat epoch is later in time than the data.
360 cat_pm = loader.loadSkyCircle(
361 center,
362 30.0*lsst.geom.degrees,
363 filterName='a',
364 epoch=self.epoch - 1.0*astropy.units.yr
365 ).refCat
367 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
368 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
370 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds()
371 for i in range(len(cat))])
372 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds()
373 for i in range(len(cat))])
374 reverse_proper_motion_dir = self.properMotionDir + 180 * lsst.geom.degrees
375 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10)
376 self.assertFloatsAlmostEqual(bearings, reverse_proper_motion_dir.asArcseconds(), rtol=1.0e-10)
378 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
379 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
380 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
381 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
383 def test_requireProperMotion(self):
384 """Tests of the requireProperMotion config field."""
385 loaderConfig = ReferenceObjectLoader.ConfigClass()
386 loaderConfig.requireProperMotion = True
387 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
388 self.handles,
389 name="testrefcat",
390 config=loaderConfig)
391 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
393 # Test that we require an epoch set.
394 msg = 'requireProperMotion=True but epoch not provided to loader'
395 with self.assertRaisesRegex(RuntimeError, msg):
396 loader.loadSkyCircle(
397 center,
398 30.0*lsst.geom.degrees,
399 filterName='a'
400 )
403class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase):
404 """Test cases for reading version 0 and version 1 catalogs."""
405 def testLoadVersion0(self):
406 """Attempting to read version 0 refcats should raise.
407 """
408 path = os.path.join(
409 os.path.dirname(os.path.abspath(__file__)),
410 'data',
411 'version0',
412 'ref_cats',
413 'cal_ref_cat'
414 )
416 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
418 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
419 with self.assertRaisesRegex(ValueError, "Version 0 refcats are no longer supported"):
420 loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20),
421 5*lsst.geom.degrees,
422 'a')
424 def testLoadVersion1(self):
425 """Test reading a format_version=1 catalog (fluxes unchanged)."""
426 path = os.path.join(
427 os.path.dirname(os.path.abspath(__file__)),
428 'data',
429 'version1',
430 'ref_cats',
431 'cal_ref_cat'
432 )
434 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
436 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
437 result = loader.loadSkyCircle(convertReferenceCatalogTestBase.make_coord(10, 20),
438 5*lsst.geom.degrees,
439 'a')
441 # version>=1 should not change units on read (they're already nJy).
442 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
443 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
444 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
445 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
446 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
449class TestMemory(lsst.utils.tests.MemoryTestCase):
450 pass
453def setup_module(module):
454 lsst.utils.tests.init()
457if __name__ == "__main__": 457 ↛ 458line 457 didn't jump to line 458, because the condition on line 457 was never true
458 lsst.utils.tests.init()
459 unittest.main()