Coverage for tests / test_convertReferenceCatalog.py: 17%
231 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-23 08:32 +0000
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import astropy.units as u
23import numpy as np
24import os.path
25import sys
26import unittest
27import unittest.mock
28import tempfile
29import itertools
30import logging
32from lsst.afw.table import SimpleCatalog
33from lsst.pex.config import FieldValidationError
34from lsst.meas.algorithms import (convertReferenceCatalog, ConvertReferenceCatalogTask, getRefFluxField)
35from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask
36from lsst.meas.algorithms.htmIndexer import HtmIndexer
37from lsst.meas.algorithms.convertRefcatManager import ConvertGaiaManager
38from lsst.meas.algorithms.convertReferenceCatalog import addRefCatMetadata, _makeSchema
40import lsst.utils
42from convertReferenceCatalogTestBase import makeConvertConfig
43import convertReferenceCatalogTestBase
46class TestMain(lsst.utils.tests.TestCase):
47 """Test mocking commandline arguments and calling
48 ``convertReferenceCatalog.main()``.
49 """
50 def setUp(self):
51 self.inpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data/mockrefcat/")
52 self.expected_files = [os.path.join(self.inpath, "123.fits"),
53 os.path.join(self.inpath, "124.fits"),
54 os.path.join(self.inpath, "125.fits")]
56 def test_main_args(self):
57 """Test that main configures the task and calls run() with the correct
58 file list.
59 """
60 outdir = tempfile.TemporaryDirectory()
61 outpath = outdir.name
62 args = ["convertReferenceCatalog",
63 outpath,
64 os.path.join(self.inpath, "mock_config.py"),
65 os.path.join(self.inpath, "*.fits")]
66 with unittest.mock.patch.object(convertReferenceCatalog.ConvertReferenceCatalogTask, "run") as run, \
67 unittest.mock.patch.object(sys, "argv", args):
68 convertReferenceCatalog.main()
69 # Test with sets because the glob can come out in any order.
70 self.assertEqual(set(run.call_args.args[0]), set(self.expected_files))
71 # This is necessary to avoid a ResourceWarning.
72 outdir.cleanup()
74 def test_main_args_bad_config(self):
75 """Test that a bad config file produces a useful error, i.e. that
76 main() validates the config.
77 """
78 outdir = tempfile.TemporaryDirectory()
79 outpath = outdir.name
80 args = ["convertReferenceCatalog",
81 outpath,
82 os.path.join(self.inpath, "bad_config.py"),
83 os.path.join(self.inpath, "*.fits")]
84 with self.assertRaisesRegex(FieldValidationError, "Field 'ra_name' failed validation"), \
85 unittest.mock.patch.object(sys, "argv", args):
86 convertReferenceCatalog.main()
87 # This is necessary to avoid a ResourceWarning.
88 outdir.cleanup()
90 def test_main_args_expanded_glob(self):
91 """Test that an un-quoted glob (i.e. list of files) fails with a
92 useful error.
93 """
94 outdir = tempfile.TemporaryDirectory()
95 outpath = outdir.name
96 args = ["convertReferenceCatalog",
97 outpath,
98 os.path.join(self.inpath, "mock_config.py"),
99 # an un-quoted glob will be shell-expanded to a list of files.
100 "file1", "file2", "file3"]
101 msg = "Final argument must be a quoted file glob, not a shell-expanded list of files."
102 with self.assertRaisesRegex(RuntimeError, msg), \
103 unittest.mock.patch.object(sys, "argv", args):
104 convertReferenceCatalog.main()
105 # This is necessary to avoid a ResourceWarning.
106 outdir.cleanup()
109class MakeSchemaTestCase(lsst.utils.tests.TestCase):
110 """Test the function to make reference catalog schemas.
111 """
112 def testMakeSchema(self):
113 """Make a schema and check it."""
114 for filterNameList in (["r"], ["foo", "_bar"]):
115 for (addIsPhotometric, addIsResolved, addIsVariable) in itertools.product((False, True),
116 (False, True),
117 (False, True)):
118 argDict = dict(
119 filterNameList=filterNameList,
120 addIsPhotometric=addIsPhotometric,
121 addIsResolved=addIsResolved,
122 addIsVariable=addIsVariable,
123 )
124 refSchema = _makeSchema(**argDict)
125 self.assertTrue("coord_ra" in refSchema)
126 self.assertTrue("coord_dec" in refSchema)
127 self.assertTrue("coord_raErr" in refSchema)
128 self.assertTrue("coord_decErr" in refSchema)
129 for filterName in filterNameList:
130 fluxField = filterName + "_flux"
131 self.assertIn(fluxField, refSchema)
132 self.assertNotIn("x" + fluxField, refSchema)
133 fluxErrField = fluxField + "Err"
134 self.assertIn(fluxErrField, refSchema)
135 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux")
136 self.assertEqual("resolved" in refSchema, addIsResolved)
137 self.assertEqual("variable" in refSchema, addIsVariable)
138 self.assertEqual("photometric" in refSchema, addIsPhotometric)
139 self.assertEqual("photometric" in refSchema, addIsPhotometric)
141 # The default for `fullPositionInformation` is False, so none
142 # of the following should be included. We test setting these
143 # all together below.
144 self.assertNotIn("epoch", refSchema)
145 self.assertNotIn("pm_ra", refSchema)
146 self.assertNotIn("pm_dec", refSchema)
147 self.assertNotIn("pm_flag", refSchema)
148 self.assertNotIn("parallax", refSchema)
149 self.assertNotIn("parallax_flag", refSchema)
151 def testMakeSchema_fullCovariance(self):
152 """Make a schema with full position information and coordinate
153 covariance and test it."""
154 refSchema = _makeSchema(filterNameList=["r"], fullPositionInformation=True)
155 # Test that the epoch, proper motion and parallax terms are included in
156 # the schema.
157 self.assertIn("epoch", refSchema)
158 self.assertIn("pm_ra", refSchema)
159 self.assertIn("pm_dec", refSchema)
160 self.assertIn("pm_flag", refSchema)
161 self.assertIn("parallax", refSchema)
162 self.assertIn("parallax_flag", refSchema)
163 # Test that a sample of the 15 covariance terms are included in the schema.
164 self.assertIn("coord_raErr", refSchema)
165 self.assertIn("coord_decErr", refSchema)
166 self.assertIn("coord_ra_coord_dec_Cov", refSchema)
167 self.assertIn("pm_raErr", refSchema)
168 self.assertIn("pm_ra_parallax_Cov", refSchema)
169 self.assertIn("parallaxErr", refSchema)
170 self.assertEqual(refSchema['coord_raErr'].asField().getUnits(), "rad")
171 self.assertEqual(refSchema['coord_ra_coord_dec_Cov'].asField().getUnits(), "rad^2")
172 self.assertEqual(refSchema['pm_raErr'].asField().getUnits(), "rad/year")
173 self.assertEqual(refSchema['pm_dec_parallax_Cov'].asField().getUnits(), "rad^2/year")
176class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase):
177 """Test validation of ConvertReferenceCatalogConfig."""
178 def testValidateRaDecMag(self):
179 config = makeConvertConfig()
180 config.validate()
182 for name in ("ra_name", "dec_name", "mag_column_list"):
183 with self.subTest(name=name):
184 config = makeConvertConfig()
185 setattr(config, name, None)
186 with self.assertRaises(ValueError):
187 config.validate()
189 def testValidateRaDecErr(self):
190 # check that a basic config validates
191 config = makeConvertConfig(withRaDecErr=True)
192 config.validate()
194 # check that a config with any of these fields missing does not validate
195 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"):
196 with self.subTest(name=name):
197 config = makeConvertConfig(withRaDecErr=True)
198 setattr(config, name, None)
199 with self.assertRaises(ValueError):
200 config.validate()
202 # check that coord_err_unit must be an astropy unit
203 config = makeConvertConfig(withRaDecErr=True)
204 config.coord_err_unit = "nonsense unit"
205 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"):
206 config.validate()
208 def testValidateMagErr(self):
209 config = makeConvertConfig(withMagErr=True)
210 config.validate()
212 # test for missing names
213 for name in config.mag_column_list:
214 with self.subTest(name=name):
215 config = makeConvertConfig(withMagErr=True)
216 del config.mag_err_column_map[name]
217 with self.assertRaises(ValueError):
218 config.validate()
220 # test for incorrect names
221 for name in config.mag_column_list:
222 with self.subTest(name=name):
223 config = makeConvertConfig(withMagErr=True)
224 config.mag_err_column_map["badName"] = config.mag_err_column_map[name]
225 del config.mag_err_column_map[name]
226 with self.assertRaises(ValueError):
227 config.validate()
229 def testValidatePm(self):
230 names = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale",
231 "pm_ra_err_name", "pm_dec_err_name"]
233 config = makeConvertConfig(withPm=True)
234 config.validate()
235 del config
237 for name in names:
238 with self.subTest(name=name):
239 config = makeConvertConfig(withPm=True)
240 setattr(config, name, None)
241 with self.assertRaises(ValueError):
242 config.validate()
244 def testValidateParallax(self):
245 """Validation should fail if any parallax-related fields are missing.
246 """
247 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"]
249 config = makeConvertConfig(withParallax=True)
250 config.validate()
251 del config
253 for name in names:
254 with self.subTest(name=name):
255 config = makeConvertConfig(withParallax=True)
256 setattr(config, name, None)
257 with self.assertRaises(ValueError, msg=name):
258 config.validate()
260 def testValidateCovariance(self):
261 """Validation should fail if any position-related fields are empty if
262 full_position_information is set.
263 """
264 names = ["ra_err_name", "dec_err_name", "coord_err_unit",
265 "parallax_name", "parallax_err_name",
266 "epoch_name", "epoch_format", "epoch_scale",
267 "pm_ra_name", "pm_dec_name", "pm_ra_err_name", "pm_dec_err_name"]
269 for name in names:
270 with self.subTest(name=name):
271 config = makeConvertConfig(withRaDecErr=True, withParallax=True, withPm=True)
272 config.full_position_information = True
273 config.manager.retarget(ConvertGaiaManager)
274 setattr(config, name, None)
275 with self.assertRaises(ValueError, msg=name):
276 config.validate()
279class ConvertGaiaManagerTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase,
280 lsst.utils.tests.TestCase):
281 """Unittests specific to the Gaia catalog.
282 """
283 def setUp(self):
284 self.tempDir = tempfile.TemporaryDirectory()
285 tempPath = self.tempDir.name
286 self.log = logging.getLogger("lsst.TestConvertRefcatManager")
287 self.config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True)
288 self.config.id_name = 'id'
289 self.config.full_position_information = True
290 self.config.manager.retarget(ConvertGaiaManager)
291 self.config.coord_err_unit = 'milliarcsecond'
292 self.config.ra_err_name = 'ra_error'
293 self.config.dec_err_name = 'dec_error'
294 self.config.pm_ra_name = 'pmra'
295 self.config.pm_dec_name = 'pmdec'
296 self.config.pm_ra_err_name = 'pmra_error'
297 self.config.pm_dec_err_name = 'pmdec_error'
298 self.config.parallax_name = 'parallax'
299 self.config.parallax_err_name = 'parallax_error'
300 self.config.epoch_name = 'unixtime'
301 self.config.epoch_format = 'unix'
302 self.config.epoch_scale = 'tai'
303 self.depth = 2 # very small depth, for as few pixels as possible.
304 self.indexer = HtmIndexer(self.depth)
305 self.htm = lsst.sphgeom.HtmPixelization(self.depth)
306 converter = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config)
307 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
308 ('a', '<f8'), ('a_err', '<f8')]
309 self.schema, self.key_map = converter.makeSchema(dtype)
310 self.fileReader = ReadTextCatalogTask()
312 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
313 self.matchedPixels = np.array([1, 1, 2, 2, 3])
314 self.tempDir2 = tempfile.TemporaryDirectory()
315 tempPath = self.tempDir2.name
316 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)}
318 self.worker = ConvertGaiaManager(self.filenames,
319 self.config,
320 self.fileReader,
321 self.indexer,
322 self.schema,
323 self.key_map,
324 self.htm.universe()[0],
325 addRefCatMetadata,
326 self.log)
328 def tearDown(self):
329 self.tempDir.cleanup()
330 self.tempDir2.cleanup()
332 def test_positionSetting(self):
333 """Test the _setProperMotion, _setParallax, and
334 _setCoordinateCovariance methods.
335 """
336 outputCatalog = SimpleCatalog(self.worker.schema)
337 outputCatalog.resize(len(self.fakeInput))
339 # Set coordinate errors and covariances:
340 coordErr = self.worker._getCoordErr(self.fakeInput)
341 for name, array in coordErr.items():
342 outputCatalog[name] = array
344 for outputRow, inputRow in zip(outputCatalog, self.fakeInput):
345 self.worker._setProperMotion(outputRow, inputRow)
346 self.worker._setParallax(outputRow, inputRow)
347 self.worker._setCoordinateCovariance(outputRow, inputRow)
349 coordConvert = (self.worker.coord_err_unit).to(u.radian)
350 pmConvert = (self.worker.config.pm_scale * u.milliarcsecond).to_value(u.radian)
351 parallaxConvert = (self.worker.config.parallax_scale * u.milliarcsecond).to_value(u.radian)
353 # Test a few combinations of coordinates, proper motion, and parallax.
354 # Check that the covariance in the output catalog matches the
355 # covariance calculated from the input, and also matches the covariance
356 # calculated from the output catalog errors with the input correlation.
357 ra_pmra_cov1 = (self.fakeInput['ra_error'] * self.fakeInput['pmra_error']
358 * self.fakeInput['ra_pmra_corr']) * coordConvert * pmConvert
359 ra_pmra_cov2 = (outputCatalog['coord_raErr'] * outputCatalog['pm_raErr']
360 * self.fakeInput['ra_pmra_corr'])
361 np.testing.assert_allclose(ra_pmra_cov1, outputCatalog['coord_ra_pm_ra_Cov'])
362 np.testing.assert_allclose(ra_pmra_cov2, outputCatalog['coord_ra_pm_ra_Cov'])
364 dec_parallax_cov1 = (self.fakeInput['dec_error'] * self.fakeInput['parallax_error']
365 * self.fakeInput['dec_parallax_corr']) * coordConvert * parallaxConvert
366 dec_parallax_cov2 = (outputCatalog['coord_decErr'] * outputCatalog['parallaxErr']
367 * self.fakeInput['dec_parallax_corr'])
368 np.testing.assert_allclose(dec_parallax_cov1, outputCatalog['coord_dec_parallax_Cov'])
369 np.testing.assert_allclose(dec_parallax_cov2, outputCatalog['coord_dec_parallax_Cov'])
371 pmdec_parallax_cov1 = (self.fakeInput['pmdec_error'] * self.fakeInput['parallax_error']
372 * self.fakeInput['parallax_pmdec_corr']) * pmConvert * parallaxConvert
373 pmdec_parallax_cov2 = (outputCatalog['pm_decErr'] * outputCatalog['parallaxErr']
374 * self.fakeInput['parallax_pmdec_corr'])
375 np.testing.assert_allclose(pmdec_parallax_cov1, outputCatalog['pm_dec_parallax_Cov'])
376 np.testing.assert_allclose(pmdec_parallax_cov2, outputCatalog['pm_dec_parallax_Cov'])
379class TestMemory(lsst.utils.tests.MemoryTestCase):
380 pass
383def setup_module(module):
384 lsst.utils.tests.init()
387if __name__ == "__main__": 387 ↛ 388line 387 didn't jump to line 388 because the condition on line 387 was never true
388 lsst.utils.tests.init()
389 unittest.main()