Coverage for tests/test_convertReferenceCatalog.py: 14%
231 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-10 10:43 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-10 10:43 +0000
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import astropy.units as u
23import numpy as np
24import os.path
25import sys
26import unittest
27import unittest.mock
28import tempfile
29import itertools
31from lsst.afw.table import SimpleCatalog
32from lsst.pex.config import FieldValidationError
33from lsst.meas.algorithms import (convertReferenceCatalog, ConvertReferenceCatalogTask, getRefFluxField)
34from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask
35from lsst.meas.algorithms.htmIndexer import HtmIndexer
36from lsst.meas.algorithms.convertRefcatManager import ConvertGaiaManager
37from lsst.meas.algorithms.convertReferenceCatalog import addRefCatMetadata, _makeSchema
39import lsst.utils
41from convertReferenceCatalogTestBase import makeConvertConfig
42import convertReferenceCatalogTestBase
45class TestMain(lsst.utils.tests.TestCase):
46 """Test mocking commandline arguments and calling
47 ``convertReferenceCatalog.main()``.
48 """
49 def setUp(self):
50 self.inpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data/mockrefcat/")
51 self.expected_files = [os.path.join(self.inpath, "123.fits"),
52 os.path.join(self.inpath, "124.fits"),
53 os.path.join(self.inpath, "125.fits")]
55 def test_main_args(self):
56 """Test that main configures the task and calls run() with the correct
57 file list.
58 """
59 outdir = tempfile.TemporaryDirectory()
60 outpath = outdir.name
61 args = ["convertReferenceCatalog",
62 outpath,
63 os.path.join(self.inpath, "mock_config.py"),
64 os.path.join(self.inpath, "*.fits")]
65 with unittest.mock.patch.object(convertReferenceCatalog.ConvertReferenceCatalogTask, "run") as run, \
66 unittest.mock.patch.object(sys, "argv", args):
67 convertReferenceCatalog.main()
68 # Test with sets because the glob can come out in any order.
69 self.assertEqual(set(run.call_args.args[0]), set(self.expected_files))
70 # This is necessary to avoid a ResourceWarning.
71 outdir.cleanup()
73 def test_main_args_bad_config(self):
74 """Test that a bad config file produces a useful error, i.e. that
75 main() validates the config.
76 """
77 outdir = tempfile.TemporaryDirectory()
78 outpath = outdir.name
79 args = ["convertReferenceCatalog",
80 outpath,
81 os.path.join(self.inpath, "bad_config.py"),
82 os.path.join(self.inpath, "*.fits")]
83 with self.assertRaisesRegex(FieldValidationError, "Field 'ra_name' failed validation"), \
84 unittest.mock.patch.object(sys, "argv", args):
85 convertReferenceCatalog.main()
86 # This is necessary to avoid a ResourceWarning.
87 outdir.cleanup()
89 def test_main_args_expanded_glob(self):
90 """Test that an un-quoted glob (i.e. list of files) fails with a
91 useful error.
92 """
93 outdir = tempfile.TemporaryDirectory()
94 outpath = outdir.name
95 args = ["convertReferenceCatalog",
96 outpath,
97 os.path.join(self.inpath, "mock_config.py"),
98 # an un-quoted glob will be shell-expanded to a list of files.
99 "file1", "file2", "file3"]
100 msg = "Final argument must be a quoted file glob, not a shell-expanded list of files."
101 with self.assertRaisesRegex(RuntimeError, msg), \
102 unittest.mock.patch.object(sys, "argv", args):
103 convertReferenceCatalog.main()
104 # This is necessary to avoid a ResourceWarning.
105 outdir.cleanup()
108class MakeSchemaTestCase(lsst.utils.tests.TestCase):
109 """Test the function to make reference catalog schemas.
110 """
111 def testMakeSchema(self):
112 """Make a schema and check it."""
113 for filterNameList in (["r"], ["foo", "_bar"]):
114 for (addIsPhotometric, addIsResolved, addIsVariable) in itertools.product((False, True),
115 (False, True),
116 (False, True)):
117 argDict = dict(
118 filterNameList=filterNameList,
119 addIsPhotometric=addIsPhotometric,
120 addIsResolved=addIsResolved,
121 addIsVariable=addIsVariable,
122 )
123 refSchema = _makeSchema(**argDict)
124 self.assertTrue("coord_ra" in refSchema)
125 self.assertTrue("coord_dec" in refSchema)
126 self.assertTrue("coord_raErr" in refSchema)
127 self.assertTrue("coord_decErr" in refSchema)
128 for filterName in filterNameList:
129 fluxField = filterName + "_flux"
130 self.assertIn(fluxField, refSchema)
131 self.assertNotIn("x" + fluxField, refSchema)
132 fluxErrField = fluxField + "Err"
133 self.assertIn(fluxErrField, refSchema)
134 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux")
135 self.assertEqual("resolved" in refSchema, addIsResolved)
136 self.assertEqual("variable" in refSchema, addIsVariable)
137 self.assertEqual("photometric" in refSchema, addIsPhotometric)
138 self.assertEqual("photometric" in refSchema, addIsPhotometric)
140 # The default for `fullPositionInformation` is False, so none
141 # of the following should be included. We test setting these
142 # all together below.
143 self.assertNotIn("epoch", refSchema)
144 self.assertNotIn("pm_ra", refSchema)
145 self.assertNotIn("pm_dec", refSchema)
146 self.assertNotIn("pm_flag", refSchema)
147 self.assertNotIn("parallax", refSchema)
148 self.assertNotIn("parallax_flag", refSchema)
150 def testMakeSchema_fullCovariance(self):
151 """Make a schema with full position information and coordinate
152 covariance and test it."""
153 refSchema = _makeSchema(filterNameList=["r"], fullPositionInformation=True)
154 # Test that the epoch, proper motion and parallax terms are included in
155 # the schema.
156 self.assertIn("epoch", refSchema)
157 self.assertIn("pm_ra", refSchema)
158 self.assertIn("pm_dec", refSchema)
159 self.assertIn("pm_flag", refSchema)
160 self.assertIn("parallax", refSchema)
161 self.assertIn("parallax_flag", refSchema)
162 # Test that a sample of the 15 covariance terms are included in the schema.
163 self.assertIn("coord_raErr", refSchema)
164 self.assertIn("coord_decErr", refSchema)
165 self.assertIn("coord_ra_coord_dec_Cov", refSchema)
166 self.assertIn("pm_raErr", refSchema)
167 self.assertIn("pm_ra_parallax_Cov", refSchema)
168 self.assertIn("parallaxErr", refSchema)
169 self.assertEqual(refSchema['coord_raErr'].asField().getUnits(), "rad")
170 self.assertEqual(refSchema['coord_ra_coord_dec_Cov'].asField().getUnits(), "rad^2")
171 self.assertEqual(refSchema['pm_raErr'].asField().getUnits(), "rad/year")
172 self.assertEqual(refSchema['pm_dec_parallax_Cov'].asField().getUnits(), "rad^2/year")
175class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase):
176 """Test validation of ConvertReferenceCatalogConfig."""
177 def testValidateRaDecMag(self):
178 config = makeConvertConfig()
179 config.validate()
181 for name in ("ra_name", "dec_name", "mag_column_list"):
182 with self.subTest(name=name):
183 config = makeConvertConfig()
184 setattr(config, name, None)
185 with self.assertRaises(ValueError):
186 config.validate()
188 def testValidateRaDecErr(self):
189 # check that a basic config validates
190 config = makeConvertConfig(withRaDecErr=True)
191 config.validate()
193 # check that a config with any of these fields missing does not validate
194 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"):
195 with self.subTest(name=name):
196 config = makeConvertConfig(withRaDecErr=True)
197 setattr(config, name, None)
198 with self.assertRaises(ValueError):
199 config.validate()
201 # check that coord_err_unit must be an astropy unit
202 config = makeConvertConfig(withRaDecErr=True)
203 config.coord_err_unit = "nonsense unit"
204 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"):
205 config.validate()
207 def testValidateMagErr(self):
208 config = makeConvertConfig(withMagErr=True)
209 config.validate()
211 # test for missing names
212 for name in config.mag_column_list:
213 with self.subTest(name=name):
214 config = makeConvertConfig(withMagErr=True)
215 del config.mag_err_column_map[name]
216 with self.assertRaises(ValueError):
217 config.validate()
219 # test for incorrect names
220 for name in config.mag_column_list:
221 with self.subTest(name=name):
222 config = makeConvertConfig(withMagErr=True)
223 config.mag_err_column_map["badName"] = config.mag_err_column_map[name]
224 del config.mag_err_column_map[name]
225 with self.assertRaises(ValueError):
226 config.validate()
228 def testValidatePm(self):
229 names = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale",
230 "pm_ra_err_name", "pm_dec_err_name"]
232 config = makeConvertConfig(withPm=True)
233 config.validate()
234 del config
236 for name in names:
237 with self.subTest(name=name):
238 config = makeConvertConfig(withPm=True)
239 setattr(config, name, None)
240 with self.assertRaises(ValueError):
241 config.validate()
243 def testValidateParallax(self):
244 """Validation should fail if any parallax-related fields are missing.
245 """
246 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"]
248 config = makeConvertConfig(withParallax=True)
249 config.validate()
250 del config
252 for name in names:
253 with self.subTest(name=name):
254 config = makeConvertConfig(withParallax=True)
255 setattr(config, name, None)
256 with self.assertRaises(ValueError, msg=name):
257 config.validate()
259 def testValidateCovariance(self):
260 """Validation should fail if any position-related fields are empty if
261 full_position_information is set.
262 """
263 names = ["ra_err_name", "dec_err_name", "coord_err_unit",
264 "parallax_name", "parallax_err_name",
265 "epoch_name", "epoch_format", "epoch_scale",
266 "pm_ra_name", "pm_dec_name", "pm_ra_err_name", "pm_dec_err_name"]
268 for name in names:
269 with self.subTest(name=name):
270 config = makeConvertConfig(withRaDecErr=True, withParallax=True, withPm=True)
271 config.full_position_information = True
272 config.manager.retarget(ConvertGaiaManager)
273 setattr(config, name, None)
274 with self.assertRaises(ValueError, msg=name):
275 config.validate()
278class ConvertGaiaManagerTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase,
279 lsst.utils.tests.TestCase):
280 """Unittests specific to the Gaia catalog.
281 """
282 def setUp(self):
284 np.random.seed(10)
286 self.tempDir = tempfile.TemporaryDirectory()
287 tempPath = self.tempDir.name
288 self.log = lsst.log.Log.getLogger("lsst.TestConvertRefcatManager")
289 self.config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True)
290 self.config.id_name = 'id'
291 self.config.full_position_information = True
292 self.config.manager.retarget(ConvertGaiaManager)
293 self.config.coord_err_unit = 'milliarcsecond'
294 self.config.ra_err_name = 'ra_error'
295 self.config.dec_err_name = 'dec_error'
296 self.config.pm_ra_name = 'pmra'
297 self.config.pm_dec_name = 'pmdec'
298 self.config.pm_ra_err_name = 'pmra_error'
299 self.config.pm_dec_err_name = 'pmdec_error'
300 self.config.parallax_name = 'parallax'
301 self.config.parallax_err_name = 'parallax_error'
302 self.config.epoch_name = 'unixtime'
303 self.config.epoch_format = 'unix'
304 self.config.epoch_scale = 'tai'
305 self.depth = 2 # very small depth, for as few pixels as possible.
306 self.indexer = HtmIndexer(self.depth)
307 self.htm = lsst.sphgeom.HtmPixelization(self.depth)
308 converter = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config)
309 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'),
310 ('a', '<f8'), ('a_err', '<f8')]
311 self.schema, self.key_map = converter.makeSchema(dtype)
312 self.fileReader = ReadTextCatalogTask()
314 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543)
315 self.matchedPixels = np.array([1, 1, 2, 2, 3])
316 self.tempDir2 = tempfile.TemporaryDirectory()
317 tempPath = self.tempDir2.name
318 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)}
320 self.worker = ConvertGaiaManager(self.filenames,
321 self.config,
322 self.fileReader,
323 self.indexer,
324 self.schema,
325 self.key_map,
326 self.htm.universe()[0],
327 addRefCatMetadata,
328 self.log)
330 def tearDown(self):
331 self.tempDir.cleanup()
332 self.tempDir2.cleanup()
334 def test_positionSetting(self):
335 """Test the _setProperMotion, _setParallax, and
336 _setCoordinateCovariance methods.
337 """
338 outputCatalog = SimpleCatalog(self.worker.schema)
339 outputCatalog.resize(len(self.fakeInput))
341 # Set coordinate errors and covariances:
342 coordErr = self.worker._getCoordErr(self.fakeInput)
343 for name, array in coordErr.items():
344 outputCatalog[name] = array
346 for outputRow, inputRow in zip(outputCatalog, self.fakeInput):
347 self.worker._setProperMotion(outputRow, inputRow)
348 self.worker._setParallax(outputRow, inputRow)
349 self.worker._setCoordinateCovariance(outputRow, inputRow)
351 coordConvert = (self.worker.coord_err_unit).to(u.radian)
352 pmConvert = (self.worker.config.pm_scale * u.milliarcsecond).to_value(u.radian)
353 parallaxConvert = (self.worker.config.parallax_scale * u.milliarcsecond).to_value(u.radian)
355 # Test a few combinations of coordinates, proper motion, and parallax.
356 # Check that the covariance in the output catalog matches the
357 # covariance calculated from the input, and also matches the covariance
358 # calculated from the output catalog errors with the input correlation.
359 ra_pmra_cov1 = (self.fakeInput['ra_error'] * self.fakeInput['pmra_error']
360 * self.fakeInput['ra_pmra_corr']) * coordConvert * pmConvert
361 ra_pmra_cov2 = (outputCatalog['coord_raErr'] * outputCatalog['pm_raErr']
362 * self.fakeInput['ra_pmra_corr'])
363 np.testing.assert_allclose(ra_pmra_cov1, outputCatalog['coord_ra_pm_ra_Cov'])
364 np.testing.assert_allclose(ra_pmra_cov2, outputCatalog['coord_ra_pm_ra_Cov'])
366 dec_parallax_cov1 = (self.fakeInput['dec_error'] * self.fakeInput['parallax_error']
367 * self.fakeInput['dec_parallax_corr']) * coordConvert * parallaxConvert
368 dec_parallax_cov2 = (outputCatalog['coord_decErr'] * outputCatalog['parallaxErr']
369 * self.fakeInput['dec_parallax_corr'])
370 np.testing.assert_allclose(dec_parallax_cov1, outputCatalog['coord_dec_parallax_Cov'])
371 np.testing.assert_allclose(dec_parallax_cov2, outputCatalog['coord_dec_parallax_Cov'])
373 pmdec_parallax_cov1 = (self.fakeInput['pmdec_error'] * self.fakeInput['parallax_error']
374 * self.fakeInput['parallax_pmdec_corr']) * pmConvert * parallaxConvert
375 pmdec_parallax_cov2 = (outputCatalog['pm_decErr'] * outputCatalog['parallaxErr']
376 * self.fakeInput['parallax_pmdec_corr'])
377 np.testing.assert_allclose(pmdec_parallax_cov1, outputCatalog['pm_dec_parallax_Cov'])
378 np.testing.assert_allclose(pmdec_parallax_cov2, outputCatalog['pm_dec_parallax_Cov'])
381class TestMemory(lsst.utils.tests.MemoryTestCase):
382 pass
385def setup_module(module):
386 lsst.utils.tests.init()
389if __name__ == "__main__": 389 ↛ 390line 389 didn't jump to line 390, because the condition on line 389 was never true
390 lsst.utils.tests.init()
391 unittest.main()