Coverage for tests/test_htmIndex.py: 16%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3#
4# Copyright 2008-2016 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24import os
25import unittest
26from collections import Counter
28import astropy.time
29import astropy.units
30import numpy as np
32import lsst.geom
33import lsst.afw.table as afwTable
34import lsst.afw.geom as afwGeom
35import lsst.daf.persistence as dafPersist
36from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask,
37 LoadIndexedReferenceObjectsConfig, getRefFluxField)
38from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits
39import lsst.utils
41from ingestIndexTestBase import (makeConvertConfig, ConvertReferenceCatalogTestBase,
42 make_coord)
45class IngestIndexTaskValidateTestCase(lsst.utils.tests.TestCase):
46 """Test validation of IngestIndexReferenceConfig."""
47 def testValidateRaDecMag(self):
48 config = makeConvertConfig()
49 config.validate()
51 for name in ("ra_name", "dec_name", "mag_column_list"):
52 with self.subTest(name=name):
53 config = makeConvertConfig()
54 setattr(config, name, None)
55 with self.assertRaises(ValueError):
56 config.validate()
58 def testValidateRaDecErr(self):
59 # check that a basic config validates
60 config = makeConvertConfig(withRaDecErr=True)
61 config.validate()
63 # check that a config with any of these fields missing does not validate
64 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"):
65 with self.subTest(name=name):
66 config = makeConvertConfig(withRaDecErr=True)
67 setattr(config, name, None)
68 with self.assertRaises(ValueError):
69 config.validate()
71 # check that coord_err_unit must be an astropy unit
72 config = makeConvertConfig(withRaDecErr=True)
73 config.coord_err_unit = "nonsense unit"
74 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"):
75 config.validate()
77 def testValidateMagErr(self):
78 config = makeConvertConfig(withMagErr=True)
79 config.validate()
81 # test for missing names
82 for name in config.mag_column_list:
83 with self.subTest(name=name):
84 config = makeConvertConfig(withMagErr=True)
85 del config.mag_err_column_map[name]
86 with self.assertRaises(ValueError):
87 config.validate()
89 # test for incorrect names
90 for name in config.mag_column_list:
91 with self.subTest(name=name):
92 config = makeConvertConfig(withMagErr=True)
93 config.mag_err_column_map["badName"] = config.mag_err_column_map[name]
94 del config.mag_err_column_map[name]
95 with self.assertRaises(ValueError):
96 config.validate()
98 def testValidatePm(self):
99 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"]
101 for withPmErr in (False, True):
102 config = makeConvertConfig(withPm=True, withPmErr=withPmErr)
103 config.validate()
104 del config
106 if withPmErr:
107 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"]
108 else:
109 names = basicNames
110 for name in names:
111 with self.subTest(name=name, withPmErr=withPmErr):
112 config = makeConvertConfig(withPm=True, withPmErr=withPmErr)
113 setattr(config, name, None)
114 with self.assertRaises(ValueError):
115 config.validate()
117 def testValidateParallax(self):
118 """Validation should fail if any parallax-related fields are missing.
119 """
120 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"]
122 config = makeConvertConfig(withParallax=True)
123 config.validate()
124 del config
126 for name in names:
127 with self.subTest(name=name):
128 config = makeConvertConfig(withParallax=True)
129 setattr(config, name, None)
130 with self.assertRaises(ValueError, msg=name):
131 config.validate()
134class ReferenceCatalogIngestAndLoadTestCase(ConvertReferenceCatalogTestBase, lsst.utils.tests.TestCase):
135 """Tests of converting, ingesting, loading and validating an HTM Indexed
136 Reference Catalog (gen2 code path).
137 """
138 @classmethod
139 def setUpClass(cls):
140 super().setUpClass()
141 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test')
142 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input")
144 # Run the ingest once to create a butler repo we can compare to
145 config = makeConvertConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True,
146 withParallax=True)
147 # Pregenerated gen2 test refcats have the "cal_ref_cat" name.
148 config.dataset_config.ref_dataset_name = "cal_ref_cat"
149 config.dataset_config.indexer.active.depth = cls.depth
150 config.id_name = 'id'
151 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr
152 config.parallax_scale = 1e3 # arcsec -> milliarcsec
153 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
154 config.file_reader.format = 'ascii.commented_header'
155 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath,
156 cls.skyCatalogFile], config=config)
157 cls.testButler = dafPersist.Butler(cls.testRepoPath)
159 @classmethod
160 def tearDownClass(cls):
161 del cls.testButler
163 def testSanity(self):
164 """Sanity-check that compCats contains some entries with sources."""
165 numWithSources = 0
166 for idList in self.compCats.values():
167 if len(idList) > 0:
168 numWithSources += 1
169 self.assertGreater(numWithSources, 0)
171 def testIngestSetsVersion(self):
172 """Test that newly ingested catalogs get the correct version number set.
173 """
174 def runTest(withRaDecErr):
175 outputPath = os.path.join(self.outPath, "output_setsVersion"
176 + "_withRaDecErr" if withRaDecErr else "")
177 # Test with multiple files and standard config
178 config = makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True,
179 withPm=True, withPmErr=True)
180 # Pregenerated gen2 test refcats have the "cal_ref_cat" name.
181 config.dataset_config.ref_dataset_name = "cal_ref_cat"
182 # don't use the default depth, to avoid taking the time to create thousands of file locks
183 config.dataset_config.indexer.active.depth = self.depth
184 IngestIndexedReferenceTask.parseAndRun(
185 args=[self.input_dir, "--output", outputPath, self.skyCatalogFile],
186 config=config)
187 # A newly-ingested refcat should be marked format_version=1.
188 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath))
189 self.assertEqual(loader.dataset_config.format_version, 1)
191 runTest(withRaDecErr=True)
192 runTest(withRaDecErr=False)
194 def testLoadSkyCircle(self):
195 """Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config."""
196 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
197 for tupl, idList in self.compCats.items():
198 cent = make_coord(*tupl)
199 lcat = loader.loadSkyCircle(cent, self.searchRadius, 'a')
200 self.assertTrue(lcat.refCat.isContiguous())
201 self.assertFalse("camFlux" in lcat.refCat.schema)
202 self.assertEqual(Counter(lcat.refCat['id']), Counter(idList))
203 if len(lcat.refCat) > 0:
204 # make sure there are no duplicate ids
205 self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1)
206 self.assertEqual(len(set(Counter(idList).values())), 1)
207 # A default-loaded sky circle should not have centroids
208 self.assertNotIn("centroid_x", lcat.refCat.schema)
209 self.assertNotIn("centroid_y", lcat.refCat.schema)
210 self.assertNotIn("hasCentroid", lcat.refCat.schema)
211 else:
212 self.assertEqual(len(idList), 0)
214 def testLoadPixelBox(self):
215 """Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config."""
216 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
217 numFound = 0
218 for tupl, idList in self.compCats.items():
219 cent = make_coord(*tupl)
220 bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary
221 ctr_pix = bbox.getCenter()
222 # catalog is sparse, so set pixel scale such that bbox encloses region
223 # used to generate compCats
224 pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth())
225 cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale)
226 wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix)
227 result = loader.loadPixelBox(bbox, wcs, "a")
228 # The following is to ensure the reference catalog coords are
229 # getting corrected for proper motion when an epoch is provided.
230 # Use an extreme epoch so that differences in corrected coords
231 # will be significant. Note that this simply tests that the coords
232 # do indeed change when the epoch is passed. It makes no attempt
233 # at assessing the correctness of the change. This is left to the
234 # explicit testProperMotion() test below.
235 resultWithEpoch = loader.loadPixelBox(bbox, wcs, "a",
236 epoch=astropy.time.Time(30000, format='mjd', scale="tai"))
237 self.assertFloatsNotEqual(result.refCat["coord_ra"], resultWithEpoch.refCat["coord_ra"],
238 rtol=1.0e-4)
239 self.assertFloatsNotEqual(result.refCat["coord_dec"], resultWithEpoch.refCat["coord_dec"],
240 rtol=1.0e-4)
241 self.assertFalse("camFlux" in result.refCat.schema)
242 self.assertGreaterEqual(len(result.refCat), len(idList))
243 numFound += len(result.refCat)
244 self.assertGreater(numFound, 0)
246 def testFilterMap(self):
247 """Test filterMap parameters of LoadIndexedReferenceObjectsConfig."""
248 config = LoadIndexedReferenceObjectsConfig()
249 config.filterMap = {"aprime": "a"}
250 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
251 for tupl, idList in self.compCats.items():
252 cent = make_coord(*tupl)
253 lcat = loader.loadSkyCircle(cent, self.searchRadius, "a")
254 self.assertEqual(lcat.fluxField, "a_flux")
255 if len(idList) > 0:
256 aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
257 self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
258 break # just need one test
260 def testProperMotion(self):
261 """Test proper motion correction"""
262 center = make_coord(93.0, -90.0)
263 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
264 references = loader.loadSkyCircle(center, self.searchRadius, 'a').refCat
265 original = references.copy(True)
267 # Zero epoch change --> no proper motion correction (except minor numerical effects)
268 loader.applyProperMotions(references, self.epoch)
269 self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14)
270 self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14)
271 self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"])
272 self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"])
274 # One year difference
275 loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr)
276 self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"])
277 self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"])
278 for orig, ref in zip(original, references):
279 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
280 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
281 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
282 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
283 predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"])
284 predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"])
285 self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr)
286 self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr)
288 def testRequireProperMotion(self):
289 """Tests of the requireProperMotion config field.
291 Requiring proper motion corrections for a catalog that does not
292 contain valid PM data should result in an exception.
294 `data/testHtmIndex-ps1-bad-pm.fits` is a random shard taken from the
295 ps1_pv3_3pi_20170110 refcat (that has the unitless PM fields),
296 stripped to only 2 rows: we patch it in here to simplify test setup.
297 """
298 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/testHtmIndex-ps1-bad-pm.fits')
299 refcatData = lsst.afw.table.SimpleCatalog.readFits(path)
300 center = make_coord(93.0, -90.0)
301 epoch = self.epoch + 1.0*astropy.units.yr
303 # malformatted catalogs should warn and raise if we require proper motion corrections
304 config = LoadIndexedReferenceObjectsConfig()
305 config.requireProperMotion = True
306 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
307 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
308 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData):
309 msg = "requireProperMotion=True but refcat pm_ra field is not an Angle"
310 with self.assertRaisesRegex(RuntimeError, msg):
311 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=epoch)
313 # not specifying `epoch` with requireProperMotion=True should raise for any catalog
314 config = LoadIndexedReferenceObjectsConfig()
315 config.requireProperMotion = True
316 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
317 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
318 msg = "requireProperMotion=True but epoch not provided to loader"
319 with self.assertRaisesRegex(RuntimeError, msg):
320 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=None)
322 # malformatted catalogs should just warn if we do not require proper motion corrections
323 config = LoadIndexedReferenceObjectsConfig()
324 config.requireProperMotion = False
325 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
326 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
327 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData):
328 with self.assertLogs("lsst.LoadIndexedReferenceObjectsTask", level="WARNING") as cm:
329 loader.loadSkyCircle(center, self.searchRadius, "g", epoch=epoch)
330 warnLog1 = "Reference catalog pm_ra field is not an Angle; cannot apply proper motion."
331 self.assertEqual(cm.records[0].message, warnLog1)
333 def testLoadVersion0(self):
334 """Test reading a pre-written format_version=0 (Jy flux) catalog.
335 It should be converted to have nJy fluxes.
336 """
337 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version0')
338 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
339 self.assertEqual(loader.dataset_config.format_version, 0)
340 result = loader.loadSkyCircle(make_coord(10, 20), 5*lsst.geom.degrees, 'a')
341 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
342 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
343 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
344 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
345 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
346 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
348 def testLoadVersion1(self):
349 """Test reading a format_version=1 catalog (fluxes unchanged)."""
350 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1')
351 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
352 self.assertEqual(loader.dataset_config.format_version, 1)
353 result = loader.loadSkyCircle(make_coord(10, 20), 5*lsst.geom.degrees, 'a')
354 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
355 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
356 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
357 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
358 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
359 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
362class TestMemory(lsst.utils.tests.MemoryTestCase):
363 pass
366def setup_module(module):
367 lsst.utils.tests.init()
370if __name__ == "__main__": 370 ↛ 371line 370 didn't jump to line 371, because the condition on line 370 was never true
371 lsst.utils.tests.init()
372 unittest.main()