Coverage for tests/test_htmIndex.py: 16%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3#
4# Copyright 2008-2016 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24import os
25import unittest
26from collections import Counter
28import astropy.time
29import astropy.units
30import numpy as np
32import lsst.geom
33import lsst.afw.table as afwTable
34import lsst.afw.geom as afwGeom
35import lsst.daf.persistence as dafPersist
36from lsst.meas.algorithms import (IngestIndexedReferenceTask, LoadIndexedReferenceObjectsTask,
37 LoadIndexedReferenceObjectsConfig, getRefFluxField)
38from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits
39import lsst.utils
41from ingestIndexTestBase import (makeConvertConfig, ConvertReferenceCatalogTestBase,
42 make_coord)
45class IngestIndexTaskValidateTestCase(lsst.utils.tests.TestCase):
46 """Test validation of IngestIndexReferenceConfig."""
47 def testValidateRaDecMag(self):
48 config = makeConvertConfig()
49 config.validate()
51 for name in ("ra_name", "dec_name", "mag_column_list"):
52 with self.subTest(name=name):
53 config = makeConvertConfig()
54 setattr(config, name, None)
55 with self.assertRaises(ValueError):
56 config.validate()
58 def testValidateRaDecErr(self):
59 # check that a basic config validates
60 config = makeConvertConfig(withRaDecErr=True)
61 config.validate()
63 # check that a config with any of these fields missing does not validate
64 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"):
65 with self.subTest(name=name):
66 config = makeConvertConfig(withRaDecErr=True)
67 setattr(config, name, None)
68 with self.assertRaises(ValueError):
69 config.validate()
71 # check that coord_err_unit must be an astropy unit
72 config = makeConvertConfig(withRaDecErr=True)
73 config.coord_err_unit = "nonsense unit"
74 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"):
75 config.validate()
77 def testValidateMagErr(self):
78 config = makeConvertConfig(withMagErr=True)
79 config.validate()
81 # test for missing names
82 for name in config.mag_column_list:
83 with self.subTest(name=name):
84 config = makeConvertConfig(withMagErr=True)
85 del config.mag_err_column_map[name]
86 with self.assertRaises(ValueError):
87 config.validate()
89 # test for incorrect names
90 for name in config.mag_column_list:
91 with self.subTest(name=name):
92 config = makeConvertConfig(withMagErr=True)
93 config.mag_err_column_map["badName"] = config.mag_err_column_map[name]
94 del config.mag_err_column_map[name]
95 with self.assertRaises(ValueError):
96 config.validate()
98 def testValidatePm(self):
99 basicNames = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale"]
101 for withPmErr in (False, True):
102 config = makeConvertConfig(withPm=True, withPmErr=withPmErr)
103 config.validate()
104 del config
106 if withPmErr:
107 names = basicNames + ["pm_ra_err_name", "pm_dec_err_name"]
108 else:
109 names = basicNames
110 for name in names:
111 with self.subTest(name=name, withPmErr=withPmErr):
112 config = makeConvertConfig(withPm=True, withPmErr=withPmErr)
113 setattr(config, name, None)
114 with self.assertRaises(ValueError):
115 config.validate()
117 def testValidateParallax(self):
118 """Validation should fail if any parallax-related fields are missing.
119 """
120 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"]
122 config = makeConvertConfig(withParallax=True)
123 config.validate()
124 del config
126 for name in names:
127 with self.subTest(name=name):
128 config = makeConvertConfig(withParallax=True)
129 setattr(config, name, None)
130 with self.assertRaises(ValueError, msg=name):
131 config.validate()
134class ReferenceCatalogIngestAndLoadTestCase(ConvertReferenceCatalogTestBase, lsst.utils.tests.TestCase):
135 """Tests of converting, ingesting, loading and validating an HTM Indexed
136 Reference Catalog (gen2 code path).
137 """
138 @classmethod
139 def setUpClass(cls):
140 super().setUpClass()
141 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test')
142 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input")
144 # Run the ingest once to create a butler repo we can compare to
145 config = makeConvertConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True,
146 withParallax=True)
147 # Pregenerated gen2 test refcats have the "cal_ref_cat" name.
148 config.dataset_config.ref_dataset_name = "cal_ref_cat"
149 config.dataset_config.indexer.active.depth = cls.depth
150 config.id_name = 'id'
151 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr
152 config.parallax_scale = 1e3 # arcsec -> milliarcsec
153 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
154 config.file_reader.format = 'ascii.commented_header'
155 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath,
156 cls.skyCatalogFile], config=config)
157 cls.testButler = dafPersist.Butler(cls.testRepoPath)
159 @classmethod
160 def tearDownClass(cls):
161 del cls.testButler
163 def testSanity(self):
164 """Sanity-check that compCats contains some entries with sources."""
165 numWithSources = 0
166 for idList in self.compCats.values():
167 if len(idList) > 0:
168 numWithSources += 1
169 self.assertGreater(numWithSources, 0)
171 def testIngestSetsVersion(self):
172 """Test that newly ingested catalogs get the correct version number set.
173 """
174 def runTest(withRaDecErr):
175 outputPath = os.path.join(self.outPath, "output_setsVersion"
176 + "_withRaDecErr" if withRaDecErr else "")
177 # Test with multiple files and standard config
178 config = makeConvertConfig(withRaDecErr=withRaDecErr, withMagErr=True,
179 withPm=True, withPmErr=True)
180 # Pregenerated gen2 test refcats have the "cal_ref_cat" name.
181 config.dataset_config.ref_dataset_name = "cal_ref_cat"
182 # don't use the default depth, to avoid taking the time to create thousands of file locks
183 config.dataset_config.indexer.active.depth = self.depth
184 IngestIndexedReferenceTask.parseAndRun(
185 args=[self.input_dir, "--output", outputPath, self.skyCatalogFile],
186 config=config)
187 # A newly-ingested refcat should be marked format_version=1.
188 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(outputPath))
189 self.assertEqual(loader.dataset_config.format_version, 1)
191 runTest(withRaDecErr=True)
192 runTest(withRaDecErr=False)
194 def testLoadSkyCircle(self):
195 """Test LoadIndexedReferenceObjectsTask.loadSkyCircle with default config."""
196 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
197 for tupl, idList in self.compCats.items():
198 cent = make_coord(*tupl)
199 lcat = loader.loadSkyCircle(cent, self.searchRadius, filterName='a')
200 self.assertTrue(lcat.refCat.isContiguous())
201 self.assertFalse("camFlux" in lcat.refCat.schema)
202 self.assertEqual(Counter(lcat.refCat['id']), Counter(idList))
203 if len(lcat.refCat) > 0:
204 # make sure there are no duplicate ids
205 self.assertEqual(len(set(Counter(lcat.refCat['id']).values())), 1)
206 self.assertEqual(len(set(Counter(idList).values())), 1)
207 # A default-loaded sky circle should not have centroids
208 self.assertNotIn("centroid_x", lcat.refCat.schema)
209 self.assertNotIn("centroid_y", lcat.refCat.schema)
210 self.assertNotIn("hasCentroid", lcat.refCat.schema)
211 else:
212 self.assertEqual(len(idList), 0)
214 def testLoadPixelBox(self):
215 """Test LoadIndexedReferenceObjectsTask.loadPixelBox with default config."""
216 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
217 numFound = 0
218 for tupl, idList in self.compCats.items():
219 cent = make_coord(*tupl)
220 bbox = lsst.geom.Box2I(lsst.geom.Point2I(30, -5), lsst.geom.Extent2I(1000, 1004)) # arbitrary
221 ctr_pix = bbox.getCenter()
222 # catalog is sparse, so set pixel scale such that bbox encloses region
223 # used to generate compCats
224 pixel_scale = 2*self.searchRadius/max(bbox.getHeight(), bbox.getWidth())
225 cdMatrix = afwGeom.makeCdMatrix(scale=pixel_scale)
226 wcs = afwGeom.makeSkyWcs(crval=cent, crpix=ctr_pix, cdMatrix=cdMatrix)
227 result = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a")
228 # The following is to ensure the reference catalog coords are
229 # getting corrected for proper motion when an epoch is provided.
230 # Use an extreme epoch so that differences in corrected coords
231 # will be significant. Note that this simply tests that the coords
232 # do indeed change when the epoch is passed. It makes no attempt
233 # at assessing the correctness of the change. This is left to the
234 # explicit testProperMotion() test below.
235 resultWithEpoch = loader.loadPixelBox(bbox=bbox, wcs=wcs, filterName="a",
236 epoch=astropy.time.Time(30000, format='mjd', scale="tai"))
237 self.assertFloatsNotEqual(result.refCat["coord_ra"], resultWithEpoch.refCat["coord_ra"],
238 rtol=1.0e-4)
239 self.assertFloatsNotEqual(result.refCat["coord_dec"], resultWithEpoch.refCat["coord_dec"],
240 rtol=1.0e-4)
241 self.assertFalse("camFlux" in result.refCat.schema)
242 self.assertGreaterEqual(len(result.refCat), len(idList))
243 numFound += len(result.refCat)
244 self.assertGreater(numFound, 0)
246 def testDefaultFilterAndFilterMap(self):
247 """Test defaultFilter and filterMap parameters of LoadIndexedReferenceObjectsConfig."""
248 config = LoadIndexedReferenceObjectsConfig()
249 config.defaultFilter = "b"
250 config.filterMap = {"aprime": "a"}
251 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
252 for tupl, idList in self.compCats.items():
253 cent = make_coord(*tupl)
254 lcat = loader.loadSkyCircle(cent, self.searchRadius)
255 self.assertEqual(lcat.fluxField, "camFlux")
256 if len(idList) > 0:
257 defFluxFieldName = getRefFluxField(lcat.refCat.schema, None)
258 self.assertTrue(defFluxFieldName in lcat.refCat.schema)
259 aprimeFluxFieldName = getRefFluxField(lcat.refCat.schema, "aprime")
260 self.assertTrue(aprimeFluxFieldName in lcat.refCat.schema)
261 break # just need one test
263 def testProperMotion(self):
264 """Test proper motion correction"""
265 center = make_coord(93.0, -90.0)
266 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler)
267 references = loader.loadSkyCircle(center, self.searchRadius, filterName='a').refCat
268 original = references.copy(True)
270 # Zero epoch change --> no proper motion correction (except minor numerical effects)
271 loader.applyProperMotions(references, self.epoch)
272 self.assertFloatsAlmostEqual(references["coord_ra"], original["coord_ra"], rtol=1.0e-14)
273 self.assertFloatsAlmostEqual(references["coord_dec"], original["coord_dec"], rtol=1.0e-14)
274 self.assertFloatsEqual(references["coord_raErr"], original["coord_raErr"])
275 self.assertFloatsEqual(references["coord_decErr"], original["coord_decErr"])
277 # One year difference
278 loader.applyProperMotions(references, self.epoch + 1.0*astropy.units.yr)
279 self.assertFloatsEqual(references["pm_raErr"], original["pm_raErr"])
280 self.assertFloatsEqual(references["pm_decErr"], original["pm_decErr"])
281 for orig, ref in zip(original, references):
282 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
283 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
284 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
285 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
286 predictedRaErr = np.hypot(original["coord_raErr"], original["pm_raErr"])
287 predictedDecErr = np.hypot(original["coord_decErr"], original["pm_decErr"])
288 self.assertFloatsAlmostEqual(references["coord_raErr"], predictedRaErr)
289 self.assertFloatsAlmostEqual(references["coord_decErr"], predictedDecErr)
291 def testRequireProperMotion(self):
292 """Tests of the requireProperMotion config field.
294 Requiring proper motion corrections for a catalog that does not
295 contain valid PM data should result in an exception.
297 `data/testHtmIndex-ps1-bad-pm.fits` is a random shard taken from the
298 ps1_pv3_3pi_20170110 refcat (that has the unitless PM fields),
299 stripped to only 2 rows: we patch it in here to simplify test setup.
300 """
301 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/testHtmIndex-ps1-bad-pm.fits')
302 refcatData = lsst.afw.table.SimpleCatalog.readFits(path)
303 center = make_coord(93.0, -90.0)
304 epoch = self.epoch + 1.0*astropy.units.yr
306 # malformatted catalogs should warn and raise if we require proper motion corrections
307 config = LoadIndexedReferenceObjectsConfig()
308 config.requireProperMotion = True
309 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
310 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
311 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData):
312 msg = "requireProperMotion=True but refcat pm_ra field is not an Angle"
313 with self.assertRaisesRegex(RuntimeError, msg):
314 loader.loadSkyCircle(center, self.searchRadius, epoch=epoch)
316 # not specifying `epoch` with requireProperMotion=True should raise for any catalog
317 config = LoadIndexedReferenceObjectsConfig()
318 config.requireProperMotion = True
319 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
320 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
321 msg = "requireProperMotion=True but epoch not provided to loader"
322 with self.assertRaisesRegex(RuntimeError, msg):
323 loader.loadSkyCircle(center, self.searchRadius, epoch=None)
325 # malformatted catalogs should just warn if we do not require proper motion corrections
326 config = LoadIndexedReferenceObjectsConfig()
327 config.requireProperMotion = False
328 config.anyFilterMapsToThis = "g" # to use a catalog not made for obs_test
329 loader = LoadIndexedReferenceObjectsTask(butler=self.testButler, config=config)
330 with unittest.mock.patch.object(self.testButler, 'get', return_value=refcatData):
331 with self.assertLogs("lsst.LoadIndexedReferenceObjectsTask", level="WARNING") as cm:
332 loader.loadSkyCircle(center, self.searchRadius, epoch=epoch)
333 warnLog1 = "Reference catalog pm_ra field is not an Angle; cannot apply proper motion."
334 self.assertEqual(cm.records[0].message, warnLog1)
336 def testLoadVersion0(self):
337 """Test reading a pre-written format_version=0 (Jy flux) catalog.
338 It should be converted to have nJy fluxes.
339 """
340 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version0')
341 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
342 self.assertEqual(loader.dataset_config.format_version, 0)
343 result = loader.loadSkyCircle(make_coord(10, 20),
344 5*lsst.geom.degrees, filterName='a')
345 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
346 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
347 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
348 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
349 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
350 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
352 def testLoadVersion1(self):
353 """Test reading a format_version=1 catalog (fluxes unchanged)."""
354 path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data/version1')
355 loader = LoadIndexedReferenceObjectsTask(butler=dafPersist.Butler(path))
356 self.assertEqual(loader.dataset_config.format_version, 1)
357 result = loader.loadSkyCircle(make_coord(10, 20),
358 5*lsst.geom.degrees, filterName='a')
359 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
360 catalog = afwTable.SimpleCatalog.readFits(os.path.join(path, 'ref_cats/cal_ref_cat/4022.fits'))
361 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
362 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
363 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
364 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
367class TestMemory(lsst.utils.tests.MemoryTestCase):
368 pass
371def setup_module(module):
372 lsst.utils.tests.init()
375if __name__ == "__main__": 375 ↛ 376line 375 didn't jump to line 376, because the condition on line 375 was never true
376 lsst.utils.tests.init()
377 unittest.main()