Coverage for tests/test_referenceObjectLoader.py: 23%
152 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:39 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:39 -0700
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import os.path
22import tempfile
23import unittest
24import glob
26import numpy as np
27from smatch.matcher import sphdist
28import astropy.time
30import lsst.daf.butler
31import lsst.afw.geom as afwGeom
32import lsst.afw.table as afwTable
33from lsst.daf.butler import DatasetType, DeferredDatasetHandle
34from lsst.daf.butler.script import ingest_files
35from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader)
36from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles
37from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits
38import lsst.utils
39import lsst.geom
41import ingestIndexTestBase
44class ReferenceObjectLoaderTestCase(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
45 lsst.utils.tests.TestCase):
46 """Test case for ReferenceObjectLoader."""
47 @classmethod
48 def setUpClass(cls):
49 super().setUpClass()
51 # Generate a catalog, with arbitrary ids
52 inTempDir = tempfile.TemporaryDirectory()
53 inPath = inTempDir.name
54 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123)
56 cls.skyCatalog = skyCatalog
58 # override some field names.
59 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True,
60 withPm=True, withPmErr=True)
61 # use a very small HTM pixelization depth
62 depth = 2
63 config.dataset_config.indexer.active.depth = depth
64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
65 config.file_reader.format = 'ascii.commented_header'
66 config.n_processes = 1
67 config.id_name = 'id' # Use the ids from the generated catalogs
68 cls.repoTempDir = tempfile.TemporaryDirectory()
69 repoPath = cls.repoTempDir.name
71 # Convert the input data files to our HTM indexed format.
72 dataTempDir = tempfile.TemporaryDirectory()
73 dataPath = dataTempDir.name
74 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
75 converter.run([skyCatalogFile])
77 # Make a temporary butler to ingest them into.
78 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
79 dimensions = [f"htm{depth}"]
80 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
81 dimensions,
82 "SimpleCatalog",
83 universe=butler.registry.dimensions,
84 isCalibration=False)
85 butler.registry.registerDatasetType(datasetType)
87 # Ingest the files into the new butler.
88 run = "testingRun"
89 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
90 ingest_files(repoPath,
91 config.dataset_config.ref_dataset_name,
92 run,
93 htmTableFile,
94 transfer="auto")
96 # Test if we can get back the catalogs, with a new butler.
97 butler = lsst.daf.butler.Butler(repoPath)
98 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
99 collections=[run]).expanded())
100 handles = []
101 for dataRef in datasetRefs:
102 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
104 cls.datasetRefs = datasetRefs
105 cls.handles = handles
107 inTempDir.cleanup()
108 dataTempDir.cleanup()
110 def test_loadSkyCircle(self):
111 """Test the loadSkyCircle routine."""
112 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
113 self.handles,
114 name="testrefcat")
115 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
116 cat = loader.loadSkyCircle(
117 center,
118 30.0*lsst.geom.degrees,
119 filterName='a',
120 ).refCat
121 # Check that the max distance is less than the radius
122 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec']))
123 self.assertLess(np.max(dist), 30.0)
125 # Check that all the objects from the two catalogs are here.
126 dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'], self.skyCatalog['dec_icrs'])
127 inside, = (dist < 30.0).nonzero()
128 self.assertEqual(len(cat), len(inside))
130 self.assertTrue(cat.isContiguous())
131 self.assertEqual(len(np.unique(cat['id'])), len(cat))
132 # A default-loaded sky circle should not have centroids
133 self.assertNotIn('centroid_x', cat.schema)
134 self.assertNotIn('centroid_y', cat.schema)
135 self.assertNotIn('hasCentroid', cat.schema)
137 def test_loadPixelBox(self):
138 """Test the loadPixelBox routine."""
139 # This will create a box 50 degrees on a side.
140 loaderConfig = ReferenceObjectLoader.ConfigClass()
141 loaderConfig.pixelMargin = 0
142 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
143 self.handles,
144 name="testrefcat",
145 config=loaderConfig)
146 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000))
147 crpix = lsst.geom.Point2D(500, 500)
148 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
149 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees)
150 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
152 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
154 # This is a sanity check on the ranges; the exact selection depends
155 # on cos(dec) and the tangent-plane projection.
156 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
157 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
158 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
159 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
161 # The following is to ensure the reference catalog coords are
162 # getting corrected for proper motion when an epoch is provided.
163 # Use an extreme epoch so that differences in corrected coords
164 # will be significant. Note that this simply tests that the coords
165 # do indeed change when the epoch is passed. It makes no attempt
166 # at assessing the correctness of the change. This is left to the
167 # explicit testProperMotion() test below.
168 catWithEpoch = loader.loadPixelBox(
169 bbox,
170 wcs,
171 'a',
172 bboxToSpherePadding=0,
173 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat
175 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4)
176 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4)
178 def test_filterMap(self):
179 """Test filterMap parameters."""
180 loaderConfig = ReferenceObjectLoader.ConfigClass()
181 loaderConfig.filterMap = {'aprime': 'a'}
182 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
183 self.handles,
184 name="testrefcat",
185 config=loaderConfig)
186 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
187 result = loader.loadSkyCircle(
188 center,
189 30.0*lsst.geom.degrees,
190 filterName='aprime',
191 )
192 self.assertEqual(result.fluxField, 'aprime_camFlux')
193 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux'])
195 def test_properMotion(self):
196 """Test proper motion correction."""
197 loaderConfig = ReferenceObjectLoader.ConfigClass()
198 loaderConfig.filterMap = {'aprime': 'a'}
199 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
200 self.handles,
201 name="testrefcat",
202 config=loaderConfig)
203 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
204 cat = loader.loadSkyCircle(
205 center,
206 30.0*lsst.geom.degrees,
207 filterName='a'
208 ).refCat
210 # Zero epoch change --> no proper motion correction (except minor numerical effects)
211 cat_pm = loader.loadSkyCircle(
212 center,
213 30.0*lsst.geom.degrees,
214 filterName='a',
215 epoch=self.epoch
216 ).refCat
218 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14)
219 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14)
220 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
221 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
223 # One year difference
224 cat_pm = loader.loadSkyCircle(
225 center,
226 30.0*lsst.geom.degrees,
227 filterName='a',
228 epoch=self.epoch + 1.0*astropy.units.yr
229 ).refCat
231 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
232 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
233 for orig, ref in zip(cat, cat_pm):
234 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
235 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
236 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
237 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
238 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
239 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
240 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
241 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
243 def test_requireProperMotion(self):
244 """Tests of the requireProperMotion config field."""
245 loaderConfig = ReferenceObjectLoader.ConfigClass()
246 loaderConfig.requireProperMotion = True
247 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
248 self.handles,
249 name="testrefcat",
250 config=loaderConfig)
251 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
253 # Test that we require an epoch set.
254 msg = 'requireProperMotion=True but epoch not provided to loader'
255 with self.assertRaisesRegex(RuntimeError, msg):
256 loader.loadSkyCircle(
257 center,
258 30.0*lsst.geom.degrees,
259 filterName='a'
260 )
263class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase):
264 """Test cases for reading version 0 and version 1 catalogs."""
265 def testLoadVersion0(self):
266 """Test reading a pre-written format_version=0 (Jy flux) catalog.
267 It should be converted to have nJy fluxes.
268 """
269 path = os.path.join(
270 os.path.dirname(os.path.abspath(__file__)),
271 'data',
272 'version0',
273 'ref_cats',
274 'cal_ref_cat'
275 )
277 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
279 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
280 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a')
282 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
283 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
284 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
285 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
286 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
287 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
289 def testLoadVersion1(self):
290 """Test reading a format_version=1 catalog (fluxes unchanged)."""
291 path = os.path.join(
292 os.path.dirname(os.path.abspath(__file__)),
293 'data',
294 'version1',
295 'ref_cats',
296 'cal_ref_cat'
297 )
299 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
301 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
302 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a')
304 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
305 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
306 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
307 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
308 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
309 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
312class TestMemory(lsst.utils.tests.MemoryTestCase):
313 pass
316def setup_module(module):
317 lsst.utils.tests.init()
320if __name__ == "__main__": 320 ↛ 321line 320 didn't jump to line 321, because the condition on line 320 was never true
321 lsst.utils.tests.init()
322 unittest.main()