Coverage for tests/test_referenceObjectLoader.py: 23%
152 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-07-09 06:31 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-07-09 06:31 -0700
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21import os.path
22import tempfile
23import unittest
24import glob
26import numpy as np
27from smatch.matcher import sphdist
28import astropy.time
30import lsst.daf.butler
31import lsst.afw.geom as afwGeom
32import lsst.afw.table as afwTable
33from lsst.daf.butler import DatasetType, DeferredDatasetHandle
34from lsst.daf.butler.script import ingest_files
35from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader)
36from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles
37from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits
38import lsst.utils
39import lsst.geom
41import ingestIndexTestBase
44class ReferenceObjectLoaderTestCase(ingestIndexTestBase.ConvertReferenceCatalogTestBase,
45 lsst.utils.tests.TestCase):
46 """Test case for ReferenceObjectLoader."""
47 @classmethod
48 def setUpClass(cls):
49 super().setUpClass()
51 # Generate a catalog, with arbitrary ids
52 inTempDir = tempfile.TemporaryDirectory()
53 inPath = inTempDir.name
54 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123)
56 cls.skyCatalog = skyCatalog
58 # override some field names.
59 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True,
60 withPm=True, withPmErr=True)
61 # use a very small HTM pixelization depth
62 depth = 2
63 config.dataset_config.indexer.active.depth = depth
64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
65 config.file_reader.format = 'ascii.commented_header'
66 config.n_processes = 1
67 config.id_name = 'id' # Use the ids from the generated catalogs
68 cls.repoTempDir = tempfile.TemporaryDirectory()
69 repoPath = cls.repoTempDir.name
71 # Convert the input data files to our HTM indexed format.
72 dataTempDir = tempfile.TemporaryDirectory()
73 dataPath = dataTempDir.name
74 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config)
75 converter.run([skyCatalogFile])
77 # Make a temporary butler to ingest them into.
78 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth)
79 dimensions = [f"htm{depth}"]
80 datasetType = DatasetType(config.dataset_config.ref_dataset_name,
81 dimensions,
82 "SimpleCatalog",
83 universe=butler.registry.dimensions,
84 isCalibration=False)
85 butler.registry.registerDatasetType(datasetType)
87 # Ingest the files into the new butler.
88 run = "testingRun"
89 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv")
90 ingest_files(repoPath,
91 config.dataset_config.ref_dataset_name,
92 run,
93 htmTableFile,
94 transfer="auto")
96 # Test if we can get back the catalogs, with a new butler.
97 butler = lsst.daf.butler.Butler(repoPath)
98 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name,
99 collections=[run]).expanded())
100 handles = []
101 for dataRef in datasetRefs:
102 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None))
104 cls.datasetRefs = datasetRefs
105 cls.handles = handles
107 inTempDir.cleanup()
108 dataTempDir.cleanup()
110 def test_loadSkyCircle(self):
111 """Test the loadSkyCircle routine."""
112 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
113 self.handles)
114 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
115 cat = loader.loadSkyCircle(
116 center,
117 30.0*lsst.geom.degrees,
118 filterName='a',
119 ).refCat
120 # Check that the max distance is less than the radius
121 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec']))
122 self.assertLess(np.max(dist), 30.0)
124 # Check that all the objects from the two catalogs are here.
125 dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'], self.skyCatalog['dec_icrs'])
126 inside, = (dist < 30.0).nonzero()
127 self.assertEqual(len(cat), len(inside))
129 self.assertTrue(cat.isContiguous())
130 self.assertEqual(len(np.unique(cat['id'])), len(cat))
131 # A default-loaded sky circle should not have centroids
132 self.assertNotIn('centroid_x', cat.schema)
133 self.assertNotIn('centroid_y', cat.schema)
134 self.assertNotIn('hasCentroid', cat.schema)
136 def test_loadPixelBox(self):
137 """Test the loadPixelBox routine."""
138 # This will create a box 50 degrees on a side.
139 loaderConfig = ReferenceObjectLoader.ConfigClass()
140 loaderConfig.pixelMargin = 0
141 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
142 self.handles,
143 config=loaderConfig)
144 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000))
145 crpix = lsst.geom.Point2D(500, 500)
146 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
147 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees)
148 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
150 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat
152 # This is a sanity check on the ranges; the exact selection depends
153 # on cos(dec) and the tangent-plane projection.
154 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0)
155 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0)
156 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0)
157 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0)
159 # The following is to ensure the reference catalog coords are
160 # getting corrected for proper motion when an epoch is provided.
161 # Use an extreme epoch so that differences in corrected coords
162 # will be significant. Note that this simply tests that the coords
163 # do indeed change when the epoch is passed. It makes no attempt
164 # at assessing the correctness of the change. This is left to the
165 # explicit testProperMotion() test below.
166 catWithEpoch = loader.loadPixelBox(
167 bbox,
168 wcs,
169 'a',
170 bboxToSpherePadding=0,
171 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat
173 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4)
174 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4)
176 def test_filterMap(self):
177 """Test filterMap parameters."""
178 loaderConfig = ReferenceObjectLoader.ConfigClass()
179 loaderConfig.filterMap = {'aprime': 'a'}
180 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
181 self.handles,
182 config=loaderConfig)
183 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
184 result = loader.loadSkyCircle(
185 center,
186 30.0*lsst.geom.degrees,
187 filterName='aprime',
188 )
189 self.assertEqual(result.fluxField, 'aprime_camFlux')
190 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux'])
192 def test_properMotion(self):
193 """Test proper motion correction."""
194 loaderConfig = ReferenceObjectLoader.ConfigClass()
195 loaderConfig.filterMap = {'aprime': 'a'}
196 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
197 self.handles,
198 config=loaderConfig)
199 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
200 cat = loader.loadSkyCircle(
201 center,
202 30.0*lsst.geom.degrees,
203 filterName='a'
204 ).refCat
206 # Zero epoch change --> no proper motion correction (except minor numerical effects)
207 cat_pm = loader.loadSkyCircle(
208 center,
209 30.0*lsst.geom.degrees,
210 filterName='a',
211 epoch=self.epoch
212 ).refCat
214 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14)
215 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14)
216 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr'])
217 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr'])
219 # One year difference
220 cat_pm = loader.loadSkyCircle(
221 center,
222 30.0*lsst.geom.degrees,
223 filterName='a',
224 epoch=self.epoch + 1.0*astropy.units.yr
225 ).refCat
227 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr'])
228 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr'])
229 for orig, ref in zip(cat, cat_pm):
230 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()),
231 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds)
232 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()),
233 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds)
234 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"])
235 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"])
236 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr)
237 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr)
239 def test_requireProperMotion(self):
240 """Tests of the requireProperMotion config field."""
241 loaderConfig = ReferenceObjectLoader.ConfigClass()
242 loaderConfig.requireProperMotion = True
243 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs],
244 self.handles,
245 config=loaderConfig)
246 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees)
248 # Test that we require an epoch set.
249 msg = 'requireProperMotion=True but epoch not provided to loader'
250 with self.assertRaisesRegex(RuntimeError, msg):
251 loader.loadSkyCircle(
252 center,
253 30.0*lsst.geom.degrees,
254 filterName='a'
255 )
258class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase):
259 """Test cases for reading version 0 and version 1 catalogs."""
260 def testLoadVersion0(self):
261 """Test reading a pre-written format_version=0 (Jy flux) catalog.
262 It should be converted to have nJy fluxes.
263 """
264 path = os.path.join(
265 os.path.dirname(os.path.abspath(__file__)),
266 'data',
267 'version0',
268 'ref_cats',
269 'cal_ref_cat'
270 )
272 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
274 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
275 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a')
277 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
278 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
279 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux'])
280 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr'])
281 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux'])
282 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr'])
284 def testLoadVersion1(self):
285 """Test reading a format_version=1 catalog (fluxes unchanged)."""
286 path = os.path.join(
287 os.path.dirname(os.path.abspath(__file__)),
288 'data',
289 'version1',
290 'ref_cats',
291 'cal_ref_cat'
292 )
294 filenames = sorted(glob.glob(os.path.join(path, '????.fits')))
296 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4)
297 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a')
299 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema))
300 catalog = afwTable.SimpleCatalog.readFits(filenames[0])
301 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux'])
302 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr'])
303 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux'])
304 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr'])
307class TestMemory(lsst.utils.tests.MemoryTestCase):
308 pass
311def setup_module(module):
312 lsst.utils.tests.init()
315if __name__ == "__main__": 315 ↛ 316line 315 didn't jump to line 316, because the condition on line 315 was never true
316 lsst.utils.tests.init()
317 unittest.main()