Coverage for tests/ingestIndexTestBase.py: 97%
149 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-04 10:04 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-04 10:04 +0000
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["ConvertReferenceCatalogTestBase", "make_coord", "makeConvertConfig"]
24import logging
25import math
26import string
27import tempfile
29import numpy as np
30import astropy
31import astropy.units as u
33import lsst.daf.butler
34from lsst.meas.algorithms import IndexerRegistry
35from lsst.meas.algorithms import ConvertReferenceCatalogConfig
36import lsst.utils
39def make_coord(ra, dec):
40 """Make an ICRS coord given its RA, Dec in degrees."""
41 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees)
44def makeConvertConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False,
45 withParallax=False):
46 """Make a config for ConvertReferenceCatalogTask
48 This is primarily intended to simplify tests of config validation,
49 so fields that are not validated are not set.
50 However, it can calso be used to reduce boilerplate in other tests.
51 """
52 config = ConvertReferenceCatalogConfig()
53 config.dataset_config.ref_dataset_name = "testRefCat"
54 config.pm_scale = 1000.0
55 config.parallax_scale = 1e3
56 config.ra_name = 'ra_icrs'
57 config.dec_name = 'dec_icrs'
58 config.mag_column_list = ['a', 'b']
60 if withMagErr:
61 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
63 if withRaDecErr:
64 config.ra_err_name = "ra_err"
65 config.dec_err_name = "dec_err"
66 config.coord_err_unit = "arcsecond"
68 if withPm:
69 config.pm_ra_name = "pm_ra"
70 config.pm_dec_name = "pm_dec"
72 if withPmErr:
73 config.pm_ra_err_name = "pm_ra_err"
74 config.pm_dec_err_name = "pm_dec_err"
76 if withParallax: 76 ↛ 77line 76 didn't jump to line 77, because the condition on line 76 was never true
77 config.parallax_name = "parallax"
78 config.parallax_err_name = "parallax_err"
80 if withPm or withParallax:
81 config.epoch_name = "unixtime"
82 config.epoch_format = "unix"
83 config.epoch_scale = "utc"
85 return config
88class ConvertReferenceCatalogTestBase:
89 """Base class for tests involving ConvertReferenceCatalogTask
90 """
91 @classmethod
92 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123):
93 """Make an on-sky catalog, and save it to a text file.
95 Parameters
96 ----------
97 outPath : `str` or None
98 The directory to write the catalog to.
99 Specify None to not write any output.
100 size : `int`, (optional)
101 Number of items to add to the catalog.
102 idStart : `int`, (optional)
103 First id number to put in the catalog.
104 seed : `float`, (optional)
105 Random seed for ``np.random``.
107 Returns
108 -------
109 refCatPath : `str`
110 Path to the created on-sky catalog.
111 refCatOtherDelimiterPath : `str`
112 Path to the created on-sky catalog with a different delimiter.
113 refCatData : `np.ndarray`
114 The data contained in the on-sky catalog files.
115 """
116 np.random.seed(seed)
117 ident = np.arange(idStart, size + idStart, dtype=int)
118 ra = np.random.random(size)*360.
119 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.))
120 dec -= 90.
121 ra_err = np.ones(size)*0.1 # arcsec
122 dec_err = np.ones(size)*0.1 # arcsec
123 a_mag = 16. + np.random.random(size)*4.
124 a_mag_err = 0.01 + np.random.random(size)*0.2
125 b_mag = 17. + np.random.random(size)*5.
126 b_mag_err = 0.02 + np.random.random(size)*0.3
127 is_photometric = np.random.randint(2, size=size)
128 is_resolved = np.random.randint(2, size=size)
129 is_variable = np.random.randint(2, size=size)
130 extra_col1 = np.random.normal(size=size)
131 extra_col2 = np.random.normal(1000., 100., size=size)
132 # compute proper motion and PM error in arcseconds/year
133 # and let the ingest task scale them to radians
134 pm_amt_arcsec = cls.properMotionAmt.asArcseconds()
135 pm_dir_rad = cls.properMotionDir.asRadians()
136 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad)
137 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad)
138 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad))
139 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad))
140 parallax = np.ones(size)*0.1 # arcseconds
141 parallax_error = np.ones(size)*0.003 # arcseconds
142 unixtime = np.ones(size)*cls.epoch.unix
144 def get_word(word_len):
145 return "".join(np.random.choice([s for s in string.ascii_letters], word_len))
146 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)])
148 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float),
149 ('ra_err', float), ('dec_err', float), ('a', float),
150 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int),
151 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float),
152 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float),
153 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float),
154 ('unixtime', float)])
156 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err,
157 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3,
158 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)),
159 dtype=dtype)
160 if outPath is not None:
161 # write the data with full precision; this is not realistic for
162 # real catalogs, but simplifies tests based on round tripped data
163 saveKwargs = dict(
164 header="id,ra_icrs,dec_icrs,ra_err,dec_err,"
165 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3,"
166 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime",
167 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g",
168 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s",
169 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"]
170 )
172 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs)
173 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs)
174 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr
175 else:
176 return arr
178 @classmethod
179 def tearDownClass(cls):
180 cls.outDir.cleanup()
181 del cls.outPath
182 del cls.skyCatalogFile
183 del cls.skyCatalogFileDelim
184 del cls.skyCatalog
185 del cls.testRas
186 del cls.testDecs
187 del cls.searchRadius
188 del cls.compCats
190 @classmethod
191 def setUpClass(cls):
192 cls.outDir = tempfile.TemporaryDirectory()
193 cls.outPath = cls.outDir.name
194 # arbitrary, but reasonable, amount of proper motion (angle/year)
195 # and direction of proper motion
196 cls.properMotionAmt = 3.0*lsst.geom.arcseconds
197 cls.properMotionDir = 45*lsst.geom.degrees
198 cls.properMotionErr = 1e-3*lsst.geom.arcseconds
199 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd")
200 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath)
201 cls.testRas = [210., 14.5, 93., 180., 286., 0.]
202 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
203 cls.searchRadius = 3. * lsst.geom.degrees
204 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center
205 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
207 config = IndexerRegistry['HTM'].ConfigClass()
208 # Match on disk comparison file
209 config.depth = cls.depth
210 cls.indexer = IndexerRegistry['HTM'](config)
211 for ra in cls.testRas:
212 for dec in cls.testDecs:
213 tupl = (ra, dec)
214 cent = make_coord(*tupl)
215 cls.compCats[tupl] = []
216 for rec in cls.skyCatalog:
217 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius:
218 cls.compCats[tupl].append(rec['id'])
220 cls.testRepoPath = cls.outPath+"/test_repo"
222 def setUp(self):
223 self.repoPath = tempfile.TemporaryDirectory() # cleaned up automatically when test ends
224 self.butler = self.makeTemporaryRepo(self.repoPath.name, self.depth)
225 self.logger = logging.getLogger('lsst.ReferenceObjectLoader')
227 def tearDown(self):
228 self.repoPath.cleanup()
230 @staticmethod
231 def makeTemporaryRepo(rootPath, depth):
232 """Create a temporary butler repository, configured to support a given
233 htm pixel depth, to use for a single test.
235 Parameters
236 ----------
237 rootPath : `str`
238 Root path for butler.
239 depth : `int`
240 HTM pixel depth to be used in this test.
242 Returns
243 -------
244 butler : `lsst.daf.butler.Butler`
245 The newly created and instantiated butler.
246 """
247 dimensionConfig = lsst.daf.butler.DimensionConfig()
248 dimensionConfig['skypix']['common'] = f'htm{depth}'
249 lsst.daf.butler.Butler.makeRepo(rootPath, dimensionConfig=dimensionConfig)
250 return lsst.daf.butler.Butler(rootPath, writeable=True)
252 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config):
253 """Check that every item in ``skyCatalog`` is in the ingested catalog,
254 and check that fields are correct in it.
256 Parameters
257 ----------
258 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask`
259 A reference object loader to use to search for rows from
260 ``skyCatalog``.
261 skyCatalog : `np.ndarray`
262 The original data to compare with.
263 config : `lsst.meas.algorithms.LoadIndexedReferenceObjectsConfig`
264 The Config that was used to generate the refcat.
265 """
266 for row in skyCatalog:
267 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees)
268 with self.assertLogs(self.logger.name, level="INFO") as cm:
269 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat
270 self.assertIn("Loading reference objects from testRefCat in region", cm.output[0])
271 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.")
272 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}"
273 self.assertEqual(row['id'], cat[0]['id'], msg)
274 # coordinates won't match perfectly due to rounding in radian/degree conversions
275 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(),
276 rtol=1e-14, msg=msg)
277 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(),
278 rtol=1e-14, msg=msg)
279 if config.coord_err_unit is not None:
280 # coordinate errors are not lsst.geom.Angle, so we have to use the
281 # `units` field to convert them, and they are float32, so the tolerance is wider.
282 raErr = cat[0]['coord_raErr']*u.Unit(cat.schema['coord_raErr'].asField().getUnits())
283 decErr = cat[0]['coord_decErr']*u.Unit(cat.schema['coord_decErr'].asField().getUnits())
284 self.assertFloatsAlmostEqual(row['ra_err'], raErr.to_value(config.coord_err_unit),
285 rtol=1e-7, msg=msg)
286 self.assertFloatsAlmostEqual(row['dec_err'], decErr.to_value(config.coord_err_unit),
287 rtol=1e-7, msg=msg)
289 if config.parallax_name is not None: 289 ↛ 290line 289 didn't jump to line 290, because the condition on line 289 was never true
290 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds())
291 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())