Coverage for tests/ingestIndexTestBase.py: 15%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["ConvertReferenceCatalogTestBase", "make_coord", "makeConvertConfig"]
24import logging
25import math
26import shutil
27import string
28import tempfile
30import numpy as np
31import astropy
32import astropy.units as u
34import lsst.daf.butler
35from lsst.meas.algorithms import IndexerRegistry
36from lsst.meas.algorithms import ConvertReferenceCatalogConfig
37import lsst.utils
40def make_coord(ra, dec):
41 """Make an ICRS coord given its RA, Dec in degrees."""
42 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees)
45def makeConvertConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False,
46 withParallax=False):
47 """Make a config for ConvertReferenceCatalogTask
49 This is primarily intended to simplify tests of config validation,
50 so fields that are not validated are not set.
51 However, it can calso be used to reduce boilerplate in other tests.
52 """
53 config = ConvertReferenceCatalogConfig()
54 config.dataset_config.ref_dataset_name = "testRefCat"
55 config.pm_scale = 1000.0
56 config.parallax_scale = 1e3
57 config.ra_name = 'ra_icrs'
58 config.dec_name = 'dec_icrs'
59 config.mag_column_list = ['a', 'b']
61 if withMagErr:
62 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
64 if withRaDecErr:
65 config.ra_err_name = "ra_err"
66 config.dec_err_name = "dec_err"
67 config.coord_err_unit = "arcsecond"
69 if withPm:
70 config.pm_ra_name = "pm_ra"
71 config.pm_dec_name = "pm_dec"
73 if withPmErr:
74 config.pm_ra_err_name = "pm_ra_err"
75 config.pm_dec_err_name = "pm_dec_err"
77 if withParallax:
78 config.parallax_name = "parallax"
79 config.parallax_err_name = "parallax_err"
81 if withPm or withParallax:
82 config.epoch_name = "unixtime"
83 config.epoch_format = "unix"
84 config.epoch_scale = "utc"
86 return config
89class ConvertReferenceCatalogTestBase:
90 """Base class for tests involving ConvertReferenceCatalogTask
91 """
92 @classmethod
93 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123):
94 """Make an on-sky catalog, and save it to a text file.
96 Parameters
97 ----------
98 outPath : `str` or None
99 The directory to write the catalog to.
100 Specify None to not write any output.
101 size : `int`, (optional)
102 Number of items to add to the catalog.
103 idStart : `int`, (optional)
104 First id number to put in the catalog.
105 seed : `float`, (optional)
106 Random seed for ``np.random``.
108 Returns
109 -------
110 refCatPath : `str`
111 Path to the created on-sky catalog.
112 refCatOtherDelimiterPath : `str`
113 Path to the created on-sky catalog with a different delimiter.
114 refCatData : `np.ndarray`
115 The data contained in the on-sky catalog files.
116 """
117 np.random.seed(seed)
118 ident = np.arange(idStart, size + idStart, dtype=int)
119 ra = np.random.random(size)*360.
120 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.))
121 dec -= 90.
122 ra_err = np.ones(size)*0.1 # arcsec
123 dec_err = np.ones(size)*0.1 # arcsec
124 a_mag = 16. + np.random.random(size)*4.
125 a_mag_err = 0.01 + np.random.random(size)*0.2
126 b_mag = 17. + np.random.random(size)*5.
127 b_mag_err = 0.02 + np.random.random(size)*0.3
128 is_photometric = np.random.randint(2, size=size)
129 is_resolved = np.random.randint(2, size=size)
130 is_variable = np.random.randint(2, size=size)
131 extra_col1 = np.random.normal(size=size)
132 extra_col2 = np.random.normal(1000., 100., size=size)
133 # compute proper motion and PM error in arcseconds/year
134 # and let the ingest task scale them to radians
135 pm_amt_arcsec = cls.properMotionAmt.asArcseconds()
136 pm_dir_rad = cls.properMotionDir.asRadians()
137 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad)
138 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad)
139 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad))
140 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad))
141 parallax = np.ones(size)*0.1 # arcseconds
142 parallax_error = np.ones(size)*0.003 # arcseconds
143 unixtime = np.ones(size)*cls.epoch.unix
145 def get_word(word_len):
146 return "".join(np.random.choice([s for s in string.ascii_letters], word_len))
147 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)])
149 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float),
150 ('ra_err', float), ('dec_err', float), ('a', float),
151 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int),
152 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float),
153 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float),
154 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float),
155 ('unixtime', float)])
157 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err,
158 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3,
159 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)),
160 dtype=dtype)
161 if outPath is not None:
162 # write the data with full precision; this is not realistic for
163 # real catalogs, but simplifies tests based on round tripped data
164 saveKwargs = dict(
165 header="id,ra_icrs,dec_icrs,ra_err,dec_err,"
166 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3,"
167 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime",
168 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g",
169 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s",
170 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"]
171 )
173 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs)
174 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs)
175 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr
176 else:
177 return arr
179 @classmethod
180 def tearDownClass(cls):
181 try:
182 shutil.rmtree(cls.outPath)
183 except Exception:
184 print("WARNING: failed to remove temporary dir %r" % (cls.outPath,))
185 del cls.outPath
186 del cls.skyCatalogFile
187 del cls.skyCatalogFileDelim
188 del cls.skyCatalog
189 del cls.testRas
190 del cls.testDecs
191 del cls.searchRadius
192 del cls.compCats
194 @classmethod
195 def setUpClass(cls):
196 cls.outPath = tempfile.mkdtemp()
197 # arbitrary, but reasonable, amount of proper motion (angle/year)
198 # and direction of proper motion
199 cls.properMotionAmt = 3.0*lsst.geom.arcseconds
200 cls.properMotionDir = 45*lsst.geom.degrees
201 cls.properMotionErr = 1e-3*lsst.geom.arcseconds
202 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd")
203 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath)
204 cls.testRas = [210., 14.5, 93., 180., 286., 0.]
205 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
206 cls.searchRadius = 3. * lsst.geom.degrees
207 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center
208 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
210 config = IndexerRegistry['HTM'].ConfigClass()
211 # Match on disk comparison file
212 config.depth = cls.depth
213 cls.indexer = IndexerRegistry['HTM'](config)
214 for ra in cls.testRas:
215 for dec in cls.testDecs:
216 tupl = (ra, dec)
217 cent = make_coord(*tupl)
218 cls.compCats[tupl] = []
219 for rec in cls.skyCatalog:
220 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius:
221 cls.compCats[tupl].append(rec['id'])
223 cls.testRepoPath = cls.outPath+"/test_repo"
225 def setUp(self):
226 self.repoPath = tempfile.TemporaryDirectory() # cleaned up automatically when test ends
227 self.butler = self.makeTemporaryRepo(self.repoPath.name, self.depth)
228 self.logger = logging.getLogger('lsst.ReferenceObjectLoader')
230 @staticmethod
231 def makeTemporaryRepo(rootPath, depth):
232 """Create a temporary butler repository, configured to support a given
233 htm pixel depth, to use for a single test.
235 Parameters
236 ----------
237 rootPath : `str`
238 Root path for butler.
239 depth : `int`
240 HTM pixel depth to be used in this test.
242 Returns
243 -------
244 butler : `lsst.daf.butler.Butler`
245 The newly created and instantiated butler.
246 """
247 dimensionConfig = lsst.daf.butler.DimensionConfig()
248 dimensionConfig['skypix']['common'] = f'htm{depth}'
249 lsst.daf.butler.Butler.makeRepo(rootPath, dimensionConfig=dimensionConfig)
250 return lsst.daf.butler.Butler(rootPath, writeable=True)
252 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config):
253 """Check that every item in ``skyCatalog`` is in the ingested catalog,
254 and check that fields are correct in it.
256 Parameters
257 ----------
258 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask`
259 A reference object loader to use to search for rows from
260 ``skyCatalog``.
261 skyCatalog : `np.ndarray`
262 The original data to compare with.
263 config : `lsst.meas.algorithms.LoadIndexedReferenceObjectsConfig`
264 The Config that was used to generate the refcat.
265 """
266 for row in skyCatalog:
267 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees)
268 with self.assertLogs(self.logger.name, level="INFO") as cm:
269 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat
270 self.assertIn("Loading reference objects from testRefCat in region", cm.output[0])
271 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.")
272 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}"
273 self.assertEqual(row['id'], cat[0]['id'], msg)
274 # coordinates won't match perfectly due to rounding in radian/degree conversions
275 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(),
276 rtol=1e-14, msg=msg)
277 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(),
278 rtol=1e-14, msg=msg)
279 if config.coord_err_unit is not None:
280 # coordinate errors are not lsst.geom.Angle, so we have to use the
281 # `units` field to convert them, and they are float32, so the tolerance is wider.
282 raErr = cat[0]['coord_raErr']*u.Unit(cat.schema['coord_raErr'].asField().getUnits())
283 decErr = cat[0]['coord_decErr']*u.Unit(cat.schema['coord_decErr'].asField().getUnits())
284 self.assertFloatsAlmostEqual(row['ra_err'], raErr.to_value(config.coord_err_unit),
285 rtol=1e-7, msg=msg)
286 self.assertFloatsAlmostEqual(row['dec_err'], decErr.to_value(config.coord_err_unit),
287 rtol=1e-7, msg=msg)
289 if config.parallax_name is not None:
290 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds())
291 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())