Coverage for tests/ingestIndexTestBase.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_algorithms.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["IngestIndexCatalogTestBase", "make_coord"]
24import math
25import os.path
26import shutil
27import string
28import tempfile
30import numpy as np
31import astropy
33import lsst.daf.persistence as dafPersist
34from lsst.meas.algorithms import IndexerRegistry
35from lsst.meas.algorithms import IngestIndexedReferenceTask
36import lsst.utils
39def make_coord(ra, dec):
40 """Make an ICRS coord given its RA, Dec in degrees."""
41 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees)
44class IngestIndexCatalogTestBase:
45 """Base class for tests involving IngestIndexedReferenceTask
46 """
47 @classmethod
48 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123):
49 """Make an on-sky catalog, and save it to a text file.
51 Parameters
52 ----------
53 outPath : `str` or None
54 The directory to write the catalog to.
55 Specify None to not write any output.
56 size : `int`, (optional)
57 Number of items to add to the catalog.
58 idStart : `int`, (optional)
59 First id number to put in the catalog.
60 seed : `float`, (optional)
61 Random seed for ``np.random``.
63 Returns
64 -------
65 refCatPath : `str`
66 Path to the created on-sky catalog.
67 refCatOtherDelimiterPath : `str`
68 Path to the created on-sky catalog with a different delimiter.
69 refCatData : `np.ndarray`
70 The data contained in the on-sky catalog files.
71 """
72 np.random.seed(seed)
73 ident = np.arange(idStart, size + idStart, dtype=int)
74 ra = np.random.random(size)*360.
75 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.))
76 dec -= 90.
77 ra_err = np.ones(size)*0.1 # arcsec
78 dec_err = np.ones(size)*0.1 # arcsec
79 a_mag = 16. + np.random.random(size)*4.
80 a_mag_err = 0.01 + np.random.random(size)*0.2
81 b_mag = 17. + np.random.random(size)*5.
82 b_mag_err = 0.02 + np.random.random(size)*0.3
83 is_photometric = np.random.randint(2, size=size)
84 is_resolved = np.random.randint(2, size=size)
85 is_variable = np.random.randint(2, size=size)
86 extra_col1 = np.random.normal(size=size)
87 extra_col2 = np.random.normal(1000., 100., size=size)
88 # compute proper motion and PM error in arcseconds/year
89 # and let the ingest task scale them to radians
90 pm_amt_arcsec = cls.properMotionAmt.asArcseconds()
91 pm_dir_rad = cls.properMotionDir.asRadians()
92 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad)
93 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad)
94 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad))
95 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad))
96 parallax = np.ones(size)*0.1 # arcseconds
97 parallax_error = np.ones(size)*0.003 # arcseconds
98 unixtime = np.ones(size)*cls.epoch.unix
100 def get_word(word_len):
101 return "".join(np.random.choice([s for s in string.ascii_letters], word_len))
102 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)])
104 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float),
105 ('ra_err', float), ('dec_err', float), ('a', float),
106 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int),
107 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float),
108 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float),
109 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float),
110 ('unixtime', float)])
112 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err,
113 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3,
114 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)),
115 dtype=dtype)
116 if outPath is not None:
117 # write the data with full precision; this is not realistic for
118 # real catalogs, but simplifies tests based on round tripped data
119 saveKwargs = dict(
120 header="id,ra_icrs,dec_icrs,ra_err,dec_err,"
121 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3,"
122 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime",
123 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g",
124 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s",
125 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"]
126 )
128 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs)
129 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs)
130 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr
131 else:
132 return arr
134 @classmethod
135 def tearDownClass(cls):
136 try:
137 shutil.rmtree(cls.outPath)
138 except Exception:
139 print("WARNING: failed to remove temporary dir %r" % (cls.outPath,))
140 del cls.outPath
141 del cls.skyCatalogFile
142 del cls.skyCatalogFileDelim
143 del cls.skyCatalog
144 del cls.testRas
145 del cls.testDecs
146 del cls.searchRadius
147 del cls.compCats
148 del cls.testButler
150 @classmethod
151 def setUpClass(cls):
152 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test')
153 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input")
155 cls.outPath = tempfile.mkdtemp()
156 cls.testCatPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data",
157 "testHtmIndex.fits")
158 # arbitrary, but reasonable, amount of proper motion (angle/year)
159 # and direction of proper motion
160 cls.properMotionAmt = 3.0*lsst.geom.arcseconds
161 cls.properMotionDir = 45*lsst.geom.degrees
162 cls.properMotionErr = 1e-3*lsst.geom.arcseconds
163 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd")
164 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath)
165 cls.testRas = [210., 14.5, 93., 180., 286., 0.]
166 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.]
167 cls.searchRadius = 3. * lsst.geom.degrees
168 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center
169 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius
171 config = IndexerRegistry['HTM'].ConfigClass()
172 # Match on disk comparison file
173 config.depth = cls.depth
174 cls.indexer = IndexerRegistry['HTM'](config)
175 for ra in cls.testRas:
176 for dec in cls.testDecs:
177 tupl = (ra, dec)
178 cent = make_coord(*tupl)
179 cls.compCats[tupl] = []
180 for rec in cls.skyCatalog:
181 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius:
182 cls.compCats[tupl].append(rec['id'])
184 cls.testRepoPath = cls.outPath+"/test_repo"
185 config = cls.makeConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True,
186 withParallax=True)
187 # To match on disk test data
188 config.dataset_config.indexer.active.depth = cls.depth
189 config.id_name = 'id'
190 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr
191 config.parallax_scale = 1e3 # arcsec -> milliarcsec
192 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that
193 config.file_reader.format = 'ascii.commented_header'
194 # run the intest once to create a butler repo we can compare to
195 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath,
196 cls.skyCatalogFile], config=config)
197 cls.defaultDatasetName = config.dataset_config.ref_dataset_name
198 cls.testDatasetName = 'diff_ref_name'
199 cls.testButler = dafPersist.Butler(cls.testRepoPath)
200 os.symlink(os.path.join(cls.testRepoPath, 'ref_cats', cls.defaultDatasetName),
201 os.path.join(cls.testRepoPath, 'ref_cats', cls.testDatasetName))
203 @staticmethod
204 def makeConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False,
205 withParallax=False):
206 """Make a config for IngestIndexedReferenceTask
208 This is primarily intended to simplify tests of config validation,
209 so fields that are not validated are not set.
210 However, it can calso be used to reduce boilerplate in other tests.
211 """
212 config = IngestIndexedReferenceTask.ConfigClass()
213 config.pm_scale = 1000.0
214 config.parallax_scale = 1e3
215 config.ra_name = 'ra_icrs'
216 config.dec_name = 'dec_icrs'
217 config.mag_column_list = ['a', 'b']
219 if withMagErr:
220 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'}
222 if withRaDecErr:
223 config.ra_err_name = "ra_err"
224 config.dec_err_name = "dec_err"
226 if withPm:
227 config.pm_ra_name = "pm_ra"
228 config.pm_dec_name = "pm_dec"
230 if withPmErr:
231 config.pm_ra_err_name = "pm_ra_err"
232 config.pm_dec_err_name = "pm_dec_err"
234 if withParallax:
235 config.parallax_name = "parallax"
236 config.parallax_err_name = "parallax_err"
238 if withPm or withParallax:
239 config.epoch_name = "unixtime"
240 config.epoch_format = "unix"
241 config.epoch_scale = "utc"
243 return config
245 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config):
246 """Check that every item in ``skyCatalog`` is in the ingested catalog.
248 Parameters
249 ----------
250 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask`
251 A reference object loader to use to search for rows from
252 ``skyCatalog``.
253 skyCatalog : `np.ndarray`
254 The original data to compare with.
255 """
256 for row in skyCatalog:
257 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees)
258 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat
259 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.")
260 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}"
261 self.assertEqual(row['id'], cat[0]['id'], msg)
262 # coordinates won't match perfectly due to rounding in radian/degree conversions
263 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(),
264 rtol=1e-14, msg=msg)
265 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(),
266 rtol=1e-14, msg=msg)
267 if config.parallax_name is not None:
268 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds())
269 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())