Coverage for tests/ingestIndexTestBase.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

150 statements  

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ConvertReferenceCatalogTestBase", "make_coord", "makeConvertConfig"] 

23 

24import logging 

25import math 

26import shutil 

27import string 

28import tempfile 

29 

30import numpy as np 

31import astropy 

32import astropy.units as u 

33 

34import lsst.daf.butler 

35from lsst.meas.algorithms import IndexerRegistry 

36from lsst.meas.algorithms import ConvertReferenceCatalogConfig 

37import lsst.utils 

38 

39 

40def make_coord(ra, dec): 

41 """Make an ICRS coord given its RA, Dec in degrees.""" 

42 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees) 

43 

44 

45def makeConvertConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False, 

46 withParallax=False): 

47 """Make a config for ConvertReferenceCatalogTask 

48 

49 This is primarily intended to simplify tests of config validation, 

50 so fields that are not validated are not set. 

51 However, it can calso be used to reduce boilerplate in other tests. 

52 """ 

53 config = ConvertReferenceCatalogConfig() 

54 config.dataset_config.ref_dataset_name = "testRefCat" 

55 config.pm_scale = 1000.0 

56 config.parallax_scale = 1e3 

57 config.ra_name = 'ra_icrs' 

58 config.dec_name = 'dec_icrs' 

59 config.mag_column_list = ['a', 'b'] 

60 

61 if withMagErr: 

62 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} 

63 

64 if withRaDecErr: 

65 config.ra_err_name = "ra_err" 

66 config.dec_err_name = "dec_err" 

67 config.coord_err_unit = "arcsecond" 

68 

69 if withPm: 

70 config.pm_ra_name = "pm_ra" 

71 config.pm_dec_name = "pm_dec" 

72 

73 if withPmErr: 

74 config.pm_ra_err_name = "pm_ra_err" 

75 config.pm_dec_err_name = "pm_dec_err" 

76 

77 if withParallax: 

78 config.parallax_name = "parallax" 

79 config.parallax_err_name = "parallax_err" 

80 

81 if withPm or withParallax: 

82 config.epoch_name = "unixtime" 

83 config.epoch_format = "unix" 

84 config.epoch_scale = "utc" 

85 

86 return config 

87 

88 

89class ConvertReferenceCatalogTestBase: 

90 """Base class for tests involving ConvertReferenceCatalogTask 

91 """ 

92 @classmethod 

93 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123): 

94 """Make an on-sky catalog, and save it to a text file. 

95 

96 Parameters 

97 ---------- 

98 outPath : `str` or None 

99 The directory to write the catalog to. 

100 Specify None to not write any output. 

101 size : `int`, (optional) 

102 Number of items to add to the catalog. 

103 idStart : `int`, (optional) 

104 First id number to put in the catalog. 

105 seed : `float`, (optional) 

106 Random seed for ``np.random``. 

107 

108 Returns 

109 ------- 

110 refCatPath : `str` 

111 Path to the created on-sky catalog. 

112 refCatOtherDelimiterPath : `str` 

113 Path to the created on-sky catalog with a different delimiter. 

114 refCatData : `np.ndarray` 

115 The data contained in the on-sky catalog files. 

116 """ 

117 np.random.seed(seed) 

118 ident = np.arange(idStart, size + idStart, dtype=int) 

119 ra = np.random.random(size)*360. 

120 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.)) 

121 dec -= 90. 

122 ra_err = np.ones(size)*0.1 # arcsec 

123 dec_err = np.ones(size)*0.1 # arcsec 

124 a_mag = 16. + np.random.random(size)*4. 

125 a_mag_err = 0.01 + np.random.random(size)*0.2 

126 b_mag = 17. + np.random.random(size)*5. 

127 b_mag_err = 0.02 + np.random.random(size)*0.3 

128 is_photometric = np.random.randint(2, size=size) 

129 is_resolved = np.random.randint(2, size=size) 

130 is_variable = np.random.randint(2, size=size) 

131 extra_col1 = np.random.normal(size=size) 

132 extra_col2 = np.random.normal(1000., 100., size=size) 

133 # compute proper motion and PM error in arcseconds/year 

134 # and let the ingest task scale them to radians 

135 pm_amt_arcsec = cls.properMotionAmt.asArcseconds() 

136 pm_dir_rad = cls.properMotionDir.asRadians() 

137 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad) 

138 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad) 

139 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad)) 

140 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad)) 

141 parallax = np.ones(size)*0.1 # arcseconds 

142 parallax_error = np.ones(size)*0.003 # arcseconds 

143 unixtime = np.ones(size)*cls.epoch.unix 

144 

145 def get_word(word_len): 

146 return "".join(np.random.choice([s for s in string.ascii_letters], word_len)) 

147 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)]) 

148 

149 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float), 

150 ('ra_err', float), ('dec_err', float), ('a', float), 

151 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int), 

152 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float), 

153 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float), 

154 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float), 

155 ('unixtime', float)]) 

156 

157 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err, 

158 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3, 

159 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)), 

160 dtype=dtype) 

161 if outPath is not None: 

162 # write the data with full precision; this is not realistic for 

163 # real catalogs, but simplifies tests based on round tripped data 

164 saveKwargs = dict( 

165 header="id,ra_icrs,dec_icrs,ra_err,dec_err," 

166 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3," 

167 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime", 

168 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g", 

169 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s", 

170 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"] 

171 ) 

172 

173 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs) 

174 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs) 

175 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr 

176 else: 

177 return arr 

178 

179 @classmethod 

180 def tearDownClass(cls): 

181 try: 

182 shutil.rmtree(cls.outPath) 

183 except Exception: 

184 print("WARNING: failed to remove temporary dir %r" % (cls.outPath,)) 

185 del cls.outPath 

186 del cls.skyCatalogFile 

187 del cls.skyCatalogFileDelim 

188 del cls.skyCatalog 

189 del cls.testRas 

190 del cls.testDecs 

191 del cls.searchRadius 

192 del cls.compCats 

193 

194 @classmethod 

195 def setUpClass(cls): 

196 cls.outPath = tempfile.mkdtemp() 

197 # arbitrary, but reasonable, amount of proper motion (angle/year) 

198 # and direction of proper motion 

199 cls.properMotionAmt = 3.0*lsst.geom.arcseconds 

200 cls.properMotionDir = 45*lsst.geom.degrees 

201 cls.properMotionErr = 1e-3*lsst.geom.arcseconds 

202 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd") 

203 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath) 

204 cls.testRas = [210., 14.5, 93., 180., 286., 0.] 

205 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.] 

206 cls.searchRadius = 3. * lsst.geom.degrees 

207 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center 

208 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius 

209 

210 config = IndexerRegistry['HTM'].ConfigClass() 

211 # Match on disk comparison file 

212 config.depth = cls.depth 

213 cls.indexer = IndexerRegistry['HTM'](config) 

214 for ra in cls.testRas: 

215 for dec in cls.testDecs: 

216 tupl = (ra, dec) 

217 cent = make_coord(*tupl) 

218 cls.compCats[tupl] = [] 

219 for rec in cls.skyCatalog: 

220 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius: 

221 cls.compCats[tupl].append(rec['id']) 

222 

223 cls.testRepoPath = cls.outPath+"/test_repo" 

224 

225 def setUp(self): 

226 self.repoPath = tempfile.TemporaryDirectory() # cleaned up automatically when test ends 

227 self.butler = self.makeTemporaryRepo(self.repoPath.name, self.depth) 

228 self.logger = logging.getLogger('lsst.ReferenceObjectLoader') 

229 

230 @staticmethod 

231 def makeTemporaryRepo(rootPath, depth): 

232 """Create a temporary butler repository, configured to support a given 

233 htm pixel depth, to use for a single test. 

234 

235 Parameters 

236 ---------- 

237 rootPath : `str` 

238 Root path for butler. 

239 depth : `int` 

240 HTM pixel depth to be used in this test. 

241 

242 Returns 

243 ------- 

244 butler : `lsst.daf.butler.Butler` 

245 The newly created and instantiated butler. 

246 """ 

247 dimensionConfig = lsst.daf.butler.DimensionConfig() 

248 dimensionConfig['skypix']['common'] = f'htm{depth}' 

249 lsst.daf.butler.Butler.makeRepo(rootPath, dimensionConfig=dimensionConfig) 

250 return lsst.daf.butler.Butler(rootPath, writeable=True) 

251 

252 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config): 

253 """Check that every item in ``skyCatalog`` is in the ingested catalog, 

254 and check that fields are correct in it. 

255 

256 Parameters 

257 ---------- 

258 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask` 

259 A reference object loader to use to search for rows from 

260 ``skyCatalog``. 

261 skyCatalog : `np.ndarray` 

262 The original data to compare with. 

263 config : `lsst.meas.algorithms.LoadIndexedReferenceObjectsConfig` 

264 The Config that was used to generate the refcat. 

265 """ 

266 for row in skyCatalog: 

267 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees) 

268 with self.assertLogs(self.logger.name, level="INFO") as cm: 

269 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat 

270 self.assertIn("Loading reference objects from testRefCat in region", cm.output[0]) 

271 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.") 

272 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}" 

273 self.assertEqual(row['id'], cat[0]['id'], msg) 

274 # coordinates won't match perfectly due to rounding in radian/degree conversions 

275 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(), 

276 rtol=1e-14, msg=msg) 

277 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(), 

278 rtol=1e-14, msg=msg) 

279 if config.coord_err_unit is not None: 

280 # coordinate errors are not lsst.geom.Angle, so we have to use the 

281 # `units` field to convert them, and they are float32, so the tolerance is wider. 

282 raErr = cat[0]['coord_raErr']*u.Unit(cat.schema['coord_raErr'].asField().getUnits()) 

283 decErr = cat[0]['coord_decErr']*u.Unit(cat.schema['coord_decErr'].asField().getUnits()) 

284 self.assertFloatsAlmostEqual(row['ra_err'], raErr.to_value(config.coord_err_unit), 

285 rtol=1e-7, msg=msg) 

286 self.assertFloatsAlmostEqual(row['dec_err'], decErr.to_value(config.coord_err_unit), 

287 rtol=1e-7, msg=msg) 

288 

289 if config.parallax_name is not None: 

290 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds()) 

291 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())