Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IngestIndexCatalogTestBase", "make_coord", "makeIngestIndexConfig"] 

23 

24import math 

25import os.path 

26import shutil 

27import string 

28import tempfile 

29 

30import numpy as np 

31import astropy 

32import astropy.units as u 

33 

34import lsst.daf.persistence as dafPersist 

35from lsst.meas.algorithms import IndexerRegistry 

36from lsst.meas.algorithms import IngestIndexedReferenceTask 

37import lsst.utils 

38 

39 

40def make_coord(ra, dec): 

41 """Make an ICRS coord given its RA, Dec in degrees.""" 

42 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees) 

43 

44 

45def makeIngestIndexConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False, 

46 withParallax=False): 

47 """Make a config for IngestIndexedReferenceTask 

48 

49 This is primarily intended to simplify tests of config validation, 

50 so fields that are not validated are not set. 

51 However, it can calso be used to reduce boilerplate in other tests. 

52 """ 

53 config = IngestIndexedReferenceTask.ConfigClass() 

54 config.pm_scale = 1000.0 

55 config.parallax_scale = 1e3 

56 config.ra_name = 'ra_icrs' 

57 config.dec_name = 'dec_icrs' 

58 config.mag_column_list = ['a', 'b'] 

59 

60 if withMagErr: 

61 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} 

62 

63 if withRaDecErr: 

64 config.ra_err_name = "ra_err" 

65 config.dec_err_name = "dec_err" 

66 config.coord_err_unit = "arcsecond" 

67 

68 if withPm: 

69 config.pm_ra_name = "pm_ra" 

70 config.pm_dec_name = "pm_dec" 

71 

72 if withPmErr: 

73 config.pm_ra_err_name = "pm_ra_err" 

74 config.pm_dec_err_name = "pm_dec_err" 

75 

76 if withParallax: 

77 config.parallax_name = "parallax" 

78 config.parallax_err_name = "parallax_err" 

79 

80 if withPm or withParallax: 

81 config.epoch_name = "unixtime" 

82 config.epoch_format = "unix" 

83 config.epoch_scale = "utc" 

84 

85 return config 

86 

87 

88class IngestIndexCatalogTestBase: 

89 """Base class for tests involving IngestIndexedReferenceTask 

90 """ 

91 @classmethod 

92 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123): 

93 """Make an on-sky catalog, and save it to a text file. 

94 

95 Parameters 

96 ---------- 

97 outPath : `str` or None 

98 The directory to write the catalog to. 

99 Specify None to not write any output. 

100 size : `int`, (optional) 

101 Number of items to add to the catalog. 

102 idStart : `int`, (optional) 

103 First id number to put in the catalog. 

104 seed : `float`, (optional) 

105 Random seed for ``np.random``. 

106 

107 Returns 

108 ------- 

109 refCatPath : `str` 

110 Path to the created on-sky catalog. 

111 refCatOtherDelimiterPath : `str` 

112 Path to the created on-sky catalog with a different delimiter. 

113 refCatData : `np.ndarray` 

114 The data contained in the on-sky catalog files. 

115 """ 

116 np.random.seed(seed) 

117 ident = np.arange(idStart, size + idStart, dtype=int) 

118 ra = np.random.random(size)*360. 

119 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.)) 

120 dec -= 90. 

121 ra_err = np.ones(size)*0.1 # arcsec 

122 dec_err = np.ones(size)*0.1 # arcsec 

123 a_mag = 16. + np.random.random(size)*4. 

124 a_mag_err = 0.01 + np.random.random(size)*0.2 

125 b_mag = 17. + np.random.random(size)*5. 

126 b_mag_err = 0.02 + np.random.random(size)*0.3 

127 is_photometric = np.random.randint(2, size=size) 

128 is_resolved = np.random.randint(2, size=size) 

129 is_variable = np.random.randint(2, size=size) 

130 extra_col1 = np.random.normal(size=size) 

131 extra_col2 = np.random.normal(1000., 100., size=size) 

132 # compute proper motion and PM error in arcseconds/year 

133 # and let the ingest task scale them to radians 

134 pm_amt_arcsec = cls.properMotionAmt.asArcseconds() 

135 pm_dir_rad = cls.properMotionDir.asRadians() 

136 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad) 

137 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad) 

138 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad)) 

139 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad)) 

140 parallax = np.ones(size)*0.1 # arcseconds 

141 parallax_error = np.ones(size)*0.003 # arcseconds 

142 unixtime = np.ones(size)*cls.epoch.unix 

143 

144 def get_word(word_len): 

145 return "".join(np.random.choice([s for s in string.ascii_letters], word_len)) 

146 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)]) 

147 

148 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float), 

149 ('ra_err', float), ('dec_err', float), ('a', float), 

150 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int), 

151 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float), 

152 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float), 

153 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float), 

154 ('unixtime', float)]) 

155 

156 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err, 

157 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3, 

158 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)), 

159 dtype=dtype) 

160 if outPath is not None: 

161 # write the data with full precision; this is not realistic for 

162 # real catalogs, but simplifies tests based on round tripped data 

163 saveKwargs = dict( 

164 header="id,ra_icrs,dec_icrs,ra_err,dec_err," 

165 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3," 

166 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime", 

167 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g", 

168 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s", 

169 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"] 

170 ) 

171 

172 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs) 

173 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs) 

174 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr 

175 else: 

176 return arr 

177 

178 @classmethod 

179 def tearDownClass(cls): 

180 try: 

181 shutil.rmtree(cls.outPath) 

182 except Exception: 

183 print("WARNING: failed to remove temporary dir %r" % (cls.outPath,)) 

184 del cls.outPath 

185 del cls.skyCatalogFile 

186 del cls.skyCatalogFileDelim 

187 del cls.skyCatalog 

188 del cls.testRas 

189 del cls.testDecs 

190 del cls.searchRadius 

191 del cls.compCats 

192 del cls.testButler 

193 

194 @classmethod 

195 def setUpClass(cls): 

196 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test') 

197 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input") 

198 

199 cls.outPath = tempfile.mkdtemp() 

200 cls.testCatPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", 

201 "testHtmIndex.fits") 

202 # arbitrary, but reasonable, amount of proper motion (angle/year) 

203 # and direction of proper motion 

204 cls.properMotionAmt = 3.0*lsst.geom.arcseconds 

205 cls.properMotionDir = 45*lsst.geom.degrees 

206 cls.properMotionErr = 1e-3*lsst.geom.arcseconds 

207 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd") 

208 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath) 

209 cls.testRas = [210., 14.5, 93., 180., 286., 0.] 

210 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.] 

211 cls.searchRadius = 3. * lsst.geom.degrees 

212 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center 

213 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius 

214 

215 config = IndexerRegistry['HTM'].ConfigClass() 

216 # Match on disk comparison file 

217 config.depth = cls.depth 

218 cls.indexer = IndexerRegistry['HTM'](config) 

219 for ra in cls.testRas: 

220 for dec in cls.testDecs: 

221 tupl = (ra, dec) 

222 cent = make_coord(*tupl) 

223 cls.compCats[tupl] = [] 

224 for rec in cls.skyCatalog: 

225 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius: 

226 cls.compCats[tupl].append(rec['id']) 

227 

228 cls.testRepoPath = cls.outPath+"/test_repo" 

229 config = makeIngestIndexConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True, 

230 withParallax=True) 

231 # To match on disk test data 

232 config.dataset_config.indexer.active.depth = cls.depth 

233 config.id_name = 'id' 

234 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr 

235 config.parallax_scale = 1e3 # arcsec -> milliarcsec 

236 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

237 config.file_reader.format = 'ascii.commented_header' 

238 # run the intest once to create a butler repo we can compare to 

239 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath, 

240 cls.skyCatalogFile], config=config) 

241 cls.defaultDatasetName = config.dataset_config.ref_dataset_name 

242 cls.testDatasetName = 'diff_ref_name' 

243 cls.testButler = dafPersist.Butler(cls.testRepoPath) 

244 os.symlink(os.path.join(cls.testRepoPath, 'ref_cats', cls.defaultDatasetName), 

245 os.path.join(cls.testRepoPath, 'ref_cats', cls.testDatasetName)) 

246 

247 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config): 

248 """Check that every item in ``skyCatalog`` is in the ingested catalog, 

249 and check that fields are correct in it. 

250 

251 Parameters 

252 ---------- 

253 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask` 

254 A reference object loader to use to search for rows from 

255 ``skyCatalog``. 

256 skyCatalog : `np.ndarray` 

257 The original data to compare with. 

258 config : `lsst.meas.algorithms.LoadIndexedReferenceObjectsConfig` 

259 The Config that was used to generate the refcat. 

260 """ 

261 for row in skyCatalog: 

262 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees) 

263 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat 

264 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.") 

265 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}" 

266 self.assertEqual(row['id'], cat[0]['id'], msg) 

267 # coordinates won't match perfectly due to rounding in radian/degree conversions 

268 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(), 

269 rtol=1e-14, msg=msg) 

270 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(), 

271 rtol=1e-14, msg=msg) 

272 if config.coord_err_unit is not None: 

273 # coordinate errors are not lsst.geom.Angle, so we have to use the 

274 # `units` field to convert them, and they are float32, so the tolerance is wider. 

275 raErr = cat[0]['coord_raErr']*u.Unit(cat.schema['coord_raErr'].asField().getUnits()) 

276 decErr = cat[0]['coord_decErr']*u.Unit(cat.schema['coord_decErr'].asField().getUnits()) 

277 self.assertFloatsAlmostEqual(row['ra_err'], raErr.to_value(config.coord_err_unit), 

278 rtol=1e-7, msg=msg) 

279 self.assertFloatsAlmostEqual(row['dec_err'], decErr.to_value(config.coord_err_unit), 

280 rtol=1e-7, msg=msg) 

281 

282 if config.parallax_name is not None: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds()) 

284 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())