Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["IngestIndexCatalogTestBase", "make_coord"] 

23 

24import math 

25import os.path 

26import shutil 

27import string 

28import tempfile 

29 

30import numpy as np 

31import astropy 

32 

33import lsst.daf.persistence as dafPersist 

34from lsst.meas.algorithms import IndexerRegistry 

35from lsst.meas.algorithms import IngestIndexedReferenceTask 

36import lsst.utils 

37 

38 

39def make_coord(ra, dec): 

40 """Make an ICRS coord given its RA, Dec in degrees.""" 

41 return lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees) 

42 

43 

44class IngestIndexCatalogTestBase: 

45 """Base class for tests involving IngestIndexedReferenceTask 

46 """ 

47 @classmethod 

48 def makeSkyCatalog(cls, outPath, size=1000, idStart=1, seed=123): 

49 """Make an on-sky catalog, and save it to a text file. 

50 

51 Parameters 

52 ---------- 

53 outPath : `str` or None 

54 The directory to write the catalog to. 

55 Specify None to not write any output. 

56 size : `int`, (optional) 

57 Number of items to add to the catalog. 

58 idStart : `int`, (optional) 

59 First id number to put in the catalog. 

60 seed : `float`, (optional) 

61 Random seed for ``np.random``. 

62 

63 Returns 

64 ------- 

65 refCatPath : `str` 

66 Path to the created on-sky catalog. 

67 refCatOtherDelimiterPath : `str` 

68 Path to the created on-sky catalog with a different delimiter. 

69 refCatData : `np.ndarray` 

70 The data contained in the on-sky catalog files. 

71 """ 

72 np.random.seed(seed) 

73 ident = np.arange(idStart, size + idStart, dtype=int) 

74 ra = np.random.random(size)*360. 

75 dec = np.degrees(np.arccos(2.*np.random.random(size) - 1.)) 

76 dec -= 90. 

77 ra_err = np.ones(size)*0.1 # arcsec 

78 dec_err = np.ones(size)*0.1 # arcsec 

79 a_mag = 16. + np.random.random(size)*4. 

80 a_mag_err = 0.01 + np.random.random(size)*0.2 

81 b_mag = 17. + np.random.random(size)*5. 

82 b_mag_err = 0.02 + np.random.random(size)*0.3 

83 is_photometric = np.random.randint(2, size=size) 

84 is_resolved = np.random.randint(2, size=size) 

85 is_variable = np.random.randint(2, size=size) 

86 extra_col1 = np.random.normal(size=size) 

87 extra_col2 = np.random.normal(1000., 100., size=size) 

88 # compute proper motion and PM error in arcseconds/year 

89 # and let the ingest task scale them to radians 

90 pm_amt_arcsec = cls.properMotionAmt.asArcseconds() 

91 pm_dir_rad = cls.properMotionDir.asRadians() 

92 pm_ra = np.ones(size)*pm_amt_arcsec*math.cos(pm_dir_rad) 

93 pm_dec = np.ones(size)*pm_amt_arcsec*math.sin(pm_dir_rad) 

94 pm_ra_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.cos(pm_dir_rad)) 

95 pm_dec_err = np.ones(size)*cls.properMotionErr.asArcseconds()*abs(math.sin(pm_dir_rad)) 

96 parallax = np.ones(size)*0.1 # arcseconds 

97 parallax_error = np.ones(size)*0.003 # arcseconds 

98 unixtime = np.ones(size)*cls.epoch.unix 

99 

100 def get_word(word_len): 

101 return "".join(np.random.choice([s for s in string.ascii_letters], word_len)) 

102 extra_col3 = np.array([get_word(num) for num in np.random.randint(11, size=size)]) 

103 

104 dtype = np.dtype([('id', float), ('ra_icrs', float), ('dec_icrs', float), 

105 ('ra_err', float), ('dec_err', float), ('a', float), 

106 ('a_err', float), ('b', float), ('b_err', float), ('is_phot', int), 

107 ('is_res', int), ('is_var', int), ('val1', float), ('val2', float), 

108 ('val3', '|S11'), ('pm_ra', float), ('pm_dec', float), ('pm_ra_err', float), 

109 ('pm_dec_err', float), ('parallax', float), ('parallax_error', float), 

110 ('unixtime', float)]) 

111 

112 arr = np.array(list(zip(ident, ra, dec, ra_err, dec_err, a_mag, a_mag_err, b_mag, b_mag_err, 

113 is_photometric, is_resolved, is_variable, extra_col1, extra_col2, extra_col3, 

114 pm_ra, pm_dec, pm_ra_err, pm_dec_err, parallax, parallax_error, unixtime)), 

115 dtype=dtype) 

116 if outPath is not None: 

117 # write the data with full precision; this is not realistic for 

118 # real catalogs, but simplifies tests based on round tripped data 

119 saveKwargs = dict( 

120 header="id,ra_icrs,dec_icrs,ra_err,dec_err," 

121 "a,a_err,b,b_err,is_phot,is_res,is_var,val1,val2,val3," 

122 "pm_ra,pm_dec,pm_ra_err,pm_dec_err,parallax,parallax_err,unixtime", 

123 fmt=["%i", "%.15g", "%.15g", "%.15g", "%.15g", 

124 "%.15g", "%.15g", "%.15g", "%.15g", "%i", "%i", "%i", "%.15g", "%.15g", "%s", 

125 "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g", "%.15g"] 

126 ) 

127 

128 np.savetxt(outPath+"/ref.txt", arr, delimiter=",", **saveKwargs) 

129 np.savetxt(outPath+"/ref_test_delim.txt", arr, delimiter="|", **saveKwargs) 

130 return outPath+"/ref.txt", outPath+"/ref_test_delim.txt", arr 

131 else: 

132 return arr 

133 

134 @classmethod 

135 def tearDownClass(cls): 

136 try: 

137 shutil.rmtree(cls.outPath) 

138 except Exception: 

139 print("WARNING: failed to remove temporary dir %r" % (cls.outPath,)) 

140 del cls.outPath 

141 del cls.skyCatalogFile 

142 del cls.skyCatalogFileDelim 

143 del cls.skyCatalog 

144 del cls.testRas 

145 del cls.testDecs 

146 del cls.searchRadius 

147 del cls.compCats 

148 del cls.testButler 

149 

150 @classmethod 

151 def setUpClass(cls): 

152 cls.obs_test_dir = lsst.utils.getPackageDir('obs_test') 

153 cls.input_dir = os.path.join(cls.obs_test_dir, "data", "input") 

154 

155 cls.outPath = tempfile.mkdtemp() 

156 cls.testCatPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data", 

157 "testHtmIndex.fits") 

158 # arbitrary, but reasonable, amount of proper motion (angle/year) 

159 # and direction of proper motion 

160 cls.properMotionAmt = 3.0*lsst.geom.arcseconds 

161 cls.properMotionDir = 45*lsst.geom.degrees 

162 cls.properMotionErr = 1e-3*lsst.geom.arcseconds 

163 cls.epoch = astropy.time.Time(58206.861330339219, scale="tai", format="mjd") 

164 cls.skyCatalogFile, cls.skyCatalogFileDelim, cls.skyCatalog = cls.makeSkyCatalog(cls.outPath) 

165 cls.testRas = [210., 14.5, 93., 180., 286., 0.] 

166 cls.testDecs = [-90., -51., -30.1, 0., 27.3, 62., 90.] 

167 cls.searchRadius = 3. * lsst.geom.degrees 

168 cls.compCats = {} # dict of center coord: list of IDs of stars within cls.searchRadius of center 

169 cls.depth = 4 # gives a mean area of 20 deg^2 per pixel, roughly matching a 3 deg search radius 

170 

171 config = IndexerRegistry['HTM'].ConfigClass() 

172 # Match on disk comparison file 

173 config.depth = cls.depth 

174 cls.indexer = IndexerRegistry['HTM'](config) 

175 for ra in cls.testRas: 

176 for dec in cls.testDecs: 

177 tupl = (ra, dec) 

178 cent = make_coord(*tupl) 

179 cls.compCats[tupl] = [] 

180 for rec in cls.skyCatalog: 

181 if make_coord(rec['ra_icrs'], rec['dec_icrs']).separation(cent) < cls.searchRadius: 

182 cls.compCats[tupl].append(rec['id']) 

183 

184 cls.testRepoPath = cls.outPath+"/test_repo" 

185 config = cls.makeConfig(withMagErr=True, withRaDecErr=True, withPm=True, withPmErr=True, 

186 withParallax=True) 

187 # To match on disk test data 

188 config.dataset_config.indexer.active.depth = cls.depth 

189 config.id_name = 'id' 

190 config.pm_scale = 1000.0 # arcsec/yr --> mas/yr 

191 config.parallax_scale = 1e3 # arcsec -> milliarcsec 

192 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

193 config.file_reader.format = 'ascii.commented_header' 

194 # run the intest once to create a butler repo we can compare to 

195 IngestIndexedReferenceTask.parseAndRun(args=[cls.input_dir, "--output", cls.testRepoPath, 

196 cls.skyCatalogFile], config=config) 

197 cls.defaultDatasetName = config.dataset_config.ref_dataset_name 

198 cls.testDatasetName = 'diff_ref_name' 

199 cls.testButler = dafPersist.Butler(cls.testRepoPath) 

200 os.symlink(os.path.join(cls.testRepoPath, 'ref_cats', cls.defaultDatasetName), 

201 os.path.join(cls.testRepoPath, 'ref_cats', cls.testDatasetName)) 

202 

203 @staticmethod 

204 def makeConfig(withMagErr=False, withRaDecErr=False, withPm=False, withPmErr=False, 

205 withParallax=False): 

206 """Make a config for IngestIndexedReferenceTask 

207 

208 This is primarily intended to simplify tests of config validation, 

209 so fields that are not validated are not set. 

210 However, it can calso be used to reduce boilerplate in other tests. 

211 """ 

212 config = IngestIndexedReferenceTask.ConfigClass() 

213 config.pm_scale = 1000.0 

214 config.parallax_scale = 1e3 

215 config.ra_name = 'ra_icrs' 

216 config.dec_name = 'dec_icrs' 

217 config.mag_column_list = ['a', 'b'] 

218 

219 if withMagErr: 

220 config.mag_err_column_map = {'a': 'a_err', 'b': 'b_err'} 

221 

222 if withRaDecErr: 

223 config.ra_err_name = "ra_err" 

224 config.dec_err_name = "dec_err" 

225 

226 if withPm: 

227 config.pm_ra_name = "pm_ra" 

228 config.pm_dec_name = "pm_dec" 

229 

230 if withPmErr: 

231 config.pm_ra_err_name = "pm_ra_err" 

232 config.pm_dec_err_name = "pm_dec_err" 

233 

234 if withParallax: 

235 config.parallax_name = "parallax" 

236 config.parallax_err_name = "parallax_err" 

237 

238 if withPm or withParallax: 

239 config.epoch_name = "unixtime" 

240 config.epoch_format = "unix" 

241 config.epoch_scale = "utc" 

242 

243 return config 

244 

245 def checkAllRowsInRefcat(self, refObjLoader, skyCatalog, config): 

246 """Check that every item in ``skyCatalog`` is in the ingested catalog. 

247 

248 Parameters 

249 ---------- 

250 refObjLoader : `lsst.meas.algorithms.LoadIndexedReferenceObjectsTask` 

251 A reference object loader to use to search for rows from 

252 ``skyCatalog``. 

253 skyCatalog : `np.ndarray` 

254 The original data to compare with. 

255 """ 

256 for row in skyCatalog: 

257 center = lsst.geom.SpherePoint(row['ra_icrs'], row['dec_icrs'], lsst.geom.degrees) 

258 cat = refObjLoader.loadSkyCircle(center, 2*lsst.geom.arcseconds, filterName='a').refCat 

259 self.assertGreater(len(cat), 0, "No objects found in loaded catalog.") 

260 msg = f"input row not found in loaded catalog:\nrow:\n{row}\n{row.dtype}\n\ncatalog:\n{cat[0]}" 

261 self.assertEqual(row['id'], cat[0]['id'], msg) 

262 # coordinates won't match perfectly due to rounding in radian/degree conversions 

263 self.assertFloatsAlmostEqual(row['ra_icrs'], cat[0]['coord_ra'].asDegrees(), 

264 rtol=1e-14, msg=msg) 

265 self.assertFloatsAlmostEqual(row['dec_icrs'], cat[0]['coord_dec'].asDegrees(), 

266 rtol=1e-14, msg=msg) 

267 if config.parallax_name is not None: 

268 self.assertFloatsAlmostEqual(row['parallax'], cat[0]['parallax'].asArcseconds()) 

269 self.assertFloatsAlmostEqual(row['parallax_error'], cat[0]['parallaxErr'].asArcseconds())