Coverage for tests/test_convertReferenceCatalog.py: 14%

231 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-08 03:03 -0800

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import astropy.units as u 

23import numpy as np 

24import os.path 

25import sys 

26import unittest 

27import unittest.mock 

28import tempfile 

29import itertools 

30 

31from lsst.afw.table import SimpleCatalog 

32from lsst.pex.config import FieldValidationError 

33from lsst.meas.algorithms import (convertReferenceCatalog, ConvertReferenceCatalogTask, getRefFluxField) 

34from lsst.meas.algorithms.readTextCatalogTask import ReadTextCatalogTask 

35from lsst.meas.algorithms.htmIndexer import HtmIndexer 

36from lsst.meas.algorithms.convertRefcatManager import ConvertGaiaManager 

37from lsst.meas.algorithms.convertReferenceCatalog import addRefCatMetadata, _makeSchema 

38 

39import lsst.utils 

40 

41from convertReferenceCatalogTestBase import makeConvertConfig 

42import convertReferenceCatalogTestBase 

43 

44 

45class TestMain(lsst.utils.tests.TestCase): 

46 """Test mocking commandline arguments and calling 

47 ``convertReferenceCatalog.main()``. 

48 """ 

49 def setUp(self): 

50 self.inpath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data/mockrefcat/") 

51 self.expected_files = [os.path.join(self.inpath, "123.fits"), 

52 os.path.join(self.inpath, "124.fits"), 

53 os.path.join(self.inpath, "125.fits")] 

54 

55 def test_main_args(self): 

56 """Test that main configures the task and calls run() with the correct 

57 file list. 

58 """ 

59 outdir = tempfile.TemporaryDirectory() 

60 outpath = outdir.name 

61 args = ["convertReferenceCatalog", 

62 outpath, 

63 os.path.join(self.inpath, "mock_config.py"), 

64 os.path.join(self.inpath, "*.fits")] 

65 with unittest.mock.patch.object(convertReferenceCatalog.ConvertReferenceCatalogTask, "run") as run, \ 

66 unittest.mock.patch.object(sys, "argv", args): 

67 convertReferenceCatalog.main() 

68 # Test with sets because the glob can come out in any order. 

69 self.assertEqual(set(run.call_args.args[0]), set(self.expected_files)) 

70 # This is necessary to avoid a ResourceWarning. 

71 outdir.cleanup() 

72 

73 def test_main_args_bad_config(self): 

74 """Test that a bad config file produces a useful error, i.e. that 

75 main() validates the config. 

76 """ 

77 outdir = tempfile.TemporaryDirectory() 

78 outpath = outdir.name 

79 args = ["convertReferenceCatalog", 

80 outpath, 

81 os.path.join(self.inpath, "bad_config.py"), 

82 os.path.join(self.inpath, "*.fits")] 

83 with self.assertRaisesRegex(FieldValidationError, "Field 'ra_name' failed validation"), \ 

84 unittest.mock.patch.object(sys, "argv", args): 

85 convertReferenceCatalog.main() 

86 # This is necessary to avoid a ResourceWarning. 

87 outdir.cleanup() 

88 

89 def test_main_args_expanded_glob(self): 

90 """Test that an un-quoted glob (i.e. list of files) fails with a 

91 useful error. 

92 """ 

93 outdir = tempfile.TemporaryDirectory() 

94 outpath = outdir.name 

95 args = ["convertReferenceCatalog", 

96 outpath, 

97 os.path.join(self.inpath, "mock_config.py"), 

98 # an un-quoted glob will be shell-expanded to a list of files. 

99 "file1", "file2", "file3"] 

100 msg = "Final argument must be a quoted file glob, not a shell-expanded list of files." 

101 with self.assertRaisesRegex(RuntimeError, msg), \ 

102 unittest.mock.patch.object(sys, "argv", args): 

103 convertReferenceCatalog.main() 

104 # This is necessary to avoid a ResourceWarning. 

105 outdir.cleanup() 

106 

107 

108class MakeSchemaTestCase(lsst.utils.tests.TestCase): 

109 """Test the function to make reference catalog schemas. 

110 """ 

111 def testMakeSchema(self): 

112 """Make a schema and check it.""" 

113 for filterNameList in (["r"], ["foo", "_bar"]): 

114 for (addIsPhotometric, addIsResolved, addIsVariable) in itertools.product((False, True), 

115 (False, True), 

116 (False, True)): 

117 argDict = dict( 

118 filterNameList=filterNameList, 

119 addIsPhotometric=addIsPhotometric, 

120 addIsResolved=addIsResolved, 

121 addIsVariable=addIsVariable, 

122 ) 

123 refSchema = _makeSchema(**argDict) 

124 self.assertTrue("coord_ra" in refSchema) 

125 self.assertTrue("coord_dec" in refSchema) 

126 self.assertTrue("coord_raErr" in refSchema) 

127 self.assertTrue("coord_decErr" in refSchema) 

128 for filterName in filterNameList: 

129 fluxField = filterName + "_flux" 

130 self.assertIn(fluxField, refSchema) 

131 self.assertNotIn("x" + fluxField, refSchema) 

132 fluxErrField = fluxField + "Err" 

133 self.assertIn(fluxErrField, refSchema) 

134 self.assertEqual(getRefFluxField(refSchema, filterName), filterName + "_flux") 

135 self.assertEqual("resolved" in refSchema, addIsResolved) 

136 self.assertEqual("variable" in refSchema, addIsVariable) 

137 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

138 self.assertEqual("photometric" in refSchema, addIsPhotometric) 

139 

140 # The default for `fullPositionInformation` is False, so none 

141 # of the following should be included. We test setting these 

142 # all together below. 

143 self.assertNotIn("epoch", refSchema) 

144 self.assertNotIn("pm_ra", refSchema) 

145 self.assertNotIn("pm_dec", refSchema) 

146 self.assertNotIn("pm_flag", refSchema) 

147 self.assertNotIn("parallax", refSchema) 

148 self.assertNotIn("parallax_flag", refSchema) 

149 

150 def testMakeSchema_fullCovariance(self): 

151 """Make a schema with full position information and coordinate 

152 covariance and test it.""" 

153 refSchema = _makeSchema(filterNameList=["r"], fullPositionInformation=True) 

154 # Test that the epoch, proper motion and parallax terms are included in 

155 # the schema. 

156 self.assertIn("epoch", refSchema) 

157 self.assertIn("pm_ra", refSchema) 

158 self.assertIn("pm_dec", refSchema) 

159 self.assertIn("pm_flag", refSchema) 

160 self.assertIn("parallax", refSchema) 

161 self.assertIn("parallax_flag", refSchema) 

162 # Test that a sample of the 15 covariance terms are included in the schema. 

163 self.assertIn("coord_raErr", refSchema) 

164 self.assertIn("coord_decErr", refSchema) 

165 self.assertIn("coord_ra_coord_dec_Cov", refSchema) 

166 self.assertIn("pm_raErr", refSchema) 

167 self.assertIn("pm_ra_parallax_Cov", refSchema) 

168 self.assertIn("parallaxErr", refSchema) 

169 self.assertEqual(refSchema['coord_raErr'].asField().getUnits(), "rad") 

170 self.assertEqual(refSchema['coord_ra_coord_dec_Cov'].asField().getUnits(), "rad^2") 

171 self.assertEqual(refSchema['pm_raErr'].asField().getUnits(), "rad/year") 

172 self.assertEqual(refSchema['pm_dec_parallax_Cov'].asField().getUnits(), "rad^2/year") 

173 

174 

175class ConvertReferenceCatalogConfigValidateTestCase(lsst.utils.tests.TestCase): 

176 """Test validation of ConvertReferenceCatalogConfig.""" 

177 def testValidateRaDecMag(self): 

178 config = makeConvertConfig() 

179 config.validate() 

180 

181 for name in ("ra_name", "dec_name", "mag_column_list"): 

182 with self.subTest(name=name): 

183 config = makeConvertConfig() 

184 setattr(config, name, None) 

185 with self.assertRaises(ValueError): 

186 config.validate() 

187 

188 def testValidateRaDecErr(self): 

189 # check that a basic config validates 

190 config = makeConvertConfig(withRaDecErr=True) 

191 config.validate() 

192 

193 # check that a config with any of these fields missing does not validate 

194 for name in ("ra_err_name", "dec_err_name", "coord_err_unit"): 

195 with self.subTest(name=name): 

196 config = makeConvertConfig(withRaDecErr=True) 

197 setattr(config, name, None) 

198 with self.assertRaises(ValueError): 

199 config.validate() 

200 

201 # check that coord_err_unit must be an astropy unit 

202 config = makeConvertConfig(withRaDecErr=True) 

203 config.coord_err_unit = "nonsense unit" 

204 with self.assertRaisesRegex(ValueError, "is not a valid astropy unit string"): 

205 config.validate() 

206 

207 def testValidateMagErr(self): 

208 config = makeConvertConfig(withMagErr=True) 

209 config.validate() 

210 

211 # test for missing names 

212 for name in config.mag_column_list: 

213 with self.subTest(name=name): 

214 config = makeConvertConfig(withMagErr=True) 

215 del config.mag_err_column_map[name] 

216 with self.assertRaises(ValueError): 

217 config.validate() 

218 

219 # test for incorrect names 

220 for name in config.mag_column_list: 

221 with self.subTest(name=name): 

222 config = makeConvertConfig(withMagErr=True) 

223 config.mag_err_column_map["badName"] = config.mag_err_column_map[name] 

224 del config.mag_err_column_map[name] 

225 with self.assertRaises(ValueError): 

226 config.validate() 

227 

228 def testValidatePm(self): 

229 names = ["pm_ra_name", "pm_dec_name", "epoch_name", "epoch_format", "epoch_scale", 

230 "pm_ra_err_name", "pm_dec_err_name"] 

231 

232 config = makeConvertConfig(withPm=True) 

233 config.validate() 

234 del config 

235 

236 for name in names: 

237 with self.subTest(name=name): 

238 config = makeConvertConfig(withPm=True) 

239 setattr(config, name, None) 

240 with self.assertRaises(ValueError): 

241 config.validate() 

242 

243 def testValidateParallax(self): 

244 """Validation should fail if any parallax-related fields are missing. 

245 """ 

246 names = ["parallax_name", "epoch_name", "epoch_format", "epoch_scale", "parallax_err_name"] 

247 

248 config = makeConvertConfig(withParallax=True) 

249 config.validate() 

250 del config 

251 

252 for name in names: 

253 with self.subTest(name=name): 

254 config = makeConvertConfig(withParallax=True) 

255 setattr(config, name, None) 

256 with self.assertRaises(ValueError, msg=name): 

257 config.validate() 

258 

259 def testValidateCovariance(self): 

260 """Validation should fail if any position-related fields are empty if 

261 full_position_information is set. 

262 """ 

263 names = ["ra_err_name", "dec_err_name", "coord_err_unit", 

264 "parallax_name", "parallax_err_name", 

265 "epoch_name", "epoch_format", "epoch_scale", 

266 "pm_ra_name", "pm_dec_name", "pm_ra_err_name", "pm_dec_err_name"] 

267 

268 for name in names: 

269 with self.subTest(name=name): 

270 config = makeConvertConfig(withRaDecErr=True, withParallax=True, withPm=True) 

271 config.full_position_information = True 

272 config.manager.retarget(ConvertGaiaManager) 

273 setattr(config, name, None) 

274 with self.assertRaises(ValueError, msg=name): 

275 config.validate() 

276 

277 

278class ConvertGaiaManagerTests(convertReferenceCatalogTestBase.ConvertReferenceCatalogTestBase, 

279 lsst.utils.tests.TestCase): 

280 """Unittests specific to the Gaia catalog. 

281 """ 

282 def setUp(self): 

283 

284 np.random.seed(10) 

285 

286 self.tempDir = tempfile.TemporaryDirectory() 

287 tempPath = self.tempDir.name 

288 self.log = lsst.log.Log.getLogger("lsst.TestConvertRefcatManager") 

289 self.config = convertReferenceCatalogTestBase.makeConvertConfig(withRaDecErr=True) 

290 self.config.id_name = 'id' 

291 self.config.full_position_information = True 

292 self.config.manager.retarget(ConvertGaiaManager) 

293 self.config.coord_err_unit = 'milliarcsecond' 

294 self.config.ra_err_name = 'ra_error' 

295 self.config.dec_err_name = 'dec_error' 

296 self.config.pm_ra_name = 'pmra' 

297 self.config.pm_dec_name = 'pmdec' 

298 self.config.pm_ra_err_name = 'pmra_error' 

299 self.config.pm_dec_err_name = 'pmdec_error' 

300 self.config.parallax_name = 'parallax' 

301 self.config.parallax_err_name = 'parallax_error' 

302 self.config.epoch_name = 'unixtime' 

303 self.config.epoch_format = 'unix' 

304 self.config.epoch_scale = 'tai' 

305 self.depth = 2 # very small depth, for as few pixels as possible. 

306 self.indexer = HtmIndexer(self.depth) 

307 self.htm = lsst.sphgeom.HtmPixelization(self.depth) 

308 converter = ConvertReferenceCatalogTask(output_dir=tempPath, config=self.config) 

309 dtype = [('id', '<f8'), ('ra', '<f8'), ('dec', '<f8'), ('ra_err', '<f8'), ('dec_err', '<f8'), 

310 ('a', '<f8'), ('a_err', '<f8')] 

311 self.schema, self.key_map = converter.makeSchema(dtype) 

312 self.fileReader = ReadTextCatalogTask() 

313 

314 self.fakeInput = self.makeSkyCatalog(outPath=None, size=5, idStart=6543) 

315 self.matchedPixels = np.array([1, 1, 2, 2, 3]) 

316 self.tempDir2 = tempfile.TemporaryDirectory() 

317 tempPath = self.tempDir2.name 

318 self.filenames = {x: os.path.join(tempPath, "%d.fits" % x) for x in set(self.matchedPixels)} 

319 

320 self.worker = ConvertGaiaManager(self.filenames, 

321 self.config, 

322 self.fileReader, 

323 self.indexer, 

324 self.schema, 

325 self.key_map, 

326 self.htm.universe()[0], 

327 addRefCatMetadata, 

328 self.log) 

329 

330 def tearDown(self): 

331 self.tempDir.cleanup() 

332 self.tempDir2.cleanup() 

333 

334 def test_positionSetting(self): 

335 """Test the _setProperMotion, _setParallax, and 

336 _setCoordinateCovariance methods. 

337 """ 

338 outputCatalog = SimpleCatalog(self.worker.schema) 

339 outputCatalog.resize(len(self.fakeInput)) 

340 

341 # Set coordinate errors and covariances: 

342 coordErr = self.worker._getCoordErr(self.fakeInput) 

343 for name, array in coordErr.items(): 

344 outputCatalog[name] = array 

345 

346 for outputRow, inputRow in zip(outputCatalog, self.fakeInput): 

347 self.worker._setProperMotion(outputRow, inputRow) 

348 self.worker._setParallax(outputRow, inputRow) 

349 self.worker._setCoordinateCovariance(outputRow, inputRow) 

350 

351 coordConvert = (self.worker.coord_err_unit).to(u.radian) 

352 pmConvert = (self.worker.config.pm_scale * u.milliarcsecond).to_value(u.radian) 

353 parallaxConvert = (self.worker.config.parallax_scale * u.milliarcsecond).to_value(u.radian) 

354 

355 # Test a few combinations of coordinates, proper motion, and parallax. 

356 # Check that the covariance in the output catalog matches the 

357 # covariance calculated from the input, and also matches the covariance 

358 # calculated from the output catalog errors with the input correlation. 

359 ra_pmra_cov1 = (self.fakeInput['ra_error'] * self.fakeInput['pmra_error'] 

360 * self.fakeInput['ra_pmra_corr']) * coordConvert * pmConvert 

361 ra_pmra_cov2 = (outputCatalog['coord_raErr'] * outputCatalog['pm_raErr'] 

362 * self.fakeInput['ra_pmra_corr']) 

363 np.testing.assert_allclose(ra_pmra_cov1, outputCatalog['coord_ra_pm_ra_Cov']) 

364 np.testing.assert_allclose(ra_pmra_cov2, outputCatalog['coord_ra_pm_ra_Cov']) 

365 

366 dec_parallax_cov1 = (self.fakeInput['dec_error'] * self.fakeInput['parallax_error'] 

367 * self.fakeInput['dec_parallax_corr']) * coordConvert * parallaxConvert 

368 dec_parallax_cov2 = (outputCatalog['coord_decErr'] * outputCatalog['parallaxErr'] 

369 * self.fakeInput['dec_parallax_corr']) 

370 np.testing.assert_allclose(dec_parallax_cov1, outputCatalog['coord_dec_parallax_Cov']) 

371 np.testing.assert_allclose(dec_parallax_cov2, outputCatalog['coord_dec_parallax_Cov']) 

372 

373 pmdec_parallax_cov1 = (self.fakeInput['pmdec_error'] * self.fakeInput['parallax_error'] 

374 * self.fakeInput['parallax_pmdec_corr']) * pmConvert * parallaxConvert 

375 pmdec_parallax_cov2 = (outputCatalog['pm_decErr'] * outputCatalog['parallaxErr'] 

376 * self.fakeInput['parallax_pmdec_corr']) 

377 np.testing.assert_allclose(pmdec_parallax_cov1, outputCatalog['pm_dec_parallax_Cov']) 

378 np.testing.assert_allclose(pmdec_parallax_cov2, outputCatalog['pm_dec_parallax_Cov']) 

379 

380 

381class TestMemory(lsst.utils.tests.MemoryTestCase): 

382 pass 

383 

384 

385def setup_module(module): 

386 lsst.utils.tests.init() 

387 

388 

389if __name__ == "__main__": 389 ↛ 390line 389 didn't jump to line 390, because the condition on line 389 was never true

390 lsst.utils.tests.init() 

391 unittest.main()