Coverage for tests/test_referenceObjectLoader.py: 18%

165 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-25 00:24 -0700

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import os.path 

22import tempfile 

23import unittest 

24import glob 

25 

26import numpy as np 

27from smatch.matcher import sphdist 

28import astropy.time 

29 

30import lsst.daf.butler 

31import lsst.afw.geom as afwGeom 

32import lsst.afw.table as afwTable 

33from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

34from lsst.daf.butler.script import ingest_files 

35from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader) 

36from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles 

37from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits 

38import lsst.utils 

39import lsst.geom 

40 

41import ingestIndexTestBase 

42 

43 

44class ReferenceObjectLoaderTestCase(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

45 lsst.utils.tests.TestCase): 

46 """Test case for ReferenceObjectLoader.""" 

47 @classmethod 

48 def setUpClass(cls): 

49 super().setUpClass() 

50 

51 # Generate a catalog, with arbitrary ids 

52 inTempDir = tempfile.TemporaryDirectory() 

53 inPath = inTempDir.name 

54 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123) 

55 

56 cls.skyCatalog = skyCatalog 

57 

58 # override some field names. 

59 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True, 

60 withPm=True, withPmErr=True) 

61 # use a very small HTM pixelization depth 

62 depth = 2 

63 config.dataset_config.indexer.active.depth = depth 

64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

65 config.file_reader.format = 'ascii.commented_header' 

66 config.n_processes = 1 

67 config.id_name = 'id' # Use the ids from the generated catalogs 

68 cls.repoTempDir = tempfile.TemporaryDirectory() 

69 repoPath = cls.repoTempDir.name 

70 

71 # Convert the input data files to our HTM indexed format. 

72 dataTempDir = tempfile.TemporaryDirectory() 

73 dataPath = dataTempDir.name 

74 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

75 converter.run([skyCatalogFile]) 

76 

77 # Make a temporary butler to ingest them into. 

78 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

79 dimensions = [f"htm{depth}"] 

80 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

81 dimensions, 

82 "SimpleCatalog", 

83 universe=butler.registry.dimensions, 

84 isCalibration=False) 

85 butler.registry.registerDatasetType(datasetType) 

86 

87 # Ingest the files into the new butler. 

88 run = "testingRun" 

89 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

90 ingest_files(repoPath, 

91 config.dataset_config.ref_dataset_name, 

92 run, 

93 htmTableFile, 

94 transfer="auto") 

95 

96 # Test if we can get back the catalogs, with a new butler. 

97 butler = lsst.daf.butler.Butler(repoPath) 

98 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

99 collections=[run]).expanded()) 

100 handles = [] 

101 for dataRef in datasetRefs: 

102 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

103 

104 cls.datasetRefs = datasetRefs 

105 cls.handles = handles 

106 

107 inTempDir.cleanup() 

108 dataTempDir.cleanup() 

109 

110 def test_loadSkyCircle(self): 

111 """Test the loadSkyCircle routine.""" 

112 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

113 self.handles) 

114 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

115 cat = loader.loadSkyCircle( 

116 center, 

117 30.0*lsst.geom.degrees, 

118 filterName='a', 

119 ).refCat 

120 # Check that the max distance is less than the radius 

121 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec'])) 

122 self.assertLess(np.max(dist), 30.0) 

123 

124 # Check that all the objects from the two catalogs are here. 

125 dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'], self.skyCatalog['dec_icrs']) 

126 inside, = (dist < 30.0).nonzero() 

127 self.assertEqual(len(cat), len(inside)) 

128 

129 self.assertTrue(cat.isContiguous()) 

130 self.assertEqual(len(np.unique(cat['id'])), len(cat)) 

131 # A default-loaded sky circle should not have centroids 

132 self.assertNotIn('centroid_x', cat.schema) 

133 self.assertNotIn('centroid_y', cat.schema) 

134 self.assertNotIn('hasCentroid', cat.schema) 

135 

136 def test_loadPixelBox(self): 

137 """Test the loadPixelBox routine.""" 

138 # This will create a box 50 degrees on a side. 

139 loaderConfig = ReferenceObjectLoader.ConfigClass() 

140 loaderConfig.pixelMargin = 0 

141 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

142 self.handles, 

143 config=loaderConfig) 

144 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000)) 

145 crpix = lsst.geom.Point2D(500, 500) 

146 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

147 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees) 

148 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

149 

150 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat 

151 

152 # This is a sanity check on the ranges; the exact selection depends 

153 # on cos(dec) and the tangent-plane projection. 

154 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0) 

155 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0) 

156 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0) 

157 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0) 

158 

159 # The following is to ensure the reference catalog coords are 

160 # getting corrected for proper motion when an epoch is provided. 

161 # Use an extreme epoch so that differences in corrected coords 

162 # will be significant. Note that this simply tests that the coords 

163 # do indeed change when the epoch is passed. It makes no attempt 

164 # at assessing the correctness of the change. This is left to the 

165 # explicit testProperMotion() test below. 

166 catWithEpoch = loader.loadPixelBox( 

167 bbox, 

168 wcs, 

169 'a', 

170 bboxToSpherePadding=0, 

171 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat 

172 

173 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4) 

174 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4) 

175 

176 def test_filterMap(self): 

177 """Test filterMap parameters.""" 

178 loaderConfig = ReferenceObjectLoader.ConfigClass() 

179 loaderConfig.filterMap = {'aprime': 'a'} 

180 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

181 self.handles, 

182 config=loaderConfig) 

183 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

184 result = loader.loadSkyCircle( 

185 center, 

186 30.0*lsst.geom.degrees, 

187 filterName='aprime', 

188 ) 

189 self.assertEqual(result.fluxField, 'aprime_camFlux') 

190 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux']) 

191 

192 def test_properMotion(self): 

193 """Test proper motion correction.""" 

194 loaderConfig = ReferenceObjectLoader.ConfigClass() 

195 loaderConfig.filterMap = {'aprime': 'a'} 

196 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

197 self.handles, 

198 config=loaderConfig) 

199 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

200 cat = loader.loadSkyCircle( 

201 center, 

202 30.0*lsst.geom.degrees, 

203 filterName='a' 

204 ).refCat 

205 

206 # Zero epoch change --> no proper motion correction (except minor numerical effects) 

207 cat_pm = loader.loadSkyCircle( 

208 center, 

209 30.0*lsst.geom.degrees, 

210 filterName='a', 

211 epoch=self.epoch 

212 ).refCat 

213 

214 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14) 

215 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14) 

216 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr']) 

217 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr']) 

218 

219 # One year difference 

220 cat_pm = loader.loadSkyCircle( 

221 center, 

222 30.0*lsst.geom.degrees, 

223 filterName='a', 

224 epoch=self.epoch + 1.0*astropy.units.yr 

225 ).refCat 

226 

227 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr']) 

228 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr']) 

229 

230 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds() 

231 for i in range(len(cat))]) 

232 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds() 

233 for i in range(len(cat))]) 

234 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10) 

235 self.assertFloatsAlmostEqual(bearings, self.properMotionDir.asArcseconds(), rtol=1.0e-10) 

236 

237 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"]) 

238 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"]) 

239 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr) 

240 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr) 

241 

242 # One year negative difference. This demonstrates a fix for DM-38808, 

243 # when the refcat epoch is later in time than the data. 

244 cat_pm = loader.loadSkyCircle( 

245 center, 

246 30.0*lsst.geom.degrees, 

247 filterName='a', 

248 epoch=self.epoch - 1.0*astropy.units.yr 

249 ).refCat 

250 

251 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr']) 

252 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr']) 

253 

254 separations = np.array([cat[i].getCoord().separation(cat_pm[i].getCoord()).asArcseconds() 

255 for i in range(len(cat))]) 

256 bearings = np.array([cat[i].getCoord().bearingTo(cat_pm[i].getCoord()).asArcseconds() 

257 for i in range(len(cat))]) 

258 reverse_proper_motion_dir = self.properMotionDir + 180 * lsst.geom.degrees 

259 self.assertFloatsAlmostEqual(separations, self.properMotionAmt.asArcseconds(), rtol=1.0e-10) 

260 self.assertFloatsAlmostEqual(bearings, reverse_proper_motion_dir.asArcseconds(), rtol=1.0e-10) 

261 

262 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"]) 

263 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"]) 

264 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr) 

265 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr) 

266 

267 def test_requireProperMotion(self): 

268 """Tests of the requireProperMotion config field.""" 

269 loaderConfig = ReferenceObjectLoader.ConfigClass() 

270 loaderConfig.requireProperMotion = True 

271 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

272 self.handles, 

273 config=loaderConfig) 

274 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

275 

276 # Test that we require an epoch set. 

277 msg = 'requireProperMotion=True but epoch not provided to loader' 

278 with self.assertRaisesRegex(RuntimeError, msg): 

279 loader.loadSkyCircle( 

280 center, 

281 30.0*lsst.geom.degrees, 

282 filterName='a' 

283 ) 

284 

285 

286class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase): 

287 """Test cases for reading version 0 and version 1 catalogs.""" 

288 def testLoadVersion0(self): 

289 """Test reading a pre-written format_version=0 (Jy flux) catalog. 

290 It should be converted to have nJy fluxes. 

291 """ 

292 path = os.path.join( 

293 os.path.dirname(os.path.abspath(__file__)), 

294 'data', 

295 'version0', 

296 'ref_cats', 

297 'cal_ref_cat' 

298 ) 

299 

300 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

301 

302 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

303 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

304 

305 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

306 catalog = afwTable.SimpleCatalog.readFits(filenames[0]) 

307 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux']) 

308 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr']) 

309 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux']) 

310 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr']) 

311 

312 def testLoadVersion1(self): 

313 """Test reading a format_version=1 catalog (fluxes unchanged).""" 

314 path = os.path.join( 

315 os.path.dirname(os.path.abspath(__file__)), 

316 'data', 

317 'version1', 

318 'ref_cats', 

319 'cal_ref_cat' 

320 ) 

321 

322 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

323 

324 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

325 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

326 

327 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

328 catalog = afwTable.SimpleCatalog.readFits(filenames[0]) 

329 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux']) 

330 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr']) 

331 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux']) 

332 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr']) 

333 

334 

335class TestMemory(lsst.utils.tests.MemoryTestCase): 

336 pass 

337 

338 

339def setup_module(module): 

340 lsst.utils.tests.init() 

341 

342 

343if __name__ == "__main__": 343 ↛ 344line 343 didn't jump to line 344, because the condition on line 343 was never true

344 lsst.utils.tests.init() 

345 unittest.main()