Coverage for tests/test_referenceObjectLoader.py: 23%

152 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-27 02:14 -0700

1# This file is part of meas_algorithms. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import os.path 

22import tempfile 

23import unittest 

24import glob 

25 

26import numpy as np 

27from smatch.matcher import sphdist 

28import astropy.time 

29 

30import lsst.daf.butler 

31import lsst.afw.geom as afwGeom 

32import lsst.afw.table as afwTable 

33from lsst.daf.butler import DatasetType, DeferredDatasetHandle 

34from lsst.daf.butler.script import ingest_files 

35from lsst.meas.algorithms import (ConvertReferenceCatalogTask, ReferenceObjectLoader) 

36from lsst.meas.algorithms.testUtils import MockReferenceObjectLoaderFromFiles 

37from lsst.meas.algorithms.loadReferenceObjects import hasNanojanskyFluxUnits 

38import lsst.utils 

39import lsst.geom 

40 

41import ingestIndexTestBase 

42 

43 

44class ReferenceObjectLoaderTestCase(ingestIndexTestBase.ConvertReferenceCatalogTestBase, 

45 lsst.utils.tests.TestCase): 

46 """Test case for ReferenceObjectLoader.""" 

47 @classmethod 

48 def setUpClass(cls): 

49 super().setUpClass() 

50 

51 # Generate a catalog, with arbitrary ids 

52 inTempDir = tempfile.TemporaryDirectory() 

53 inPath = inTempDir.name 

54 skyCatalogFile, _, skyCatalog = cls.makeSkyCatalog(inPath, idStart=25, seed=123) 

55 

56 cls.skyCatalog = skyCatalog 

57 

58 # override some field names. 

59 config = ingestIndexTestBase.makeConvertConfig(withRaDecErr=True, withMagErr=True, 

60 withPm=True, withPmErr=True) 

61 # use a very small HTM pixelization depth 

62 depth = 2 

63 config.dataset_config.indexer.active.depth = depth 

64 # np.savetxt prepends '# ' to the header lines, so use a reader that understands that 

65 config.file_reader.format = 'ascii.commented_header' 

66 config.n_processes = 1 

67 config.id_name = 'id' # Use the ids from the generated catalogs 

68 cls.repoTempDir = tempfile.TemporaryDirectory() 

69 repoPath = cls.repoTempDir.name 

70 

71 # Convert the input data files to our HTM indexed format. 

72 dataTempDir = tempfile.TemporaryDirectory() 

73 dataPath = dataTempDir.name 

74 converter = ConvertReferenceCatalogTask(output_dir=dataPath, config=config) 

75 converter.run([skyCatalogFile]) 

76 

77 # Make a temporary butler to ingest them into. 

78 butler = cls.makeTemporaryRepo(repoPath, config.dataset_config.indexer.active.depth) 

79 dimensions = [f"htm{depth}"] 

80 datasetType = DatasetType(config.dataset_config.ref_dataset_name, 

81 dimensions, 

82 "SimpleCatalog", 

83 universe=butler.registry.dimensions, 

84 isCalibration=False) 

85 butler.registry.registerDatasetType(datasetType) 

86 

87 # Ingest the files into the new butler. 

88 run = "testingRun" 

89 htmTableFile = os.path.join(dataPath, "filename_to_htm.ecsv") 

90 ingest_files(repoPath, 

91 config.dataset_config.ref_dataset_name, 

92 run, 

93 htmTableFile, 

94 transfer="auto") 

95 

96 # Test if we can get back the catalogs, with a new butler. 

97 butler = lsst.daf.butler.Butler(repoPath) 

98 datasetRefs = list(butler.registry.queryDatasets(config.dataset_config.ref_dataset_name, 

99 collections=[run]).expanded()) 

100 handles = [] 

101 for dataRef in datasetRefs: 

102 handles.append(DeferredDatasetHandle(butler=butler, ref=dataRef, parameters=None)) 

103 

104 cls.datasetRefs = datasetRefs 

105 cls.handles = handles 

106 

107 inTempDir.cleanup() 

108 dataTempDir.cleanup() 

109 

110 def test_loadSkyCircle(self): 

111 """Test the loadSkyCircle routine.""" 

112 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

113 self.handles, 

114 name="testrefcat") 

115 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

116 cat = loader.loadSkyCircle( 

117 center, 

118 30.0*lsst.geom.degrees, 

119 filterName='a', 

120 ).refCat 

121 # Check that the max distance is less than the radius 

122 dist = sphdist(180.0, 0.0, np.rad2deg(cat['coord_ra']), np.rad2deg(cat['coord_dec'])) 

123 self.assertLess(np.max(dist), 30.0) 

124 

125 # Check that all the objects from the two catalogs are here. 

126 dist = sphdist(180.0, 0.0, self.skyCatalog['ra_icrs'], self.skyCatalog['dec_icrs']) 

127 inside, = (dist < 30.0).nonzero() 

128 self.assertEqual(len(cat), len(inside)) 

129 

130 self.assertTrue(cat.isContiguous()) 

131 self.assertEqual(len(np.unique(cat['id'])), len(cat)) 

132 # A default-loaded sky circle should not have centroids 

133 self.assertNotIn('centroid_x', cat.schema) 

134 self.assertNotIn('centroid_y', cat.schema) 

135 self.assertNotIn('hasCentroid', cat.schema) 

136 

137 def test_loadPixelBox(self): 

138 """Test the loadPixelBox routine.""" 

139 # This will create a box 50 degrees on a side. 

140 loaderConfig = ReferenceObjectLoader.ConfigClass() 

141 loaderConfig.pixelMargin = 0 

142 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

143 self.handles, 

144 name="testrefcat", 

145 config=loaderConfig) 

146 bbox = lsst.geom.Box2I(corner=lsst.geom.Point2I(0, 0), dimensions=lsst.geom.Extent2I(1000, 1000)) 

147 crpix = lsst.geom.Point2D(500, 500) 

148 crval = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

149 cdMatrix = afwGeom.makeCdMatrix(scale=0.05*lsst.geom.degrees) 

150 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

151 

152 cat = loader.loadPixelBox(bbox, wcs, 'a', bboxToSpherePadding=0).refCat 

153 

154 # This is a sanity check on the ranges; the exact selection depends 

155 # on cos(dec) and the tangent-plane projection. 

156 self.assertLess(np.max(np.rad2deg(cat['coord_ra'])), 180.0 + 25.0) 

157 self.assertGreater(np.max(np.rad2deg(cat['coord_ra'])), 180.0 - 25.0) 

158 self.assertLess(np.max(np.rad2deg(cat['coord_dec'])), 25.0) 

159 self.assertGreater(np.min(np.rad2deg(cat['coord_dec'])), -25.0) 

160 

161 # The following is to ensure the reference catalog coords are 

162 # getting corrected for proper motion when an epoch is provided. 

163 # Use an extreme epoch so that differences in corrected coords 

164 # will be significant. Note that this simply tests that the coords 

165 # do indeed change when the epoch is passed. It makes no attempt 

166 # at assessing the correctness of the change. This is left to the 

167 # explicit testProperMotion() test below. 

168 catWithEpoch = loader.loadPixelBox( 

169 bbox, 

170 wcs, 

171 'a', 

172 bboxToSpherePadding=0, 

173 epoch=astropy.time.Time(30000, format='mjd', scale='tai')).refCat 

174 

175 self.assertFloatsNotEqual(cat['coord_ra'], catWithEpoch['coord_ra'], rtol=1.0e-4) 

176 self.assertFloatsNotEqual(cat['coord_dec'], catWithEpoch['coord_dec'], rtol=1.0e-4) 

177 

178 def test_filterMap(self): 

179 """Test filterMap parameters.""" 

180 loaderConfig = ReferenceObjectLoader.ConfigClass() 

181 loaderConfig.filterMap = {'aprime': 'a'} 

182 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

183 self.handles, 

184 name="testrefcat", 

185 config=loaderConfig) 

186 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

187 result = loader.loadSkyCircle( 

188 center, 

189 30.0*lsst.geom.degrees, 

190 filterName='aprime', 

191 ) 

192 self.assertEqual(result.fluxField, 'aprime_camFlux') 

193 self.assertFloatsEqual(result.refCat['aprime_camFlux'], result.refCat['a_flux']) 

194 

195 def test_properMotion(self): 

196 """Test proper motion correction.""" 

197 loaderConfig = ReferenceObjectLoader.ConfigClass() 

198 loaderConfig.filterMap = {'aprime': 'a'} 

199 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

200 self.handles, 

201 name="testrefcat", 

202 config=loaderConfig) 

203 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

204 cat = loader.loadSkyCircle( 

205 center, 

206 30.0*lsst.geom.degrees, 

207 filterName='a' 

208 ).refCat 

209 

210 # Zero epoch change --> no proper motion correction (except minor numerical effects) 

211 cat_pm = loader.loadSkyCircle( 

212 center, 

213 30.0*lsst.geom.degrees, 

214 filterName='a', 

215 epoch=self.epoch 

216 ).refCat 

217 

218 self.assertFloatsAlmostEqual(cat_pm['coord_ra'], cat['coord_ra'], rtol=1.0e-14) 

219 self.assertFloatsAlmostEqual(cat_pm['coord_dec'], cat['coord_dec'], rtol=1.0e-14) 

220 self.assertFloatsEqual(cat_pm['coord_raErr'], cat['coord_raErr']) 

221 self.assertFloatsEqual(cat_pm['coord_decErr'], cat['coord_decErr']) 

222 

223 # One year difference 

224 cat_pm = loader.loadSkyCircle( 

225 center, 

226 30.0*lsst.geom.degrees, 

227 filterName='a', 

228 epoch=self.epoch + 1.0*astropy.units.yr 

229 ).refCat 

230 

231 self.assertFloatsEqual(cat_pm['pm_raErr'], cat['pm_raErr']) 

232 self.assertFloatsEqual(cat_pm['pm_decErr'], cat['pm_decErr']) 

233 for orig, ref in zip(cat, cat_pm): 

234 self.assertAnglesAlmostEqual(orig.getCoord().separation(ref.getCoord()), 

235 self.properMotionAmt, maxDiff=1.0e-6*lsst.geom.arcseconds) 

236 self.assertAnglesAlmostEqual(orig.getCoord().bearingTo(ref.getCoord()), 

237 self.properMotionDir, maxDiff=1.0e-4*lsst.geom.arcseconds) 

238 predictedRaErr = np.hypot(cat["coord_raErr"], cat["pm_raErr"]) 

239 predictedDecErr = np.hypot(cat["coord_decErr"], cat["pm_decErr"]) 

240 self.assertFloatsAlmostEqual(cat_pm["coord_raErr"], predictedRaErr) 

241 self.assertFloatsAlmostEqual(cat_pm["coord_decErr"], predictedDecErr) 

242 

243 def test_requireProperMotion(self): 

244 """Tests of the requireProperMotion config field.""" 

245 loaderConfig = ReferenceObjectLoader.ConfigClass() 

246 loaderConfig.requireProperMotion = True 

247 loader = ReferenceObjectLoader([dataRef.dataId for dataRef in self.datasetRefs], 

248 self.handles, 

249 name="testrefcat", 

250 config=loaderConfig) 

251 center = lsst.geom.SpherePoint(180.0*lsst.geom.degrees, 0.0*lsst.geom.degrees) 

252 

253 # Test that we require an epoch set. 

254 msg = 'requireProperMotion=True but epoch not provided to loader' 

255 with self.assertRaisesRegex(RuntimeError, msg): 

256 loader.loadSkyCircle( 

257 center, 

258 30.0*lsst.geom.degrees, 

259 filterName='a' 

260 ) 

261 

262 

263class Version0Version1ReferenceObjectLoaderTestCase(lsst.utils.tests.TestCase): 

264 """Test cases for reading version 0 and version 1 catalogs.""" 

265 def testLoadVersion0(self): 

266 """Test reading a pre-written format_version=0 (Jy flux) catalog. 

267 It should be converted to have nJy fluxes. 

268 """ 

269 path = os.path.join( 

270 os.path.dirname(os.path.abspath(__file__)), 

271 'data', 

272 'version0', 

273 'ref_cats', 

274 'cal_ref_cat' 

275 ) 

276 

277 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

278 

279 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

280 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

281 

282 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

283 catalog = afwTable.SimpleCatalog.readFits(filenames[0]) 

284 self.assertFloatsEqual(catalog['a_flux']*1e9, result.refCat['a_flux']) 

285 self.assertFloatsEqual(catalog['a_fluxSigma']*1e9, result.refCat['a_fluxErr']) 

286 self.assertFloatsEqual(catalog['b_flux']*1e9, result.refCat['b_flux']) 

287 self.assertFloatsEqual(catalog['b_fluxSigma']*1e9, result.refCat['b_fluxErr']) 

288 

289 def testLoadVersion1(self): 

290 """Test reading a format_version=1 catalog (fluxes unchanged).""" 

291 path = os.path.join( 

292 os.path.dirname(os.path.abspath(__file__)), 

293 'data', 

294 'version1', 

295 'ref_cats', 

296 'cal_ref_cat' 

297 ) 

298 

299 filenames = sorted(glob.glob(os.path.join(path, '????.fits'))) 

300 

301 loader = MockReferenceObjectLoaderFromFiles(filenames, name='cal_ref_cat', htmLevel=4) 

302 result = loader.loadSkyCircle(ingestIndexTestBase.make_coord(10, 20), 5*lsst.geom.degrees, 'a') 

303 

304 self.assertTrue(hasNanojanskyFluxUnits(result.refCat.schema)) 

305 catalog = afwTable.SimpleCatalog.readFits(filenames[0]) 

306 self.assertFloatsEqual(catalog['a_flux'], result.refCat['a_flux']) 

307 self.assertFloatsEqual(catalog['a_fluxErr'], result.refCat['a_fluxErr']) 

308 self.assertFloatsEqual(catalog['b_flux'], result.refCat['b_flux']) 

309 self.assertFloatsEqual(catalog['b_fluxErr'], result.refCat['b_fluxErr']) 

310 

311 

312class TestMemory(lsst.utils.tests.MemoryTestCase): 

313 pass 

314 

315 

316def setup_module(module): 

317 lsst.utils.tests.init() 

318 

319 

320if __name__ == "__main__": 320 ↛ 321line 320 didn't jump to line 321, because the condition on line 320 was never true

321 lsst.utils.tests.init() 

322 unittest.main()