Coverage for tests/test_gbdesAstrometricFit.py: 11%

484 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-29 04:30 -0700

1# This file is part of drp_tasks 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os.path 

23import unittest 

24 

25import astropy.units as u 

26import lsst.afw.geom as afwgeom 

27import lsst.afw.table as afwTable 

28import lsst.geom 

29import lsst.utils 

30import numpy as np 

31import pandas as pd 

32import wcsfit 

33import yaml 

34from lsst import sphgeom 

35from lsst.daf.base import PropertyList 

36from lsst.drp.tasks.gbdesAstrometricFit import ( 

37 GbdesAstrometricFitConfig, 

38 GbdesAstrometricFitTask, 

39 GbdesGlobalAstrometricFitConfig, 

40 GbdesGlobalAstrometricFitTask, 

41) 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43from lsst.meas.algorithms.testUtils import MockRefcatDataId 

44from lsst.pipe.base import InMemoryDatasetHandle 

45from smatch.matcher import Matcher 

46 

47TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

48 

49 

50class TestGbdesAstrometricFit(lsst.utils.tests.TestCase): 

51 """This class tests `GbdesAstrometricFit` using real `visitSummaryTable`s 

52 from HSC RC2 processing, with simulated sources for those visits. 

53 """ 

54 

55 @classmethod 

56 def setUpClass(cls): 

57 # Set random seed 

58 np.random.seed(1234) 

59 

60 # Make fake data 

61 cls.datadir = os.path.join(TESTDIR, "data") 

62 

63 cls.fieldNumber = 0 

64 cls.nFields = 1 

65 cls.instrumentName = "HSC" 

66 cls.instrument = wcsfit.Instrument(cls.instrumentName) 

67 cls.refEpoch = 57205.5 

68 

69 # Make test inputVisitSummary. VisitSummaryTables are taken from 

70 # collection HSC/runs/RC2/w_2022_20/DM-34794 

71 cls.testVisits = [1176, 17900, 17930, 17934] 

72 cls.inputVisitSummary = [] 

73 for testVisit in cls.testVisits: 

74 visSum = afwTable.ExposureCatalog.readFits( 

75 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits") 

76 ) 

77 cls.inputVisitSummary.append(visSum) 

78 

79 cls.config = GbdesAstrometricFitConfig() 

80 cls.config.systematicError = 0 

81 cls.config.devicePolyOrder = 4 

82 cls.config.exposurePolyOrder = 6 

83 cls.config.fitReserveFraction = 0 

84 cls.config.fitReserveRandomSeed = 1234 

85 cls.config.saveModelParams = True 

86 cls.config.allowSelfMatches = True 

87 cls.task = GbdesAstrometricFitTask(config=cls.config) 

88 

89 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info( 

90 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch 

91 ) 

92 

93 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky( 

94 cls.inputVisitSummary, cls.exposureInfo.medianEpoch 

95 ) 

96 

97 # Bounding box of observations: 

98 raMins, raMaxs = [], [] 

99 decMins, decMaxs = [], [] 

100 for visSum in cls.inputVisitSummary: 

101 raMins.append(visSum["raCorners"].min()) 

102 raMaxs.append(visSum["raCorners"].max()) 

103 decMins.append(visSum["decCorners"].min()) 

104 decMaxs.append(visSum["decCorners"].max()) 

105 raMin = min(raMins) 

106 raMax = max(raMaxs) 

107 decMin = min(decMins) 

108 decMax = max(decMaxs) 

109 

110 corners = [ 

111 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(), 

112 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(), 

113 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(), 

114 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(), 

115 ] 

116 cls.boundingPolygon = sphgeom.ConvexPolygon(corners) 

117 

118 # Make random set of data in a bounding box determined by input visits 

119 # Make wcs objects for the "true" model 

120 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars( 

121 raMin, raMax, decMin, decMax, cls.config.matchRadius 

122 ) 

123 

124 # Fraction of simulated stars in the reference catalog and science 

125 # exposures 

126 inReferenceFraction = 1 

127 inScienceFraction = 1 

128 

129 # Make a reference catalog and load it into ReferenceObjectLoader 

130 refDataId, deferredRefCat = cls._make_refCat( 

131 starIds, starRAs, starDecs, inReferenceFraction, cls.boundingPolygon 

132 ) 

133 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat]) 

134 cls.refObjectLoader.config.requireProperMotion = False 

135 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter" 

136 

137 cls.task.refObjectLoader = cls.refObjectLoader 

138 

139 # Get True WCS for stars: 

140 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f: 

141 cls.trueModel = yaml.load(f, Loader=yaml.Loader) 

142 

143 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary) 

144 

145 # Make source catalogs: 

146 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction) 

147 

148 cls.outputs = cls.task.run( 

149 cls.inputCatalogRefs, 

150 cls.inputVisitSummary, 

151 instrumentName=cls.instrumentName, 

152 refEpoch=cls.refEpoch, 

153 refObjectLoader=cls.refObjectLoader, 

154 ) 

155 

156 @staticmethod 

157 def _make_simulated_stars(raMin, raMax, decMin, decMax, matchRadius, nStars=10000): 

158 """Generate random positions for "stars" in an RA/Dec box. 

159 

160 Parameters 

161 ---------- 

162 raMin : `float` 

163 Minimum RA for simulated stars. 

164 raMax : `float` 

165 Maximum RA for simulated stars. 

166 decMin : `float` 

167 Minimum Dec for simulated stars. 

168 decMax : `float` 

169 Maximum Dec for simulated stars. 

170 matchRadius : `float` 

171 Minimum allowed distance in arcsec between stars. 

172 nStars : `int`, optional 

173 Number of stars to simulate. Final number will be lower if any 

174 too-close stars are dropped. 

175 

176 Returns 

177 ------- 

178 nStars : `int` 

179 Number of stars simulated. 

180 starIds: `np.ndarray` 

181 Unique identification number for stars. 

182 starRAs: `np.ndarray` 

183 Simulated Right Ascensions. 

184 starDecs: `np.ndarray` 

185 Simulated Declination. 

186 """ 

187 starIds = np.arange(nStars) 

188 starRAs = np.random.random(nStars) * (raMax - raMin) + raMin 

189 starDecs = np.random.random(nStars) * (decMax - decMin) + decMin 

190 # Remove neighbors: 

191 with Matcher(starRAs, starDecs) as matcher: 

192 idx = matcher.query_groups(matchRadius / 3600.0, min_match=2) 

193 if len(idx) > 0: 

194 neighbors = np.unique(np.concatenate(idx)) 

195 starRAs = np.delete(starRAs, neighbors) 

196 starDecs = np.delete(starDecs, neighbors) 

197 nStars = len(starRAs) 

198 starIds = np.arange(nStars) 

199 

200 return nStars, starIds, starRAs, starDecs 

201 

202 @classmethod 

203 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction, bounds): 

204 """Make reference catalog from a subset of the simulated data 

205 

206 Parameters 

207 ---------- 

208 starIds : `np.ndarray` [`int`] 

209 Source ids for the simulated stars 

210 starRas : `np.ndarray` [`float`] 

211 RAs of the simulated stars 

212 starDecs : `np.ndarray` [`float`] 

213 Decs of the simulated stars 

214 inReferenceFraction : float 

215 Percentage of simulated stars to include in reference catalog 

216 bounds : `lsst.sphgeom.ConvexPolygon` 

217 Boundary of the reference catalog region 

218 

219 Returns 

220 ------- 

221 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId` 

222 Object that replicates the functionality of a dataId. 

223 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle` 

224 Dataset handle for reference catalog. 

225 """ 

226 nRefs = int(cls.nStars * inReferenceFraction) 

227 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False) 

228 # Make simpleCatalog to hold data, create datasetRef with `region` 

229 # determined by bounding box used in above simulate. 

230 refSchema = afwTable.SimpleTable.makeMinimalSchema() 

231 idKey = refSchema.addField("sourceId", type="I") 

232 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64) 

233 raErrKey = refSchema.addField("coord_raErr", units="rad", type=np.float64) 

234 decErrKey = refSchema.addField("coord_decErr", units="rad", type=np.float64) 

235 pmraErrKey = refSchema.addField("pm_raErr", units="rad2 / yr", type=np.float64) 

236 pmdecErrKey = refSchema.addField("pm_decErr", units="rad2 / yr", type=np.float64) 

237 refCat = afwTable.SimpleCatalog(refSchema) 

238 ref_md = PropertyList() 

239 ref_md.set("REFCAT_FORMAT_VERSION", 1) 

240 refCat.table.setMetadata(ref_md) 

241 for i in refStarIndices: 

242 record = refCat.addNew() 

243 record.set(idKey, starIds[i]) 

244 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees)) 

245 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees)) 

246 record.set(fluxKey, 1) 

247 record.set(raErrKey, 0.00001) 

248 record.set(decErrKey, 0.00001) 

249 record.set(pmraErrKey, 1e-9) 

250 record.set(pmdecErrKey, 1e-9) 

251 refDataId = MockRefcatDataId(bounds) 

252 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat") 

253 

254 return refDataId, deferredRefCat 

255 

256 @classmethod 

257 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction): 

258 """Make a `pd.DataFrame` catalog with the columns needed for the 

259 object selector. 

260 

261 Parameters 

262 ---------- 

263 starIds : `np.ndarray` [`int`] 

264 Source ids for the simulated stars 

265 starRas : `np.ndarray` [`float`] 

266 RAs of the simulated stars 

267 starDecs : `np.ndarray` [`float`] 

268 Decs of the simulated stars 

269 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`] 

270 WCS with which to simulate the source pixel coordinates 

271 inReferenceFraction : float 

272 Percentage of simulated stars to include in reference catalog 

273 

274 Returns 

275 ------- 

276 sourceCat : `list` [`lsst.pipe.base.InMemoryDatasetHandle`] 

277 List of reference to source catalogs. 

278 """ 

279 inputCatalogRefs = [] 

280 # Take a subset of the simulated data 

281 # Use true wcs objects to put simulated data into ccds 

282 bbox = lsst.geom.BoxD( 

283 lsst.geom.Point2D( 

284 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"] 

285 ), 

286 lsst.geom.Point2D( 

287 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"] 

288 ), 

289 ) 

290 bboxCorners = bbox.getCorners() 

291 cls.inputCatalogRefs = [] 

292 for v, visit in enumerate(cls.testVisits): 

293 nVisStars = int(cls.nStars * inScienceFraction) 

294 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False) 

295 visitStarIds = starIds[visitStarIndices] 

296 visitStarRas = starRas[visitStarIndices] 

297 visitStarDecs = starDecs[visitStarIndices] 

298 sourceCats = [] 

299 for detector in trueWCSs[visit]: 

300 detWcs = detector.getWcs() 

301 detectorId = detector["id"] 

302 radecCorners = detWcs.pixelToSky(bboxCorners) 

303 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners]) 

304 detectorIndices = detectorFootprint.contains( 

305 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian) 

306 ) 

307 nDetectorStars = detectorIndices.sum() 

308 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"] 

309 

310 ones_like = np.ones(nDetectorStars) 

311 zeros_like = np.zeros(nDetectorStars, dtype=bool) 

312 

313 x, y = detWcs.skyToPixelArray( 

314 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True 

315 ) 

316 

317 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs() 

318 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True) 

319 

320 sourceDict = {} 

321 sourceDict["detector"] = detectorArray 

322 sourceDict["sourceId"] = visitStarIds[detectorIndices] 

323 sourceDict["x"] = x 

324 sourceDict["y"] = y 

325 sourceDict["xErr"] = 1e-3 * ones_like 

326 sourceDict["yErr"] = 1e-3 * ones_like 

327 sourceDict["inputRA"] = inputRa 

328 sourceDict["inputDec"] = inputDec 

329 sourceDict["trueRA"] = visitStarRas[detectorIndices] 

330 sourceDict["trueDec"] = visitStarDecs[detectorIndices] 

331 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]: 

332 sourceDict[key] = ones_like 

333 for key in [ 

334 "pixelFlags_edge", 

335 "pixelFlags_saturated", 

336 "pixelFlags_interpolatedCenter", 

337 "pixelFlags_interpolated", 

338 "pixelFlags_crCenter", 

339 "pixelFlags_bad", 

340 "hsmPsfMoments_flag", 

341 "apFlux_12_0_flag", 

342 "extendedness", 

343 "sizeExtendedness", 

344 "parentSourceId", 

345 "deblend_nChild", 

346 "ixy", 

347 ]: 

348 sourceDict[key] = zeros_like 

349 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like 

350 sourceDict["detect_isPrimary"] = ones_like.astype(bool) 

351 

352 sourceCat = pd.DataFrame(sourceDict) 

353 sourceCats.append(sourceCat) 

354 

355 visitSourceTable = pd.concat(sourceCats) 

356 

357 inputCatalogRef = InMemoryDatasetHandle( 

358 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit} 

359 ) 

360 

361 inputCatalogRefs.append(inputCatalogRef) 

362 

363 return inputCatalogRefs 

364 

365 @classmethod 

366 def _make_wcs(cls, model, inputVisitSummaries): 

367 """Make a `lsst.afw.geom.SkyWcs` from given model parameters 

368 

369 Parameters 

370 ---------- 

371 model : `dict` 

372 Dictionary with WCS model parameters 

373 inputVisitSummaries : `list` [`lsst.afw.table.ExposureCatalog`] 

374 Visit summary catalogs 

375 Returns 

376 ------- 

377 catalogs : `dict` [`int`, `lsst.afw.table.ExposureCatalog`] 

378 Visit summary catalogs with WCS set to input model 

379 """ 

380 

381 # Pixels will need to be rescaled before going into the mappings 

382 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"] 

383 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"] 

384 

385 catalogs = {} 

386 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

387 schema.addField("visit", type="L", doc="Visit number") 

388 for visitSum in inputVisitSummaries: 

389 visit = visitSum[0]["visit"] 

390 visitMapName = f"{visit}/poly" 

391 visitModel = model[visitMapName] 

392 

393 catalog = lsst.afw.table.ExposureCatalog(schema) 

394 catalog.resize(len(visitSum)) 

395 catalog["visit"] = visit 

396 

397 raDec = visitSum[0].getVisitInfo().getBoresightRaDec() 

398 

399 visitMapType = visitModel["Type"] 

400 visitDict = {"Type": visitMapType} 

401 if visitMapType == "Poly": 

402 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"] 

403 visitDict["Coefficients"] = mapCoefficients 

404 

405 for d, detector in enumerate(visitSum): 

406 detectorId = detector["id"] 

407 detectorMapName = f"HSC/{detectorId}/poly" 

408 detectorModel = model[detectorMapName] 

409 

410 detectorMapType = detectorModel["Type"] 

411 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict} 

412 if detectorMapType == "Poly": 

413 mapCoefficients = ( 

414 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"] 

415 ) 

416 mapDict[detectorMapName]["Coefficients"] = mapCoefficients 

417 

418 outWCS = cls.task._make_afw_wcs( 

419 mapDict, 

420 raDec.getRa(), 

421 raDec.getDec(), 

422 doNormalizePixels=True, 

423 xScale=xscale, 

424 yScale=yscale, 

425 ) 

426 catalog[d].setId(detectorId) 

427 catalog[d].setWcs(outWCS) 

428 

429 catalog.sort() 

430 catalogs[visit] = catalog 

431 

432 return catalogs 

433 

434 def test_get_exposure_info(self): 

435 """Test that information for input exposures is as expected and that 

436 the WCS in the class object gives approximately the same results as the 

437 input `lsst.afw.geom.SkyWcs`. 

438 """ 

439 

440 # The total number of extensions is the number of detectors for each 

441 # visit plus one for the reference catalog for each field. 

442 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + self.nFields 

443 

444 self.assertEqual(totalExtensions, len(self.extensionInfo.visit)) 

445 

446 taskVisits = set(self.extensionInfo.visit) 

447 refVisits = np.arange(0, -1 * self.nFields, -1).tolist() 

448 self.assertEqual(taskVisits, set(self.testVisits + refVisits)) 

449 

450 xx = np.linspace(0, 2000, 3) 

451 yy = np.linspace(0, 4000, 6) 

452 xgrid, ygrid = np.meshgrid(xx, yy) 

453 for visSum in self.inputVisitSummary: 

454 visit = visSum[0]["visit"] 

455 for detectorInfo in visSum: 

456 detector = detectorInfo["id"] 

457 extensionIndex = np.flatnonzero( 

458 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector) 

459 )[0] 

460 fitWcs = self.extensionInfo.wcs[extensionIndex] 

461 calexpWcs = detectorInfo.getWcs() 

462 

463 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())]) 

464 

465 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True) 

466 

467 # The pixel origin maps to a position slightly off from the 

468 # tangent plane origin, so we want to use this value for the 

469 # tangent-plane-to-sky part of the mapping. 

470 tangentPlaneCenter = fitWcs.toWorld( 

471 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY() 

472 ) 

473 tangentPlaneOrigin = lsst.geom.Point2D(tangentPlaneCenter) 

474 skyOrigin = calexpWcs.pixelToSky( 

475 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY() 

476 ) 

477 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0.0 * lsst.geom.degrees, True) 

478 iwcToSkyWcs = afwgeom.makeSkyWcs(tangentPlaneOrigin, skyOrigin, cdMatrix) 

479 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray( 

480 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True 

481 ) 

482 

483 np.testing.assert_allclose(calexpra, newRAdeg) 

484 np.testing.assert_allclose(calexpdec, newDecdeg) 

485 

486 def test_refCatLoader(self): 

487 """Test that we can load objects from refCat""" 

488 

489 tmpAssociations = wcsfit.FoFClass( 

490 self.fields, 

491 [self.instrument], 

492 self.exposuresHelper, 

493 [self.fieldRadius.asDegrees()], 

494 (self.task.config.matchRadius * u.arcsec).to(u.degree).value, 

495 ) 

496 

497 self.task._load_refcat( 

498 self.refObjectLoader, 

499 self.extensionInfo, 

500 associations=tmpAssociations, 

501 center=self.fieldCenter, 

502 radius=self.fieldRadius, 

503 epoch=self.exposureInfo.medianEpoch, 

504 ) 

505 

506 # We have only loaded one catalog, so getting the 'matches' should just 

507 # return the same objects we put in, except some random objects that 

508 # are too close together. 

509 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1) 

510 

511 nMatches = (np.array(tmpAssociations.sequence) == 0).sum() 

512 

513 self.assertLessEqual(nMatches, self.nStars) 

514 self.assertGreater(nMatches, self.nStars * 0.9) 

515 

516 def test_loading_and_association(self): 

517 """Test that objects can be loaded and correctly associated.""" 

518 # Running `_load_catalogs_and_associate` changes the input WCSs, so 

519 # recalculate them here so that the variables shared among tests are 

520 # not affected. 

521 instrument = wcsfit.Instrument(self.instrumentName) 

522 _, exposuresHelper, extensionInfo = self.task._get_exposure_info( 

523 self.inputVisitSummary, instrument, refEpoch=self.refEpoch 

524 ) 

525 

526 tmpAssociations = wcsfit.FoFClass( 

527 self.fields, 

528 [self.instrument], 

529 exposuresHelper, 

530 [self.fieldRadius.asDegrees()], 

531 (self.task.config.matchRadius * u.arcsec).to(u.degree).value, 

532 ) 

533 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, extensionInfo) 

534 

535 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2) 

536 

537 matchIds = [] 

538 correctMatches = [] 

539 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj): 

540 objVisitInd = extensionInfo.visitIndex[e] 

541 objDet = extensionInfo.detector[e] 

542 extnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet 

543 objInfo = self.inputCatalogRefs[objVisitInd].get()[extnInds].iloc[o] 

544 if s == 0: 

545 if len(matchIds) > 0: 

546 correctMatches.append(len(set(matchIds)) == 1) 

547 matchIds = [] 

548 

549 matchIds.append(objInfo["sourceId"]) 

550 

551 # A few matches may incorrectly associate sources because of the random 

552 # positions 

553 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95) 

554 

555 def test_make_outputs(self): 

556 """Test that the run method recovers the input model parameters.""" 

557 for v, visit in enumerate(self.testVisits): 

558 visitSummary = self.inputVisitSummary[v] 

559 outputWcsCatalog = self.outputs.outputWcss[visit] 

560 visitSources = self.inputCatalogRefs[v].get() 

561 for d, detectorRow in enumerate(visitSummary): 

562 detectorId = detectorRow["id"] 

563 fitwcs = outputWcsCatalog[d].getWcs() 

564 detSources = visitSources[visitSources["detector"] == detectorId] 

565 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True) 

566 dRA = fitRA - detSources["trueRA"] 

567 dDec = fitDec - detSources["trueDec"] 

568 # Check that input coordinates match the output coordinates 

569 self.assertAlmostEqual(np.mean(dRA), 0) 

570 self.assertAlmostEqual(np.std(dRA), 0) 

571 self.assertAlmostEqual(np.mean(dDec), 0) 

572 self.assertAlmostEqual(np.std(dDec), 0) 

573 

574 def test_compute_model_params(self): 

575 """Test the optional model parameters and covariance output.""" 

576 modelParams = pd.DataFrame(self.outputs.modelParams) 

577 # Check that DataFrame is the expected size. 

578 shape = modelParams.shape 

579 self.assertEqual(shape[0] + 4, shape[1]) 

580 # Check that covariance matrix is symmetric. 

581 covariance = (modelParams.iloc[:, 4:]).to_numpy() 

582 np.testing.assert_allclose(covariance, covariance.T, atol=1e-18) 

583 

584 def test_run(self): 

585 """Test that run method recovers the input model parameters""" 

586 outputMaps = self.outputs.fitModel.mapCollection.getParamDict() 

587 

588 for v, visit in enumerate(self.testVisits): 

589 visitSummary = self.inputVisitSummary[v] 

590 visitMapName = f"{visit}/poly" 

591 

592 origModel = self.trueModel[visitMapName] 

593 if origModel["Type"] != "Identity": 

594 fitModel = outputMaps[visitMapName] 

595 origXPoly = origModel["XPoly"]["Coefficients"] 

596 origYPoly = origModel["YPoly"]["Coefficients"] 

597 fitXPoly = fitModel[: len(origXPoly)] 

598 fitYPoly = fitModel[len(origXPoly) :] 

599 

600 absDiffX = abs(fitXPoly - origXPoly) 

601 absDiffY = abs(fitYPoly - origYPoly) 

602 # Check that input visit model matches fit 

603 np.testing.assert_array_less(absDiffX, 1e-6) 

604 np.testing.assert_array_less(absDiffY, 1e-6) 

605 for d, detectorRow in enumerate(visitSummary): 

606 detectorId = detectorRow["id"] 

607 detectorMapName = f"HSC/{detectorId}/poly" 

608 origModel = self.trueModel[detectorMapName] 

609 if (origModel["Type"] != "Identity") and (v == 0): 

610 fitModel = outputMaps[detectorMapName] 

611 origXPoly = origModel["XPoly"]["Coefficients"] 

612 origYPoly = origModel["YPoly"]["Coefficients"] 

613 fitXPoly = fitModel[: len(origXPoly)] 

614 fitYPoly = fitModel[len(origXPoly) :] 

615 absDiffX = abs(fitXPoly - origXPoly) 

616 absDiffY = abs(fitYPoly - origYPoly) 

617 # Check that input detector model matches fit 

618 np.testing.assert_array_less(absDiffX, 1e-7) 

619 np.testing.assert_array_less(absDiffY, 1e-7) 

620 

621 def test_missingWcs(self): 

622 """Test that task does not fail when the input WCS is None for one 

623 extension and that the fit WCS for that extension returns a finite 

624 result. 

625 """ 

626 inputVisitSummary = self.inputVisitSummary.copy() 

627 # Set one WCS to be None 

628 testVisit = 0 

629 testDetector = 20 

630 inputVisitSummary[testVisit][testDetector].setWcs(None) 

631 

632 outputs = self.task.run( 

633 self.inputCatalogRefs, 

634 inputVisitSummary, 

635 instrumentName=self.instrumentName, 

636 refEpoch=self.refEpoch, 

637 refObjectLoader=self.refObjectLoader, 

638 ) 

639 

640 # Check that the fit WCS for the extension with input WCS=None returns 

641 # finite sky values. 

642 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs() 

643 testSky = testWcs.pixelToSky(0, 0) 

644 self.assertTrue(testSky.isFinite()) 

645 

646 

647class TestGbdesGlobalAstrometricFit(TestGbdesAstrometricFit): 

648 @classmethod 

649 def setUpClass(cls): 

650 # Set random seed 

651 np.random.seed(1234) 

652 

653 # Fraction of simulated stars in the reference catalog and science 

654 # exposures 

655 inReferenceFraction = 1 

656 inScienceFraction = 1 

657 

658 # Make fake data 

659 cls.datadir = os.path.join(TESTDIR, "data") 

660 

661 cls.nFields = 2 

662 cls.instrumentName = "HSC" 

663 cls.instrument = wcsfit.Instrument(cls.instrumentName) 

664 cls.refEpoch = 57205.5 

665 

666 # Make test inputVisitSummary. VisitSummaryTables are taken from 

667 # collection HSC/runs/RC2/w_2022_20/DM-34794 

668 cls.testVisits = [1176, 17900, 17930, 17934, 36434, 36460, 36494, 36446] 

669 cls.inputVisitSummary = [] 

670 for testVisit in cls.testVisits: 

671 visSum = afwTable.ExposureCatalog.readFits( 

672 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits") 

673 ) 

674 cls.inputVisitSummary.append(visSum) 

675 

676 cls.config = GbdesGlobalAstrometricFitConfig() 

677 cls.config.systematicError = 0 

678 cls.config.devicePolyOrder = 4 

679 cls.config.exposurePolyOrder = 6 

680 cls.config.fitReserveFraction = 0 

681 cls.config.fitReserveRandomSeed = 1234 

682 cls.config.saveModelParams = True 

683 cls.task = GbdesGlobalAstrometricFitTask(config=cls.config) 

684 

685 cls.fields, cls.fieldRegions = cls.task._prep_sky(cls.inputVisitSummary) 

686 

687 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info( 

688 cls.inputVisitSummary, cls.instrument, fieldRegions=cls.fieldRegions 

689 ) 

690 

691 refDataIds, deferredRefCats = [], [] 

692 allStarIds = [] 

693 allStarRAs = [] 

694 allStarDecs = [] 

695 for region in cls.fieldRegions.values(): 

696 # Bounding box of observations: 

697 bbox = region.getBoundingBox() 

698 raMin = bbox.getLon().getA().asDegrees() 

699 raMax = bbox.getLon().getB().asDegrees() 

700 decMin = bbox.getLat().getA().asDegrees() 

701 decMax = bbox.getLat().getB().asDegrees() 

702 

703 # Make random set of data in a bounding box determined by input 

704 # visits 

705 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars( 

706 raMin, raMax, decMin, decMax, cls.config.matchRadius 

707 ) 

708 

709 allStarIds.append(starIds) 

710 allStarRAs.append(starRAs) 

711 allStarDecs.append(starDecs) 

712 

713 corners = [ 

714 lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees).getVector() 

715 for (ra, dec) in zip([raMin, raMax, raMax, raMin], [decMin, decMin, decMax, decMax]) 

716 ] 

717 conv_box = lsst.sphgeom.ConvexPolygon.convexHull(corners) 

718 # Make a reference catalog that will be loaded into 

719 # ReferenceObjectLoader 

720 refDataId, deferredRefCat = cls._make_refCat( 

721 starIds, starRAs, starDecs, inReferenceFraction, conv_box 

722 ) 

723 refDataIds.append(refDataId) 

724 deferredRefCats.append(deferredRefCat) 

725 

726 cls.refObjectLoader = ReferenceObjectLoader(refDataIds, deferredRefCats) 

727 cls.refObjectLoader.config.requireProperMotion = False 

728 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter" 

729 

730 cls.task.refObjectLoader = cls.refObjectLoader 

731 

732 allRefObjects, allRefCovariances = {}, {} 

733 for f, fieldRegion in cls.fieldRegions.items(): 

734 refObjects, refCovariance = cls.task._load_refcat( 

735 cls.refObjectLoader, cls.extensionInfo, epoch=cls.exposureInfo.medianEpoch, region=fieldRegion 

736 ) 

737 allRefObjects[f] = refObjects 

738 allRefCovariances[f] = refCovariance 

739 cls.refObjects = allRefObjects 

740 

741 # Get True WCS for stars: 

742 with open(os.path.join(cls.datadir, "sample_global_wcs.yaml"), "r") as f: 

743 cls.trueModel = yaml.load(f, Loader=yaml.Loader) 

744 

745 cls.trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary) 

746 

747 cls.isolatedStarCatalogs, cls.isolatedStarSources = cls._make_isolatedStars( 

748 allStarIds, allStarRAs, allStarDecs, cls.trueWCSs, inScienceFraction 

749 ) 

750 

751 cls.outputs = cls.task.run( 

752 cls.inputVisitSummary, 

753 cls.isolatedStarSources, 

754 cls.isolatedStarCatalogs, 

755 instrumentName=cls.instrumentName, 

756 refEpoch=cls.refEpoch, 

757 refObjectLoader=cls.refObjectLoader, 

758 ) 

759 

760 @classmethod 

761 def _make_isolatedStars(cls, allStarIds, allStarRAs, allStarDecs, trueWCSs, inScienceFraction): 

762 """Given a subset of the simulated data, make source catalogs and star 

763 catalogs. 

764 

765 This takes the true WCSs to go from the RA and Decs of the simulated 

766 stars to pixel coordinates for a given visit and detector. If those 

767 pixel coordinates are within the bounding box of the detector, the 

768 source and visit information is put in the corresponding catalog of 

769 isolated star sources. 

770 

771 Parameters 

772 ---------- 

773 allStarIds : `np.ndarray` [`int`] 

774 Source ids for the simulated stars 

775 allStarRas : `np.ndarray` [`float`] 

776 RAs of the simulated stars 

777 allStarDecs : `np.ndarray` [`float`] 

778 Decs of the simulated stars 

779 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`] 

780 WCS with which to simulate the source pixel coordinates 

781 inReferenceFraction : float 

782 Percentage of simulated stars to include in reference catalog 

783 

784 Returns 

785 ------- 

786 isolatedStarCatalogRefs : `list` 

787 [`lsst.pipe.base.InMemoryDatasetHandle`] 

788 List of references to isolated star catalogs. 

789 isolatedStarSourceRefs : `list` 

790 [`lsst.pipe.base.InMemoryDatasetHandle`] 

791 List of references to isolated star sources. 

792 """ 

793 bbox = lsst.geom.BoxD( 

794 lsst.geom.Point2D( 

795 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"] 

796 ), 

797 lsst.geom.Point2D( 

798 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"] 

799 ), 

800 ) 

801 bboxCorners = bbox.getCorners() 

802 

803 isolatedStarCatalogRefs = [] 

804 isolatedStarSourceRefs = [] 

805 for i in range(len(allStarIds)): 

806 starIds = allStarIds[i] 

807 starRAs = allStarRAs[i] 

808 starDecs = allStarDecs[i] 

809 isolatedStarCatalog = pd.DataFrame({"ra": starRAs, "dec": starDecs}, index=starIds) 

810 isolatedStarCatalogRefs.append( 

811 InMemoryDatasetHandle(isolatedStarCatalog, storageClass="DataFrame", dataId={"tract": 0}) 

812 ) 

813 sourceCats = [] 

814 for v, visit in enumerate(cls.testVisits): 

815 nVisStars = int(cls.nStars * inScienceFraction) 

816 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False) 

817 visitStarIds = starIds[visitStarIndices] 

818 visitStarRas = starRAs[visitStarIndices] 

819 visitStarDecs = starDecs[visitStarIndices] 

820 for detector in trueWCSs[visit]: 

821 detWcs = detector.getWcs() 

822 detectorId = detector["id"] 

823 radecCorners = detWcs.pixelToSky(bboxCorners) 

824 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners]) 

825 detectorIndices = detectorFootprint.contains( 

826 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian) 

827 ) 

828 nDetectorStars = detectorIndices.sum() 

829 detectorArray = np.ones(nDetectorStars, dtype=int) * detector["id"] 

830 visitArray = np.ones(nDetectorStars, dtype=int) * visit 

831 

832 ones_like = np.ones(nDetectorStars) 

833 zeros_like = np.zeros(nDetectorStars, dtype=bool) 

834 

835 x, y = detWcs.skyToPixelArray( 

836 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True 

837 ) 

838 

839 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[ 

840 0 

841 ].getWcs() 

842 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True) 

843 

844 sourceDict = {} 

845 sourceDict["detector"] = detectorArray 

846 sourceDict["visit"] = visitArray 

847 sourceDict["obj_index"] = visitStarIds[detectorIndices] 

848 sourceDict["x"] = x 

849 sourceDict["y"] = y 

850 sourceDict["xErr"] = 1e-3 * ones_like 

851 sourceDict["yErr"] = 1e-3 * ones_like 

852 sourceDict["inputRA"] = inputRa 

853 sourceDict["inputDec"] = inputDec 

854 sourceDict["trueRA"] = visitStarRas[detectorIndices] 

855 sourceDict["trueDec"] = visitStarDecs[detectorIndices] 

856 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]: 

857 sourceDict[key] = ones_like 

858 for key in [ 

859 "pixelFlags_edge", 

860 "pixelFlags_saturated", 

861 "pixelFlags_interpolatedCenter", 

862 "pixelFlags_interpolated", 

863 "pixelFlags_crCenter", 

864 "pixelFlags_bad", 

865 "hsmPsfMoments_flag", 

866 "apFlux_12_0_flag", 

867 "extendedness", 

868 "parentSourceId", 

869 "deblend_nChild", 

870 "ixy", 

871 ]: 

872 sourceDict[key] = zeros_like 

873 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like 

874 sourceDict["detect_isPrimary"] = ones_like.astype(bool) 

875 

876 sourceCat = pd.DataFrame(sourceDict) 

877 sourceCats.append(sourceCat) 

878 

879 isolatedStarSourceTable = pd.concat(sourceCats, ignore_index=True) 

880 isolatedStarSourceTable = isolatedStarSourceTable.sort_values(by=["obj_index"]) 

881 isolatedStarSourceRefs.append( 

882 InMemoryDatasetHandle(isolatedStarSourceTable, storageClass="DataFrame", dataId={"tract": 0}) 

883 ) 

884 

885 return isolatedStarCatalogRefs, isolatedStarSourceRefs 

886 

887 def test_loading_and_association(self): 

888 """Test that associated objects actually correspond to the same 

889 simulated object.""" 

890 associations, sourceDict = self.task._associate_from_isolated_sources( 

891 self.isolatedStarSources, self.isolatedStarCatalogs, self.extensionInfo, self.refObjects 

892 ) 

893 

894 object_groups = np.flatnonzero(np.array(associations.sequence) == 0) 

895 for i in range(len(object_groups) - 1)[:10]: 

896 ras, decs = [], [] 

897 for ind in np.arange(object_groups[i], object_groups[i + 1]): 

898 visit = self.extensionInfo.visit[associations.extn[ind]] 

899 detectorInd = self.extensionInfo.detectorIndex[associations.extn[ind]] 

900 detector = self.extensionInfo.detector[associations.extn[ind]] 

901 if detectorInd == -1: 

902 ra = self.refObjects[visit * -1]["ra"][associations.obj[ind]] 

903 dec = self.refObjects[visit * -1]["dec"][associations.obj[ind]] 

904 ras.append(ra) 

905 decs.append(dec) 

906 else: 

907 x = sourceDict[visit][detector]["x"][associations.obj[ind]] 

908 y = sourceDict[visit][detector]["y"][associations.obj[ind]] 

909 ra, dec = self.trueWCSs[visit][detectorInd].getWcs().pixelToSky(x, y) 

910 ras.append(ra.asDegrees()) 

911 decs.append(dec.asDegrees()) 

912 np.testing.assert_allclose(ras, ras[0]) 

913 np.testing.assert_allclose(decs, decs[0]) 

914 

915 def test_refCatLoader(self): 

916 """Test loading objects from the refCat in each of the fields.""" 

917 

918 for region in self.fieldRegions.values(): 

919 refCat, refCov = self.task._load_refcat( 

920 self.refObjectLoader, self.extensionInfo, region=region, epoch=self.exposureInfo.medianEpoch 

921 ) 

922 assert len(refCat) > 0 

923 

924 def test_make_outputs(self): 

925 """Test that the run method recovers the input model parameters.""" 

926 for isolatedStarSourceRef in self.isolatedStarSources: 

927 iss = isolatedStarSourceRef.get() 

928 visits = np.unique(iss["visit"]) 

929 for v, visit in enumerate(visits): 

930 outputWcsCatalog = self.outputs.outputWcss[visit] 

931 visitSources = iss[iss["visit"] == visit] 

932 detectors = outputWcsCatalog["id"] 

933 for d, detectorId in enumerate(detectors): 

934 fitwcs = outputWcsCatalog[d].getWcs() 

935 detSources = visitSources[visitSources["detector"] == detectorId] 

936 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True) 

937 dRA = fitRA - detSources["trueRA"] 

938 dDec = fitDec - detSources["trueDec"] 

939 # Check that input coordinates match the output coordinates 

940 self.assertAlmostEqual(np.mean(dRA), 0) 

941 self.assertAlmostEqual(np.std(dRA), 0) 

942 self.assertAlmostEqual(np.mean(dDec), 0) 

943 self.assertAlmostEqual(np.std(dDec), 0) 

944 

945 def test_missingWcs(self): 

946 """Test that task does not fail when the input WCS is None for one 

947 extension and that the fit WCS for that extension returns a finite 

948 result. 

949 """ 

950 inputVisitSummary = self.inputVisitSummary.copy() 

951 # Set one WCS to be None 

952 testVisit = 0 

953 testDetector = 20 

954 inputVisitSummary[testVisit][testDetector].setWcs(None) 

955 

956 outputs = self.task.run( 

957 inputVisitSummary, 

958 self.isolatedStarSources, 

959 self.isolatedStarCatalogs, 

960 instrumentName=self.instrumentName, 

961 refEpoch=self.refEpoch, 

962 refObjectLoader=self.refObjectLoader, 

963 ) 

964 

965 # Check that the fit WCS for the extension with input WCS=None returns 

966 # finite sky values. 

967 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs() 

968 testSky = testWcs.pixelToSky(0, 0) 

969 self.assertTrue(testSky.isFinite()) 

970 

971 

972def setup_module(module): 

973 lsst.utils.tests.init() 

974 

975 

976if __name__ == "__main__": 976 ↛ 977line 976 didn't jump to line 977, because the condition on line 976 was never true

977 lsst.utils.tests.init() 

978 unittest.main()