Coverage for tests/test_gbdesAstrometricFit.py: 11%

484 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-18 11:45 +0000

1# This file is part of drp_tasks 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os.path 

23import unittest 

24 

25import astropy.units as u 

26import lsst.afw.geom as afwgeom 

27import lsst.afw.table as afwTable 

28import lsst.geom 

29import lsst.utils 

30import numpy as np 

31import pandas as pd 

32import wcsfit 

33import yaml 

34from lsst import sphgeom 

35from lsst.daf.base import PropertyList 

36from lsst.drp.tasks.gbdesAstrometricFit import ( 

37 GbdesAstrometricFitConfig, 

38 GbdesAstrometricFitTask, 

39 GbdesGlobalAstrometricFitConfig, 

40 GbdesGlobalAstrometricFitTask, 

41) 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43from lsst.meas.algorithms.testUtils import MockRefcatDataId 

44from lsst.pipe.base import InMemoryDatasetHandle 

45from smatch.matcher import Matcher 

46 

47TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

48 

49 

50class TestGbdesAstrometricFit(lsst.utils.tests.TestCase): 

51 """This class tests `GbdesAstrometricFit` using real `visitSummaryTable`s 

52 from HSC RC2 processing, with simulated sources for those visits. 

53 """ 

54 

55 @classmethod 

56 def setUpClass(cls): 

57 # Set random seed 

58 np.random.seed(1234) 

59 

60 # Make fake data 

61 cls.datadir = os.path.join(TESTDIR, "data") 

62 

63 cls.fieldNumber = 0 

64 cls.nFields = 1 

65 cls.instrumentName = "HSC" 

66 cls.instrument = wcsfit.Instrument(cls.instrumentName) 

67 cls.refEpoch = 57205.5 

68 

69 # Make test inputVisitSummary. VisitSummaryTables are taken from 

70 # collection HSC/runs/RC2/w_2022_20/DM-34794 

71 cls.testVisits = [1176, 17900, 17930, 17934] 

72 cls.inputVisitSummary = [] 

73 for testVisit in cls.testVisits: 

74 visSum = afwTable.ExposureCatalog.readFits( 

75 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits") 

76 ) 

77 cls.inputVisitSummary.append(visSum) 

78 

79 cls.config = GbdesAstrometricFitConfig() 

80 cls.config.systematicError = 0 

81 cls.config.devicePolyOrder = 4 

82 cls.config.exposurePolyOrder = 6 

83 cls.config.fitReserveFraction = 0 

84 cls.config.fitReserveRandomSeed = 1234 

85 cls.config.saveModelParams = True 

86 cls.config.allowSelfMatches = True 

87 cls.task = GbdesAstrometricFitTask(config=cls.config) 

88 

89 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info( 

90 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch 

91 ) 

92 

93 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky( 

94 cls.inputVisitSummary, cls.exposureInfo.medianEpoch 

95 ) 

96 

97 # Bounding box of observations: 

98 raMins, raMaxs = [], [] 

99 decMins, decMaxs = [], [] 

100 for visSum in cls.inputVisitSummary: 

101 raMins.append(visSum["raCorners"].min()) 

102 raMaxs.append(visSum["raCorners"].max()) 

103 decMins.append(visSum["decCorners"].min()) 

104 decMaxs.append(visSum["decCorners"].max()) 

105 raMin = min(raMins) 

106 raMax = max(raMaxs) 

107 decMin = min(decMins) 

108 decMax = max(decMaxs) 

109 

110 corners = [ 

111 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(), 

112 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(), 

113 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(), 

114 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(), 

115 ] 

116 cls.boundingPolygon = sphgeom.ConvexPolygon(corners) 

117 

118 # Make random set of data in a bounding box determined by input visits 

119 # Make wcs objects for the "true" model 

120 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars( 

121 raMin, raMax, decMin, decMax, cls.config.matchRadius 

122 ) 

123 

124 # Fraction of simulated stars in the reference catalog and science 

125 # exposures 

126 inReferenceFraction = 1 

127 inScienceFraction = 1 

128 

129 # Make a reference catalog and load it into ReferenceObjectLoader 

130 refDataId, deferredRefCat = cls._make_refCat( 

131 starIds, starRAs, starDecs, inReferenceFraction, cls.boundingPolygon 

132 ) 

133 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat]) 

134 cls.refObjectLoader.config.requireProperMotion = False 

135 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter" 

136 

137 cls.task.refObjectLoader = cls.refObjectLoader 

138 

139 # Get True WCS for stars: 

140 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f: 

141 cls.trueModel = yaml.load(f, Loader=yaml.Loader) 

142 

143 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary) 

144 

145 # Make source catalogs: 

146 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction) 

147 

148 cls.outputs = cls.task.run( 

149 cls.inputCatalogRefs, 

150 cls.inputVisitSummary, 

151 instrumentName=cls.instrumentName, 

152 refEpoch=cls.refEpoch, 

153 refObjectLoader=cls.refObjectLoader, 

154 ) 

155 

156 @staticmethod 

157 def _make_simulated_stars(raMin, raMax, decMin, decMax, matchRadius, nStars=10000): 

158 """Generate random positions for "stars" in an RA/Dec box. 

159 

160 Parameters 

161 ---------- 

162 raMin : `float` 

163 Minimum RA for simulated stars. 

164 raMax : `float` 

165 Maximum RA for simulated stars. 

166 decMin : `float` 

167 Minimum Dec for simulated stars. 

168 decMax : `float` 

169 Maximum Dec for simulated stars. 

170 matchRadius : `float` 

171 Minimum allowed distance in arcsec between stars. 

172 nStars : `int`, optional 

173 Number of stars to simulate. Final number will be lower if any 

174 too-close stars are dropped. 

175 

176 Returns 

177 ------- 

178 nStars : `int` 

179 Number of stars simulated. 

180 starIds: `np.ndarray` 

181 Unique identification number for stars. 

182 starRAs: `np.ndarray` 

183 Simulated Right Ascensions. 

184 starDecs: `np.ndarray` 

185 Simulated Declination. 

186 """ 

187 starIds = np.arange(nStars) 

188 starRAs = np.random.random(nStars) * (raMax - raMin) + raMin 

189 starDecs = np.random.random(nStars) * (decMax - decMin) + decMin 

190 # Remove neighbors: 

191 with Matcher(starRAs, starDecs) as matcher: 

192 idx = matcher.query_groups(matchRadius / 3600.0, min_match=2) 

193 if len(idx) > 0: 

194 neighbors = np.unique(np.concatenate(idx)) 

195 starRAs = np.delete(starRAs, neighbors) 

196 starDecs = np.delete(starDecs, neighbors) 

197 nStars = len(starRAs) 

198 starIds = np.arange(nStars) 

199 

200 return nStars, starIds, starRAs, starDecs 

201 

202 @classmethod 

203 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction, bounds): 

204 """Make reference catalog from a subset of the simulated data 

205 

206 Parameters 

207 ---------- 

208 starIds : `np.ndarray` [`int`] 

209 Source ids for the simulated stars 

210 starRas : `np.ndarray` [`float`] 

211 RAs of the simulated stars 

212 starDecs : `np.ndarray` [`float`] 

213 Decs of the simulated stars 

214 inReferenceFraction : float 

215 Percentage of simulated stars to include in reference catalog 

216 bounds : `lsst.sphgeom.ConvexPolygon` 

217 Boundary of the reference catalog region 

218 

219 Returns 

220 ------- 

221 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId` 

222 Object that replicates the functionality of a dataId. 

223 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle` 

224 Dataset handle for reference catalog. 

225 """ 

226 nRefs = int(cls.nStars * inReferenceFraction) 

227 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False) 

228 # Make simpleCatalog to hold data, create datasetRef with `region` 

229 # determined by bounding box used in above simulate. 

230 refSchema = afwTable.SimpleTable.makeMinimalSchema() 

231 idKey = refSchema.addField("sourceId", type="I") 

232 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64) 

233 raErrKey = refSchema.addField("coord_raErr", units="rad", type=np.float64) 

234 decErrKey = refSchema.addField("coord_decErr", units="rad", type=np.float64) 

235 pmraErrKey = refSchema.addField("pm_raErr", units="rad2 / yr", type=np.float64) 

236 pmdecErrKey = refSchema.addField("pm_decErr", units="rad2 / yr", type=np.float64) 

237 refCat = afwTable.SimpleCatalog(refSchema) 

238 ref_md = PropertyList() 

239 ref_md.set("REFCAT_FORMAT_VERSION", 1) 

240 refCat.table.setMetadata(ref_md) 

241 for i in refStarIndices: 

242 record = refCat.addNew() 

243 record.set(idKey, starIds[i]) 

244 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees)) 

245 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees)) 

246 record.set(fluxKey, 1) 

247 record.set(raErrKey, 0.00001) 

248 record.set(decErrKey, 0.00001) 

249 record.set(pmraErrKey, 1e-9) 

250 record.set(pmdecErrKey, 1e-9) 

251 refDataId = MockRefcatDataId(bounds) 

252 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat") 

253 

254 return refDataId, deferredRefCat 

255 

256 @classmethod 

257 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction): 

258 """Make a `pd.DataFrame` catalog with the columns needed for the 

259 object selector. 

260 

261 Parameters 

262 ---------- 

263 starIds : `np.ndarray` [`int`] 

264 Source ids for the simulated stars 

265 starRas : `np.ndarray` [`float`] 

266 RAs of the simulated stars 

267 starDecs : `np.ndarray` [`float`] 

268 Decs of the simulated stars 

269 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`] 

270 WCS with which to simulate the source pixel coordinates 

271 inReferenceFraction : float 

272 Percentage of simulated stars to include in reference catalog 

273 

274 Returns 

275 ------- 

276 sourceCat : `list` [`lsst.pipe.base.InMemoryDatasetHandle`] 

277 List of reference to source catalogs. 

278 """ 

279 inputCatalogRefs = [] 

280 # Take a subset of the simulated data 

281 # Use true wcs objects to put simulated data into ccds 

282 bbox = lsst.geom.BoxD( 

283 lsst.geom.Point2D( 

284 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"] 

285 ), 

286 lsst.geom.Point2D( 

287 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"] 

288 ), 

289 ) 

290 bboxCorners = bbox.getCorners() 

291 cls.inputCatalogRefs = [] 

292 for v, visit in enumerate(cls.testVisits): 

293 nVisStars = int(cls.nStars * inScienceFraction) 

294 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False) 

295 visitStarIds = starIds[visitStarIndices] 

296 visitStarRas = starRas[visitStarIndices] 

297 visitStarDecs = starDecs[visitStarIndices] 

298 sourceCats = [] 

299 for detector in trueWCSs[visit]: 

300 detWcs = detector.getWcs() 

301 detectorId = detector["id"] 

302 radecCorners = detWcs.pixelToSky(bboxCorners) 

303 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners]) 

304 detectorIndices = detectorFootprint.contains( 

305 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian) 

306 ) 

307 nDetectorStars = detectorIndices.sum() 

308 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"] 

309 

310 ones_like = np.ones(nDetectorStars) 

311 zeros_like = np.zeros(nDetectorStars, dtype=bool) 

312 

313 x, y = detWcs.skyToPixelArray( 

314 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True 

315 ) 

316 

317 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs() 

318 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True) 

319 

320 sourceDict = {} 

321 sourceDict["detector"] = detectorArray 

322 sourceDict["sourceId"] = visitStarIds[detectorIndices] 

323 sourceDict["x"] = x 

324 sourceDict["y"] = y 

325 sourceDict["xErr"] = 1e-3 * ones_like 

326 sourceDict["yErr"] = 1e-3 * ones_like 

327 sourceDict["inputRA"] = inputRa 

328 sourceDict["inputDec"] = inputDec 

329 sourceDict["trueRA"] = visitStarRas[detectorIndices] 

330 sourceDict["trueDec"] = visitStarDecs[detectorIndices] 

331 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]: 

332 sourceDict[key] = ones_like 

333 for key in [ 

334 "pixelFlags_edge", 

335 "pixelFlags_saturated", 

336 "pixelFlags_interpolatedCenter", 

337 "pixelFlags_interpolated", 

338 "pixelFlags_crCenter", 

339 "pixelFlags_bad", 

340 "hsmPsfMoments_flag", 

341 "apFlux_12_0_flag", 

342 "extendedness", 

343 "parentSourceId", 

344 "deblend_nChild", 

345 "ixy", 

346 ]: 

347 sourceDict[key] = zeros_like 

348 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like 

349 sourceDict["detect_isPrimary"] = ones_like.astype(bool) 

350 

351 sourceCat = pd.DataFrame(sourceDict) 

352 sourceCats.append(sourceCat) 

353 

354 visitSourceTable = pd.concat(sourceCats) 

355 

356 inputCatalogRef = InMemoryDatasetHandle( 

357 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit} 

358 ) 

359 

360 inputCatalogRefs.append(inputCatalogRef) 

361 

362 return inputCatalogRefs 

363 

364 @classmethod 

365 def _make_wcs(cls, model, inputVisitSummaries): 

366 """Make a `lsst.afw.geom.SkyWcs` from given model parameters 

367 

368 Parameters 

369 ---------- 

370 model : `dict` 

371 Dictionary with WCS model parameters 

372 inputVisitSummaries : `list` [`lsst.afw.table.ExposureCatalog`] 

373 Visit summary catalogs 

374 Returns 

375 ------- 

376 catalogs : `dict` [`int`, `lsst.afw.table.ExposureCatalog`] 

377 Visit summary catalogs with WCS set to input model 

378 """ 

379 

380 # Pixels will need to be rescaled before going into the mappings 

381 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"] 

382 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"] 

383 

384 catalogs = {} 

385 schema = lsst.afw.table.ExposureTable.makeMinimalSchema() 

386 schema.addField("visit", type="L", doc="Visit number") 

387 for visitSum in inputVisitSummaries: 

388 visit = visitSum[0]["visit"] 

389 visitMapName = f"{visit}/poly" 

390 visitModel = model[visitMapName] 

391 

392 catalog = lsst.afw.table.ExposureCatalog(schema) 

393 catalog.resize(len(visitSum)) 

394 catalog["visit"] = visit 

395 

396 raDec = visitSum[0].getVisitInfo().getBoresightRaDec() 

397 

398 visitMapType = visitModel["Type"] 

399 visitDict = {"Type": visitMapType} 

400 if visitMapType == "Poly": 

401 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"] 

402 visitDict["Coefficients"] = mapCoefficients 

403 

404 for d, detector in enumerate(visitSum): 

405 detectorId = detector["id"] 

406 detectorMapName = f"HSC/{detectorId}/poly" 

407 detectorModel = model[detectorMapName] 

408 

409 detectorMapType = detectorModel["Type"] 

410 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict} 

411 if detectorMapType == "Poly": 

412 mapCoefficients = ( 

413 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"] 

414 ) 

415 mapDict[detectorMapName]["Coefficients"] = mapCoefficients 

416 

417 outWCS = cls.task._make_afw_wcs( 

418 mapDict, 

419 raDec.getRa(), 

420 raDec.getDec(), 

421 doNormalizePixels=True, 

422 xScale=xscale, 

423 yScale=yscale, 

424 ) 

425 catalog[d].setId(detectorId) 

426 catalog[d].setWcs(outWCS) 

427 

428 catalog.sort() 

429 catalogs[visit] = catalog 

430 

431 return catalogs 

432 

433 def test_get_exposure_info(self): 

434 """Test that information for input exposures is as expected and that 

435 the WCS in the class object gives approximately the same results as the 

436 input `lsst.afw.geom.SkyWcs`. 

437 """ 

438 

439 # The total number of extensions is the number of detectors for each 

440 # visit plus one for the reference catalog for each field. 

441 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + self.nFields 

442 

443 self.assertEqual(totalExtensions, len(self.extensionInfo.visit)) 

444 

445 taskVisits = set(self.extensionInfo.visit) 

446 refVisits = np.arange(0, -1 * self.nFields, -1).tolist() 

447 self.assertEqual(taskVisits, set(self.testVisits + refVisits)) 

448 

449 xx = np.linspace(0, 2000, 3) 

450 yy = np.linspace(0, 4000, 6) 

451 xgrid, ygrid = np.meshgrid(xx, yy) 

452 for visSum in self.inputVisitSummary: 

453 visit = visSum[0]["visit"] 

454 for detectorInfo in visSum: 

455 detector = detectorInfo["id"] 

456 extensionIndex = np.flatnonzero( 

457 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector) 

458 )[0] 

459 fitWcs = self.extensionInfo.wcs[extensionIndex] 

460 calexpWcs = detectorInfo.getWcs() 

461 

462 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())]) 

463 

464 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True) 

465 

466 # The pixel origin maps to a position slightly off from the 

467 # tangent plane origin, so we want to use this value for the 

468 # tangent-plane-to-sky part of the mapping. 

469 tangentPlaneCenter = fitWcs.toWorld( 

470 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY() 

471 ) 

472 tangentPlaneOrigin = lsst.geom.Point2D(tangentPlaneCenter) 

473 skyOrigin = calexpWcs.pixelToSky( 

474 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY() 

475 ) 

476 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0.0 * lsst.geom.degrees, True) 

477 iwcToSkyWcs = afwgeom.makeSkyWcs(tangentPlaneOrigin, skyOrigin, cdMatrix) 

478 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray( 

479 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True 

480 ) 

481 

482 np.testing.assert_allclose(calexpra, newRAdeg) 

483 np.testing.assert_allclose(calexpdec, newDecdeg) 

484 

485 def test_refCatLoader(self): 

486 """Test that we can load objects from refCat""" 

487 

488 tmpAssociations = wcsfit.FoFClass( 

489 self.fields, 

490 [self.instrument], 

491 self.exposuresHelper, 

492 [self.fieldRadius.asDegrees()], 

493 (self.task.config.matchRadius * u.arcsec).to(u.degree).value, 

494 ) 

495 

496 self.task._load_refcat( 

497 self.refObjectLoader, 

498 self.extensionInfo, 

499 associations=tmpAssociations, 

500 center=self.fieldCenter, 

501 radius=self.fieldRadius, 

502 epoch=self.exposureInfo.medianEpoch, 

503 ) 

504 

505 # We have only loaded one catalog, so getting the 'matches' should just 

506 # return the same objects we put in, except some random objects that 

507 # are too close together. 

508 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1) 

509 

510 nMatches = (np.array(tmpAssociations.sequence) == 0).sum() 

511 

512 self.assertLessEqual(nMatches, self.nStars) 

513 self.assertGreater(nMatches, self.nStars * 0.9) 

514 

515 def test_loading_and_association(self): 

516 """Test that objects can be loaded and correctly associated.""" 

517 # Running `_load_catalogs_and_associate` changes the input WCSs, so 

518 # recalculate them here so that the variables shared among tests are 

519 # not affected. 

520 instrument = wcsfit.Instrument(self.instrumentName) 

521 _, exposuresHelper, extensionInfo = self.task._get_exposure_info( 

522 self.inputVisitSummary, instrument, refEpoch=self.refEpoch 

523 ) 

524 

525 tmpAssociations = wcsfit.FoFClass( 

526 self.fields, 

527 [self.instrument], 

528 exposuresHelper, 

529 [self.fieldRadius.asDegrees()], 

530 (self.task.config.matchRadius * u.arcsec).to(u.degree).value, 

531 ) 

532 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, extensionInfo) 

533 

534 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2) 

535 

536 matchIds = [] 

537 correctMatches = [] 

538 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj): 

539 objVisitInd = extensionInfo.visitIndex[e] 

540 objDet = extensionInfo.detector[e] 

541 extnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet 

542 objInfo = self.inputCatalogRefs[objVisitInd].get()[extnInds].iloc[o] 

543 if s == 0: 

544 if len(matchIds) > 0: 

545 correctMatches.append(len(set(matchIds)) == 1) 

546 matchIds = [] 

547 

548 matchIds.append(objInfo["sourceId"]) 

549 

550 # A few matches may incorrectly associate sources because of the random 

551 # positions 

552 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95) 

553 

554 def test_make_outputs(self): 

555 """Test that the run method recovers the input model parameters.""" 

556 for v, visit in enumerate(self.testVisits): 

557 visitSummary = self.inputVisitSummary[v] 

558 outputWcsCatalog = self.outputs.outputWcss[visit] 

559 visitSources = self.inputCatalogRefs[v].get() 

560 for d, detectorRow in enumerate(visitSummary): 

561 detectorId = detectorRow["id"] 

562 fitwcs = outputWcsCatalog[d].getWcs() 

563 detSources = visitSources[visitSources["detector"] == detectorId] 

564 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True) 

565 dRA = fitRA - detSources["trueRA"] 

566 dDec = fitDec - detSources["trueDec"] 

567 # Check that input coordinates match the output coordinates 

568 self.assertAlmostEqual(np.mean(dRA), 0) 

569 self.assertAlmostEqual(np.std(dRA), 0) 

570 self.assertAlmostEqual(np.mean(dDec), 0) 

571 self.assertAlmostEqual(np.std(dDec), 0) 

572 

573 def test_compute_model_params(self): 

574 """Test the optional model parameters and covariance output.""" 

575 modelParams = pd.DataFrame(self.outputs.modelParams) 

576 # Check that DataFrame is the expected size. 

577 shape = modelParams.shape 

578 self.assertEqual(shape[0] + 4, shape[1]) 

579 # Check that covariance matrix is symmetric. 

580 covariance = (modelParams.iloc[:, 4:]).to_numpy() 

581 np.testing.assert_allclose(covariance, covariance.T, atol=1e-18) 

582 

583 def test_run(self): 

584 """Test that run method recovers the input model parameters""" 

585 outputMaps = self.outputs.fitModel.mapCollection.getParamDict() 

586 

587 for v, visit in enumerate(self.testVisits): 

588 visitSummary = self.inputVisitSummary[v] 

589 visitMapName = f"{visit}/poly" 

590 

591 origModel = self.trueModel[visitMapName] 

592 if origModel["Type"] != "Identity": 

593 fitModel = outputMaps[visitMapName] 

594 origXPoly = origModel["XPoly"]["Coefficients"] 

595 origYPoly = origModel["YPoly"]["Coefficients"] 

596 fitXPoly = fitModel[: len(origXPoly)] 

597 fitYPoly = fitModel[len(origXPoly) :] 

598 

599 absDiffX = abs(fitXPoly - origXPoly) 

600 absDiffY = abs(fitYPoly - origYPoly) 

601 # Check that input visit model matches fit 

602 np.testing.assert_array_less(absDiffX, 1e-6) 

603 np.testing.assert_array_less(absDiffY, 1e-6) 

604 for d, detectorRow in enumerate(visitSummary): 

605 detectorId = detectorRow["id"] 

606 detectorMapName = f"HSC/{detectorId}/poly" 

607 origModel = self.trueModel[detectorMapName] 

608 if (origModel["Type"] != "Identity") and (v == 0): 

609 fitModel = outputMaps[detectorMapName] 

610 origXPoly = origModel["XPoly"]["Coefficients"] 

611 origYPoly = origModel["YPoly"]["Coefficients"] 

612 fitXPoly = fitModel[: len(origXPoly)] 

613 fitYPoly = fitModel[len(origXPoly) :] 

614 absDiffX = abs(fitXPoly - origXPoly) 

615 absDiffY = abs(fitYPoly - origYPoly) 

616 # Check that input detector model matches fit 

617 np.testing.assert_array_less(absDiffX, 1e-7) 

618 np.testing.assert_array_less(absDiffY, 1e-7) 

619 

620 def test_missingWcs(self): 

621 """Test that task does not fail when the input WCS is None for one 

622 extension and that the fit WCS for that extension returns a finite 

623 result. 

624 """ 

625 inputVisitSummary = self.inputVisitSummary.copy() 

626 # Set one WCS to be None 

627 testVisit = 0 

628 testDetector = 20 

629 inputVisitSummary[testVisit][testDetector].setWcs(None) 

630 

631 outputs = self.task.run( 

632 self.inputCatalogRefs, 

633 inputVisitSummary, 

634 instrumentName=self.instrumentName, 

635 refEpoch=self.refEpoch, 

636 refObjectLoader=self.refObjectLoader, 

637 ) 

638 

639 # Check that the fit WCS for the extension with input WCS=None returns 

640 # finite sky values. 

641 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs() 

642 testSky = testWcs.pixelToSky(0, 0) 

643 self.assertTrue(testSky.isFinite()) 

644 

645 

646class TestGbdesGlobalAstrometricFit(TestGbdesAstrometricFit): 

647 @classmethod 

648 def setUpClass(cls): 

649 # Set random seed 

650 np.random.seed(1234) 

651 

652 # Fraction of simulated stars in the reference catalog and science 

653 # exposures 

654 inReferenceFraction = 1 

655 inScienceFraction = 1 

656 

657 # Make fake data 

658 cls.datadir = os.path.join(TESTDIR, "data") 

659 

660 cls.nFields = 2 

661 cls.instrumentName = "HSC" 

662 cls.instrument = wcsfit.Instrument(cls.instrumentName) 

663 cls.refEpoch = 57205.5 

664 

665 # Make test inputVisitSummary. VisitSummaryTables are taken from 

666 # collection HSC/runs/RC2/w_2022_20/DM-34794 

667 cls.testVisits = [1176, 17900, 17930, 17934, 36434, 36460, 36494, 36446] 

668 cls.inputVisitSummary = [] 

669 for testVisit in cls.testVisits: 

670 visSum = afwTable.ExposureCatalog.readFits( 

671 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits") 

672 ) 

673 cls.inputVisitSummary.append(visSum) 

674 

675 cls.config = GbdesGlobalAstrometricFitConfig() 

676 cls.config.systematicError = 0 

677 cls.config.devicePolyOrder = 4 

678 cls.config.exposurePolyOrder = 6 

679 cls.config.fitReserveFraction = 0 

680 cls.config.fitReserveRandomSeed = 1234 

681 cls.config.saveModelParams = True 

682 cls.task = GbdesGlobalAstrometricFitTask(config=cls.config) 

683 

684 cls.fields, cls.fieldRegions = cls.task._prep_sky(cls.inputVisitSummary) 

685 

686 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info( 

687 cls.inputVisitSummary, cls.instrument, fieldRegions=cls.fieldRegions 

688 ) 

689 

690 refDataIds, deferredRefCats = [], [] 

691 allStarIds = [] 

692 allStarRAs = [] 

693 allStarDecs = [] 

694 for region in cls.fieldRegions.values(): 

695 # Bounding box of observations: 

696 bbox = region.getBoundingBox() 

697 raMin = bbox.getLon().getA().asDegrees() 

698 raMax = bbox.getLon().getB().asDegrees() 

699 decMin = bbox.getLat().getA().asDegrees() 

700 decMax = bbox.getLat().getB().asDegrees() 

701 

702 # Make random set of data in a bounding box determined by input 

703 # visits 

704 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars( 

705 raMin, raMax, decMin, decMax, cls.config.matchRadius 

706 ) 

707 

708 allStarIds.append(starIds) 

709 allStarRAs.append(starRAs) 

710 allStarDecs.append(starDecs) 

711 

712 corners = [ 

713 lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees).getVector() 

714 for (ra, dec) in zip([raMin, raMax, raMax, raMin], [decMin, decMin, decMax, decMax]) 

715 ] 

716 conv_box = lsst.sphgeom.ConvexPolygon.convexHull(corners) 

717 # Make a reference catalog that will be loaded into 

718 # ReferenceObjectLoader 

719 refDataId, deferredRefCat = cls._make_refCat( 

720 starIds, starRAs, starDecs, inReferenceFraction, conv_box 

721 ) 

722 refDataIds.append(refDataId) 

723 deferredRefCats.append(deferredRefCat) 

724 

725 cls.refObjectLoader = ReferenceObjectLoader(refDataIds, deferredRefCats) 

726 cls.refObjectLoader.config.requireProperMotion = False 

727 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter" 

728 

729 cls.task.refObjectLoader = cls.refObjectLoader 

730 

731 allRefObjects, allRefCovariances = {}, {} 

732 for f, fieldRegion in cls.fieldRegions.items(): 

733 refObjects, refCovariance = cls.task._load_refcat( 

734 cls.refObjectLoader, cls.extensionInfo, epoch=cls.exposureInfo.medianEpoch, region=fieldRegion 

735 ) 

736 allRefObjects[f] = refObjects 

737 allRefCovariances[f] = refCovariance 

738 cls.refObjects = allRefObjects 

739 

740 # Get True WCS for stars: 

741 with open(os.path.join(cls.datadir, "sample_global_wcs.yaml"), "r") as f: 

742 cls.trueModel = yaml.load(f, Loader=yaml.Loader) 

743 

744 cls.trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary) 

745 

746 cls.isolatedStarCatalogs, cls.isolatedStarSources = cls._make_isolatedStars( 

747 allStarIds, allStarRAs, allStarDecs, cls.trueWCSs, inScienceFraction 

748 ) 

749 

750 cls.outputs = cls.task.run( 

751 cls.inputVisitSummary, 

752 cls.isolatedStarSources, 

753 cls.isolatedStarCatalogs, 

754 instrumentName=cls.instrumentName, 

755 refEpoch=cls.refEpoch, 

756 refObjectLoader=cls.refObjectLoader, 

757 ) 

758 

759 @classmethod 

760 def _make_isolatedStars(cls, allStarIds, allStarRAs, allStarDecs, trueWCSs, inScienceFraction): 

761 """Given a subset of the simulated data, make source catalogs and star 

762 catalogs. 

763 

764 This takes the true WCSs to go from the RA and Decs of the simulated 

765 stars to pixel coordinates for a given visit and detector. If those 

766 pixel coordinates are within the bounding box of the detector, the 

767 source and visit information is put in the corresponding catalog of 

768 isolated star sources. 

769 

770 Parameters 

771 ---------- 

772 allStarIds : `np.ndarray` [`int`] 

773 Source ids for the simulated stars 

774 allStarRas : `np.ndarray` [`float`] 

775 RAs of the simulated stars 

776 allStarDecs : `np.ndarray` [`float`] 

777 Decs of the simulated stars 

778 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`] 

779 WCS with which to simulate the source pixel coordinates 

780 inReferenceFraction : float 

781 Percentage of simulated stars to include in reference catalog 

782 

783 Returns 

784 ------- 

785 isolatedStarCatalogRefs : `list` 

786 [`lsst.pipe.base.InMemoryDatasetHandle`] 

787 List of references to isolated star catalogs. 

788 isolatedStarSourceRefs : `list` 

789 [`lsst.pipe.base.InMemoryDatasetHandle`] 

790 List of references to isolated star sources. 

791 """ 

792 bbox = lsst.geom.BoxD( 

793 lsst.geom.Point2D( 

794 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"] 

795 ), 

796 lsst.geom.Point2D( 

797 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"] 

798 ), 

799 ) 

800 bboxCorners = bbox.getCorners() 

801 

802 isolatedStarCatalogRefs = [] 

803 isolatedStarSourceRefs = [] 

804 for i in range(len(allStarIds)): 

805 starIds = allStarIds[i] 

806 starRAs = allStarRAs[i] 

807 starDecs = allStarDecs[i] 

808 isolatedStarCatalog = pd.DataFrame({"ra": starRAs, "dec": starDecs}, index=starIds) 

809 isolatedStarCatalogRefs.append( 

810 InMemoryDatasetHandle(isolatedStarCatalog, storageClass="DataFrame", dataId={"tract": 0}) 

811 ) 

812 sourceCats = [] 

813 for v, visit in enumerate(cls.testVisits): 

814 nVisStars = int(cls.nStars * inScienceFraction) 

815 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False) 

816 visitStarIds = starIds[visitStarIndices] 

817 visitStarRas = starRAs[visitStarIndices] 

818 visitStarDecs = starDecs[visitStarIndices] 

819 for detector in trueWCSs[visit]: 

820 detWcs = detector.getWcs() 

821 detectorId = detector["id"] 

822 radecCorners = detWcs.pixelToSky(bboxCorners) 

823 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners]) 

824 detectorIndices = detectorFootprint.contains( 

825 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian) 

826 ) 

827 nDetectorStars = detectorIndices.sum() 

828 detectorArray = np.ones(nDetectorStars, dtype=int) * detector["id"] 

829 visitArray = np.ones(nDetectorStars, dtype=int) * visit 

830 

831 ones_like = np.ones(nDetectorStars) 

832 zeros_like = np.zeros(nDetectorStars, dtype=bool) 

833 

834 x, y = detWcs.skyToPixelArray( 

835 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True 

836 ) 

837 

838 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[ 

839 0 

840 ].getWcs() 

841 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True) 

842 

843 sourceDict = {} 

844 sourceDict["detector"] = detectorArray 

845 sourceDict["visit"] = visitArray 

846 sourceDict["obj_index"] = visitStarIds[detectorIndices] 

847 sourceDict["x"] = x 

848 sourceDict["y"] = y 

849 sourceDict["xErr"] = 1e-3 * ones_like 

850 sourceDict["yErr"] = 1e-3 * ones_like 

851 sourceDict["inputRA"] = inputRa 

852 sourceDict["inputDec"] = inputDec 

853 sourceDict["trueRA"] = visitStarRas[detectorIndices] 

854 sourceDict["trueDec"] = visitStarDecs[detectorIndices] 

855 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]: 

856 sourceDict[key] = ones_like 

857 for key in [ 

858 "pixelFlags_edge", 

859 "pixelFlags_saturated", 

860 "pixelFlags_interpolatedCenter", 

861 "pixelFlags_interpolated", 

862 "pixelFlags_crCenter", 

863 "pixelFlags_bad", 

864 "hsmPsfMoments_flag", 

865 "apFlux_12_0_flag", 

866 "extendedness", 

867 "parentSourceId", 

868 "deblend_nChild", 

869 "ixy", 

870 ]: 

871 sourceDict[key] = zeros_like 

872 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like 

873 sourceDict["detect_isPrimary"] = ones_like.astype(bool) 

874 

875 sourceCat = pd.DataFrame(sourceDict) 

876 sourceCats.append(sourceCat) 

877 

878 isolatedStarSourceTable = pd.concat(sourceCats, ignore_index=True) 

879 isolatedStarSourceTable = isolatedStarSourceTable.sort_values(by=["obj_index"]) 

880 isolatedStarSourceRefs.append( 

881 InMemoryDatasetHandle(isolatedStarSourceTable, storageClass="DataFrame", dataId={"tract": 0}) 

882 ) 

883 

884 return isolatedStarCatalogRefs, isolatedStarSourceRefs 

885 

886 def test_loading_and_association(self): 

887 """Test that associated objects actually correspond to the same 

888 simulated object.""" 

889 associations, sourceDict = self.task._associate_from_isolated_sources( 

890 self.isolatedStarSources, self.isolatedStarCatalogs, self.extensionInfo, self.refObjects 

891 ) 

892 

893 object_groups = np.flatnonzero(np.array(associations.sequence) == 0) 

894 for i in range(len(object_groups) - 1)[:10]: 

895 ras, decs = [], [] 

896 for ind in np.arange(object_groups[i], object_groups[i + 1]): 

897 visit = self.extensionInfo.visit[associations.extn[ind]] 

898 detectorInd = self.extensionInfo.detectorIndex[associations.extn[ind]] 

899 detector = self.extensionInfo.detector[associations.extn[ind]] 

900 if detectorInd == -1: 

901 ra = self.refObjects[visit * -1]["ra"][associations.obj[ind]] 

902 dec = self.refObjects[visit * -1]["dec"][associations.obj[ind]] 

903 ras.append(ra) 

904 decs.append(dec) 

905 else: 

906 x = sourceDict[visit][detector]["x"][associations.obj[ind]] 

907 y = sourceDict[visit][detector]["y"][associations.obj[ind]] 

908 ra, dec = self.trueWCSs[visit][detectorInd].getWcs().pixelToSky(x, y) 

909 ras.append(ra.asDegrees()) 

910 decs.append(dec.asDegrees()) 

911 np.testing.assert_allclose(ras, ras[0]) 

912 np.testing.assert_allclose(decs, decs[0]) 

913 

914 def test_refCatLoader(self): 

915 """Test loading objects from the refCat in each of the fields.""" 

916 

917 for region in self.fieldRegions.values(): 

918 refCat, refCov = self.task._load_refcat( 

919 self.refObjectLoader, self.extensionInfo, region=region, epoch=self.exposureInfo.medianEpoch 

920 ) 

921 assert len(refCat) > 0 

922 

923 def test_make_outputs(self): 

924 """Test that the run method recovers the input model parameters.""" 

925 for isolatedStarSourceRef in self.isolatedStarSources: 

926 iss = isolatedStarSourceRef.get() 

927 visits = np.unique(iss["visit"]) 

928 for v, visit in enumerate(visits): 

929 outputWcsCatalog = self.outputs.outputWcss[visit] 

930 visitSources = iss[iss["visit"] == visit] 

931 detectors = outputWcsCatalog["id"] 

932 for d, detectorId in enumerate(detectors): 

933 fitwcs = outputWcsCatalog[d].getWcs() 

934 detSources = visitSources[visitSources["detector"] == detectorId] 

935 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True) 

936 dRA = fitRA - detSources["trueRA"] 

937 dDec = fitDec - detSources["trueDec"] 

938 # Check that input coordinates match the output coordinates 

939 self.assertAlmostEqual(np.mean(dRA), 0) 

940 self.assertAlmostEqual(np.std(dRA), 0) 

941 self.assertAlmostEqual(np.mean(dDec), 0) 

942 self.assertAlmostEqual(np.std(dDec), 0) 

943 

944 def test_missingWcs(self): 

945 """Test that task does not fail when the input WCS is None for one 

946 extension and that the fit WCS for that extension returns a finite 

947 result. 

948 """ 

949 inputVisitSummary = self.inputVisitSummary.copy() 

950 # Set one WCS to be None 

951 testVisit = 0 

952 testDetector = 20 

953 inputVisitSummary[testVisit][testDetector].setWcs(None) 

954 

955 outputs = self.task.run( 

956 inputVisitSummary, 

957 self.isolatedStarSources, 

958 self.isolatedStarCatalogs, 

959 instrumentName=self.instrumentName, 

960 refEpoch=self.refEpoch, 

961 refObjectLoader=self.refObjectLoader, 

962 ) 

963 

964 # Check that the fit WCS for the extension with input WCS=None returns 

965 # finite sky values. 

966 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs() 

967 testSky = testWcs.pixelToSky(0, 0) 

968 self.assertTrue(testSky.isFinite()) 

969 

970 

971def setup_module(module): 

972 lsst.utils.tests.init() 

973 

974 

975if __name__ == "__main__": 975 ↛ 976line 975 didn't jump to line 976, because the condition on line 975 was never true

976 lsst.utils.tests.init() 

977 unittest.main()