Coverage for tests/test_gbdesAstrometricFit.py: 10%
535 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 05:05 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-11 05:05 -0700
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import unittest
24from copy import copy
26import astropy.units as u
27import lsst.afw.geom as afwgeom
28import lsst.afw.table as afwTable
29import lsst.geom
30import lsst.utils
31import numpy as np
32import pandas as pd
33import wcsfit
34import yaml
35from lsst import sphgeom
36from lsst.daf.base import PropertyList
37from lsst.drp.tasks.gbdesAstrometricFit import (
38 GbdesAstrometricFitConfig,
39 GbdesAstrometricFitTask,
40 GbdesGlobalAstrometricFitConfig,
41 GbdesGlobalAstrometricFitTask,
42)
43from lsst.meas.algorithms import ReferenceObjectLoader
44from lsst.meas.algorithms.testUtils import MockRefcatDataId
45from lsst.pipe.base import InMemoryDatasetHandle
46from smatch.matcher import Matcher
48TESTDIR = os.path.abspath(os.path.dirname(__file__))
51class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
52 """This class tests `GbdesAstrometricFit` using real `visitSummaryTable`s
53 from HSC RC2 processing, with simulated sources for those visits.
54 """
56 @classmethod
57 def setUpClass(cls):
58 # Set random seed
59 np.random.seed(1234)
61 # Make fake data
62 cls.datadir = os.path.join(TESTDIR, "data")
64 cls.fieldNumber = 0
65 cls.nFields = 1
66 cls.instrumentName = "HSC"
67 cls.instrument = wcsfit.Instrument(cls.instrumentName)
68 cls.refEpoch = 57205.5
70 # Make test inputVisitSummary. VisitSummaryTables are taken from
71 # collection HSC/runs/RC2/w_2022_20/DM-34794
72 cls.testVisits = [1176, 17900, 17930, 17934]
73 cls.inputVisitSummary = []
74 for testVisit in cls.testVisits:
75 visSum = afwTable.ExposureCatalog.readFits(
76 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
77 )
78 cls.inputVisitSummary.append(visSum)
80 cls.config = GbdesAstrometricFitConfig()
81 cls.config.systematicError = 0
82 cls.config.devicePolyOrder = 4
83 cls.config.exposurePolyOrder = 6
84 cls.config.fitReserveFraction = 0
85 cls.config.fitReserveRandomSeed = 1234
86 cls.config.saveModelParams = True
87 cls.config.allowSelfMatches = True
88 cls.config.saveCameraModel = True
89 cls.task = GbdesAstrometricFitTask(config=cls.config)
91 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
92 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch
93 )
95 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
96 cls.inputVisitSummary, cls.exposureInfo.medianEpoch
97 )
99 # Bounding box of observations:
100 raMins, raMaxs = [], []
101 decMins, decMaxs = [], []
102 for visSum in cls.inputVisitSummary:
103 raMins.append(visSum["raCorners"].min())
104 raMaxs.append(visSum["raCorners"].max())
105 decMins.append(visSum["decCorners"].min())
106 decMaxs.append(visSum["decCorners"].max())
107 raMin = min(raMins)
108 raMax = max(raMaxs)
109 decMin = min(decMins)
110 decMax = max(decMaxs)
112 corners = [
113 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
114 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
115 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
116 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(),
117 ]
118 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
120 # Make random set of data in a bounding box determined by input visits
121 # Make wcs objects for the "true" model
122 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
123 raMin, raMax, decMin, decMax, cls.config.matchRadius
124 )
126 # Fraction of simulated stars in the reference catalog and science
127 # exposures
128 inReferenceFraction = 1
129 inScienceFraction = 1
131 # Make a reference catalog and load it into ReferenceObjectLoader
132 refDataId, deferredRefCat = cls._make_refCat(
133 starIds, starRAs, starDecs, inReferenceFraction, cls.boundingPolygon
134 )
135 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
136 cls.refObjectLoader.config.requireProperMotion = False
137 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
139 cls.task.refObjectLoader = cls.refObjectLoader
141 # Get True WCS for stars:
142 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f:
143 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
145 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
147 # Make source catalogs:
148 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction)
150 cls.outputs = cls.task.run(
151 cls.inputCatalogRefs,
152 cls.inputVisitSummary,
153 instrumentName=cls.instrumentName,
154 refEpoch=cls.refEpoch,
155 refObjectLoader=cls.refObjectLoader,
156 )
158 @staticmethod
159 def _make_simulated_stars(raMin, raMax, decMin, decMax, matchRadius, nStars=10000):
160 """Generate random positions for "stars" in an RA/Dec box.
162 Parameters
163 ----------
164 raMin : `float`
165 Minimum RA for simulated stars.
166 raMax : `float`
167 Maximum RA for simulated stars.
168 decMin : `float`
169 Minimum Dec for simulated stars.
170 decMax : `float`
171 Maximum Dec for simulated stars.
172 matchRadius : `float`
173 Minimum allowed distance in arcsec between stars.
174 nStars : `int`, optional
175 Number of stars to simulate. Final number will be lower if any
176 too-close stars are dropped.
178 Returns
179 -------
180 nStars : `int`
181 Number of stars simulated.
182 starIds: `np.ndarray`
183 Unique identification number for stars.
184 starRAs: `np.ndarray`
185 Simulated Right Ascensions.
186 starDecs: `np.ndarray`
187 Simulated Declination.
188 """
189 starIds = np.arange(nStars)
190 starRAs = np.random.random(nStars) * (raMax - raMin) + raMin
191 starDecs = np.random.random(nStars) * (decMax - decMin) + decMin
192 # Remove neighbors:
193 with Matcher(starRAs, starDecs) as matcher:
194 idx = matcher.query_groups(matchRadius / 3600.0, min_match=2)
195 if len(idx) > 0:
196 neighbors = np.unique(np.concatenate(idx))
197 starRAs = np.delete(starRAs, neighbors)
198 starDecs = np.delete(starDecs, neighbors)
199 nStars = len(starRAs)
200 starIds = np.arange(nStars)
202 return nStars, starIds, starRAs, starDecs
204 @classmethod
205 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction, bounds):
206 """Make reference catalog from a subset of the simulated data
208 Parameters
209 ----------
210 starIds : `np.ndarray` [`int`]
211 Source ids for the simulated stars
212 starRas : `np.ndarray` [`float`]
213 RAs of the simulated stars
214 starDecs : `np.ndarray` [`float`]
215 Decs of the simulated stars
216 inReferenceFraction : float
217 Percentage of simulated stars to include in reference catalog
218 bounds : `lsst.sphgeom.ConvexPolygon`
219 Boundary of the reference catalog region
221 Returns
222 -------
223 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId`
224 Object that replicates the functionality of a dataId.
225 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
226 Dataset handle for reference catalog.
227 """
228 nRefs = int(cls.nStars * inReferenceFraction)
229 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
230 # Make simpleCatalog to hold data, create datasetRef with `region`
231 # determined by bounding box used in above simulate.
232 refSchema = afwTable.SimpleTable.makeMinimalSchema()
233 idKey = refSchema.addField("sourceId", type="I")
234 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64)
235 raErrKey = refSchema.addField("coord_raErr", units="rad", type=np.float64)
236 decErrKey = refSchema.addField("coord_decErr", units="rad", type=np.float64)
237 pmraErrKey = refSchema.addField("pm_raErr", units="rad2 / yr", type=np.float64)
238 pmdecErrKey = refSchema.addField("pm_decErr", units="rad2 / yr", type=np.float64)
239 refCat = afwTable.SimpleCatalog(refSchema)
240 ref_md = PropertyList()
241 ref_md.set("REFCAT_FORMAT_VERSION", 1)
242 refCat.table.setMetadata(ref_md)
243 for i in refStarIndices:
244 record = refCat.addNew()
245 record.set(idKey, starIds[i])
246 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
247 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
248 record.set(fluxKey, 1)
249 record.set(raErrKey, 0.00001)
250 record.set(decErrKey, 0.00001)
251 record.set(pmraErrKey, 1e-9)
252 record.set(pmdecErrKey, 1e-9)
253 refDataId = MockRefcatDataId(bounds)
254 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
256 return refDataId, deferredRefCat
258 @classmethod
259 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
260 """Make a `pd.DataFrame` catalog with the columns needed for the
261 object selector.
263 Parameters
264 ----------
265 starIds : `np.ndarray` [`int`]
266 Source ids for the simulated stars
267 starRas : `np.ndarray` [`float`]
268 RAs of the simulated stars
269 starDecs : `np.ndarray` [`float`]
270 Decs of the simulated stars
271 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
272 WCS with which to simulate the source pixel coordinates
273 inReferenceFraction : float
274 Percentage of simulated stars to include in reference catalog
276 Returns
277 -------
278 sourceCat : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
279 List of reference to source catalogs.
280 """
281 inputCatalogRefs = []
282 # Take a subset of the simulated data
283 # Use true wcs objects to put simulated data into ccds
284 bbox = lsst.geom.BoxD(
285 lsst.geom.Point2D(
286 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
287 ),
288 lsst.geom.Point2D(
289 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
290 ),
291 )
292 bboxCorners = bbox.getCorners()
293 cls.inputCatalogRefs = []
294 for v, visit in enumerate(cls.testVisits):
295 nVisStars = int(cls.nStars * inScienceFraction)
296 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
297 visitStarIds = starIds[visitStarIndices]
298 visitStarRas = starRas[visitStarIndices]
299 visitStarDecs = starDecs[visitStarIndices]
300 sourceCats = []
301 for detector in trueWCSs[visit]:
302 detWcs = detector.getWcs()
303 detectorId = detector["id"]
304 radecCorners = detWcs.pixelToSky(bboxCorners)
305 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
306 detectorIndices = detectorFootprint.contains(
307 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
308 )
309 nDetectorStars = detectorIndices.sum()
310 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"]
312 ones_like = np.ones(nDetectorStars)
313 zeros_like = np.zeros(nDetectorStars, dtype=bool)
315 x, y = detWcs.skyToPixelArray(
316 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
317 )
319 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs()
320 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
322 sourceDict = {}
323 sourceDict["detector"] = detectorArray
324 sourceDict["sourceId"] = visitStarIds[detectorIndices]
325 sourceDict["x"] = x
326 sourceDict["y"] = y
327 sourceDict["xErr"] = 1e-3 * ones_like
328 sourceDict["yErr"] = 1e-3 * ones_like
329 sourceDict["inputRA"] = inputRa
330 sourceDict["inputDec"] = inputDec
331 sourceDict["trueRA"] = visitStarRas[detectorIndices]
332 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
333 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
334 sourceDict[key] = ones_like
335 for key in [
336 "pixelFlags_edge",
337 "pixelFlags_saturated",
338 "pixelFlags_interpolatedCenter",
339 "pixelFlags_interpolated",
340 "pixelFlags_crCenter",
341 "pixelFlags_bad",
342 "hsmPsfMoments_flag",
343 "apFlux_12_0_flag",
344 "extendedness",
345 "sizeExtendedness",
346 "parentSourceId",
347 "deblend_nChild",
348 "ixy",
349 ]:
350 sourceDict[key] = zeros_like
351 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
352 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
354 sourceCat = pd.DataFrame(sourceDict)
355 sourceCats.append(sourceCat)
357 visitSourceTable = pd.concat(sourceCats)
359 inputCatalogRef = InMemoryDatasetHandle(
360 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit}
361 )
363 inputCatalogRefs.append(inputCatalogRef)
365 return inputCatalogRefs
367 @classmethod
368 def _make_wcs(cls, model, inputVisitSummaries):
369 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
371 Parameters
372 ----------
373 model : `dict`
374 Dictionary with WCS model parameters
375 inputVisitSummaries : `list` [`lsst.afw.table.ExposureCatalog`]
376 Visit summary catalogs
377 Returns
378 -------
379 catalogs : `dict` [`int`, `lsst.afw.table.ExposureCatalog`]
380 Visit summary catalogs with WCS set to input model
381 """
383 # Pixels will need to be rescaled before going into the mappings
384 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"]
385 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"]
387 catalogs = {}
388 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
389 schema.addField("visit", type="L", doc="Visit number")
390 for visitSum in inputVisitSummaries:
391 visit = visitSum[0]["visit"]
392 visitMapName = f"{visit}/poly"
393 visitModel = model[visitMapName]
395 catalog = lsst.afw.table.ExposureCatalog(schema)
396 catalog.resize(len(visitSum))
397 catalog["visit"] = visit
399 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
401 visitMapType = visitModel["Type"]
402 visitDict = {"Type": visitMapType}
403 if visitMapType == "Poly":
404 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"]
405 visitDict["Coefficients"] = mapCoefficients
407 for d, detector in enumerate(visitSum):
408 detectorId = detector["id"]
409 detectorMapName = f"HSC/{detectorId}/poly"
410 detectorModel = model[detectorMapName]
412 detectorMapType = detectorModel["Type"]
413 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict}
414 if detectorMapType == "Poly":
415 mapCoefficients = (
416 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"]
417 )
418 mapDict[detectorMapName]["Coefficients"] = mapCoefficients
420 outWCS = cls.task._make_afw_wcs(
421 mapDict,
422 raDec.getRa(),
423 raDec.getDec(),
424 doNormalizePixels=True,
425 xScale=xscale,
426 yScale=yscale,
427 )
428 catalog[d].setId(detectorId)
429 catalog[d].setWcs(outWCS)
431 catalog.sort()
432 catalogs[visit] = catalog
434 return catalogs
436 def test_get_exposure_info(self):
437 """Test that information for input exposures is as expected and that
438 the WCS in the class object gives approximately the same results as the
439 input `lsst.afw.geom.SkyWcs`.
440 """
442 # The total number of extensions is the number of detectors for each
443 # visit plus one for the reference catalog for each field.
444 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + self.nFields
446 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
448 taskVisits = set(self.extensionInfo.visit)
449 refVisits = np.arange(0, -1 * self.nFields, -1).tolist()
450 self.assertEqual(taskVisits, set(self.testVisits + refVisits))
452 xx = np.linspace(0, 2000, 3)
453 yy = np.linspace(0, 4000, 6)
454 xgrid, ygrid = np.meshgrid(xx, yy)
455 for visSum in self.inputVisitSummary:
456 visit = visSum[0]["visit"]
457 for detectorInfo in visSum:
458 detector = detectorInfo["id"]
459 extensionIndex = np.flatnonzero(
460 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector)
461 )[0]
462 fitWcs = self.extensionInfo.wcs[extensionIndex]
463 calexpWcs = detectorInfo.getWcs()
465 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())])
467 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
469 # The pixel origin maps to a position slightly off from the
470 # tangent plane origin, so we want to use this value for the
471 # tangent-plane-to-sky part of the mapping.
472 tangentPlaneCenter = fitWcs.toWorld(
473 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
474 )
475 tangentPlaneOrigin = lsst.geom.Point2D(tangentPlaneCenter)
476 skyOrigin = calexpWcs.pixelToSky(
477 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
478 )
479 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0.0 * lsst.geom.degrees, True)
480 iwcToSkyWcs = afwgeom.makeSkyWcs(tangentPlaneOrigin, skyOrigin, cdMatrix)
481 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(
482 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True
483 )
485 np.testing.assert_allclose(calexpra, newRAdeg)
486 np.testing.assert_allclose(calexpdec, newDecdeg)
488 def test_refCatLoader(self):
489 """Test that we can load objects from refCat"""
491 tmpAssociations = wcsfit.FoFClass(
492 self.fields,
493 [self.instrument],
494 self.exposuresHelper,
495 [self.fieldRadius.asDegrees()],
496 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
497 )
499 self.task._load_refcat(
500 self.refObjectLoader,
501 self.extensionInfo,
502 associations=tmpAssociations,
503 center=self.fieldCenter,
504 radius=self.fieldRadius,
505 epoch=self.exposureInfo.medianEpoch,
506 )
508 # We have only loaded one catalog, so getting the 'matches' should just
509 # return the same objects we put in, except some random objects that
510 # are too close together.
511 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
513 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
515 self.assertLessEqual(nMatches, self.nStars)
516 self.assertGreater(nMatches, self.nStars * 0.9)
518 def test_loading_and_association(self):
519 """Test that objects can be loaded and correctly associated."""
520 # Running `_load_catalogs_and_associate` changes the input WCSs, so
521 # recalculate them here so that the variables shared among tests are
522 # not affected.
523 instrument = wcsfit.Instrument(self.instrumentName)
524 _, exposuresHelper, extensionInfo = self.task._get_exposure_info(
525 self.inputVisitSummary, instrument, refEpoch=self.refEpoch
526 )
528 tmpAssociations = wcsfit.FoFClass(
529 self.fields,
530 [self.instrument],
531 exposuresHelper,
532 [self.fieldRadius.asDegrees()],
533 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
534 )
535 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, extensionInfo)
537 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
539 matchIds = []
540 correctMatches = []
541 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
542 objVisitInd = extensionInfo.visitIndex[e]
543 objDet = extensionInfo.detector[e]
544 extnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet
545 objInfo = self.inputCatalogRefs[objVisitInd].get()[extnInds].iloc[o]
546 if s == 0:
547 if len(matchIds) > 0:
548 correctMatches.append(len(set(matchIds)) == 1)
549 matchIds = []
551 matchIds.append(objInfo["sourceId"])
553 # A few matches may incorrectly associate sources because of the random
554 # positions
555 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
557 def test_make_outputs(self):
558 """Test that the run method recovers the input model parameters."""
559 for v, visit in enumerate(self.testVisits):
560 visitSummary = self.inputVisitSummary[v]
561 outputWcsCatalog = self.outputs.outputWcss[visit]
562 visitSources = self.inputCatalogRefs[v].get()
563 for d, detectorRow in enumerate(visitSummary):
564 detectorId = detectorRow["id"]
565 fitwcs = outputWcsCatalog[d].getWcs()
566 detSources = visitSources[visitSources["detector"] == detectorId]
567 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
568 dRA = fitRA - detSources["trueRA"]
569 dDec = fitDec - detSources["trueDec"]
570 # Check that input coordinates match the output coordinates
571 self.assertAlmostEqual(np.mean(dRA), 0)
572 self.assertAlmostEqual(np.std(dRA), 0)
573 self.assertAlmostEqual(np.mean(dDec), 0)
574 self.assertAlmostEqual(np.std(dDec), 0)
576 def test_compute_model_params(self):
577 """Test the optional model parameters and covariance output."""
578 modelParams = pd.DataFrame(self.outputs.modelParams)
579 # Check that DataFrame is the expected size.
580 shape = modelParams.shape
581 self.assertEqual(shape[0] + 4, shape[1])
582 # Check that covariance matrix is symmetric.
583 covariance = (modelParams.iloc[:, 4:]).to_numpy()
584 np.testing.assert_allclose(covariance, covariance.T, atol=1e-18)
586 def test_run(self):
587 """Test that run method recovers the input model parameters"""
588 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
590 for v, visit in enumerate(self.testVisits):
591 visitSummary = self.inputVisitSummary[v]
592 visitMapName = f"{visit}/poly"
594 origModel = self.trueModel[visitMapName]
595 if origModel["Type"] != "Identity":
596 fitModel = outputMaps[visitMapName]
597 origXPoly = origModel["XPoly"]["Coefficients"]
598 origYPoly = origModel["YPoly"]["Coefficients"]
599 fitXPoly = fitModel[: len(origXPoly)]
600 fitYPoly = fitModel[len(origXPoly) :]
602 absDiffX = abs(fitXPoly - origXPoly)
603 absDiffY = abs(fitYPoly - origYPoly)
604 # Check that input visit model matches fit
605 np.testing.assert_array_less(absDiffX, 1e-6)
606 np.testing.assert_array_less(absDiffY, 1e-6)
607 for d, detectorRow in enumerate(visitSummary):
608 detectorId = detectorRow["id"]
609 detectorMapName = f"HSC/{detectorId}/poly"
610 origModel = self.trueModel[detectorMapName]
611 if (origModel["Type"] != "Identity") and (v == 0):
612 fitModel = outputMaps[detectorMapName]
613 origXPoly = origModel["XPoly"]["Coefficients"]
614 origYPoly = origModel["YPoly"]["Coefficients"]
615 fitXPoly = fitModel[: len(origXPoly)]
616 fitYPoly = fitModel[len(origXPoly) :]
617 absDiffX = abs(fitXPoly - origXPoly)
618 absDiffY = abs(fitYPoly - origYPoly)
619 # Check that input detector model matches fit
620 np.testing.assert_array_less(absDiffX, 1e-7)
621 np.testing.assert_array_less(absDiffY, 1e-7)
623 def test_missingWcs(self):
624 """Test that task does not fail when the input WCS is None for one
625 extension and that the fit WCS for that extension returns a finite
626 result.
627 """
628 inputVisitSummary = self.inputVisitSummary.copy()
629 # Set one WCS to be None
630 testVisit = 0
631 testDetector = 20
632 inputVisitSummary[testVisit][testDetector].setWcs(None)
634 outputs = self.task.run(
635 self.inputCatalogRefs,
636 inputVisitSummary,
637 instrumentName=self.instrumentName,
638 refEpoch=self.refEpoch,
639 refObjectLoader=self.refObjectLoader,
640 )
642 # Check that the fit WCS for the extension with input WCS=None returns
643 # finite sky values.
644 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
645 testSky = testWcs.pixelToSky(0, 0)
646 self.assertTrue(testSky.isFinite())
648 def test_inputCameraModel(self):
649 """Test running task with an input camera model, and check that true
650 object coordinates are recovered.
651 """
652 config = copy(self.config)
653 config.saveCameraModel = False
654 config.useInputCameraModel = True
655 task = GbdesAstrometricFitTask(config=config)
656 with open(os.path.join(self.datadir, "sample_camera_model.yaml"), "r") as f:
657 cameraModel = yaml.load(f, Loader=yaml.Loader)
659 outputs = task.run(
660 self.inputCatalogRefs,
661 self.inputVisitSummary,
662 instrumentName=self.instrumentName,
663 refObjectLoader=self.refObjectLoader,
664 inputCameraModel=cameraModel,
665 )
667 # Check that the output WCS is close (not necessarily exactly equal) to
668 # the result when fitting the full model.
669 for v, visit in enumerate(self.testVisits):
670 visitSummary = self.inputVisitSummary[v]
671 outputWcsCatalog = outputs.outputWcss[visit]
672 visitSources = self.inputCatalogRefs[v].get()
673 for d, detectorRow in enumerate(visitSummary):
674 detectorId = detectorRow["id"]
675 fitwcs = outputWcsCatalog[d].getWcs()
676 detSources = visitSources[visitSources["detector"] == detectorId]
677 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
678 dRA = fitRA - detSources["trueRA"]
679 dDec = fitDec - detSources["trueDec"]
680 # Check that input coordinates match the output coordinates
681 self.assertAlmostEqual(np.mean(dRA), 0)
682 self.assertAlmostEqual(np.std(dRA), 0)
683 self.assertAlmostEqual(np.mean(dDec), 0)
684 self.assertAlmostEqual(np.std(dDec), 0)
687class TestGbdesGlobalAstrometricFit(TestGbdesAstrometricFit):
688 @classmethod
689 def setUpClass(cls):
690 # Set random seed
691 np.random.seed(1234)
693 # Fraction of simulated stars in the reference catalog and science
694 # exposures
695 inReferenceFraction = 1
696 inScienceFraction = 1
698 # Make fake data
699 cls.datadir = os.path.join(TESTDIR, "data")
701 cls.nFields = 2
702 cls.instrumentName = "HSC"
703 cls.instrument = wcsfit.Instrument(cls.instrumentName)
704 cls.refEpoch = 57205.5
706 # Make test inputVisitSummary. VisitSummaryTables are taken from
707 # collection HSC/runs/RC2/w_2022_20/DM-34794
708 cls.testVisits = [1176, 17900, 17930, 17934, 36434, 36460, 36494, 36446]
709 cls.inputVisitSummary = []
710 for testVisit in cls.testVisits:
711 visSum = afwTable.ExposureCatalog.readFits(
712 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
713 )
714 cls.inputVisitSummary.append(visSum)
716 cls.config = GbdesGlobalAstrometricFitConfig()
717 cls.config.systematicError = 0
718 cls.config.devicePolyOrder = 4
719 cls.config.exposurePolyOrder = 6
720 cls.config.fitReserveFraction = 0
721 cls.config.fitReserveRandomSeed = 1234
722 cls.config.saveModelParams = True
723 cls.config.saveCameraModel = True
724 cls.task = GbdesGlobalAstrometricFitTask(config=cls.config)
726 cls.fields, cls.fieldRegions = cls.task._prep_sky(cls.inputVisitSummary)
728 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
729 cls.inputVisitSummary, cls.instrument, fieldRegions=cls.fieldRegions
730 )
732 refDataIds, deferredRefCats = [], []
733 allStarIds = []
734 allStarRAs = []
735 allStarDecs = []
736 for region in cls.fieldRegions.values():
737 # Bounding box of observations:
738 bbox = region.getBoundingBox()
739 raMin = bbox.getLon().getA().asDegrees()
740 raMax = bbox.getLon().getB().asDegrees()
741 decMin = bbox.getLat().getA().asDegrees()
742 decMax = bbox.getLat().getB().asDegrees()
744 # Make random set of data in a bounding box determined by input
745 # visits
746 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
747 raMin, raMax, decMin, decMax, cls.config.matchRadius
748 )
750 allStarIds.append(starIds)
751 allStarRAs.append(starRAs)
752 allStarDecs.append(starDecs)
754 corners = [
755 lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees).getVector()
756 for (ra, dec) in zip([raMin, raMax, raMax, raMin], [decMin, decMin, decMax, decMax])
757 ]
758 conv_box = lsst.sphgeom.ConvexPolygon.convexHull(corners)
759 # Make a reference catalog that will be loaded into
760 # ReferenceObjectLoader
761 refDataId, deferredRefCat = cls._make_refCat(
762 starIds, starRAs, starDecs, inReferenceFraction, conv_box
763 )
764 refDataIds.append(refDataId)
765 deferredRefCats.append(deferredRefCat)
767 cls.refObjectLoader = ReferenceObjectLoader(refDataIds, deferredRefCats)
768 cls.refObjectLoader.config.requireProperMotion = False
769 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
771 cls.task.refObjectLoader = cls.refObjectLoader
773 allRefObjects, allRefCovariances = {}, {}
774 for f, fieldRegion in cls.fieldRegions.items():
775 refObjects, refCovariance = cls.task._load_refcat(
776 cls.refObjectLoader, cls.extensionInfo, epoch=cls.exposureInfo.medianEpoch, region=fieldRegion
777 )
778 allRefObjects[f] = refObjects
779 allRefCovariances[f] = refCovariance
780 cls.refObjects = allRefObjects
782 # Get True WCS for stars:
783 with open(os.path.join(cls.datadir, "sample_global_wcs.yaml"), "r") as f:
784 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
786 cls.trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
788 cls.isolatedStarCatalogs, cls.isolatedStarSources = cls._make_isolatedStars(
789 allStarIds, allStarRAs, allStarDecs, cls.trueWCSs, inScienceFraction
790 )
792 cls.outputs = cls.task.run(
793 cls.inputVisitSummary,
794 cls.isolatedStarSources,
795 cls.isolatedStarCatalogs,
796 instrumentName=cls.instrumentName,
797 refEpoch=cls.refEpoch,
798 refObjectLoader=cls.refObjectLoader,
799 )
801 @classmethod
802 def _make_isolatedStars(cls, allStarIds, allStarRAs, allStarDecs, trueWCSs, inScienceFraction):
803 """Given a subset of the simulated data, make source catalogs and star
804 catalogs.
806 This takes the true WCSs to go from the RA and Decs of the simulated
807 stars to pixel coordinates for a given visit and detector. If those
808 pixel coordinates are within the bounding box of the detector, the
809 source and visit information is put in the corresponding catalog of
810 isolated star sources.
812 Parameters
813 ----------
814 allStarIds : `np.ndarray` [`int`]
815 Source ids for the simulated stars
816 allStarRas : `np.ndarray` [`float`]
817 RAs of the simulated stars
818 allStarDecs : `np.ndarray` [`float`]
819 Decs of the simulated stars
820 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
821 WCS with which to simulate the source pixel coordinates
822 inReferenceFraction : float
823 Percentage of simulated stars to include in reference catalog
825 Returns
826 -------
827 isolatedStarCatalogRefs : `list`
828 [`lsst.pipe.base.InMemoryDatasetHandle`]
829 List of references to isolated star catalogs.
830 isolatedStarSourceRefs : `list`
831 [`lsst.pipe.base.InMemoryDatasetHandle`]
832 List of references to isolated star sources.
833 """
834 bbox = lsst.geom.BoxD(
835 lsst.geom.Point2D(
836 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
837 ),
838 lsst.geom.Point2D(
839 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
840 ),
841 )
842 bboxCorners = bbox.getCorners()
844 isolatedStarCatalogRefs = []
845 isolatedStarSourceRefs = []
846 for i in range(len(allStarIds)):
847 starIds = allStarIds[i]
848 starRAs = allStarRAs[i]
849 starDecs = allStarDecs[i]
850 isolatedStarCatalog = pd.DataFrame({"ra": starRAs, "dec": starDecs}, index=starIds)
851 isolatedStarCatalogRefs.append(
852 InMemoryDatasetHandle(isolatedStarCatalog, storageClass="DataFrame", dataId={"tract": 0})
853 )
854 sourceCats = []
855 for v, visit in enumerate(cls.testVisits):
856 nVisStars = int(cls.nStars * inScienceFraction)
857 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
858 visitStarIds = starIds[visitStarIndices]
859 visitStarRas = starRAs[visitStarIndices]
860 visitStarDecs = starDecs[visitStarIndices]
861 for detector in trueWCSs[visit]:
862 detWcs = detector.getWcs()
863 detectorId = detector["id"]
864 radecCorners = detWcs.pixelToSky(bboxCorners)
865 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
866 detectorIndices = detectorFootprint.contains(
867 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
868 )
869 nDetectorStars = detectorIndices.sum()
870 detectorArray = np.ones(nDetectorStars, dtype=int) * detector["id"]
871 visitArray = np.ones(nDetectorStars, dtype=int) * visit
873 ones_like = np.ones(nDetectorStars)
874 zeros_like = np.zeros(nDetectorStars, dtype=bool)
876 x, y = detWcs.skyToPixelArray(
877 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
878 )
880 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[
881 0
882 ].getWcs()
883 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
885 sourceDict = {}
886 sourceDict["detector"] = detectorArray
887 sourceDict["visit"] = visitArray
888 sourceDict["obj_index"] = visitStarIds[detectorIndices]
889 sourceDict["x"] = x
890 sourceDict["y"] = y
891 sourceDict["xErr"] = 1e-3 * ones_like
892 sourceDict["yErr"] = 1e-3 * ones_like
893 sourceDict["inputRA"] = inputRa
894 sourceDict["inputDec"] = inputDec
895 sourceDict["trueRA"] = visitStarRas[detectorIndices]
896 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
897 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
898 sourceDict[key] = ones_like
899 for key in [
900 "pixelFlags_edge",
901 "pixelFlags_saturated",
902 "pixelFlags_interpolatedCenter",
903 "pixelFlags_interpolated",
904 "pixelFlags_crCenter",
905 "pixelFlags_bad",
906 "hsmPsfMoments_flag",
907 "apFlux_12_0_flag",
908 "extendedness",
909 "parentSourceId",
910 "deblend_nChild",
911 "ixy",
912 ]:
913 sourceDict[key] = zeros_like
914 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
915 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
917 sourceCat = pd.DataFrame(sourceDict)
918 sourceCats.append(sourceCat)
920 isolatedStarSourceTable = pd.concat(sourceCats, ignore_index=True)
921 isolatedStarSourceTable = isolatedStarSourceTable.sort_values(by=["obj_index"])
922 isolatedStarSourceRefs.append(
923 InMemoryDatasetHandle(isolatedStarSourceTable, storageClass="DataFrame", dataId={"tract": 0})
924 )
926 return isolatedStarCatalogRefs, isolatedStarSourceRefs
928 def test_loading_and_association(self):
929 """Test that associated objects actually correspond to the same
930 simulated object."""
931 associations, sourceDict = self.task._associate_from_isolated_sources(
932 self.isolatedStarSources, self.isolatedStarCatalogs, self.extensionInfo, self.refObjects
933 )
935 object_groups = np.flatnonzero(np.array(associations.sequence) == 0)
936 for i in range(len(object_groups) - 1)[:10]:
937 ras, decs = [], []
938 for ind in np.arange(object_groups[i], object_groups[i + 1]):
939 visit = self.extensionInfo.visit[associations.extn[ind]]
940 detectorInd = self.extensionInfo.detectorIndex[associations.extn[ind]]
941 detector = self.extensionInfo.detector[associations.extn[ind]]
942 if detectorInd == -1:
943 ra = self.refObjects[visit * -1]["ra"][associations.obj[ind]]
944 dec = self.refObjects[visit * -1]["dec"][associations.obj[ind]]
945 ras.append(ra)
946 decs.append(dec)
947 else:
948 x = sourceDict[visit][detector]["x"][associations.obj[ind]]
949 y = sourceDict[visit][detector]["y"][associations.obj[ind]]
950 ra, dec = self.trueWCSs[visit][detectorInd].getWcs().pixelToSky(x, y)
951 ras.append(ra.asDegrees())
952 decs.append(dec.asDegrees())
953 np.testing.assert_allclose(ras, ras[0])
954 np.testing.assert_allclose(decs, decs[0])
956 def test_refCatLoader(self):
957 """Test loading objects from the refCat in each of the fields."""
959 for region in self.fieldRegions.values():
960 refCat, refCov = self.task._load_refcat(
961 self.refObjectLoader, self.extensionInfo, region=region, epoch=self.exposureInfo.medianEpoch
962 )
963 assert len(refCat) > 0
965 def test_make_outputs(self):
966 """Test that the run method recovers the input model parameters."""
967 for isolatedStarSourceRef in self.isolatedStarSources:
968 iss = isolatedStarSourceRef.get()
969 visits = np.unique(iss["visit"])
970 for v, visit in enumerate(visits):
971 outputWcsCatalog = self.outputs.outputWcss[visit]
972 visitSources = iss[iss["visit"] == visit]
973 detectors = outputWcsCatalog["id"]
974 for d, detectorId in enumerate(detectors):
975 fitwcs = outputWcsCatalog[d].getWcs()
976 detSources = visitSources[visitSources["detector"] == detectorId]
977 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
978 dRA = fitRA - detSources["trueRA"]
979 dDec = fitDec - detSources["trueDec"]
980 # Check that input coordinates match the output coordinates
981 self.assertAlmostEqual(np.mean(dRA), 0)
982 self.assertAlmostEqual(np.std(dRA), 0)
983 self.assertAlmostEqual(np.mean(dDec), 0)
984 self.assertAlmostEqual(np.std(dDec), 0)
986 def test_missingWcs(self):
987 """Test that task does not fail when the input WCS is None for one
988 extension and that the fit WCS for that extension returns a finite
989 result.
990 """
991 inputVisitSummary = self.inputVisitSummary.copy()
992 # Set one WCS to be None
993 testVisit = 0
994 testDetector = 20
995 inputVisitSummary[testVisit][testDetector].setWcs(None)
997 outputs = self.task.run(
998 inputVisitSummary,
999 self.isolatedStarSources,
1000 self.isolatedStarCatalogs,
1001 instrumentName=self.instrumentName,
1002 refEpoch=self.refEpoch,
1003 refObjectLoader=self.refObjectLoader,
1004 )
1006 # Check that the fit WCS for the extension with input WCS=None returns
1007 # finite sky values.
1008 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
1009 testSky = testWcs.pixelToSky(0, 0)
1010 self.assertTrue(testSky.isFinite())
1012 def test_inputCameraModel(self):
1013 """Test running task with an input camera model, and check that true
1014 object coordinates are recovered.
1015 """
1016 config = copy(self.config)
1017 config.saveCameraModel = False
1018 config.useInputCameraModel = True
1019 task = GbdesGlobalAstrometricFitTask(config=config)
1020 with open(os.path.join(self.datadir, "sample_global_camera_model.yaml"), "r") as f:
1021 cameraModel = yaml.load(f, Loader=yaml.Loader)
1023 outputs = task.run(
1024 self.inputVisitSummary,
1025 self.isolatedStarSources,
1026 self.isolatedStarCatalogs,
1027 instrumentName=self.instrumentName,
1028 refObjectLoader=self.refObjectLoader,
1029 inputCameraModel=cameraModel,
1030 )
1032 # Check that the output WCS is close (not necessarily exactly equal) to
1033 # the result when fitting the full model.
1034 for isolatedStarSourceRef in self.isolatedStarSources:
1035 iss = isolatedStarSourceRef.get()
1036 visits = np.unique(iss["visit"])
1037 for v, visit in enumerate(visits):
1038 outputWcsCatalog = outputs.outputWcss[visit]
1039 visitSources = iss[iss["visit"] == visit]
1040 detectors = outputWcsCatalog["id"]
1041 for d, detectorId in enumerate(detectors):
1042 fitwcs = outputWcsCatalog[d].getWcs()
1043 detSources = visitSources[visitSources["detector"] == detectorId]
1044 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
1045 dRA = fitRA - detSources["trueRA"]
1046 dDec = fitDec - detSources["trueDec"]
1047 # Check that input coordinates match the output coordinates
1048 self.assertAlmostEqual(np.mean(dRA), 0, places=6)
1049 self.assertAlmostEqual(np.std(dRA), 0)
1050 self.assertAlmostEqual(np.mean(dDec), 0, places=6)
1051 self.assertAlmostEqual(np.std(dDec), 0)
1054def setup_module(module):
1055 lsst.utils.tests.init()
1058if __name__ == "__main__": 1058 ↛ 1059line 1058 didn't jump to line 1059, because the condition on line 1058 was never true
1059 lsst.utils.tests.init()
1060 unittest.main()