Coverage for tests/test_gbdesAstrometricFit.py: 11%
484 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-29 04:30 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-29 04:30 -0700
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import unittest
25import astropy.units as u
26import lsst.afw.geom as afwgeom
27import lsst.afw.table as afwTable
28import lsst.geom
29import lsst.utils
30import numpy as np
31import pandas as pd
32import wcsfit
33import yaml
34from lsst import sphgeom
35from lsst.daf.base import PropertyList
36from lsst.drp.tasks.gbdesAstrometricFit import (
37 GbdesAstrometricFitConfig,
38 GbdesAstrometricFitTask,
39 GbdesGlobalAstrometricFitConfig,
40 GbdesGlobalAstrometricFitTask,
41)
42from lsst.meas.algorithms import ReferenceObjectLoader
43from lsst.meas.algorithms.testUtils import MockRefcatDataId
44from lsst.pipe.base import InMemoryDatasetHandle
45from smatch.matcher import Matcher
47TESTDIR = os.path.abspath(os.path.dirname(__file__))
50class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
51 """This class tests `GbdesAstrometricFit` using real `visitSummaryTable`s
52 from HSC RC2 processing, with simulated sources for those visits.
53 """
55 @classmethod
56 def setUpClass(cls):
57 # Set random seed
58 np.random.seed(1234)
60 # Make fake data
61 cls.datadir = os.path.join(TESTDIR, "data")
63 cls.fieldNumber = 0
64 cls.nFields = 1
65 cls.instrumentName = "HSC"
66 cls.instrument = wcsfit.Instrument(cls.instrumentName)
67 cls.refEpoch = 57205.5
69 # Make test inputVisitSummary. VisitSummaryTables are taken from
70 # collection HSC/runs/RC2/w_2022_20/DM-34794
71 cls.testVisits = [1176, 17900, 17930, 17934]
72 cls.inputVisitSummary = []
73 for testVisit in cls.testVisits:
74 visSum = afwTable.ExposureCatalog.readFits(
75 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
76 )
77 cls.inputVisitSummary.append(visSum)
79 cls.config = GbdesAstrometricFitConfig()
80 cls.config.systematicError = 0
81 cls.config.devicePolyOrder = 4
82 cls.config.exposurePolyOrder = 6
83 cls.config.fitReserveFraction = 0
84 cls.config.fitReserveRandomSeed = 1234
85 cls.config.saveModelParams = True
86 cls.config.allowSelfMatches = True
87 cls.task = GbdesAstrometricFitTask(config=cls.config)
89 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
90 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch
91 )
93 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
94 cls.inputVisitSummary, cls.exposureInfo.medianEpoch
95 )
97 # Bounding box of observations:
98 raMins, raMaxs = [], []
99 decMins, decMaxs = [], []
100 for visSum in cls.inputVisitSummary:
101 raMins.append(visSum["raCorners"].min())
102 raMaxs.append(visSum["raCorners"].max())
103 decMins.append(visSum["decCorners"].min())
104 decMaxs.append(visSum["decCorners"].max())
105 raMin = min(raMins)
106 raMax = max(raMaxs)
107 decMin = min(decMins)
108 decMax = max(decMaxs)
110 corners = [
111 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
112 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
113 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
114 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(),
115 ]
116 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
118 # Make random set of data in a bounding box determined by input visits
119 # Make wcs objects for the "true" model
120 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
121 raMin, raMax, decMin, decMax, cls.config.matchRadius
122 )
124 # Fraction of simulated stars in the reference catalog and science
125 # exposures
126 inReferenceFraction = 1
127 inScienceFraction = 1
129 # Make a reference catalog and load it into ReferenceObjectLoader
130 refDataId, deferredRefCat = cls._make_refCat(
131 starIds, starRAs, starDecs, inReferenceFraction, cls.boundingPolygon
132 )
133 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
134 cls.refObjectLoader.config.requireProperMotion = False
135 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
137 cls.task.refObjectLoader = cls.refObjectLoader
139 # Get True WCS for stars:
140 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f:
141 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
143 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
145 # Make source catalogs:
146 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction)
148 cls.outputs = cls.task.run(
149 cls.inputCatalogRefs,
150 cls.inputVisitSummary,
151 instrumentName=cls.instrumentName,
152 refEpoch=cls.refEpoch,
153 refObjectLoader=cls.refObjectLoader,
154 )
156 @staticmethod
157 def _make_simulated_stars(raMin, raMax, decMin, decMax, matchRadius, nStars=10000):
158 """Generate random positions for "stars" in an RA/Dec box.
160 Parameters
161 ----------
162 raMin : `float`
163 Minimum RA for simulated stars.
164 raMax : `float`
165 Maximum RA for simulated stars.
166 decMin : `float`
167 Minimum Dec for simulated stars.
168 decMax : `float`
169 Maximum Dec for simulated stars.
170 matchRadius : `float`
171 Minimum allowed distance in arcsec between stars.
172 nStars : `int`, optional
173 Number of stars to simulate. Final number will be lower if any
174 too-close stars are dropped.
176 Returns
177 -------
178 nStars : `int`
179 Number of stars simulated.
180 starIds: `np.ndarray`
181 Unique identification number for stars.
182 starRAs: `np.ndarray`
183 Simulated Right Ascensions.
184 starDecs: `np.ndarray`
185 Simulated Declination.
186 """
187 starIds = np.arange(nStars)
188 starRAs = np.random.random(nStars) * (raMax - raMin) + raMin
189 starDecs = np.random.random(nStars) * (decMax - decMin) + decMin
190 # Remove neighbors:
191 with Matcher(starRAs, starDecs) as matcher:
192 idx = matcher.query_groups(matchRadius / 3600.0, min_match=2)
193 if len(idx) > 0:
194 neighbors = np.unique(np.concatenate(idx))
195 starRAs = np.delete(starRAs, neighbors)
196 starDecs = np.delete(starDecs, neighbors)
197 nStars = len(starRAs)
198 starIds = np.arange(nStars)
200 return nStars, starIds, starRAs, starDecs
202 @classmethod
203 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction, bounds):
204 """Make reference catalog from a subset of the simulated data
206 Parameters
207 ----------
208 starIds : `np.ndarray` [`int`]
209 Source ids for the simulated stars
210 starRas : `np.ndarray` [`float`]
211 RAs of the simulated stars
212 starDecs : `np.ndarray` [`float`]
213 Decs of the simulated stars
214 inReferenceFraction : float
215 Percentage of simulated stars to include in reference catalog
216 bounds : `lsst.sphgeom.ConvexPolygon`
217 Boundary of the reference catalog region
219 Returns
220 -------
221 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId`
222 Object that replicates the functionality of a dataId.
223 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
224 Dataset handle for reference catalog.
225 """
226 nRefs = int(cls.nStars * inReferenceFraction)
227 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
228 # Make simpleCatalog to hold data, create datasetRef with `region`
229 # determined by bounding box used in above simulate.
230 refSchema = afwTable.SimpleTable.makeMinimalSchema()
231 idKey = refSchema.addField("sourceId", type="I")
232 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64)
233 raErrKey = refSchema.addField("coord_raErr", units="rad", type=np.float64)
234 decErrKey = refSchema.addField("coord_decErr", units="rad", type=np.float64)
235 pmraErrKey = refSchema.addField("pm_raErr", units="rad2 / yr", type=np.float64)
236 pmdecErrKey = refSchema.addField("pm_decErr", units="rad2 / yr", type=np.float64)
237 refCat = afwTable.SimpleCatalog(refSchema)
238 ref_md = PropertyList()
239 ref_md.set("REFCAT_FORMAT_VERSION", 1)
240 refCat.table.setMetadata(ref_md)
241 for i in refStarIndices:
242 record = refCat.addNew()
243 record.set(idKey, starIds[i])
244 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
245 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
246 record.set(fluxKey, 1)
247 record.set(raErrKey, 0.00001)
248 record.set(decErrKey, 0.00001)
249 record.set(pmraErrKey, 1e-9)
250 record.set(pmdecErrKey, 1e-9)
251 refDataId = MockRefcatDataId(bounds)
252 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
254 return refDataId, deferredRefCat
256 @classmethod
257 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
258 """Make a `pd.DataFrame` catalog with the columns needed for the
259 object selector.
261 Parameters
262 ----------
263 starIds : `np.ndarray` [`int`]
264 Source ids for the simulated stars
265 starRas : `np.ndarray` [`float`]
266 RAs of the simulated stars
267 starDecs : `np.ndarray` [`float`]
268 Decs of the simulated stars
269 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
270 WCS with which to simulate the source pixel coordinates
271 inReferenceFraction : float
272 Percentage of simulated stars to include in reference catalog
274 Returns
275 -------
276 sourceCat : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
277 List of reference to source catalogs.
278 """
279 inputCatalogRefs = []
280 # Take a subset of the simulated data
281 # Use true wcs objects to put simulated data into ccds
282 bbox = lsst.geom.BoxD(
283 lsst.geom.Point2D(
284 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
285 ),
286 lsst.geom.Point2D(
287 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
288 ),
289 )
290 bboxCorners = bbox.getCorners()
291 cls.inputCatalogRefs = []
292 for v, visit in enumerate(cls.testVisits):
293 nVisStars = int(cls.nStars * inScienceFraction)
294 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
295 visitStarIds = starIds[visitStarIndices]
296 visitStarRas = starRas[visitStarIndices]
297 visitStarDecs = starDecs[visitStarIndices]
298 sourceCats = []
299 for detector in trueWCSs[visit]:
300 detWcs = detector.getWcs()
301 detectorId = detector["id"]
302 radecCorners = detWcs.pixelToSky(bboxCorners)
303 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
304 detectorIndices = detectorFootprint.contains(
305 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
306 )
307 nDetectorStars = detectorIndices.sum()
308 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"]
310 ones_like = np.ones(nDetectorStars)
311 zeros_like = np.zeros(nDetectorStars, dtype=bool)
313 x, y = detWcs.skyToPixelArray(
314 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
315 )
317 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs()
318 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
320 sourceDict = {}
321 sourceDict["detector"] = detectorArray
322 sourceDict["sourceId"] = visitStarIds[detectorIndices]
323 sourceDict["x"] = x
324 sourceDict["y"] = y
325 sourceDict["xErr"] = 1e-3 * ones_like
326 sourceDict["yErr"] = 1e-3 * ones_like
327 sourceDict["inputRA"] = inputRa
328 sourceDict["inputDec"] = inputDec
329 sourceDict["trueRA"] = visitStarRas[detectorIndices]
330 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
331 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
332 sourceDict[key] = ones_like
333 for key in [
334 "pixelFlags_edge",
335 "pixelFlags_saturated",
336 "pixelFlags_interpolatedCenter",
337 "pixelFlags_interpolated",
338 "pixelFlags_crCenter",
339 "pixelFlags_bad",
340 "hsmPsfMoments_flag",
341 "apFlux_12_0_flag",
342 "extendedness",
343 "sizeExtendedness",
344 "parentSourceId",
345 "deblend_nChild",
346 "ixy",
347 ]:
348 sourceDict[key] = zeros_like
349 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
350 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
352 sourceCat = pd.DataFrame(sourceDict)
353 sourceCats.append(sourceCat)
355 visitSourceTable = pd.concat(sourceCats)
357 inputCatalogRef = InMemoryDatasetHandle(
358 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit}
359 )
361 inputCatalogRefs.append(inputCatalogRef)
363 return inputCatalogRefs
365 @classmethod
366 def _make_wcs(cls, model, inputVisitSummaries):
367 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
369 Parameters
370 ----------
371 model : `dict`
372 Dictionary with WCS model parameters
373 inputVisitSummaries : `list` [`lsst.afw.table.ExposureCatalog`]
374 Visit summary catalogs
375 Returns
376 -------
377 catalogs : `dict` [`int`, `lsst.afw.table.ExposureCatalog`]
378 Visit summary catalogs with WCS set to input model
379 """
381 # Pixels will need to be rescaled before going into the mappings
382 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"]
383 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"]
385 catalogs = {}
386 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
387 schema.addField("visit", type="L", doc="Visit number")
388 for visitSum in inputVisitSummaries:
389 visit = visitSum[0]["visit"]
390 visitMapName = f"{visit}/poly"
391 visitModel = model[visitMapName]
393 catalog = lsst.afw.table.ExposureCatalog(schema)
394 catalog.resize(len(visitSum))
395 catalog["visit"] = visit
397 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
399 visitMapType = visitModel["Type"]
400 visitDict = {"Type": visitMapType}
401 if visitMapType == "Poly":
402 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"]
403 visitDict["Coefficients"] = mapCoefficients
405 for d, detector in enumerate(visitSum):
406 detectorId = detector["id"]
407 detectorMapName = f"HSC/{detectorId}/poly"
408 detectorModel = model[detectorMapName]
410 detectorMapType = detectorModel["Type"]
411 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict}
412 if detectorMapType == "Poly":
413 mapCoefficients = (
414 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"]
415 )
416 mapDict[detectorMapName]["Coefficients"] = mapCoefficients
418 outWCS = cls.task._make_afw_wcs(
419 mapDict,
420 raDec.getRa(),
421 raDec.getDec(),
422 doNormalizePixels=True,
423 xScale=xscale,
424 yScale=yscale,
425 )
426 catalog[d].setId(detectorId)
427 catalog[d].setWcs(outWCS)
429 catalog.sort()
430 catalogs[visit] = catalog
432 return catalogs
434 def test_get_exposure_info(self):
435 """Test that information for input exposures is as expected and that
436 the WCS in the class object gives approximately the same results as the
437 input `lsst.afw.geom.SkyWcs`.
438 """
440 # The total number of extensions is the number of detectors for each
441 # visit plus one for the reference catalog for each field.
442 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + self.nFields
444 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
446 taskVisits = set(self.extensionInfo.visit)
447 refVisits = np.arange(0, -1 * self.nFields, -1).tolist()
448 self.assertEqual(taskVisits, set(self.testVisits + refVisits))
450 xx = np.linspace(0, 2000, 3)
451 yy = np.linspace(0, 4000, 6)
452 xgrid, ygrid = np.meshgrid(xx, yy)
453 for visSum in self.inputVisitSummary:
454 visit = visSum[0]["visit"]
455 for detectorInfo in visSum:
456 detector = detectorInfo["id"]
457 extensionIndex = np.flatnonzero(
458 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector)
459 )[0]
460 fitWcs = self.extensionInfo.wcs[extensionIndex]
461 calexpWcs = detectorInfo.getWcs()
463 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())])
465 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
467 # The pixel origin maps to a position slightly off from the
468 # tangent plane origin, so we want to use this value for the
469 # tangent-plane-to-sky part of the mapping.
470 tangentPlaneCenter = fitWcs.toWorld(
471 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
472 )
473 tangentPlaneOrigin = lsst.geom.Point2D(tangentPlaneCenter)
474 skyOrigin = calexpWcs.pixelToSky(
475 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
476 )
477 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0.0 * lsst.geom.degrees, True)
478 iwcToSkyWcs = afwgeom.makeSkyWcs(tangentPlaneOrigin, skyOrigin, cdMatrix)
479 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(
480 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True
481 )
483 np.testing.assert_allclose(calexpra, newRAdeg)
484 np.testing.assert_allclose(calexpdec, newDecdeg)
486 def test_refCatLoader(self):
487 """Test that we can load objects from refCat"""
489 tmpAssociations = wcsfit.FoFClass(
490 self.fields,
491 [self.instrument],
492 self.exposuresHelper,
493 [self.fieldRadius.asDegrees()],
494 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
495 )
497 self.task._load_refcat(
498 self.refObjectLoader,
499 self.extensionInfo,
500 associations=tmpAssociations,
501 center=self.fieldCenter,
502 radius=self.fieldRadius,
503 epoch=self.exposureInfo.medianEpoch,
504 )
506 # We have only loaded one catalog, so getting the 'matches' should just
507 # return the same objects we put in, except some random objects that
508 # are too close together.
509 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
511 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
513 self.assertLessEqual(nMatches, self.nStars)
514 self.assertGreater(nMatches, self.nStars * 0.9)
516 def test_loading_and_association(self):
517 """Test that objects can be loaded and correctly associated."""
518 # Running `_load_catalogs_and_associate` changes the input WCSs, so
519 # recalculate them here so that the variables shared among tests are
520 # not affected.
521 instrument = wcsfit.Instrument(self.instrumentName)
522 _, exposuresHelper, extensionInfo = self.task._get_exposure_info(
523 self.inputVisitSummary, instrument, refEpoch=self.refEpoch
524 )
526 tmpAssociations = wcsfit.FoFClass(
527 self.fields,
528 [self.instrument],
529 exposuresHelper,
530 [self.fieldRadius.asDegrees()],
531 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
532 )
533 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, extensionInfo)
535 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
537 matchIds = []
538 correctMatches = []
539 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
540 objVisitInd = extensionInfo.visitIndex[e]
541 objDet = extensionInfo.detector[e]
542 extnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet
543 objInfo = self.inputCatalogRefs[objVisitInd].get()[extnInds].iloc[o]
544 if s == 0:
545 if len(matchIds) > 0:
546 correctMatches.append(len(set(matchIds)) == 1)
547 matchIds = []
549 matchIds.append(objInfo["sourceId"])
551 # A few matches may incorrectly associate sources because of the random
552 # positions
553 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
555 def test_make_outputs(self):
556 """Test that the run method recovers the input model parameters."""
557 for v, visit in enumerate(self.testVisits):
558 visitSummary = self.inputVisitSummary[v]
559 outputWcsCatalog = self.outputs.outputWcss[visit]
560 visitSources = self.inputCatalogRefs[v].get()
561 for d, detectorRow in enumerate(visitSummary):
562 detectorId = detectorRow["id"]
563 fitwcs = outputWcsCatalog[d].getWcs()
564 detSources = visitSources[visitSources["detector"] == detectorId]
565 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
566 dRA = fitRA - detSources["trueRA"]
567 dDec = fitDec - detSources["trueDec"]
568 # Check that input coordinates match the output coordinates
569 self.assertAlmostEqual(np.mean(dRA), 0)
570 self.assertAlmostEqual(np.std(dRA), 0)
571 self.assertAlmostEqual(np.mean(dDec), 0)
572 self.assertAlmostEqual(np.std(dDec), 0)
574 def test_compute_model_params(self):
575 """Test the optional model parameters and covariance output."""
576 modelParams = pd.DataFrame(self.outputs.modelParams)
577 # Check that DataFrame is the expected size.
578 shape = modelParams.shape
579 self.assertEqual(shape[0] + 4, shape[1])
580 # Check that covariance matrix is symmetric.
581 covariance = (modelParams.iloc[:, 4:]).to_numpy()
582 np.testing.assert_allclose(covariance, covariance.T, atol=1e-18)
584 def test_run(self):
585 """Test that run method recovers the input model parameters"""
586 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
588 for v, visit in enumerate(self.testVisits):
589 visitSummary = self.inputVisitSummary[v]
590 visitMapName = f"{visit}/poly"
592 origModel = self.trueModel[visitMapName]
593 if origModel["Type"] != "Identity":
594 fitModel = outputMaps[visitMapName]
595 origXPoly = origModel["XPoly"]["Coefficients"]
596 origYPoly = origModel["YPoly"]["Coefficients"]
597 fitXPoly = fitModel[: len(origXPoly)]
598 fitYPoly = fitModel[len(origXPoly) :]
600 absDiffX = abs(fitXPoly - origXPoly)
601 absDiffY = abs(fitYPoly - origYPoly)
602 # Check that input visit model matches fit
603 np.testing.assert_array_less(absDiffX, 1e-6)
604 np.testing.assert_array_less(absDiffY, 1e-6)
605 for d, detectorRow in enumerate(visitSummary):
606 detectorId = detectorRow["id"]
607 detectorMapName = f"HSC/{detectorId}/poly"
608 origModel = self.trueModel[detectorMapName]
609 if (origModel["Type"] != "Identity") and (v == 0):
610 fitModel = outputMaps[detectorMapName]
611 origXPoly = origModel["XPoly"]["Coefficients"]
612 origYPoly = origModel["YPoly"]["Coefficients"]
613 fitXPoly = fitModel[: len(origXPoly)]
614 fitYPoly = fitModel[len(origXPoly) :]
615 absDiffX = abs(fitXPoly - origXPoly)
616 absDiffY = abs(fitYPoly - origYPoly)
617 # Check that input detector model matches fit
618 np.testing.assert_array_less(absDiffX, 1e-7)
619 np.testing.assert_array_less(absDiffY, 1e-7)
621 def test_missingWcs(self):
622 """Test that task does not fail when the input WCS is None for one
623 extension and that the fit WCS for that extension returns a finite
624 result.
625 """
626 inputVisitSummary = self.inputVisitSummary.copy()
627 # Set one WCS to be None
628 testVisit = 0
629 testDetector = 20
630 inputVisitSummary[testVisit][testDetector].setWcs(None)
632 outputs = self.task.run(
633 self.inputCatalogRefs,
634 inputVisitSummary,
635 instrumentName=self.instrumentName,
636 refEpoch=self.refEpoch,
637 refObjectLoader=self.refObjectLoader,
638 )
640 # Check that the fit WCS for the extension with input WCS=None returns
641 # finite sky values.
642 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
643 testSky = testWcs.pixelToSky(0, 0)
644 self.assertTrue(testSky.isFinite())
647class TestGbdesGlobalAstrometricFit(TestGbdesAstrometricFit):
648 @classmethod
649 def setUpClass(cls):
650 # Set random seed
651 np.random.seed(1234)
653 # Fraction of simulated stars in the reference catalog and science
654 # exposures
655 inReferenceFraction = 1
656 inScienceFraction = 1
658 # Make fake data
659 cls.datadir = os.path.join(TESTDIR, "data")
661 cls.nFields = 2
662 cls.instrumentName = "HSC"
663 cls.instrument = wcsfit.Instrument(cls.instrumentName)
664 cls.refEpoch = 57205.5
666 # Make test inputVisitSummary. VisitSummaryTables are taken from
667 # collection HSC/runs/RC2/w_2022_20/DM-34794
668 cls.testVisits = [1176, 17900, 17930, 17934, 36434, 36460, 36494, 36446]
669 cls.inputVisitSummary = []
670 for testVisit in cls.testVisits:
671 visSum = afwTable.ExposureCatalog.readFits(
672 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
673 )
674 cls.inputVisitSummary.append(visSum)
676 cls.config = GbdesGlobalAstrometricFitConfig()
677 cls.config.systematicError = 0
678 cls.config.devicePolyOrder = 4
679 cls.config.exposurePolyOrder = 6
680 cls.config.fitReserveFraction = 0
681 cls.config.fitReserveRandomSeed = 1234
682 cls.config.saveModelParams = True
683 cls.task = GbdesGlobalAstrometricFitTask(config=cls.config)
685 cls.fields, cls.fieldRegions = cls.task._prep_sky(cls.inputVisitSummary)
687 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
688 cls.inputVisitSummary, cls.instrument, fieldRegions=cls.fieldRegions
689 )
691 refDataIds, deferredRefCats = [], []
692 allStarIds = []
693 allStarRAs = []
694 allStarDecs = []
695 for region in cls.fieldRegions.values():
696 # Bounding box of observations:
697 bbox = region.getBoundingBox()
698 raMin = bbox.getLon().getA().asDegrees()
699 raMax = bbox.getLon().getB().asDegrees()
700 decMin = bbox.getLat().getA().asDegrees()
701 decMax = bbox.getLat().getB().asDegrees()
703 # Make random set of data in a bounding box determined by input
704 # visits
705 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
706 raMin, raMax, decMin, decMax, cls.config.matchRadius
707 )
709 allStarIds.append(starIds)
710 allStarRAs.append(starRAs)
711 allStarDecs.append(starDecs)
713 corners = [
714 lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees).getVector()
715 for (ra, dec) in zip([raMin, raMax, raMax, raMin], [decMin, decMin, decMax, decMax])
716 ]
717 conv_box = lsst.sphgeom.ConvexPolygon.convexHull(corners)
718 # Make a reference catalog that will be loaded into
719 # ReferenceObjectLoader
720 refDataId, deferredRefCat = cls._make_refCat(
721 starIds, starRAs, starDecs, inReferenceFraction, conv_box
722 )
723 refDataIds.append(refDataId)
724 deferredRefCats.append(deferredRefCat)
726 cls.refObjectLoader = ReferenceObjectLoader(refDataIds, deferredRefCats)
727 cls.refObjectLoader.config.requireProperMotion = False
728 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
730 cls.task.refObjectLoader = cls.refObjectLoader
732 allRefObjects, allRefCovariances = {}, {}
733 for f, fieldRegion in cls.fieldRegions.items():
734 refObjects, refCovariance = cls.task._load_refcat(
735 cls.refObjectLoader, cls.extensionInfo, epoch=cls.exposureInfo.medianEpoch, region=fieldRegion
736 )
737 allRefObjects[f] = refObjects
738 allRefCovariances[f] = refCovariance
739 cls.refObjects = allRefObjects
741 # Get True WCS for stars:
742 with open(os.path.join(cls.datadir, "sample_global_wcs.yaml"), "r") as f:
743 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
745 cls.trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
747 cls.isolatedStarCatalogs, cls.isolatedStarSources = cls._make_isolatedStars(
748 allStarIds, allStarRAs, allStarDecs, cls.trueWCSs, inScienceFraction
749 )
751 cls.outputs = cls.task.run(
752 cls.inputVisitSummary,
753 cls.isolatedStarSources,
754 cls.isolatedStarCatalogs,
755 instrumentName=cls.instrumentName,
756 refEpoch=cls.refEpoch,
757 refObjectLoader=cls.refObjectLoader,
758 )
760 @classmethod
761 def _make_isolatedStars(cls, allStarIds, allStarRAs, allStarDecs, trueWCSs, inScienceFraction):
762 """Given a subset of the simulated data, make source catalogs and star
763 catalogs.
765 This takes the true WCSs to go from the RA and Decs of the simulated
766 stars to pixel coordinates for a given visit and detector. If those
767 pixel coordinates are within the bounding box of the detector, the
768 source and visit information is put in the corresponding catalog of
769 isolated star sources.
771 Parameters
772 ----------
773 allStarIds : `np.ndarray` [`int`]
774 Source ids for the simulated stars
775 allStarRas : `np.ndarray` [`float`]
776 RAs of the simulated stars
777 allStarDecs : `np.ndarray` [`float`]
778 Decs of the simulated stars
779 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
780 WCS with which to simulate the source pixel coordinates
781 inReferenceFraction : float
782 Percentage of simulated stars to include in reference catalog
784 Returns
785 -------
786 isolatedStarCatalogRefs : `list`
787 [`lsst.pipe.base.InMemoryDatasetHandle`]
788 List of references to isolated star catalogs.
789 isolatedStarSourceRefs : `list`
790 [`lsst.pipe.base.InMemoryDatasetHandle`]
791 List of references to isolated star sources.
792 """
793 bbox = lsst.geom.BoxD(
794 lsst.geom.Point2D(
795 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
796 ),
797 lsst.geom.Point2D(
798 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
799 ),
800 )
801 bboxCorners = bbox.getCorners()
803 isolatedStarCatalogRefs = []
804 isolatedStarSourceRefs = []
805 for i in range(len(allStarIds)):
806 starIds = allStarIds[i]
807 starRAs = allStarRAs[i]
808 starDecs = allStarDecs[i]
809 isolatedStarCatalog = pd.DataFrame({"ra": starRAs, "dec": starDecs}, index=starIds)
810 isolatedStarCatalogRefs.append(
811 InMemoryDatasetHandle(isolatedStarCatalog, storageClass="DataFrame", dataId={"tract": 0})
812 )
813 sourceCats = []
814 for v, visit in enumerate(cls.testVisits):
815 nVisStars = int(cls.nStars * inScienceFraction)
816 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
817 visitStarIds = starIds[visitStarIndices]
818 visitStarRas = starRAs[visitStarIndices]
819 visitStarDecs = starDecs[visitStarIndices]
820 for detector in trueWCSs[visit]:
821 detWcs = detector.getWcs()
822 detectorId = detector["id"]
823 radecCorners = detWcs.pixelToSky(bboxCorners)
824 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
825 detectorIndices = detectorFootprint.contains(
826 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
827 )
828 nDetectorStars = detectorIndices.sum()
829 detectorArray = np.ones(nDetectorStars, dtype=int) * detector["id"]
830 visitArray = np.ones(nDetectorStars, dtype=int) * visit
832 ones_like = np.ones(nDetectorStars)
833 zeros_like = np.zeros(nDetectorStars, dtype=bool)
835 x, y = detWcs.skyToPixelArray(
836 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
837 )
839 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[
840 0
841 ].getWcs()
842 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
844 sourceDict = {}
845 sourceDict["detector"] = detectorArray
846 sourceDict["visit"] = visitArray
847 sourceDict["obj_index"] = visitStarIds[detectorIndices]
848 sourceDict["x"] = x
849 sourceDict["y"] = y
850 sourceDict["xErr"] = 1e-3 * ones_like
851 sourceDict["yErr"] = 1e-3 * ones_like
852 sourceDict["inputRA"] = inputRa
853 sourceDict["inputDec"] = inputDec
854 sourceDict["trueRA"] = visitStarRas[detectorIndices]
855 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
856 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
857 sourceDict[key] = ones_like
858 for key in [
859 "pixelFlags_edge",
860 "pixelFlags_saturated",
861 "pixelFlags_interpolatedCenter",
862 "pixelFlags_interpolated",
863 "pixelFlags_crCenter",
864 "pixelFlags_bad",
865 "hsmPsfMoments_flag",
866 "apFlux_12_0_flag",
867 "extendedness",
868 "parentSourceId",
869 "deblend_nChild",
870 "ixy",
871 ]:
872 sourceDict[key] = zeros_like
873 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
874 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
876 sourceCat = pd.DataFrame(sourceDict)
877 sourceCats.append(sourceCat)
879 isolatedStarSourceTable = pd.concat(sourceCats, ignore_index=True)
880 isolatedStarSourceTable = isolatedStarSourceTable.sort_values(by=["obj_index"])
881 isolatedStarSourceRefs.append(
882 InMemoryDatasetHandle(isolatedStarSourceTable, storageClass="DataFrame", dataId={"tract": 0})
883 )
885 return isolatedStarCatalogRefs, isolatedStarSourceRefs
887 def test_loading_and_association(self):
888 """Test that associated objects actually correspond to the same
889 simulated object."""
890 associations, sourceDict = self.task._associate_from_isolated_sources(
891 self.isolatedStarSources, self.isolatedStarCatalogs, self.extensionInfo, self.refObjects
892 )
894 object_groups = np.flatnonzero(np.array(associations.sequence) == 0)
895 for i in range(len(object_groups) - 1)[:10]:
896 ras, decs = [], []
897 for ind in np.arange(object_groups[i], object_groups[i + 1]):
898 visit = self.extensionInfo.visit[associations.extn[ind]]
899 detectorInd = self.extensionInfo.detectorIndex[associations.extn[ind]]
900 detector = self.extensionInfo.detector[associations.extn[ind]]
901 if detectorInd == -1:
902 ra = self.refObjects[visit * -1]["ra"][associations.obj[ind]]
903 dec = self.refObjects[visit * -1]["dec"][associations.obj[ind]]
904 ras.append(ra)
905 decs.append(dec)
906 else:
907 x = sourceDict[visit][detector]["x"][associations.obj[ind]]
908 y = sourceDict[visit][detector]["y"][associations.obj[ind]]
909 ra, dec = self.trueWCSs[visit][detectorInd].getWcs().pixelToSky(x, y)
910 ras.append(ra.asDegrees())
911 decs.append(dec.asDegrees())
912 np.testing.assert_allclose(ras, ras[0])
913 np.testing.assert_allclose(decs, decs[0])
915 def test_refCatLoader(self):
916 """Test loading objects from the refCat in each of the fields."""
918 for region in self.fieldRegions.values():
919 refCat, refCov = self.task._load_refcat(
920 self.refObjectLoader, self.extensionInfo, region=region, epoch=self.exposureInfo.medianEpoch
921 )
922 assert len(refCat) > 0
924 def test_make_outputs(self):
925 """Test that the run method recovers the input model parameters."""
926 for isolatedStarSourceRef in self.isolatedStarSources:
927 iss = isolatedStarSourceRef.get()
928 visits = np.unique(iss["visit"])
929 for v, visit in enumerate(visits):
930 outputWcsCatalog = self.outputs.outputWcss[visit]
931 visitSources = iss[iss["visit"] == visit]
932 detectors = outputWcsCatalog["id"]
933 for d, detectorId in enumerate(detectors):
934 fitwcs = outputWcsCatalog[d].getWcs()
935 detSources = visitSources[visitSources["detector"] == detectorId]
936 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
937 dRA = fitRA - detSources["trueRA"]
938 dDec = fitDec - detSources["trueDec"]
939 # Check that input coordinates match the output coordinates
940 self.assertAlmostEqual(np.mean(dRA), 0)
941 self.assertAlmostEqual(np.std(dRA), 0)
942 self.assertAlmostEqual(np.mean(dDec), 0)
943 self.assertAlmostEqual(np.std(dDec), 0)
945 def test_missingWcs(self):
946 """Test that task does not fail when the input WCS is None for one
947 extension and that the fit WCS for that extension returns a finite
948 result.
949 """
950 inputVisitSummary = self.inputVisitSummary.copy()
951 # Set one WCS to be None
952 testVisit = 0
953 testDetector = 20
954 inputVisitSummary[testVisit][testDetector].setWcs(None)
956 outputs = self.task.run(
957 inputVisitSummary,
958 self.isolatedStarSources,
959 self.isolatedStarCatalogs,
960 instrumentName=self.instrumentName,
961 refEpoch=self.refEpoch,
962 refObjectLoader=self.refObjectLoader,
963 )
965 # Check that the fit WCS for the extension with input WCS=None returns
966 # finite sky values.
967 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
968 testSky = testWcs.pixelToSky(0, 0)
969 self.assertTrue(testSky.isFinite())
972def setup_module(module):
973 lsst.utils.tests.init()
976if __name__ == "__main__": 976 ↛ 977line 976 didn't jump to line 977, because the condition on line 976 was never true
977 lsst.utils.tests.init()
978 unittest.main()