Coverage for tests/test_gbdesAstrometricFit.py: 11%
484 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-16 13:25 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-16 13:25 +0000
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import unittest
25import astropy.units as u
26import lsst.afw.geom as afwgeom
27import lsst.afw.table as afwTable
28import lsst.geom
29import lsst.utils
30import numpy as np
31import pandas as pd
32import wcsfit
33import yaml
34from lsst import sphgeom
35from lsst.daf.base import PropertyList
36from lsst.drp.tasks.gbdesAstrometricFit import (
37 GbdesAstrometricFitConfig,
38 GbdesAstrometricFitTask,
39 GbdesGlobalAstrometricFitConfig,
40 GbdesGlobalAstrometricFitTask,
41)
42from lsst.meas.algorithms import ReferenceObjectLoader
43from lsst.meas.algorithms.testUtils import MockRefcatDataId
44from lsst.pipe.base import InMemoryDatasetHandle
45from smatch.matcher import Matcher
47TESTDIR = os.path.abspath(os.path.dirname(__file__))
50class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
51 """This class tests `GbdesAstrometricFit` using real `visitSummaryTable`s
52 from HSC RC2 processing, with simulated sources for those visits.
53 """
55 @classmethod
56 def setUpClass(cls):
57 # Set random seed
58 np.random.seed(1234)
60 # Make fake data
61 cls.datadir = os.path.join(TESTDIR, "data")
63 cls.fieldNumber = 0
64 cls.nFields = 1
65 cls.instrumentName = "HSC"
66 cls.instrument = wcsfit.Instrument(cls.instrumentName)
67 cls.refEpoch = 57205.5
69 # Make test inputVisitSummary. VisitSummaryTables are taken from
70 # collection HSC/runs/RC2/w_2022_20/DM-34794
71 cls.testVisits = [1176, 17900, 17930, 17934]
72 cls.inputVisitSummary = []
73 for testVisit in cls.testVisits:
74 visSum = afwTable.ExposureCatalog.readFits(
75 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
76 )
77 cls.inputVisitSummary.append(visSum)
79 cls.config = GbdesAstrometricFitConfig()
80 cls.config.systematicError = 0
81 cls.config.devicePolyOrder = 4
82 cls.config.exposurePolyOrder = 6
83 cls.config.fitReserveFraction = 0
84 cls.config.fitReserveRandomSeed = 1234
85 cls.config.saveModelParams = True
86 cls.config.allowSelfMatches = True
87 cls.task = GbdesAstrometricFitTask(config=cls.config)
89 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
90 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch
91 )
93 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
94 cls.inputVisitSummary, cls.exposureInfo.medianEpoch
95 )
97 # Bounding box of observations:
98 raMins, raMaxs = [], []
99 decMins, decMaxs = [], []
100 for visSum in cls.inputVisitSummary:
101 raMins.append(visSum["raCorners"].min())
102 raMaxs.append(visSum["raCorners"].max())
103 decMins.append(visSum["decCorners"].min())
104 decMaxs.append(visSum["decCorners"].max())
105 raMin = min(raMins)
106 raMax = max(raMaxs)
107 decMin = min(decMins)
108 decMax = max(decMaxs)
110 corners = [
111 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
112 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
113 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
114 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(),
115 ]
116 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
118 # Make random set of data in a bounding box determined by input visits
119 # Make wcs objects for the "true" model
120 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
121 raMin, raMax, decMin, decMax, cls.config.matchRadius
122 )
124 # Fraction of simulated stars in the reference catalog and science
125 # exposures
126 inReferenceFraction = 1
127 inScienceFraction = 1
129 # Make a reference catalog and load it into ReferenceObjectLoader
130 refDataId, deferredRefCat = cls._make_refCat(
131 starIds, starRAs, starDecs, inReferenceFraction, cls.boundingPolygon
132 )
133 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
134 cls.refObjectLoader.config.requireProperMotion = False
135 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
137 cls.task.refObjectLoader = cls.refObjectLoader
139 # Get True WCS for stars:
140 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f:
141 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
143 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
145 # Make source catalogs:
146 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction)
148 cls.outputs = cls.task.run(
149 cls.inputCatalogRefs,
150 cls.inputVisitSummary,
151 instrumentName=cls.instrumentName,
152 refEpoch=cls.refEpoch,
153 refObjectLoader=cls.refObjectLoader,
154 )
156 @staticmethod
157 def _make_simulated_stars(raMin, raMax, decMin, decMax, matchRadius, nStars=10000):
158 """Generate random positions for "stars" in an RA/Dec box.
160 Parameters
161 ----------
162 raMin : `float`
163 Minimum RA for simulated stars.
164 raMax : `float`
165 Maximum RA for simulated stars.
166 decMin : `float`
167 Minimum Dec for simulated stars.
168 decMax : `float`
169 Maximum Dec for simulated stars.
170 matchRadius : `float`
171 Minimum allowed distance in arcsec between stars.
172 nStars : `int`, optional
173 Number of stars to simulate. Final number will be lower if any
174 too-close stars are dropped.
176 Returns
177 -------
178 nStars : `int`
179 Number of stars simulated.
180 starIds: `np.ndarray`
181 Unique identification number for stars.
182 starRAs: `np.ndarray`
183 Simulated Right Ascensions.
184 starDecs: `np.ndarray`
185 Simulated Declination.
186 """
187 starIds = np.arange(nStars)
188 starRAs = np.random.random(nStars) * (raMax - raMin) + raMin
189 starDecs = np.random.random(nStars) * (decMax - decMin) + decMin
190 # Remove neighbors:
191 with Matcher(starRAs, starDecs) as matcher:
192 idx = matcher.query_groups(matchRadius / 3600.0, min_match=2)
193 if len(idx) > 0:
194 neighbors = np.unique(np.concatenate(idx))
195 starRAs = np.delete(starRAs, neighbors)
196 starDecs = np.delete(starDecs, neighbors)
197 nStars = len(starRAs)
198 starIds = np.arange(nStars)
200 return nStars, starIds, starRAs, starDecs
202 @classmethod
203 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction, bounds):
204 """Make reference catalog from a subset of the simulated data
206 Parameters
207 ----------
208 starIds : `np.ndarray` [`int`]
209 Source ids for the simulated stars
210 starRas : `np.ndarray` [`float`]
211 RAs of the simulated stars
212 starDecs : `np.ndarray` [`float`]
213 Decs of the simulated stars
214 inReferenceFraction : float
215 Percentage of simulated stars to include in reference catalog
216 bounds : `lsst.sphgeom.ConvexPolygon`
217 Boundary of the reference catalog region
219 Returns
220 -------
221 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId`
222 Object that replicates the functionality of a dataId.
223 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
224 Dataset handle for reference catalog.
225 """
226 nRefs = int(cls.nStars * inReferenceFraction)
227 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
228 # Make simpleCatalog to hold data, create datasetRef with `region`
229 # determined by bounding box used in above simulate.
230 refSchema = afwTable.SimpleTable.makeMinimalSchema()
231 idKey = refSchema.addField("sourceId", type="I")
232 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64)
233 raErrKey = refSchema.addField("coord_raErr", units="rad", type=np.float64)
234 decErrKey = refSchema.addField("coord_decErr", units="rad", type=np.float64)
235 pmraErrKey = refSchema.addField("pm_raErr", units="rad2 / yr", type=np.float64)
236 pmdecErrKey = refSchema.addField("pm_decErr", units="rad2 / yr", type=np.float64)
237 refCat = afwTable.SimpleCatalog(refSchema)
238 ref_md = PropertyList()
239 ref_md.set("REFCAT_FORMAT_VERSION", 1)
240 refCat.table.setMetadata(ref_md)
241 for i in refStarIndices:
242 record = refCat.addNew()
243 record.set(idKey, starIds[i])
244 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
245 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
246 record.set(fluxKey, 1)
247 record.set(raErrKey, 0.00001)
248 record.set(decErrKey, 0.00001)
249 record.set(pmraErrKey, 1e-9)
250 record.set(pmdecErrKey, 1e-9)
251 refDataId = MockRefcatDataId(bounds)
252 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
254 return refDataId, deferredRefCat
256 @classmethod
257 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
258 """Make a `pd.DataFrame` catalog with the columns needed for the
259 object selector.
261 Parameters
262 ----------
263 starIds : `np.ndarray` [`int`]
264 Source ids for the simulated stars
265 starRas : `np.ndarray` [`float`]
266 RAs of the simulated stars
267 starDecs : `np.ndarray` [`float`]
268 Decs of the simulated stars
269 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
270 WCS with which to simulate the source pixel coordinates
271 inReferenceFraction : float
272 Percentage of simulated stars to include in reference catalog
274 Returns
275 -------
276 sourceCat : `list` [`lsst.pipe.base.InMemoryDatasetHandle`]
277 List of reference to source catalogs.
278 """
279 inputCatalogRefs = []
280 # Take a subset of the simulated data
281 # Use true wcs objects to put simulated data into ccds
282 bbox = lsst.geom.BoxD(
283 lsst.geom.Point2D(
284 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
285 ),
286 lsst.geom.Point2D(
287 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
288 ),
289 )
290 bboxCorners = bbox.getCorners()
291 cls.inputCatalogRefs = []
292 for v, visit in enumerate(cls.testVisits):
293 nVisStars = int(cls.nStars * inScienceFraction)
294 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
295 visitStarIds = starIds[visitStarIndices]
296 visitStarRas = starRas[visitStarIndices]
297 visitStarDecs = starDecs[visitStarIndices]
298 sourceCats = []
299 for detector in trueWCSs[visit]:
300 detWcs = detector.getWcs()
301 detectorId = detector["id"]
302 radecCorners = detWcs.pixelToSky(bboxCorners)
303 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
304 detectorIndices = detectorFootprint.contains(
305 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
306 )
307 nDetectorStars = detectorIndices.sum()
308 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"]
310 ones_like = np.ones(nDetectorStars)
311 zeros_like = np.zeros(nDetectorStars, dtype=bool)
313 x, y = detWcs.skyToPixelArray(
314 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
315 )
317 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs()
318 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
320 sourceDict = {}
321 sourceDict["detector"] = detectorArray
322 sourceDict["sourceId"] = visitStarIds[detectorIndices]
323 sourceDict["x"] = x
324 sourceDict["y"] = y
325 sourceDict["xErr"] = 1e-3 * ones_like
326 sourceDict["yErr"] = 1e-3 * ones_like
327 sourceDict["inputRA"] = inputRa
328 sourceDict["inputDec"] = inputDec
329 sourceDict["trueRA"] = visitStarRas[detectorIndices]
330 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
331 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
332 sourceDict[key] = ones_like
333 for key in [
334 "pixelFlags_edge",
335 "pixelFlags_saturated",
336 "pixelFlags_interpolatedCenter",
337 "pixelFlags_interpolated",
338 "pixelFlags_crCenter",
339 "pixelFlags_bad",
340 "hsmPsfMoments_flag",
341 "apFlux_12_0_flag",
342 "extendedness",
343 "parentSourceId",
344 "deblend_nChild",
345 "ixy",
346 ]:
347 sourceDict[key] = zeros_like
348 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
349 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
351 sourceCat = pd.DataFrame(sourceDict)
352 sourceCats.append(sourceCat)
354 visitSourceTable = pd.concat(sourceCats)
356 inputCatalogRef = InMemoryDatasetHandle(
357 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit}
358 )
360 inputCatalogRefs.append(inputCatalogRef)
362 return inputCatalogRefs
364 @classmethod
365 def _make_wcs(cls, model, inputVisitSummaries):
366 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
368 Parameters
369 ----------
370 model : `dict`
371 Dictionary with WCS model parameters
372 inputVisitSummaries : `list` [`lsst.afw.table.ExposureCatalog`]
373 Visit summary catalogs
374 Returns
375 -------
376 catalogs : `dict` [`int`, `lsst.afw.table.ExposureCatalog`]
377 Visit summary catalogs with WCS set to input model
378 """
380 # Pixels will need to be rescaled before going into the mappings
381 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"]
382 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"]
384 catalogs = {}
385 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
386 schema.addField("visit", type="L", doc="Visit number")
387 for visitSum in inputVisitSummaries:
388 visit = visitSum[0]["visit"]
389 visitMapName = f"{visit}/poly"
390 visitModel = model[visitMapName]
392 catalog = lsst.afw.table.ExposureCatalog(schema)
393 catalog.resize(len(visitSum))
394 catalog["visit"] = visit
396 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
398 visitMapType = visitModel["Type"]
399 visitDict = {"Type": visitMapType}
400 if visitMapType == "Poly":
401 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"]
402 visitDict["Coefficients"] = mapCoefficients
404 for d, detector in enumerate(visitSum):
405 detectorId = detector["id"]
406 detectorMapName = f"HSC/{detectorId}/poly"
407 detectorModel = model[detectorMapName]
409 detectorMapType = detectorModel["Type"]
410 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict}
411 if detectorMapType == "Poly":
412 mapCoefficients = (
413 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"]
414 )
415 mapDict[detectorMapName]["Coefficients"] = mapCoefficients
417 outWCS = cls.task._make_afw_wcs(
418 mapDict,
419 raDec.getRa(),
420 raDec.getDec(),
421 doNormalizePixels=True,
422 xScale=xscale,
423 yScale=yscale,
424 )
425 catalog[d].setId(detectorId)
426 catalog[d].setWcs(outWCS)
428 catalog.sort()
429 catalogs[visit] = catalog
431 return catalogs
433 def test_get_exposure_info(self):
434 """Test that information for input exposures is as expected and that
435 the WCS in the class object gives approximately the same results as the
436 input `lsst.afw.geom.SkyWcs`.
437 """
439 # The total number of extensions is the number of detectors for each
440 # visit plus one for the reference catalog for each field.
441 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + self.nFields
443 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
445 taskVisits = set(self.extensionInfo.visit)
446 refVisits = np.arange(0, -1 * self.nFields, -1).tolist()
447 self.assertEqual(taskVisits, set(self.testVisits + refVisits))
449 xx = np.linspace(0, 2000, 3)
450 yy = np.linspace(0, 4000, 6)
451 xgrid, ygrid = np.meshgrid(xx, yy)
452 for visSum in self.inputVisitSummary:
453 visit = visSum[0]["visit"]
454 for detectorInfo in visSum:
455 detector = detectorInfo["id"]
456 extensionIndex = np.flatnonzero(
457 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector)
458 )[0]
459 fitWcs = self.extensionInfo.wcs[extensionIndex]
460 calexpWcs = detectorInfo.getWcs()
462 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())])
464 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
466 # The pixel origin maps to a position slightly off from the
467 # tangent plane origin, so we want to use this value for the
468 # tangent-plane-to-sky part of the mapping.
469 tangentPlaneCenter = fitWcs.toWorld(
470 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
471 )
472 tangentPlaneOrigin = lsst.geom.Point2D(tangentPlaneCenter)
473 skyOrigin = calexpWcs.pixelToSky(
474 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
475 )
476 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0.0 * lsst.geom.degrees, True)
477 iwcToSkyWcs = afwgeom.makeSkyWcs(tangentPlaneOrigin, skyOrigin, cdMatrix)
478 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(
479 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True
480 )
482 np.testing.assert_allclose(calexpra, newRAdeg)
483 np.testing.assert_allclose(calexpdec, newDecdeg)
485 def test_refCatLoader(self):
486 """Test that we can load objects from refCat"""
488 tmpAssociations = wcsfit.FoFClass(
489 self.fields,
490 [self.instrument],
491 self.exposuresHelper,
492 [self.fieldRadius.asDegrees()],
493 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
494 )
496 self.task._load_refcat(
497 self.refObjectLoader,
498 self.extensionInfo,
499 associations=tmpAssociations,
500 center=self.fieldCenter,
501 radius=self.fieldRadius,
502 epoch=self.exposureInfo.medianEpoch,
503 )
505 # We have only loaded one catalog, so getting the 'matches' should just
506 # return the same objects we put in, except some random objects that
507 # are too close together.
508 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
510 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
512 self.assertLessEqual(nMatches, self.nStars)
513 self.assertGreater(nMatches, self.nStars * 0.9)
515 def test_loading_and_association(self):
516 """Test that objects can be loaded and correctly associated."""
517 # Running `_load_catalogs_and_associate` changes the input WCSs, so
518 # recalculate them here so that the variables shared among tests are
519 # not affected.
520 instrument = wcsfit.Instrument(self.instrumentName)
521 _, exposuresHelper, extensionInfo = self.task._get_exposure_info(
522 self.inputVisitSummary, instrument, refEpoch=self.refEpoch
523 )
525 tmpAssociations = wcsfit.FoFClass(
526 self.fields,
527 [self.instrument],
528 exposuresHelper,
529 [self.fieldRadius.asDegrees()],
530 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
531 )
532 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, extensionInfo)
534 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
536 matchIds = []
537 correctMatches = []
538 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
539 objVisitInd = extensionInfo.visitIndex[e]
540 objDet = extensionInfo.detector[e]
541 extnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet
542 objInfo = self.inputCatalogRefs[objVisitInd].get()[extnInds].iloc[o]
543 if s == 0:
544 if len(matchIds) > 0:
545 correctMatches.append(len(set(matchIds)) == 1)
546 matchIds = []
548 matchIds.append(objInfo["sourceId"])
550 # A few matches may incorrectly associate sources because of the random
551 # positions
552 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
554 def test_make_outputs(self):
555 """Test that the run method recovers the input model parameters."""
556 for v, visit in enumerate(self.testVisits):
557 visitSummary = self.inputVisitSummary[v]
558 outputWcsCatalog = self.outputs.outputWcss[visit]
559 visitSources = self.inputCatalogRefs[v].get()
560 for d, detectorRow in enumerate(visitSummary):
561 detectorId = detectorRow["id"]
562 fitwcs = outputWcsCatalog[d].getWcs()
563 detSources = visitSources[visitSources["detector"] == detectorId]
564 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
565 dRA = fitRA - detSources["trueRA"]
566 dDec = fitDec - detSources["trueDec"]
567 # Check that input coordinates match the output coordinates
568 self.assertAlmostEqual(np.mean(dRA), 0)
569 self.assertAlmostEqual(np.std(dRA), 0)
570 self.assertAlmostEqual(np.mean(dDec), 0)
571 self.assertAlmostEqual(np.std(dDec), 0)
573 def test_compute_model_params(self):
574 """Test the optional model parameters and covariance output."""
575 modelParams = pd.DataFrame(self.outputs.modelParams)
576 # Check that DataFrame is the expected size.
577 shape = modelParams.shape
578 self.assertEqual(shape[0] + 4, shape[1])
579 # Check that covariance matrix is symmetric.
580 covariance = (modelParams.iloc[:, 4:]).to_numpy()
581 np.testing.assert_allclose(covariance, covariance.T, atol=1e-18)
583 def test_run(self):
584 """Test that run method recovers the input model parameters"""
585 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
587 for v, visit in enumerate(self.testVisits):
588 visitSummary = self.inputVisitSummary[v]
589 visitMapName = f"{visit}/poly"
591 origModel = self.trueModel[visitMapName]
592 if origModel["Type"] != "Identity":
593 fitModel = outputMaps[visitMapName]
594 origXPoly = origModel["XPoly"]["Coefficients"]
595 origYPoly = origModel["YPoly"]["Coefficients"]
596 fitXPoly = fitModel[: len(origXPoly)]
597 fitYPoly = fitModel[len(origXPoly) :]
599 absDiffX = abs(fitXPoly - origXPoly)
600 absDiffY = abs(fitYPoly - origYPoly)
601 # Check that input visit model matches fit
602 np.testing.assert_array_less(absDiffX, 1e-6)
603 np.testing.assert_array_less(absDiffY, 1e-6)
604 for d, detectorRow in enumerate(visitSummary):
605 detectorId = detectorRow["id"]
606 detectorMapName = f"HSC/{detectorId}/poly"
607 origModel = self.trueModel[detectorMapName]
608 if (origModel["Type"] != "Identity") and (v == 0):
609 fitModel = outputMaps[detectorMapName]
610 origXPoly = origModel["XPoly"]["Coefficients"]
611 origYPoly = origModel["YPoly"]["Coefficients"]
612 fitXPoly = fitModel[: len(origXPoly)]
613 fitYPoly = fitModel[len(origXPoly) :]
614 absDiffX = abs(fitXPoly - origXPoly)
615 absDiffY = abs(fitYPoly - origYPoly)
616 # Check that input detector model matches fit
617 np.testing.assert_array_less(absDiffX, 1e-7)
618 np.testing.assert_array_less(absDiffY, 1e-7)
620 def test_missingWcs(self):
621 """Test that task does not fail when the input WCS is None for one
622 extension and that the fit WCS for that extension returns a finite
623 result.
624 """
625 inputVisitSummary = self.inputVisitSummary.copy()
626 # Set one WCS to be None
627 testVisit = 0
628 testDetector = 20
629 inputVisitSummary[testVisit][testDetector].setWcs(None)
631 outputs = self.task.run(
632 self.inputCatalogRefs,
633 inputVisitSummary,
634 instrumentName=self.instrumentName,
635 refEpoch=self.refEpoch,
636 refObjectLoader=self.refObjectLoader,
637 )
639 # Check that the fit WCS for the extension with input WCS=None returns
640 # finite sky values.
641 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
642 testSky = testWcs.pixelToSky(0, 0)
643 self.assertTrue(testSky.isFinite())
646class TestGbdesGlobalAstrometricFit(TestGbdesAstrometricFit):
647 @classmethod
648 def setUpClass(cls):
649 # Set random seed
650 np.random.seed(1234)
652 # Fraction of simulated stars in the reference catalog and science
653 # exposures
654 inReferenceFraction = 1
655 inScienceFraction = 1
657 # Make fake data
658 cls.datadir = os.path.join(TESTDIR, "data")
660 cls.nFields = 2
661 cls.instrumentName = "HSC"
662 cls.instrument = wcsfit.Instrument(cls.instrumentName)
663 cls.refEpoch = 57205.5
665 # Make test inputVisitSummary. VisitSummaryTables are taken from
666 # collection HSC/runs/RC2/w_2022_20/DM-34794
667 cls.testVisits = [1176, 17900, 17930, 17934, 36434, 36460, 36494, 36446]
668 cls.inputVisitSummary = []
669 for testVisit in cls.testVisits:
670 visSum = afwTable.ExposureCatalog.readFits(
671 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
672 )
673 cls.inputVisitSummary.append(visSum)
675 cls.config = GbdesGlobalAstrometricFitConfig()
676 cls.config.systematicError = 0
677 cls.config.devicePolyOrder = 4
678 cls.config.exposurePolyOrder = 6
679 cls.config.fitReserveFraction = 0
680 cls.config.fitReserveRandomSeed = 1234
681 cls.config.saveModelParams = True
682 cls.task = GbdesGlobalAstrometricFitTask(config=cls.config)
684 cls.fields, cls.fieldRegions = cls.task._prep_sky(cls.inputVisitSummary)
686 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
687 cls.inputVisitSummary, cls.instrument, fieldRegions=cls.fieldRegions
688 )
690 refDataIds, deferredRefCats = [], []
691 allStarIds = []
692 allStarRAs = []
693 allStarDecs = []
694 for region in cls.fieldRegions.values():
695 # Bounding box of observations:
696 bbox = region.getBoundingBox()
697 raMin = bbox.getLon().getA().asDegrees()
698 raMax = bbox.getLon().getB().asDegrees()
699 decMin = bbox.getLat().getA().asDegrees()
700 decMax = bbox.getLat().getB().asDegrees()
702 # Make random set of data in a bounding box determined by input
703 # visits
704 cls.nStars, starIds, starRAs, starDecs = cls._make_simulated_stars(
705 raMin, raMax, decMin, decMax, cls.config.matchRadius
706 )
708 allStarIds.append(starIds)
709 allStarRAs.append(starRAs)
710 allStarDecs.append(starDecs)
712 corners = [
713 lsst.geom.SpherePoint(ra, dec, lsst.geom.degrees).getVector()
714 for (ra, dec) in zip([raMin, raMax, raMax, raMin], [decMin, decMin, decMax, decMax])
715 ]
716 conv_box = lsst.sphgeom.ConvexPolygon.convexHull(corners)
717 # Make a reference catalog that will be loaded into
718 # ReferenceObjectLoader
719 refDataId, deferredRefCat = cls._make_refCat(
720 starIds, starRAs, starDecs, inReferenceFraction, conv_box
721 )
722 refDataIds.append(refDataId)
723 deferredRefCats.append(deferredRefCat)
725 cls.refObjectLoader = ReferenceObjectLoader(refDataIds, deferredRefCats)
726 cls.refObjectLoader.config.requireProperMotion = False
727 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
729 cls.task.refObjectLoader = cls.refObjectLoader
731 allRefObjects, allRefCovariances = {}, {}
732 for f, fieldRegion in cls.fieldRegions.items():
733 refObjects, refCovariance = cls.task._load_refcat(
734 cls.refObjectLoader, cls.extensionInfo, epoch=cls.exposureInfo.medianEpoch, region=fieldRegion
735 )
736 allRefObjects[f] = refObjects
737 allRefCovariances[f] = refCovariance
738 cls.refObjects = allRefObjects
740 # Get True WCS for stars:
741 with open(os.path.join(cls.datadir, "sample_global_wcs.yaml"), "r") as f:
742 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
744 cls.trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
746 cls.isolatedStarCatalogs, cls.isolatedStarSources = cls._make_isolatedStars(
747 allStarIds, allStarRAs, allStarDecs, cls.trueWCSs, inScienceFraction
748 )
750 cls.outputs = cls.task.run(
751 cls.inputVisitSummary,
752 cls.isolatedStarSources,
753 cls.isolatedStarCatalogs,
754 instrumentName=cls.instrumentName,
755 refEpoch=cls.refEpoch,
756 refObjectLoader=cls.refObjectLoader,
757 )
759 @classmethod
760 def _make_isolatedStars(cls, allStarIds, allStarRAs, allStarDecs, trueWCSs, inScienceFraction):
761 """Given a subset of the simulated data, make source catalogs and star
762 catalogs.
764 This takes the true WCSs to go from the RA and Decs of the simulated
765 stars to pixel coordinates for a given visit and detector. If those
766 pixel coordinates are within the bounding box of the detector, the
767 source and visit information is put in the corresponding catalog of
768 isolated star sources.
770 Parameters
771 ----------
772 allStarIds : `np.ndarray` [`int`]
773 Source ids for the simulated stars
774 allStarRas : `np.ndarray` [`float`]
775 RAs of the simulated stars
776 allStarDecs : `np.ndarray` [`float`]
777 Decs of the simulated stars
778 trueWCSs : `list` [`lsst.afw.geom.SkyWcs`]
779 WCS with which to simulate the source pixel coordinates
780 inReferenceFraction : float
781 Percentage of simulated stars to include in reference catalog
783 Returns
784 -------
785 isolatedStarCatalogRefs : `list`
786 [`lsst.pipe.base.InMemoryDatasetHandle`]
787 List of references to isolated star catalogs.
788 isolatedStarSourceRefs : `list`
789 [`lsst.pipe.base.InMemoryDatasetHandle`]
790 List of references to isolated star sources.
791 """
792 bbox = lsst.geom.BoxD(
793 lsst.geom.Point2D(
794 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
795 ),
796 lsst.geom.Point2D(
797 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
798 ),
799 )
800 bboxCorners = bbox.getCorners()
802 isolatedStarCatalogRefs = []
803 isolatedStarSourceRefs = []
804 for i in range(len(allStarIds)):
805 starIds = allStarIds[i]
806 starRAs = allStarRAs[i]
807 starDecs = allStarDecs[i]
808 isolatedStarCatalog = pd.DataFrame({"ra": starRAs, "dec": starDecs}, index=starIds)
809 isolatedStarCatalogRefs.append(
810 InMemoryDatasetHandle(isolatedStarCatalog, storageClass="DataFrame", dataId={"tract": 0})
811 )
812 sourceCats = []
813 for v, visit in enumerate(cls.testVisits):
814 nVisStars = int(cls.nStars * inScienceFraction)
815 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
816 visitStarIds = starIds[visitStarIndices]
817 visitStarRas = starRAs[visitStarIndices]
818 visitStarDecs = starDecs[visitStarIndices]
819 for detector in trueWCSs[visit]:
820 detWcs = detector.getWcs()
821 detectorId = detector["id"]
822 radecCorners = detWcs.pixelToSky(bboxCorners)
823 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
824 detectorIndices = detectorFootprint.contains(
825 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
826 )
827 nDetectorStars = detectorIndices.sum()
828 detectorArray = np.ones(nDetectorStars, dtype=int) * detector["id"]
829 visitArray = np.ones(nDetectorStars, dtype=int) * visit
831 ones_like = np.ones(nDetectorStars)
832 zeros_like = np.zeros(nDetectorStars, dtype=bool)
834 x, y = detWcs.skyToPixelArray(
835 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
836 )
838 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[
839 0
840 ].getWcs()
841 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
843 sourceDict = {}
844 sourceDict["detector"] = detectorArray
845 sourceDict["visit"] = visitArray
846 sourceDict["obj_index"] = visitStarIds[detectorIndices]
847 sourceDict["x"] = x
848 sourceDict["y"] = y
849 sourceDict["xErr"] = 1e-3 * ones_like
850 sourceDict["yErr"] = 1e-3 * ones_like
851 sourceDict["inputRA"] = inputRa
852 sourceDict["inputDec"] = inputDec
853 sourceDict["trueRA"] = visitStarRas[detectorIndices]
854 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
855 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
856 sourceDict[key] = ones_like
857 for key in [
858 "pixelFlags_edge",
859 "pixelFlags_saturated",
860 "pixelFlags_interpolatedCenter",
861 "pixelFlags_interpolated",
862 "pixelFlags_crCenter",
863 "pixelFlags_bad",
864 "hsmPsfMoments_flag",
865 "apFlux_12_0_flag",
866 "extendedness",
867 "parentSourceId",
868 "deblend_nChild",
869 "ixy",
870 ]:
871 sourceDict[key] = zeros_like
872 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
873 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
875 sourceCat = pd.DataFrame(sourceDict)
876 sourceCats.append(sourceCat)
878 isolatedStarSourceTable = pd.concat(sourceCats, ignore_index=True)
879 isolatedStarSourceTable = isolatedStarSourceTable.sort_values(by=["obj_index"])
880 isolatedStarSourceRefs.append(
881 InMemoryDatasetHandle(isolatedStarSourceTable, storageClass="DataFrame", dataId={"tract": 0})
882 )
884 return isolatedStarCatalogRefs, isolatedStarSourceRefs
886 def test_loading_and_association(self):
887 """Test that associated objects actually correspond to the same
888 simulated object."""
889 associations, sourceDict = self.task._associate_from_isolated_sources(
890 self.isolatedStarSources, self.isolatedStarCatalogs, self.extensionInfo, self.refObjects
891 )
893 object_groups = np.flatnonzero(np.array(associations.sequence) == 0)
894 for i in range(len(object_groups) - 1)[:10]:
895 ras, decs = [], []
896 for ind in np.arange(object_groups[i], object_groups[i + 1]):
897 visit = self.extensionInfo.visit[associations.extn[ind]]
898 detectorInd = self.extensionInfo.detectorIndex[associations.extn[ind]]
899 detector = self.extensionInfo.detector[associations.extn[ind]]
900 if detectorInd == -1:
901 ra = self.refObjects[visit * -1]["ra"][associations.obj[ind]]
902 dec = self.refObjects[visit * -1]["dec"][associations.obj[ind]]
903 ras.append(ra)
904 decs.append(dec)
905 else:
906 x = sourceDict[visit][detector]["x"][associations.obj[ind]]
907 y = sourceDict[visit][detector]["y"][associations.obj[ind]]
908 ra, dec = self.trueWCSs[visit][detectorInd].getWcs().pixelToSky(x, y)
909 ras.append(ra.asDegrees())
910 decs.append(dec.asDegrees())
911 np.testing.assert_allclose(ras, ras[0])
912 np.testing.assert_allclose(decs, decs[0])
914 def test_refCatLoader(self):
915 """Test loading objects from the refCat in each of the fields."""
917 for region in self.fieldRegions.values():
918 refCat, refCov = self.task._load_refcat(
919 self.refObjectLoader, self.extensionInfo, region=region, epoch=self.exposureInfo.medianEpoch
920 )
921 assert len(refCat) > 0
923 def test_make_outputs(self):
924 """Test that the run method recovers the input model parameters."""
925 for isolatedStarSourceRef in self.isolatedStarSources:
926 iss = isolatedStarSourceRef.get()
927 visits = np.unique(iss["visit"])
928 for v, visit in enumerate(visits):
929 outputWcsCatalog = self.outputs.outputWcss[visit]
930 visitSources = iss[iss["visit"] == visit]
931 detectors = outputWcsCatalog["id"]
932 for d, detectorId in enumerate(detectors):
933 fitwcs = outputWcsCatalog[d].getWcs()
934 detSources = visitSources[visitSources["detector"] == detectorId]
935 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
936 dRA = fitRA - detSources["trueRA"]
937 dDec = fitDec - detSources["trueDec"]
938 # Check that input coordinates match the output coordinates
939 self.assertAlmostEqual(np.mean(dRA), 0)
940 self.assertAlmostEqual(np.std(dRA), 0)
941 self.assertAlmostEqual(np.mean(dDec), 0)
942 self.assertAlmostEqual(np.std(dDec), 0)
944 def test_missingWcs(self):
945 """Test that task does not fail when the input WCS is None for one
946 extension and that the fit WCS for that extension returns a finite
947 result.
948 """
949 inputVisitSummary = self.inputVisitSummary.copy()
950 # Set one WCS to be None
951 testVisit = 0
952 testDetector = 20
953 inputVisitSummary[testVisit][testDetector].setWcs(None)
955 outputs = self.task.run(
956 inputVisitSummary,
957 self.isolatedStarSources,
958 self.isolatedStarCatalogs,
959 instrumentName=self.instrumentName,
960 refEpoch=self.refEpoch,
961 refObjectLoader=self.refObjectLoader,
962 )
964 # Check that the fit WCS for the extension with input WCS=None returns
965 # finite sky values.
966 testWcs = outputs.outputWcss[self.testVisits[testVisit]][testDetector].getWcs()
967 testSky = testWcs.pixelToSky(0, 0)
968 self.assertTrue(testSky.isFinite())
971def setup_module(module):
972 lsst.utils.tests.init()
975if __name__ == "__main__": 975 ↛ 976line 975 didn't jump to line 976, because the condition on line 975 was never true
976 lsst.utils.tests.init()
977 unittest.main()