Coverage for tests/test_gbdesAstrometricFit.py: 13%
281 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-18 13:24 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-18 13:24 +0000
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import os.path
23import unittest
25import astropy.units as u
26import lsst.afw.geom as afwgeom
27import lsst.afw.table as afwTable
28import lsst.geom
29import lsst.utils
30import numpy as np
31import pandas as pd
32import wcsfit
33import yaml
34from lsst import sphgeom
35from lsst.daf.base import PropertyList
36from lsst.drp.tasks.gbdesAstrometricFit import GbdesAstrometricFitConfig, GbdesAstrometricFitTask
37from lsst.meas.algorithms import ReferenceObjectLoader
38from lsst.meas.algorithms.testUtils import MockRefcatDataId
39from lsst.pipe.base import InMemoryDatasetHandle
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
45 @classmethod
46 def setUpClass(cls):
47 # Set random seed
48 np.random.seed(1234)
50 # Fraction of simulated stars in the reference catalog and science
51 # exposures
52 inReferenceFraction = 1
53 inScienceFraction = 1
55 # Make fake data
56 cls.datadir = os.path.join(TESTDIR, "data")
58 cls.fieldNumber = 0
59 cls.instrumentName = "HSC"
60 cls.instrument = wcsfit.Instrument(cls.instrumentName)
61 cls.refEpoch = 57205.5
63 # Make test inputVisitSummary. VisitSummaryTables are taken from
64 # collection HSC/runs/RC2/w_2022_20/DM-34794
65 cls.testVisits = [1176, 17900, 17930, 17934]
66 cls.inputVisitSummary = []
67 for testVisit in cls.testVisits:
68 visSum = afwTable.ExposureCatalog.readFits(
69 os.path.join(cls.datadir, f"visitSummary_{testVisit}.fits")
70 )
71 cls.inputVisitSummary.append(visSum)
73 cls.config = GbdesAstrometricFitConfig()
74 cls.config.systematicError = 0
75 cls.config.devicePolyOrder = 4
76 cls.config.exposurePolyOrder = 6
77 cls.config.fitReserveFraction = 0
78 cls.config.fitReserveRandomSeed = 1234
79 cls.task = GbdesAstrometricFitTask(config=cls.config)
81 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
82 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch
83 )
85 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
86 cls.inputVisitSummary, cls.exposureInfo.medianEpoch
87 )
89 # Bounding box of observations:
90 raMins, raMaxs = [], []
91 decMins, decMaxs = [], []
92 for visSum in cls.inputVisitSummary:
93 raMins.append(visSum["raCorners"].min())
94 raMaxs.append(visSum["raCorners"].max())
95 decMins.append(visSum["decCorners"].min())
96 decMaxs.append(visSum["decCorners"].max())
97 raMin = min(raMins)
98 raMax = max(raMaxs)
99 decMin = min(decMins)
100 decMax = max(decMaxs)
102 corners = [
103 lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
104 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
105 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
106 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector(),
107 ]
108 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
110 # Make random set of data in a bounding box determined by input visits
111 # Make wcs objects for the "true" model
112 cls.nStars = 10000
113 starIds = np.arange(cls.nStars)
114 starRAs = np.random.random(cls.nStars) * (raMax - raMin) + raMin
115 starDecs = np.random.random(cls.nStars) * (decMax - decMin) + decMin
117 # Make a reference catalog and load it into ReferenceObjectLoader
118 refDataId, deferredRefCat = cls._make_refCat(starIds, starRAs, starDecs, inReferenceFraction)
119 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
120 cls.refObjectLoader.config.requireProperMotion = False
121 cls.refObjectLoader.config.anyFilterMapsToThis = "test_filter"
123 cls.task.refObjectLoader = cls.refObjectLoader
125 # Get True WCS for stars:
126 with open(os.path.join(cls.datadir, "sample_wcs.yaml"), "r") as f:
127 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
129 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
131 # Make source catalogs:
132 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs, inScienceFraction)
134 cls.outputs = cls.task.run(
135 cls.inputCatalogRefs,
136 cls.inputVisitSummary,
137 instrumentName=cls.instrumentName,
138 refEpoch=cls.refEpoch,
139 refObjectLoader=cls.refObjectLoader,
140 )
142 @classmethod
143 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction):
144 """Make reference catalog from a subset of the simulated data
146 Parameters
147 ----------
148 starIds : `np.ndarray` of `int`
149 Source ids for the simulated stars
150 starRas : `np.ndarray` of `float`
151 RAs of the simulated stars
152 starDecs : `np.ndarray` of `float`
153 Decs of the simulated stars
154 inReferenceFraction : float
155 Percentage of simulated stars to include in reference catalog
157 Returns
158 -------
159 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId`
160 Object that replicates the functionality of a dataId.
161 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
162 Dataset handle for reference catalog.
163 """
164 nRefs = int(cls.nStars * inReferenceFraction)
165 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
166 # Make simpleCatalog to hold data, create datasetRef with `region`
167 # determined by bounding box used in above simulate.
168 refSchema = afwTable.SimpleTable.makeMinimalSchema()
169 idKey = refSchema.addField("sourceId", type="I")
170 fluxKey = refSchema.addField("test_filter_flux", units="nJy", type=np.float64)
171 raErrKey = refSchema.addField("coord_raErr", type=np.float64)
172 decErrKey = refSchema.addField("coord_decErr", type=np.float64)
173 pmraErrKey = refSchema.addField("pm_raErr", type=np.float64)
174 pmdecErrKey = refSchema.addField("pm_decErr", type=np.float64)
175 refCat = afwTable.SimpleCatalog(refSchema)
176 ref_md = PropertyList()
177 ref_md.set("REFCAT_FORMAT_VERSION", 1)
178 refCat.table.setMetadata(ref_md)
179 for i in refStarIndices:
180 record = refCat.addNew()
181 record.set(idKey, starIds[i])
182 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
183 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
184 record.set(fluxKey, 1)
185 record.set(raErrKey, 0.00001)
186 record.set(decErrKey, 0.00001)
187 record.set(pmraErrKey, 1e-9)
188 record.set(pmdecErrKey, 1e-9)
189 refDataId = MockRefcatDataId(cls.boundingPolygon)
190 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
192 return refDataId, deferredRefCat
194 @classmethod
195 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
196 """Make a `pd.DataFrame` catalog with the columns needed for the
197 object selector.
199 Parameters
200 ----------
201 starIds : `np.ndarray` of `int`
202 Source ids for the simulated stars
203 starRas : `np.ndarray` of `float`
204 RAs of the simulated stars
205 starDecs : `np.ndarray` of `float`
206 Decs of the simulated stars
207 trueWCSs : `list` of `lsst.afw.geom.SkyWcs`
208 WCS with which to simulate the source pixel coordinates
209 inReferenceFraction : float
210 Percentage of simulated stars to include in reference catalog
212 Returns
213 -------
214 sourceCat : `list` of `lsst.pipe.base.InMemoryDatasetHandle`
215 List of reference to source catalogs.
216 """
217 inputCatalogRefs = []
218 # Take a subset of the simulated data
219 # Use true wcs objects to put simulated data into ccds
220 bbox = lsst.geom.BoxD(
221 lsst.geom.Point2D(
222 cls.inputVisitSummary[0][0]["bbox_min_x"], cls.inputVisitSummary[0][0]["bbox_min_y"]
223 ),
224 lsst.geom.Point2D(
225 cls.inputVisitSummary[0][0]["bbox_max_x"], cls.inputVisitSummary[0][0]["bbox_max_y"]
226 ),
227 )
228 bboxCorners = bbox.getCorners()
229 cls.inputCatalogRefs = []
230 for v, visit in enumerate(cls.testVisits):
231 nVisStars = int(cls.nStars * inScienceFraction)
232 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
233 visitStarIds = starIds[visitStarIndices]
234 visitStarRas = starRas[visitStarIndices]
235 visitStarDecs = starDecs[visitStarIndices]
236 sourceCats = []
237 for detector in trueWCSs[visit]:
238 detWcs = detector.getWcs()
239 detectorId = detector["id"]
240 radecCorners = detWcs.pixelToSky(bboxCorners)
241 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
242 detectorIndices = detectorFootprint.contains(
243 (visitStarRas * u.degree).to(u.radian), (visitStarDecs * u.degree).to(u.radian)
244 )
245 nDetectorStars = detectorIndices.sum()
246 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector["id"]
248 ones_like = np.ones(nDetectorStars)
249 zeros_like = np.zeros(nDetectorStars, dtype=bool)
251 x, y = detWcs.skyToPixelArray(
252 visitStarRas[detectorIndices], visitStarDecs[detectorIndices], degrees=True
253 )
255 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]["id"] == detectorId])[0].getWcs()
256 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
258 sourceDict = {}
259 sourceDict["detector"] = detectorArray
260 sourceDict["sourceId"] = visitStarIds[detectorIndices]
261 sourceDict["x"] = x
262 sourceDict["y"] = y
263 sourceDict["xErr"] = 1e-3 * ones_like
264 sourceDict["yErr"] = 1e-3 * ones_like
265 sourceDict["inputRA"] = inputRa
266 sourceDict["inputDec"] = inputDec
267 sourceDict["trueRA"] = visitStarRas[detectorIndices]
268 sourceDict["trueDec"] = visitStarDecs[detectorIndices]
269 for key in ["apFlux_12_0_flux", "apFlux_12_0_instFlux", "ixx", "iyy"]:
270 sourceDict[key] = ones_like
271 for key in [
272 "pixelFlags_edge",
273 "pixelFlags_saturated",
274 "pixelFlags_interpolatedCenter",
275 "pixelFlags_interpolated",
276 "pixelFlags_crCenter",
277 "pixelFlags_bad",
278 "hsmPsfMoments_flag",
279 "apFlux_12_0_flag",
280 "extendedness",
281 "parentSourceId",
282 "deblend_nChild",
283 "ixy",
284 ]:
285 sourceDict[key] = zeros_like
286 sourceDict["apFlux_12_0_instFluxErr"] = 1e-3 * ones_like
287 sourceDict["detect_isPrimary"] = ones_like.astype(bool)
289 sourceCat = pd.DataFrame(sourceDict)
290 sourceCats.append(sourceCat)
292 visitSourceTable = pd.concat(sourceCats)
294 inputCatalogRef = InMemoryDatasetHandle(
295 visitSourceTable, storageClass="DataFrame", dataId={"visit": visit}
296 )
298 inputCatalogRefs.append(inputCatalogRef)
300 return inputCatalogRefs
302 @classmethod
303 def _make_wcs(cls, model, inputVisitSummaries):
304 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
306 Parameters
307 ----------
308 model : `dict`
309 Dictionary with WCS model parameters
310 inputVisitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
311 Visit summary catalogs
312 Returns
313 -------
314 catalogs : `dict` of `lsst.afw.table.ExposureCatalog`
315 Visit summary catalogs with WCS set to input model
316 """
318 # Pixels will need to be rescaled before going into the mappings
319 xscale = inputVisitSummaries[0][0]["bbox_max_x"] - inputVisitSummaries[0][0]["bbox_min_x"]
320 yscale = inputVisitSummaries[0][0]["bbox_max_y"] - inputVisitSummaries[0][0]["bbox_min_y"]
322 catalogs = {}
323 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
324 schema.addField("visit", type="L", doc="Visit number")
325 for visitSum in inputVisitSummaries:
326 visit = visitSum[0]["visit"]
327 visitMapName = f"{visit}/poly"
328 visitModel = model[visitMapName]
330 catalog = lsst.afw.table.ExposureCatalog(schema)
331 catalog.resize(len(visitSum))
332 catalog["visit"] = visit
334 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
336 visitMapType = visitModel["Type"]
337 visitDict = {"Type": visitMapType}
338 if visitMapType == "Poly":
339 mapCoefficients = visitModel["XPoly"]["Coefficients"] + visitModel["YPoly"]["Coefficients"]
340 visitDict["Coefficients"] = mapCoefficients
342 for d, detector in enumerate(visitSum):
343 detectorId = detector["id"]
344 detectorMapName = f"HSC/{detectorId}/poly"
345 detectorModel = model[detectorMapName]
347 detectorMapType = detectorModel["Type"]
348 mapDict = {detectorMapName: {"Type": detectorMapType}, visitMapName: visitDict}
349 if detectorMapType == "Poly":
350 mapCoefficients = (
351 detectorModel["XPoly"]["Coefficients"] + detectorModel["YPoly"]["Coefficients"]
352 )
353 mapDict[detectorMapName]["Coefficients"] = mapCoefficients
355 outWCS = cls.task._make_afw_wcs(
356 mapDict,
357 raDec.getRa(),
358 raDec.getDec(),
359 doNormalizePixels=True,
360 xScale=xscale,
361 yScale=yscale,
362 )
363 catalog[d].setId(detectorId)
364 catalog[d].setWcs(outWCS)
366 catalog.sort()
367 catalogs[visit] = catalog
369 return catalogs
371 def test_get_exposure_info(self):
372 """Test that information for input exposures is as expected and that
373 the WCS in the class object gives approximately the same results as the
374 input `lsst.afw.geom.SkyWcs`.
375 """
377 # The total number of extensions is the number of detectors for each
378 # visit plus one for the reference catalog
379 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + 1
381 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
383 taskVisits = set(self.extensionInfo.visit)
384 self.assertEqual(taskVisits, set(self.testVisits + [-1]))
386 xx = np.linspace(0, 2000, 3)
387 yy = np.linspace(0, 4000, 6)
388 xgrid, ygrid = np.meshgrid(xx, yy)
389 for visSum in self.inputVisitSummary:
390 visit = visSum[0]["visit"]
391 for detectorInfo in visSum:
392 detector = detectorInfo["id"]
393 extensionIndex = np.flatnonzero(
394 (self.extensionInfo.visit == visit) & (self.extensionInfo.detector == detector)
395 )[0]
396 fitWcs = self.extensionInfo.wcs[extensionIndex]
397 calexpWcs = detectorInfo.getWcs()
399 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(), ygrid.ravel())])
401 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
403 tangentPoint = calexpWcs.pixelToSky(
404 calexpWcs.getPixelOrigin().getX(), calexpWcs.getPixelOrigin().getY()
405 )
406 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0 * lsst.geom.degrees, True)
407 iwcToSkyWcs = afwgeom.makeSkyWcs(lsst.geom.Point2D(0, 0), tangentPoint, cdMatrix)
408 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(
409 tanPlaneXY[:, 0], tanPlaneXY[:, 1], degrees=True
410 )
412 # One WCS is in SIP and the other is TPV. The pixel-to-sky
413 # conversion is not exactly the same but should be close.
414 # TODO: sip_tpv + astropy.wcs.WCS gets a better result here,
415 # particularly for detector # >= 100. See if we can improve/if
416 # improving is necessary. Check if matching in corner detectors
417 # is ok.
418 rtol = 1e-3 if (detector >= 100) else 1e-5
419 np.testing.assert_allclose(calexpra, newRAdeg, rtol=rtol)
420 np.testing.assert_allclose(calexpdec, newDecdeg, rtol=rtol)
422 def test_refCatLoader(self):
423 """Test that we can load objects from refCat"""
425 tmpAssociations = wcsfit.FoFClass(
426 self.fields,
427 [self.instrument],
428 self.exposuresHelper,
429 [self.fieldRadius.asDegrees()],
430 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
431 )
433 self.task._load_refcat(
434 tmpAssociations,
435 self.refObjectLoader,
436 self.fieldCenter,
437 self.fieldRadius,
438 self.extensionInfo,
439 epoch=2015,
440 )
442 # We have only loaded one catalog, so getting the 'matches' should just
443 # return the same objects we put in, except some random objects that
444 # are too close together.
445 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
447 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
449 self.assertLessEqual(nMatches, self.nStars)
450 self.assertGreater(nMatches, self.nStars * 0.9)
452 def test_load_catalogs_and_associate(self):
453 tmpAssociations = wcsfit.FoFClass(
454 self.fields,
455 [self.instrument],
456 self.exposuresHelper,
457 [self.fieldRadius.asDegrees()],
458 (self.task.config.matchRadius * u.arcsec).to(u.degree).value,
459 )
460 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, self.extensionInfo)
462 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
464 matchIds = []
465 correctMatches = []
466 for s, e, o in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
467 objVisitInd = self.extensionInfo.visitIndex[e]
468 objDet = self.extensionInfo.detector[e]
469 ExtnInds = self.inputCatalogRefs[objVisitInd].get()["detector"] == objDet
470 objInfo = self.inputCatalogRefs[objVisitInd].get()[ExtnInds].iloc[o]
471 if s == 0:
472 if len(matchIds) > 0:
473 correctMatches.append(len(set(matchIds)) == 1)
474 matchIds = []
476 matchIds.append(objInfo["sourceId"])
478 # A few matches may incorrectly associate sources because of the random
479 # positions
480 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
482 def test_make_outputs(self):
483 """Test that the run method recovers the input model parameters."""
484 for v, visit in enumerate(self.testVisits):
485 visitSummary = self.inputVisitSummary[v]
486 outputWcsCatalog = self.outputs.outputWCSs[visit]
487 visitSources = self.inputCatalogRefs[v].get()
488 for d, detectorRow in enumerate(visitSummary):
489 detectorId = detectorRow["id"]
490 fitwcs = outputWcsCatalog[d].getWcs()
491 detSources = visitSources[visitSources["detector"] == detectorId]
492 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources["x"], detSources["y"], degrees=True)
493 dRA = fitRA - detSources["trueRA"]
494 dDec = fitDec - detSources["trueDec"]
495 # Check that input coordinates match the output coordinates
496 self.assertAlmostEqual(np.mean(dRA), 0)
497 self.assertAlmostEqual(np.std(dRA), 0)
498 self.assertAlmostEqual(np.mean(dDec), 0)
499 self.assertAlmostEqual(np.std(dDec), 0)
501 def test_run(self):
502 """Test that run method recovers the input model parameters"""
503 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
505 for v, visit in enumerate(self.testVisits):
506 visitSummary = self.inputVisitSummary[v]
507 visitMapName = f"{visit}/poly"
509 origModel = self.trueModel[visitMapName]
510 if origModel["Type"] != "Identity":
511 fitModel = outputMaps[visitMapName]
512 origXPoly = origModel["XPoly"]["Coefficients"]
513 origYPoly = origModel["YPoly"]["Coefficients"]
514 fitXPoly = fitModel[: len(origXPoly)]
515 fitYPoly = fitModel[len(origXPoly) :]
517 absDiffX = abs(fitXPoly - origXPoly)
518 absDiffY = abs(fitYPoly - origYPoly)
519 # Check that input visit model matches fit
520 np.testing.assert_array_less(absDiffX, 1e-6)
521 np.testing.assert_array_less(absDiffY, 1e-6)
522 for d, detectorRow in enumerate(visitSummary):
523 detectorId = detectorRow["id"]
524 detectorMapName = f"HSC/{detectorId}/poly"
525 origModel = self.trueModel[detectorMapName]
526 if (origModel["Type"] != "Identity") and (v == 0):
527 fitModel = outputMaps[detectorMapName]
528 origXPoly = origModel["XPoly"]["Coefficients"]
529 origYPoly = origModel["YPoly"]["Coefficients"]
530 fitXPoly = fitModel[: len(origXPoly)]
531 fitYPoly = fitModel[len(origXPoly) :]
532 absDiffX = abs(fitXPoly - origXPoly)
533 absDiffY = abs(fitYPoly - origYPoly)
534 # Check that input detector model matches fit
535 np.testing.assert_array_less(absDiffX, 1e-7)
536 np.testing.assert_array_less(absDiffY, 1e-7)
539def setup_module(module):
540 lsst.utils.tests.init()
543if __name__ == "__main__": 543 ↛ 544line 543 didn't jump to line 544, because the condition on line 543 was never true
544 lsst.utils.tests.init()
545 unittest.main()