Coverage for tests/test_gbdesAstrometricFit.py: 10%
281 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-21 11:48 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-21 11:48 +0000
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
23import os.path
24import numpy as np
25import yaml
26import astropy.units as u
27import pandas as pd
28import wcsfit
30import lsst.utils
31import lsst.afw.table as afwTable
32import lsst.afw.geom as afwgeom
33from lsst.drp.tasks.gbdesAstrometricFit import GbdesAstrometricFitConfig, GbdesAstrometricFitTask
34from lsst.daf.base import PropertyList
35from lsst.meas.algorithms import ReferenceObjectLoader
36from lsst.meas.algorithms.testUtils import MockRefcatDataId
37from lsst.pipe.base import InMemoryDatasetHandle
38from lsst import sphgeom
39import lsst.geom
41TESTDIR = os.path.abspath(os.path.dirname(__file__))
44class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
46 @classmethod
47 def setUpClass(cls):
49 # Set random seed
50 np.random.seed(1234)
52 # Fraction of simulated stars in the reference catalog and science
53 # exposures
54 inReferenceFraction = 1
55 inScienceFraction = 1
57 # Make fake data
58 cls.datadir = os.path.join(TESTDIR, "data")
60 cls.fieldNumber = 0
61 cls.instrumentName = 'HSC'
62 cls.instrument = wcsfit.Instrument(cls.instrumentName)
63 cls.refEpoch = 57205.5
65 # Make test inputVisitSummary. VisitSummaryTables are taken from
66 # collection HSC/runs/RC2/w_2022_20/DM-34794
67 cls.testVisits = [1176, 17900, 17930, 17934]
68 cls.inputVisitSummary = []
69 for testVisit in cls.testVisits:
70 visSum = afwTable.ExposureCatalog.readFits(os.path.join(cls.datadir,
71 f'visitSummary_{testVisit}.fits'))
72 cls.inputVisitSummary.append(visSum)
74 cls.config = GbdesAstrometricFitConfig()
75 cls.config.systematicError = 0
76 cls.config.devicePolyOrder = 4
77 cls.config.exposurePolyOrder = 6
78 cls.config.fitReserveFraction = 0
79 cls.config.fitReserveRandomSeed = 1234
80 cls.task = GbdesAstrometricFitTask(config=cls.config)
82 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
83 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch)
85 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
86 cls.inputVisitSummary, cls.exposureInfo.medianEpoch)
88 # Bounding box of observations:
89 raMins, raMaxs = [], []
90 decMins, decMaxs = [], []
91 for visSum in cls.inputVisitSummary:
92 raMins.append(visSum['raCorners'].min())
93 raMaxs.append(visSum['raCorners'].max())
94 decMins.append(visSum['decCorners'].min())
95 decMaxs.append(visSum['decCorners'].max())
96 raMin = min(raMins)
97 raMax = max(raMaxs)
98 decMin = min(decMins)
99 decMax = max(decMaxs)
101 corners = [lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
102 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
103 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
104 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector()]
105 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
107 # Make random set of data in a bounding box determined by input visits
108 # Make wcs objects for the "true" model
109 cls.nStars = 10000
110 starIds = np.arange(cls.nStars)
111 starRAs = np.random.random(cls.nStars) * (raMax - raMin) + raMin
112 starDecs = np.random.random(cls.nStars) * (decMax - decMin) + decMin
114 # Make a reference catalog and load it into ReferenceObjectLoader
115 refDataId, deferredRefCat = cls._make_refCat(starIds, starRAs, starDecs, inReferenceFraction)
116 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
117 cls.refObjectLoader.config.requireProperMotion = False
118 cls.refObjectLoader.config.anyFilterMapsToThis = 'test_filter'
120 cls.task.refObjectLoader = cls.refObjectLoader
122 # Get True WCS for stars:
123 with open(os.path.join(cls.datadir, 'sample_wcs.yaml'), 'r') as f:
124 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
126 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
128 # Make source catalogs:
129 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs,
130 inScienceFraction)
132 cls.outputs = cls.task.run(cls.inputCatalogRefs, cls.inputVisitSummary,
133 instrumentName=cls.instrumentName, refEpoch=cls.refEpoch,
134 refObjectLoader=cls.refObjectLoader)
136 @classmethod
137 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction):
138 """Make reference catalog from a subset of the simulated data
140 Parameters
141 ----------
142 starIds : `np.ndarray` of `int`
143 Source ids for the simulated stars
144 starRas : `np.ndarray` of `float`
145 RAs of the simulated stars
146 starDecs : `np.ndarray` of `float`
147 Decs of the simulated stars
148 inReferenceFraction : float
149 Percentage of simulated stars to include in reference catalog
151 Returns
152 -------
153 refDataId : `lsst.meas.algorithms.testUtils.MockRefcatDataId`
154 Object that replicates the functionality of a dataId.
155 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
156 Dataset handle for reference catalog.
157 """
158 nRefs = int(cls.nStars * inReferenceFraction)
159 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
160 # Make simpleCatalog to hold data, create datasetRef with `region`
161 # determined by bounding box used in above simulate.
162 refSchema = afwTable.SimpleTable.makeMinimalSchema()
163 idKey = refSchema.addField("sourceId", type="I")
164 fluxKey = refSchema.addField("test_filter_flux", units='nJy', type=np.float64)
165 raErrKey = refSchema.addField("coord_raErr", type=np.float64)
166 decErrKey = refSchema.addField("coord_decErr", type=np.float64)
167 pmraErrKey = refSchema.addField("pm_raErr", type=np.float64)
168 pmdecErrKey = refSchema.addField("pm_decErr", type=np.float64)
169 refCat = afwTable.SimpleCatalog(refSchema)
170 ref_md = PropertyList()
171 ref_md.set("REFCAT_FORMAT_VERSION", 1)
172 refCat.table.setMetadata(ref_md)
173 for i in refStarIndices:
174 record = refCat.addNew()
175 record.set(idKey, starIds[i])
176 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
177 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
178 record.set(fluxKey, 1)
179 record.set(raErrKey, 0.00001)
180 record.set(decErrKey, 0.00001)
181 record.set(pmraErrKey, 1e-9)
182 record.set(pmdecErrKey, 1e-9)
183 refDataId = MockRefcatDataId(cls.boundingPolygon)
184 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
186 return refDataId, deferredRefCat
188 @classmethod
189 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
190 """Make a `pd.DataFrame` catalog with the columns needed for the
191 object selector.
193 Parameters
194 ----------
195 starIds : `np.ndarray` of `int`
196 Source ids for the simulated stars
197 starRas : `np.ndarray` of `float`
198 RAs of the simulated stars
199 starDecs : `np.ndarray` of `float`
200 Decs of the simulated stars
201 trueWCSs : `list` of `lsst.afw.geom.SkyWcs`
202 WCS with which to simulate the source pixel coordinates
203 inReferenceFraction : float
204 Percentage of simulated stars to include in reference catalog
206 Returns
207 -------
208 sourceCat : `list` of `lsst.pipe.base.InMemoryDatasetHandle`
209 List of reference to source catalogs.
210 """
211 inputCatalogRefs = []
212 # Take a subset of the simulated data
213 # Use true wcs objects to put simulated data into ccds
214 bbox = lsst.geom.BoxD(lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_min_x'],
215 cls.inputVisitSummary[0][0]['bbox_min_y']),
216 lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_max_x'],
217 cls.inputVisitSummary[0][0]['bbox_max_y']))
218 bboxCorners = bbox.getCorners()
219 cls.inputCatalogRefs = []
220 for v, visit in enumerate(cls.testVisits):
221 nVisStars = int(cls.nStars * inScienceFraction)
222 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
223 visitStarIds = starIds[visitStarIndices]
224 visitStarRas = starRas[visitStarIndices]
225 visitStarDecs = starDecs[visitStarIndices]
226 sourceCats = []
227 for detector in trueWCSs[visit]:
228 detWcs = detector.getWcs()
229 detectorId = detector['id']
230 radecCorners = detWcs.pixelToSky(bboxCorners)
231 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
232 detectorIndices = detectorFootprint.contains((visitStarRas*u.degree).to(u.radian),
233 (visitStarDecs*u.degree).to(u.radian))
234 nDetectorStars = detectorIndices.sum()
235 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector['id']
237 ones_like = np.ones(nDetectorStars)
238 zeros_like = np.zeros(nDetectorStars, dtype=bool)
240 x, y = detWcs.skyToPixelArray(visitStarRas[detectorIndices], visitStarDecs[detectorIndices],
241 degrees=True)
243 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]['id'] == detectorId])[0].getWcs()
244 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
246 sourceDict = {}
247 sourceDict['detector'] = detectorArray
248 sourceDict['sourceId'] = visitStarIds[detectorIndices]
249 sourceDict['x'] = x
250 sourceDict['y'] = y
251 sourceDict['xErr'] = 1e-3 * ones_like
252 sourceDict['yErr'] = 1e-3 * ones_like
253 sourceDict['inputRA'] = inputRa
254 sourceDict['inputDec'] = inputDec
255 sourceDict['trueRA'] = visitStarRas[detectorIndices]
256 sourceDict['trueDec'] = visitStarDecs[detectorIndices]
257 for key in ['apFlux_12_0_flux', 'apFlux_12_0_instFlux', 'ixx', 'iyy']:
258 sourceDict[key] = ones_like
259 for key in ['pixelFlags_edge', 'pixelFlags_saturated', 'pixelFlags_interpolatedCenter',
260 'pixelFlags_interpolated', 'pixelFlags_crCenter', 'pixelFlags_bad',
261 'hsmPsfMoments_flag', 'apFlux_12_0_flag', 'extendedness', 'parentSourceId',
262 'deblend_nChild', 'ixy']:
263 sourceDict[key] = zeros_like
264 sourceDict['apFlux_12_0_instFluxErr'] = 1e-3 * ones_like
265 sourceDict['detect_isPrimary'] = ones_like.astype(bool)
267 sourceCat = pd.DataFrame(sourceDict)
268 sourceCats.append(sourceCat)
270 visitSourceTable = pd.concat(sourceCats)
272 inputCatalogRef = InMemoryDatasetHandle(visitSourceTable, storageClass="DataFrame",
273 dataId={"visit": visit})
275 inputCatalogRefs.append(inputCatalogRef)
277 return inputCatalogRefs
279 @classmethod
280 def _make_wcs(cls, model, inputVisitSummaries):
281 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
283 Parameters
284 ----------
285 model : `dict`
286 Dictionary with WCS model parameters
287 inputVisitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
288 Visit summary catalogs
289 Returns
290 -------
291 catalogs : `dict` of `lsst.afw.table.ExposureCatalog`
292 Visit summary catalogs with WCS set to input model
293 """
295 # Pixels will need to be rescaled before going into the mappings
296 xscale = inputVisitSummaries[0][0]['bbox_max_x'] - inputVisitSummaries[0][0]['bbox_min_x']
297 yscale = inputVisitSummaries[0][0]['bbox_max_y'] - inputVisitSummaries[0][0]['bbox_min_y']
299 catalogs = {}
300 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
301 schema.addField('visit', type='L', doc='Visit number')
302 for visitSum in inputVisitSummaries:
303 visit = visitSum[0]['visit']
304 visitMapName = f'{visit}/poly'
305 visitModel = model[visitMapName]
307 catalog = lsst.afw.table.ExposureCatalog(schema)
308 catalog.resize(len(visitSum))
309 catalog['visit'] = visit
311 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
313 visitMapType = visitModel['Type']
314 visitDict = {'Type': visitMapType}
315 if visitMapType == 'Poly':
316 mapCoefficients = (visitModel['XPoly']['Coefficients']
317 + visitModel['YPoly']['Coefficients'])
318 visitDict["Coefficients"] = mapCoefficients
320 for d, detector in enumerate(visitSum):
321 detectorId = detector['id']
322 detectorMapName = f'HSC/{detectorId}/poly'
323 detectorModel = model[detectorMapName]
325 detectorMapType = detectorModel['Type']
326 mapDict = {detectorMapName: {'Type': detectorMapType},
327 visitMapName: visitDict}
328 if detectorMapType == 'Poly':
329 mapCoefficients = (detectorModel['XPoly']['Coefficients']
330 + detectorModel['YPoly']['Coefficients'])
331 mapDict[detectorMapName]['Coefficients'] = mapCoefficients
333 outWCS = cls.task._make_afw_wcs(mapDict, raDec.getRa(), raDec.getDec(),
334 doNormalizePixels=True, xScale=xscale, yScale=yscale)
335 catalog[d].setId(detectorId)
336 catalog[d].setWcs(outWCS)
338 catalog.sort()
339 catalogs[visit] = catalog
341 return catalogs
343 def test_get_exposure_info(self):
344 """Test that information for input exposures is as expected and that
345 the WCS in the class object gives approximately the same results as the
346 input `lsst.afw.geom.SkyWcs`.
347 """
349 # The total number of extensions is the number of detectors for each
350 # visit plus one for the reference catalog
351 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + 1
353 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
355 taskVisits = set(self.extensionInfo.visit)
356 self.assertEqual(taskVisits, set(self.testVisits + [-1]))
358 xx = np.linspace(0, 2000, 3)
359 yy = np.linspace(0, 4000, 6)
360 xgrid, ygrid = np.meshgrid(xx, yy)
361 for visSum in self.inputVisitSummary:
362 visit = visSum[0]['visit']
363 for detectorInfo in visSum:
364 detector = detectorInfo['id']
365 extensionIndex = np.flatnonzero((self.extensionInfo.visit == visit)
366 & (self.extensionInfo.detector == detector))[0]
367 fitWcs = self.extensionInfo.wcs[extensionIndex]
368 calexpWcs = detectorInfo.getWcs()
370 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(),
371 ygrid.ravel())])
373 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
375 tangentPoint = calexpWcs.pixelToSky(calexpWcs.getPixelOrigin().getX(),
376 calexpWcs.getPixelOrigin().getY())
377 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0 * lsst.geom.degrees, True)
378 iwcToSkyWcs = afwgeom.makeSkyWcs(lsst.geom.Point2D(0, 0), tangentPoint, cdMatrix)
379 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(tanPlaneXY[:, 0], tanPlaneXY[:, 1],
380 degrees=True)
382 # One WCS is in SIP and the other is TPV. The pixel-to-sky
383 # conversion is not exactly the same but should be close.
384 # TODO: sip_tpv + astropy.wcs.WCS gets a better result here,
385 # particularly for detector # >= 100. See if we can improve/if
386 # improving is necessary. Check if matching in corner detectors
387 # is ok.
388 rtol = (1e-3 if (detector >= 100) else 1e-5)
389 np.testing.assert_allclose(calexpra, newRAdeg, rtol=rtol)
390 np.testing.assert_allclose(calexpdec, newDecdeg, rtol=rtol)
392 def test_refCatLoader(self):
393 """Test that we can load objects from refCat
394 """
396 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
397 [self.fieldRadius.asDegrees()],
398 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
400 self.task._load_refcat(tmpAssociations, self.refObjectLoader, self.fieldCenter, self.fieldRadius,
401 self.extensionInfo, epoch=2015)
403 # We have only loaded one catalog, so getting the 'matches' should just
404 # return the same objects we put in, except some random objects that
405 # are too close together.
406 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
408 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
410 self.assertLessEqual(nMatches, self.nStars)
411 self.assertGreater(nMatches, self.nStars * 0.9)
413 def test_load_catalogs_and_associate(self):
415 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
416 [self.fieldRadius.asDegrees()],
417 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
418 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, self.extensionInfo)
420 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
422 matchIds = []
423 correctMatches = []
424 for (s, e, o) in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
425 objVisitInd = self.extensionInfo.visitIndex[e]
426 objDet = self.extensionInfo.detector[e]
427 ExtnInds = self.inputCatalogRefs[objVisitInd].get()['detector'] == objDet
428 objInfo = self.inputCatalogRefs[objVisitInd].get()[ExtnInds].iloc[o]
429 if s == 0:
430 if len(matchIds) > 0:
431 correctMatches.append(len(set(matchIds)) == 1)
432 matchIds = []
434 matchIds.append(objInfo['sourceId'])
436 # A few matches may incorrectly associate sources because of the random
437 # positions
438 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
440 def test_make_outputs(self):
441 """Test that the run method recovers the input model parameters.
442 """
443 for v, visit in enumerate(self.testVisits):
444 visitSummary = self.inputVisitSummary[v]
445 outputWcsCatalog = self.outputs.outputWCSs[visit]
446 visitSources = self.inputCatalogRefs[v].get()
447 for d, detectorRow in enumerate(visitSummary):
448 detectorId = detectorRow['id']
449 fitwcs = outputWcsCatalog[d].getWcs()
450 detSources = visitSources[visitSources['detector'] == detectorId]
451 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources['x'], detSources['y'], degrees=True)
452 dRA = fitRA - detSources['trueRA']
453 dDec = fitDec - detSources['trueDec']
454 # Check that input coordinates match the output coordinates
455 self.assertAlmostEqual(np.mean(dRA), 0)
456 self.assertAlmostEqual(np.std(dRA), 0)
457 self.assertAlmostEqual(np.mean(dDec), 0)
458 self.assertAlmostEqual(np.std(dDec), 0)
460 def test_run(self):
461 """Test that run method recovers the input model parameters
462 """
463 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
465 for v, visit in enumerate(self.testVisits):
466 visitSummary = self.inputVisitSummary[v]
467 visitMapName = f'{visit}/poly'
469 origModel = self.trueModel[visitMapName]
470 if origModel['Type'] != 'Identity':
471 fitModel = outputMaps[visitMapName]
472 origXPoly = origModel['XPoly']['Coefficients']
473 origYPoly = origModel['YPoly']['Coefficients']
474 fitXPoly = fitModel[:len(origXPoly)]
475 fitYPoly = fitModel[len(origXPoly):]
477 absDiffX = abs(fitXPoly - origXPoly)
478 absDiffY = abs(fitYPoly - origYPoly)
479 # Check that input visit model matches fit
480 np.testing.assert_array_less(absDiffX, 1e-6)
481 np.testing.assert_array_less(absDiffY, 1e-6)
482 for d, detectorRow in enumerate(visitSummary):
483 detectorId = detectorRow['id']
484 detectorMapName = f'HSC/{detectorId}/poly'
485 origModel = self.trueModel[detectorMapName]
486 if (origModel['Type'] != 'Identity') and (v == 0):
487 fitModel = outputMaps[detectorMapName]
488 origXPoly = origModel['XPoly']['Coefficients']
489 origYPoly = origModel['YPoly']['Coefficients']
490 fitXPoly = fitModel[:len(origXPoly)]
491 fitYPoly = fitModel[len(origXPoly):]
492 absDiffX = abs(fitXPoly - origXPoly)
493 absDiffY = abs(fitYPoly - origYPoly)
494 # Check that input detector model matches fit
495 np.testing.assert_array_less(absDiffX, 1e-7)
496 np.testing.assert_array_less(absDiffY, 1e-7)
499def setup_module(module):
500 lsst.utils.tests.init()
503if __name__ == "__main__": 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true
504 lsst.utils.tests.init()
505 unittest.main()