Coverage for tests/test_gbdesAstrometricFit.py: 10%
282 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-21 09:45 +0000
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-21 09:45 +0000
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
23import os.path
24import numpy as np
25import yaml
26import astropy.units as u
27import pandas as pd
28import wcsfit
30import lsst.utils
31import lsst.afw.table as afwTable
32import lsst.afw.geom as afwgeom
33from lsst.drp.tasks.gbdesAstrometricFit import GbdesAstrometricFitConfig, GbdesAstrometricFitTask
34from lsst.daf.base import PropertyList
35from lsst.meas.algorithms import ReferenceObjectLoader
36from lsst.pipe.base import InMemoryDatasetHandle
37from lsst import sphgeom
38import lsst.geom
41class MockRefCatDataId():
43 def __init__(self, region):
44 self.region = region
47class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
49 @classmethod
50 def setUpClass(cls):
52 # Set random seed
53 np.random.seed(1234)
55 # Fraction of simulated stars in the reference catalog and science
56 # exposures
57 inReferenceFraction = 1
58 inScienceFraction = 1
60 # Make fake data
61 packageDir = lsst.utils.getPackageDir('drp_tasks')
62 cls.datadir = os.path.join(packageDir, 'tests', "data")
64 cls.fieldNumber = 0
65 cls.instrumentName = 'HSC'
66 cls.instrument = wcsfit.Instrument(cls.instrumentName)
67 cls.refEpoch = 57205.5
69 # Make test inputVisitSummary. VisitSummaryTables are taken from
70 # collection HSC/runs/RC2/w_2022_20/DM-34794
71 cls.testVisits = [1176, 17900, 17930, 17934]
72 cls.inputVisitSummary = []
73 for testVisit in cls.testVisits:
74 visSum = afwTable.ExposureCatalog.readFits(os.path.join(cls.datadir,
75 f'visitSummary_{testVisit}.fits'))
76 cls.inputVisitSummary.append(visSum)
78 cls.config = GbdesAstrometricFitConfig()
79 cls.config.systematicError = 0
80 cls.config.devicePolyOrder = 4
81 cls.config.exposurePolyOrder = 6
82 cls.config.fitReserveFraction = 0
83 cls.config.fitReserveRandomSeed = 1234
84 cls.task = GbdesAstrometricFitTask(config=cls.config)
86 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
87 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch)
89 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
90 cls.inputVisitSummary, cls.exposureInfo.medianEpoch)
92 # Bounding box of observations:
93 raMins, raMaxs = [], []
94 decMins, decMaxs = [], []
95 for visSum in cls.inputVisitSummary:
96 raMins.append(visSum['raCorners'].min())
97 raMaxs.append(visSum['raCorners'].max())
98 decMins.append(visSum['decCorners'].min())
99 decMaxs.append(visSum['decCorners'].max())
100 raMin = min(raMins)
101 raMax = max(raMaxs)
102 decMin = min(decMins)
103 decMax = max(decMaxs)
105 corners = [lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
106 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
107 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
108 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector()]
109 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
111 # Make random set of data in a bounding box determined by input visits
112 # Make wcs objects for the "true" model
113 cls.nStars = 10000
114 starIds = np.arange(cls.nStars)
115 starRAs = np.random.random(cls.nStars) * (raMax - raMin) + raMin
116 starDecs = np.random.random(cls.nStars) * (decMax - decMin) + decMin
118 # Make a reference catalog and load it into ReferenceObjectLoader
119 refDataId, deferredRefCat = cls._make_refCat(starIds, starRAs, starDecs, inReferenceFraction)
120 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
121 cls.refObjectLoader.config.requireProperMotion = False
122 cls.refObjectLoader.config.anyFilterMapsToThis = 'test_filter'
124 cls.task.refObjectLoader = cls.refObjectLoader
126 # Get True WCS for stars:
127 with open(os.path.join(cls.datadir, 'sample_wcs.yaml'), 'r') as f:
128 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
130 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
132 # Make source catalogs:
133 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs,
134 inScienceFraction)
136 cls.outputs = cls.task.run(cls.inputCatalogRefs, cls.inputVisitSummary,
137 instrumentName=cls.instrumentName, refEpoch=cls.refEpoch,
138 refObjectLoader=cls.refObjectLoader)
140 @classmethod
141 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction):
142 """Make reference catalog from a subset of the simulated data
144 Parameters
145 ----------
146 starIds : `np.ndarray` of `int`
147 Source ids for the simulated stars
148 starRas : `np.ndarray` of `float`
149 RAs of the simulated stars
150 starDecs : `np.ndarray` of `float`
151 Decs of the simulated stars
152 inReferenceFraction : float
153 Percentage of simulated stars to include in reference catalog
155 Returns
156 -------
157 refDataId : MockRefCatDataId
158 Object that replicates the functionality of a dataId.
159 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
160 Dataset handle for reference catalog.
161 """
162 nRefs = int(cls.nStars * inReferenceFraction)
163 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
164 # Make simpleCatalog to hold data, create datasetRef with `region`
165 # determined by bounding box used in above simulate.
166 refSchema = afwTable.SimpleTable.makeMinimalSchema()
167 idKey = refSchema.addField("sourceId", type="I")
168 fluxKey = refSchema.addField("test_filter_flux", units='nJy', type=np.float64)
169 raErrKey = refSchema.addField("coord_raErr", type=np.float64)
170 decErrKey = refSchema.addField("coord_decErr", type=np.float64)
171 pmraErrKey = refSchema.addField("pm_raErr", type=np.float64)
172 pmdecErrKey = refSchema.addField("pm_decErr", type=np.float64)
173 refCat = afwTable.SimpleCatalog(refSchema)
174 ref_md = PropertyList()
175 ref_md.set("REFCAT_FORMAT_VERSION", 1)
176 refCat.table.setMetadata(ref_md)
177 for i in refStarIndices:
178 record = refCat.addNew()
179 record.set(idKey, starIds[i])
180 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
181 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
182 record.set(fluxKey, 1)
183 record.set(raErrKey, 0.00001)
184 record.set(decErrKey, 0.00001)
185 record.set(pmraErrKey, 1e-9)
186 record.set(pmdecErrKey, 1e-9)
187 refDataId = MockRefCatDataId(cls.boundingPolygon)
188 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog", htm7="mockRefCat")
190 return refDataId, deferredRefCat
192 @classmethod
193 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
194 """Make a `pd.DataFrame` catalog with the columns needed for the
195 object selector.
197 Parameters
198 ----------
199 starIds : `np.ndarray` of `int`
200 Source ids for the simulated stars
201 starRas : `np.ndarray` of `float`
202 RAs of the simulated stars
203 starDecs : `np.ndarray` of `float`
204 Decs of the simulated stars
205 trueWCSs : `list` of `lsst.afw.geom.SkyWcs`
206 WCS with which to simulate the source pixel coordinates
207 inReferenceFraction : float
208 Percentage of simulated stars to include in reference catalog
210 Returns
211 -------
212 sourceCat : `list` of `lsst.pipe.base.InMemoryDatasetHandle`
213 List of reference to source catalogs.
214 """
215 inputCatalogRefs = []
216 # Take a subset of the simulated data
217 # Use true wcs objects to put simulated data into ccds
218 bbox = lsst.geom.BoxD(lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_min_x'],
219 cls.inputVisitSummary[0][0]['bbox_min_y']),
220 lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_max_x'],
221 cls.inputVisitSummary[0][0]['bbox_max_y']))
222 bboxCorners = bbox.getCorners()
223 cls.inputCatalogRefs = []
224 for v, visit in enumerate(cls.testVisits):
225 nVisStars = int(cls.nStars * inScienceFraction)
226 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
227 visitStarIds = starIds[visitStarIndices]
228 visitStarRas = starRas[visitStarIndices]
229 visitStarDecs = starDecs[visitStarIndices]
230 sourceCats = []
231 for detector in trueWCSs[visit]:
232 detWcs = detector.getWcs()
233 detectorId = detector['id']
234 radecCorners = detWcs.pixelToSky(bboxCorners)
235 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
236 detectorIndices = detectorFootprint.contains((visitStarRas*u.degree).to(u.radian),
237 (visitStarDecs*u.degree).to(u.radian))
238 nDetectorStars = detectorIndices.sum()
239 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector['id']
241 ones_like = np.ones(nDetectorStars)
242 zeros_like = np.zeros(nDetectorStars, dtype=bool)
244 x, y = detWcs.skyToPixelArray(visitStarRas[detectorIndices], visitStarDecs[detectorIndices],
245 degrees=True)
247 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]['id'] == detectorId])[0].getWcs()
248 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
250 sourceDict = {}
251 sourceDict['detector'] = detectorArray
252 sourceDict['sourceId'] = visitStarIds[detectorIndices]
253 sourceDict['x'] = x
254 sourceDict['y'] = y
255 sourceDict['xErr'] = 1e-3 * ones_like
256 sourceDict['yErr'] = 1e-3 * ones_like
257 sourceDict['inputRA'] = inputRa
258 sourceDict['inputDec'] = inputDec
259 sourceDict['trueRA'] = visitStarRas[detectorIndices]
260 sourceDict['trueDec'] = visitStarDecs[detectorIndices]
261 for key in ['apFlux_12_0_flux', 'apFlux_12_0_instFlux', 'ixx', 'iyy']:
262 sourceDict[key] = ones_like
263 for key in ['pixelFlags_edge', 'pixelFlags_saturated', 'pixelFlags_interpolatedCenter',
264 'pixelFlags_interpolated', 'pixelFlags_crCenter', 'pixelFlags_bad',
265 'hsmPsfMoments_flag', 'apFlux_12_0_flag', 'extendedness', 'parentSourceId',
266 'deblend_nChild', 'ixy']:
267 sourceDict[key] = zeros_like
268 sourceDict['apFlux_12_0_instFluxErr'] = 1e-3 * ones_like
270 sourceCat = pd.DataFrame(sourceDict)
271 sourceCats.append(sourceCat)
273 visitSourceTable = pd.concat(sourceCats)
275 inputCatalogRef = InMemoryDatasetHandle(visitSourceTable, storageClass="DataFrame",
276 dataId={"visit": visit})
278 inputCatalogRefs.append(inputCatalogRef)
280 return inputCatalogRefs
282 @classmethod
283 def _make_wcs(cls, model, inputVisitSummaries):
284 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
286 Parameters
287 ----------
288 model : `dict`
289 Dictionary with WCS model parameters
290 inputVisitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
291 Visit summary catalogs
292 Returns
293 -------
294 catalogs : `dict` of `lsst.afw.table.ExposureCatalog`
295 Visit summary catalogs with WCS set to input model
296 """
298 # Pixels will need to be rescaled before going into the mappings
299 xscale = inputVisitSummaries[0][0]['bbox_max_x'] - inputVisitSummaries[0][0]['bbox_min_x']
300 yscale = inputVisitSummaries[0][0]['bbox_max_y'] - inputVisitSummaries[0][0]['bbox_min_y']
302 catalogs = {}
303 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
304 schema.addField('visit', type='L', doc='Visit number')
305 for visitSum in inputVisitSummaries:
306 visit = visitSum[0]['visit']
307 visitMapName = f'{visit}/poly'
308 visitModel = model[visitMapName]
310 catalog = lsst.afw.table.ExposureCatalog(schema)
311 catalog.resize(len(visitSum))
312 catalog['visit'] = visit
314 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
316 visitMapType = visitModel['Type']
317 visitDict = {'Type': visitMapType}
318 if visitMapType == 'Poly':
319 mapCoefficients = (visitModel['XPoly']['Coefficients']
320 + visitModel['YPoly']['Coefficients'])
321 visitDict["Coefficients"] = mapCoefficients
323 for d, detector in enumerate(visitSum):
324 detectorId = detector['id']
325 detectorMapName = f'HSC/{detectorId}/poly'
326 detectorModel = model[detectorMapName]
328 detectorMapType = detectorModel['Type']
329 mapDict = {detectorMapName: {'Type': detectorMapType},
330 visitMapName: visitDict}
331 if detectorMapType == 'Poly':
332 mapCoefficients = (detectorModel['XPoly']['Coefficients']
333 + detectorModel['YPoly']['Coefficients'])
334 mapDict[detectorMapName]['Coefficients'] = mapCoefficients
336 outWCS = cls.task._make_afw_wcs(mapDict, raDec.getRa(), raDec.getDec(),
337 doNormalizePixels=True, xScale=xscale, yScale=yscale)
338 catalog[d].setId(detectorId)
339 catalog[d].setWcs(outWCS)
341 catalog.sort()
342 catalogs[visit] = catalog
344 return catalogs
346 def test_get_exposure_info(self):
347 """Test that information for input exposures is as expected and that
348 the WCS in the class object gives approximately the same results as the
349 input `lsst.afw.geom.SkyWcs`.
350 """
352 # The total number of extensions is the number of detectors for each
353 # visit plus one for the reference catalog
354 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + 1
356 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
358 taskVisits = set(self.extensionInfo.visit)
359 self.assertEqual(taskVisits, set(self.testVisits + [-1]))
361 xx = np.linspace(0, 2000, 3)
362 yy = np.linspace(0, 4000, 6)
363 xgrid, ygrid = np.meshgrid(xx, yy)
364 for visSum in self.inputVisitSummary:
365 visit = visSum[0]['visit']
366 for detectorInfo in visSum:
367 detector = detectorInfo['id']
368 extensionIndex = np.flatnonzero((self.extensionInfo.visit == visit)
369 & (self.extensionInfo.detector == detector))[0]
370 fitWcs = self.extensionInfo.wcs[extensionIndex]
371 calexpWcs = detectorInfo.getWcs()
373 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(),
374 ygrid.ravel())])
376 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
378 tangentPoint = calexpWcs.pixelToSky(calexpWcs.getPixelOrigin().getX(),
379 calexpWcs.getPixelOrigin().getY())
380 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0 * lsst.geom.degrees, True)
381 iwcToSkyWcs = afwgeom.makeSkyWcs(lsst.geom.Point2D(0, 0), tangentPoint, cdMatrix)
382 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(tanPlaneXY[:, 0], tanPlaneXY[:, 1],
383 degrees=True)
385 # One WCS is in SIP and the other is TPV. The pixel-to-sky
386 # conversion is not exactly the same but should be close.
387 # TODO: sip_tpv + astropy.wcs.WCS gets a better result here,
388 # particularly for detector # >= 100. See if we can improve/if
389 # improving is necessary. Check if matching in corner detectors
390 # is ok.
391 rtol = (1e-3 if (detector >= 100) else 1e-5)
392 np.testing.assert_allclose(calexpra, newRAdeg, rtol=rtol)
393 np.testing.assert_allclose(calexpdec, newDecdeg, rtol=rtol)
395 def test_refCatLoader(self):
396 """Test that we can load objects from refCat
397 """
399 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
400 [self.fieldRadius.asDegrees()],
401 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
403 self.task._load_refcat(tmpAssociations, self.refObjectLoader, self.fieldCenter, self.fieldRadius,
404 self.extensionInfo, epoch=2015)
406 # We have only loaded one catalog, so getting the 'matches' should just
407 # return the same objects we put in, except some random objects that
408 # are too close together.
409 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
411 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
413 self.assertLessEqual(nMatches, self.nStars)
414 self.assertGreater(nMatches, self.nStars * 0.9)
416 def test_load_catalogs_and_associate(self):
418 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
419 [self.fieldRadius.asDegrees()],
420 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
421 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, self.extensionInfo)
423 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
425 matchIds = []
426 correctMatches = []
427 for (s, e, o) in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
428 objVisitInd = self.extensionInfo.visitIndex[e]
429 objDet = self.extensionInfo.detector[e]
430 ExtnInds = self.inputCatalogRefs[objVisitInd].get()['detector'] == objDet
431 objInfo = self.inputCatalogRefs[objVisitInd].get()[ExtnInds].iloc[o]
432 if s == 0:
433 if len(matchIds) > 0:
434 correctMatches.append(len(set(matchIds)) == 1)
435 matchIds = []
437 matchIds.append(objInfo['sourceId'])
439 # A few matches may incorrectly associate sources because of the random
440 # positions
441 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
443 def test_make_outputs(self):
444 """Test that the run method recovers the input model parameters.
445 """
446 for v, visit in enumerate(self.testVisits):
447 visitSummary = self.inputVisitSummary[v]
448 outputWcsCatalog = self.outputs.outputWCSs[visit]
449 visitSources = self.inputCatalogRefs[v].get()
450 for d, detectorRow in enumerate(visitSummary):
451 detectorId = detectorRow['id']
452 fitwcs = outputWcsCatalog[d].getWcs()
453 detSources = visitSources[visitSources['detector'] == detectorId]
454 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources['x'], detSources['y'], degrees=True)
455 dRA = fitRA - detSources['trueRA']
456 dDec = fitDec - detSources['trueDec']
457 # Check that input coordinates match the output coordinates
458 self.assertAlmostEqual(np.mean(dRA), 0)
459 self.assertAlmostEqual(np.std(dRA), 0)
460 self.assertAlmostEqual(np.mean(dDec), 0)
461 self.assertAlmostEqual(np.std(dDec), 0)
463 def test_run(self):
464 """Test that run method recovers the input model parameters
465 """
466 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
468 for v, visit in enumerate(self.testVisits):
469 visitSummary = self.inputVisitSummary[v]
470 visitMapName = f'{visit}/poly'
472 origModel = self.trueModel[visitMapName]
473 if origModel['Type'] != 'Identity':
474 fitModel = outputMaps[visitMapName]
475 origXPoly = origModel['XPoly']['Coefficients']
476 origYPoly = origModel['YPoly']['Coefficients']
477 fitXPoly = fitModel[:len(origXPoly)]
478 fitYPoly = fitModel[len(origXPoly):]
480 absDiffX = abs(fitXPoly - origXPoly)
481 absDiffY = abs(fitYPoly - origYPoly)
482 # Check that input visit model matches fit
483 np.testing.assert_array_less(absDiffX, 1e-6)
484 np.testing.assert_array_less(absDiffY, 1e-6)
485 for d, detectorRow in enumerate(visitSummary):
486 detectorId = detectorRow['id']
487 detectorMapName = f'HSC/{detectorId}/poly'
488 origModel = self.trueModel[detectorMapName]
489 if (origModel['Type'] != 'Identity') and (v == 0):
490 fitModel = outputMaps[detectorMapName]
491 origXPoly = origModel['XPoly']['Coefficients']
492 origYPoly = origModel['YPoly']['Coefficients']
493 fitXPoly = fitModel[:len(origXPoly)]
494 fitYPoly = fitModel[len(origXPoly):]
495 absDiffX = abs(fitXPoly - origXPoly)
496 absDiffY = abs(fitYPoly - origYPoly)
497 # Check that input detector model matches fit
498 np.testing.assert_array_less(absDiffX, 1e-7)
499 np.testing.assert_array_less(absDiffY, 1e-7)
502def setup_module(module):
503 lsst.utils.tests.init()
506if __name__ == "__main__": 506 ↛ 507line 506 didn't jump to line 507, because the condition on line 506 was never true
507 lsst.utils.tests.init()
508 unittest.main()