Coverage for tests/test_gbdesAstrometricFit.py: 11%
286 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-08 02:21 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-08 02:21 -0800
1# This file is part of drp_tasks
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
23import os.path
24import numpy as np
25import yaml
26import astropy.units as u
27import pandas as pd
28import wcsfit
30import lsst.utils
31import lsst.afw.table as afwTable
32import lsst.afw.geom as afwgeom
33from lsst.drp.tasks.gbdesAstrometricFit import GbdesAstrometricFitConfig, GbdesAstrometricFitTask
34from lsst.daf.base import PropertyList
35from lsst.daf.butler import DimensionUniverse, DatasetType, DatasetRef, StorageClass
36from lsst.meas.algorithms import ReferenceObjectLoader
37from lsst.pipe.base import InMemoryDatasetHandle
38from lsst import sphgeom
39import lsst.geom
42class MockRefCatDataId():
44 def __init__(self, region):
45 self.region = region
47 datasetDimensions = DimensionUniverse().extract(['htm7'])
48 datasetType = DatasetType('gaia_dr2_20200414', datasetDimensions, StorageClass("SimpleCatalog"))
49 self.ref = DatasetRef(datasetType, {'htm7': "mockRefCat"})
52class TestGbdesAstrometricFit(lsst.utils.tests.TestCase):
54 @classmethod
55 def setUpClass(cls):
57 # Set random seed
58 np.random.seed(1234)
60 # Fraction of simulated stars in the reference catalog and science
61 # exposures
62 inReferenceFraction = 1
63 inScienceFraction = 1
65 # Make fake data
66 packageDir = lsst.utils.getPackageDir('drp_tasks')
67 cls.datadir = os.path.join(packageDir, 'tests', "data")
69 cls.fieldNumber = 0
70 cls.instrumentName = 'HSC'
71 cls.instrument = wcsfit.Instrument(cls.instrumentName)
72 cls.refEpoch = 57205.5
74 # Make test inputVisitSummary. VisitSummaryTables are taken from
75 # collection HSC/runs/RC2/w_2022_20/DM-34794
76 cls.testVisits = [1176, 17900, 17930, 17934]
77 cls.inputVisitSummary = []
78 for testVisit in cls.testVisits:
79 visSum = afwTable.ExposureCatalog.readFits(os.path.join(cls.datadir,
80 f'visitSummary_{testVisit}.fits'))
81 cls.inputVisitSummary.append(visSum)
83 cls.config = GbdesAstrometricFitConfig()
84 cls.config.systematicError = 0
85 cls.config.devicePolyOrder = 4
86 cls.config.exposurePolyOrder = 6
87 cls.config.fitReserveFraction = 0
88 cls.config.fitReserveRandomSeed = 1234
89 cls.task = GbdesAstrometricFitTask(config=cls.config)
91 cls.exposureInfo, cls.exposuresHelper, cls.extensionInfo = cls.task._get_exposure_info(
92 cls.inputVisitSummary, cls.instrument, refEpoch=cls.refEpoch)
94 cls.fields, cls.fieldCenter, cls.fieldRadius = cls.task._prep_sky(
95 cls.inputVisitSummary, cls.exposureInfo.medianEpoch)
97 # Bounding box of observations:
98 raMins, raMaxs = [], []
99 decMins, decMaxs = [], []
100 for visSum in cls.inputVisitSummary:
101 raMins.append(visSum['raCorners'].min())
102 raMaxs.append(visSum['raCorners'].max())
103 decMins.append(visSum['decCorners'].min())
104 decMaxs.append(visSum['decCorners'].max())
105 raMin = min(raMins)
106 raMax = max(raMaxs)
107 decMin = min(decMins)
108 decMax = max(decMaxs)
110 corners = [lsst.geom.SpherePoint(raMin, decMin, lsst.geom.degrees).getVector(),
111 lsst.geom.SpherePoint(raMax, decMin, lsst.geom.degrees).getVector(),
112 lsst.geom.SpherePoint(raMax, decMax, lsst.geom.degrees).getVector(),
113 lsst.geom.SpherePoint(raMin, decMax, lsst.geom.degrees).getVector()]
114 cls.boundingPolygon = sphgeom.ConvexPolygon(corners)
116 # Make random set of data in a bounding box determined by input visits
117 # Make wcs objects for the "true" model
118 cls.nStars = 10000
119 starIds = np.arange(cls.nStars)
120 starRAs = np.random.random(cls.nStars) * (raMax - raMin) + raMin
121 starDecs = np.random.random(cls.nStars) * (decMax - decMin) + decMin
123 # Make a reference catalog and load it into ReferenceObjectLoader
124 refDataId, deferredRefCat = cls._make_refCat(starIds, starRAs, starDecs, inReferenceFraction)
125 cls.refObjectLoader = ReferenceObjectLoader([refDataId], [deferredRefCat])
126 cls.refObjectLoader.config.requireProperMotion = False
127 cls.refObjectLoader.config.anyFilterMapsToThis = 'test_filter'
129 cls.task.refObjectLoader = cls.refObjectLoader
131 # Get True WCS for stars:
132 with open(os.path.join(cls.datadir, 'sample_wcs.yaml'), 'r') as f:
133 cls.trueModel = yaml.load(f, Loader=yaml.Loader)
135 trueWCSs = cls._make_wcs(cls.trueModel, cls.inputVisitSummary)
137 # Make source catalogs:
138 cls.inputCatalogRefs = cls._make_sourceCat(starIds, starRAs, starDecs, trueWCSs,
139 inScienceFraction)
141 cls.outputs = cls.task.run(cls.inputCatalogRefs, cls.inputVisitSummary,
142 instrumentName=cls.instrumentName, refEpoch=cls.refEpoch,
143 refObjectLoader=cls.refObjectLoader)
145 @classmethod
146 def _make_refCat(cls, starIds, starRas, starDecs, inReferenceFraction):
147 """Make reference catalog from a subset of the simulated data
149 Parameters
150 ----------
151 starIds : `np.ndarray` of `int`
152 Source ids for the simulated stars
153 starRas : `np.ndarray` of `float`
154 RAs of the simulated stars
155 starDecs : `np.ndarray` of `float`
156 Decs of the simulated stars
157 inReferenceFraction : float
158 Percentage of simulated stars to include in reference catalog
160 Returns
161 -------
162 refDataId : MockRefCatDataId
163 Object that replicates the functionality of a dataId.
164 deferredRefCat : `lsst.pipe.base.InMemoryDatasetHandle`
165 Dataset handle for reference catalog.
166 """
167 nRefs = int(cls.nStars * inReferenceFraction)
168 refStarIndices = np.random.choice(cls.nStars, nRefs, replace=False)
169 # Make simpleCatalog to hold data, create datasetRef with `region`
170 # determined by bounding box used in above simulate.
171 refSchema = afwTable.SimpleTable.makeMinimalSchema()
172 idKey = refSchema.addField("sourceId", type="I")
173 fluxKey = refSchema.addField("test_filter_flux", units='nJy', type=np.float64)
174 raErrKey = refSchema.addField("coord_raErr", type=np.float64)
175 decErrKey = refSchema.addField("coord_decErr", type=np.float64)
176 pmraErrKey = refSchema.addField("pm_raErr", type=np.float64)
177 pmdecErrKey = refSchema.addField("pm_decErr", type=np.float64)
178 refCat = afwTable.SimpleCatalog(refSchema)
179 ref_md = PropertyList()
180 ref_md.set("REFCAT_FORMAT_VERSION", 1)
181 refCat.table.setMetadata(ref_md)
182 for i in refStarIndices:
183 record = refCat.addNew()
184 record.set(idKey, starIds[i])
185 record.setRa(lsst.geom.Angle(starRas[i], lsst.geom.degrees))
186 record.setDec(lsst.geom.Angle(starDecs[i], lsst.geom.degrees))
187 record.set(fluxKey, 1)
188 record.set(raErrKey, 0.00001)
189 record.set(decErrKey, 0.00001)
190 record.set(pmraErrKey, 1e-9)
191 record.set(pmdecErrKey, 1e-9)
192 refDataId = MockRefCatDataId(cls.boundingPolygon)
193 deferredRefCat = InMemoryDatasetHandle(refCat, storageClass="SourceCatalog")
195 return refDataId, deferredRefCat
197 @classmethod
198 def _make_sourceCat(cls, starIds, starRas, starDecs, trueWCSs, inScienceFraction):
199 """Make a `pd.DataFrame` catalog with the columns needed for the
200 object selector.
202 Parameters
203 ----------
204 starIds : `np.ndarray` of `int`
205 Source ids for the simulated stars
206 starRas : `np.ndarray` of `float`
207 RAs of the simulated stars
208 starDecs : `np.ndarray` of `float`
209 Decs of the simulated stars
210 trueWCSs : `list` of `lsst.afw.geom.SkyWcs`
211 WCS with which to simulate the source pixel coordinates
212 inReferenceFraction : float
213 Percentage of simulated stars to include in reference catalog
215 Returns
216 -------
217 sourceCat : `list` of `lsst.pipe.base.InMemoryDatasetHandle`
218 List of reference to source catalogs.
219 """
220 inputCatalogRefs = []
221 # Take a subset of the simulated data
222 # Use true wcs objects to put simulated data into ccds
223 bbox = lsst.geom.BoxD(lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_min_x'],
224 cls.inputVisitSummary[0][0]['bbox_min_y']),
225 lsst.geom.Point2D(cls.inputVisitSummary[0][0]['bbox_max_x'],
226 cls.inputVisitSummary[0][0]['bbox_max_y']))
227 bboxCorners = bbox.getCorners()
228 cls.inputCatalogRefs = []
229 for v, visit in enumerate(cls.testVisits):
230 nVisStars = int(cls.nStars * inScienceFraction)
231 visitStarIndices = np.random.choice(cls.nStars, nVisStars, replace=False)
232 visitStarIds = starIds[visitStarIndices]
233 visitStarRas = starRas[visitStarIndices]
234 visitStarDecs = starDecs[visitStarIndices]
235 sourceCats = []
236 for detector in trueWCSs[visit]:
237 detWcs = detector.getWcs()
238 detectorId = detector['id']
239 radecCorners = detWcs.pixelToSky(bboxCorners)
240 detectorFootprint = sphgeom.ConvexPolygon([rd.getVector() for rd in radecCorners])
241 detectorIndices = detectorFootprint.contains((visitStarRas*u.degree).to(u.radian),
242 (visitStarDecs*u.degree).to(u.radian))
243 nDetectorStars = detectorIndices.sum()
244 detectorArray = np.ones(nDetectorStars, dtype=bool) * detector['id']
246 ones_like = np.ones(nDetectorStars)
247 zeros_like = np.zeros(nDetectorStars, dtype=bool)
249 x, y = detWcs.skyToPixelArray(visitStarRas[detectorIndices], visitStarDecs[detectorIndices],
250 degrees=True)
252 origWcs = (cls.inputVisitSummary[v][cls.inputVisitSummary[v]['id'] == detectorId])[0].getWcs()
253 inputRa, inputDec = origWcs.pixelToSkyArray(x, y, degrees=True)
255 sourceDict = {}
256 sourceDict['detector'] = detectorArray
257 sourceDict['sourceId'] = visitStarIds[detectorIndices]
258 sourceDict['x'] = x
259 sourceDict['y'] = y
260 sourceDict['xErr'] = 1e-3 * ones_like
261 sourceDict['yErr'] = 1e-3 * ones_like
262 sourceDict['inputRA'] = inputRa
263 sourceDict['inputDec'] = inputDec
264 sourceDict['trueRA'] = visitStarRas[detectorIndices]
265 sourceDict['trueDec'] = visitStarDecs[detectorIndices]
266 for key in ['apFlux_12_0_flux', 'apFlux_12_0_instFlux', 'ixx', 'iyy']:
267 sourceDict[key] = ones_like
268 for key in ['pixelFlags_edge', 'pixelFlags_saturated', 'pixelFlags_interpolatedCenter',
269 'pixelFlags_interpolated', 'pixelFlags_crCenter', 'pixelFlags_bad',
270 'hsmPsfMoments_flag', 'apFlux_12_0_flag', 'extendedness', 'parentSourceId',
271 'deblend_nChild', 'ixy']:
272 sourceDict[key] = zeros_like
273 sourceDict['apFlux_12_0_instFluxErr'] = 1e-3 * ones_like
275 sourceCat = pd.DataFrame(sourceDict)
276 sourceCats.append(sourceCat)
278 visitSourceTable = pd.concat(sourceCats)
280 inputCatalogRef = InMemoryDatasetHandle(visitSourceTable, storageClass="DataFrame",
281 dataId={"visit": visit})
283 inputCatalogRefs.append(inputCatalogRef)
285 return inputCatalogRefs
287 @classmethod
288 def _make_wcs(cls, model, inputVisitSummaries):
289 """Make a `lsst.afw.geom.SkyWcs` from given model parameters
291 Parameters
292 ----------
293 model : `dict`
294 Dictionary with WCS model parameters
295 inputVisitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
296 Visit summary catalogs
297 Returns
298 -------
299 catalogs : `dict` of `lsst.afw.table.ExposureCatalog`
300 Visit summary catalogs with WCS set to input model
301 """
303 # Pixels will need to be rescaled before going into the mappings
304 xscale = inputVisitSummaries[0][0]['bbox_max_x'] - inputVisitSummaries[0][0]['bbox_min_x']
305 yscale = inputVisitSummaries[0][0]['bbox_max_y'] - inputVisitSummaries[0][0]['bbox_min_y']
307 catalogs = {}
308 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
309 schema.addField('visit', type='L', doc='Visit number')
310 for visitSum in inputVisitSummaries:
311 visit = visitSum[0]['visit']
312 visitMapName = f'{visit}/poly'
313 visitModel = model[visitMapName]
315 catalog = lsst.afw.table.ExposureCatalog(schema)
316 catalog.resize(len(visitSum))
317 catalog['visit'] = visit
319 raDec = visitSum[0].getVisitInfo().getBoresightRaDec()
321 visitMapType = visitModel['Type']
322 visitDict = {'Type': visitMapType}
323 if visitMapType == 'Poly':
324 mapCoefficients = (visitModel['XPoly']['Coefficients']
325 + visitModel['YPoly']['Coefficients'])
326 visitDict["Coefficients"] = mapCoefficients
328 for d, detector in enumerate(visitSum):
329 detectorId = detector['id']
330 detectorMapName = f'HSC/{detectorId}/poly'
331 detectorModel = model[detectorMapName]
333 detectorMapType = detectorModel['Type']
334 mapDict = {detectorMapName: {'Type': detectorMapType},
335 visitMapName: visitDict}
336 if detectorMapType == 'Poly':
337 mapCoefficients = (detectorModel['XPoly']['Coefficients']
338 + detectorModel['YPoly']['Coefficients'])
339 mapDict[detectorMapName]['Coefficients'] = mapCoefficients
341 outWCS = cls.task._make_afw_wcs(mapDict, raDec.getRa(), raDec.getDec(),
342 doNormalizePixels=True, xScale=xscale, yScale=yscale)
343 catalog[d].setId(detectorId)
344 catalog[d].setWcs(outWCS)
346 catalog.sort()
347 catalogs[visit] = catalog
349 return catalogs
351 def test_get_exposure_info(self):
352 """Test that information for input exposures is as expected and that
353 the WCS in the class object gives approximately the same results as the
354 input `lsst.afw.geom.SkyWcs`.
355 """
357 # The total number of extensions is the number of detectors for each
358 # visit plus one for the reference catalog
359 totalExtensions = sum([len(visSum) for visSum in self.inputVisitSummary]) + 1
361 self.assertEqual(totalExtensions, len(self.extensionInfo.visit))
363 taskVisits = set(self.extensionInfo.visit)
364 self.assertEqual(taskVisits, set(self.testVisits + [-1]))
366 xx = np.linspace(0, 2000, 3)
367 yy = np.linspace(0, 4000, 6)
368 xgrid, ygrid = np.meshgrid(xx, yy)
369 for visSum in self.inputVisitSummary:
370 visit = visSum[0]['visit']
371 for detectorInfo in visSum:
372 detector = detectorInfo['id']
373 extensionIndex = np.flatnonzero((self.extensionInfo.visit == visit)
374 & (self.extensionInfo.detector == detector))[0]
375 fitWcs = self.extensionInfo.wcs[extensionIndex]
376 calexpWcs = detectorInfo.getWcs()
378 tanPlaneXY = np.array([fitWcs.toWorld(x, y) for (x, y) in zip(xgrid.ravel(),
379 ygrid.ravel())])
381 calexpra, calexpdec = calexpWcs.pixelToSkyArray(xgrid.ravel(), ygrid.ravel(), degrees=True)
383 tangentPoint = calexpWcs.pixelToSky(calexpWcs.getPixelOrigin().getX(),
384 calexpWcs.getPixelOrigin().getY())
385 cdMatrix = afwgeom.makeCdMatrix(1.0 * lsst.geom.degrees, 0 * lsst.geom.degrees, True)
386 iwcToSkyWcs = afwgeom.makeSkyWcs(lsst.geom.Point2D(0, 0), tangentPoint, cdMatrix)
387 newRAdeg, newDecdeg = iwcToSkyWcs.pixelToSkyArray(tanPlaneXY[:, 0], tanPlaneXY[:, 1],
388 degrees=True)
390 # One WCS is in SIP and the other is TPV. The pixel-to-sky
391 # conversion is not exactly the same but should be close.
392 # TODO: sip_tpv + astropy.wcs.WCS gets a better result here,
393 # particularly for detector # >= 100. See if we can improve/if
394 # improving is necessary. Check if matching in corner detectors
395 # is ok.
396 rtol = (1e-3 if (detector >= 100) else 1e-5)
397 np.testing.assert_allclose(calexpra, newRAdeg, rtol=rtol)
398 np.testing.assert_allclose(calexpdec, newDecdeg, rtol=rtol)
400 def test_refCatLoader(self):
401 """Test that we can load objects from refCat
402 """
404 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
405 [self.fieldRadius.asDegrees()],
406 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
408 self.task._load_refcat(tmpAssociations, self.refObjectLoader, self.fieldCenter, self.fieldRadius,
409 self.extensionInfo, epoch=2015)
411 # We have only loaded one catalog, so getting the 'matches' should just
412 # return the same objects we put in, except some random objects that
413 # are too close together.
414 tmpAssociations.sortMatches(self.fieldNumber, minMatches=1)
416 nMatches = (np.array(tmpAssociations.sequence) == 0).sum()
418 self.assertLessEqual(nMatches, self.nStars)
419 self.assertGreater(nMatches, self.nStars * 0.9)
421 def test_load_catalogs_and_associate(self):
423 tmpAssociations = wcsfit.FoFClass(self.fields, [self.instrument], self.exposuresHelper,
424 [self.fieldRadius.asDegrees()],
425 (self.task.config.matchRadius * u.arcsec).to(u.degree).value)
426 self.task._load_catalogs_and_associate(tmpAssociations, self.inputCatalogRefs, self.extensionInfo)
428 tmpAssociations.sortMatches(self.fieldNumber, minMatches=2)
430 matchIds = []
431 correctMatches = []
432 for (s, e, o) in zip(tmpAssociations.sequence, tmpAssociations.extn, tmpAssociations.obj):
433 objVisitInd = self.extensionInfo.visitIndex[e]
434 objDet = self.extensionInfo.detector[e]
435 ExtnInds = self.inputCatalogRefs[objVisitInd].get()['detector'] == objDet
436 objInfo = self.inputCatalogRefs[objVisitInd].get()[ExtnInds].iloc[o]
437 if s == 0:
438 if len(matchIds) > 0:
439 correctMatches.append(len(set(matchIds)) == 1)
440 matchIds = []
442 matchIds.append(objInfo['sourceId'])
444 # A few matches may incorrectly associate sources because of the random
445 # positions
446 self.assertGreater(sum(correctMatches), len(correctMatches) * 0.95)
448 def test_make_outputs(self):
449 """Test that the run method recovers the input model parameters.
450 """
451 for v, visit in enumerate(self.testVisits):
452 visitSummary = self.inputVisitSummary[v]
453 outputWcsCatalog = self.outputs.outputWCSs[visit]
454 visitSources = self.inputCatalogRefs[v].get()
455 for d, detectorRow in enumerate(visitSummary):
456 detectorId = detectorRow['id']
457 fitwcs = outputWcsCatalog[d].getWcs()
458 detSources = visitSources[visitSources['detector'] == detectorId]
459 fitRA, fitDec = fitwcs.pixelToSkyArray(detSources['x'], detSources['y'], degrees=True)
460 dRA = fitRA - detSources['trueRA']
461 dDec = fitDec - detSources['trueDec']
462 # Check that input coordinates match the output coordinates
463 self.assertAlmostEqual(np.mean(dRA), 0)
464 self.assertAlmostEqual(np.std(dRA), 0)
465 self.assertAlmostEqual(np.mean(dDec), 0)
466 self.assertAlmostEqual(np.std(dDec), 0)
468 def test_run(self):
469 """Test that run method recovers the input model parameters
470 """
471 outputMaps = self.outputs.fitModel.mapCollection.getParamDict()
473 for v, visit in enumerate(self.testVisits):
474 visitSummary = self.inputVisitSummary[v]
475 visitMapName = f'{visit}/poly'
477 origModel = self.trueModel[visitMapName]
478 if origModel['Type'] != 'Identity':
479 fitModel = outputMaps[visitMapName]
480 origXPoly = origModel['XPoly']['Coefficients']
481 origYPoly = origModel['YPoly']['Coefficients']
482 fitXPoly = fitModel[:len(origXPoly)]
483 fitYPoly = fitModel[len(origXPoly):]
485 absDiffX = abs(fitXPoly - origXPoly)
486 absDiffY = abs(fitYPoly - origYPoly)
487 # Check that input visit model matches fit
488 np.testing.assert_array_less(absDiffX, 1e-6)
489 np.testing.assert_array_less(absDiffY, 1e-6)
490 for d, detectorRow in enumerate(visitSummary):
491 detectorId = detectorRow['id']
492 detectorMapName = f'HSC/{detectorId}/poly'
493 origModel = self.trueModel[detectorMapName]
494 if (origModel['Type'] != 'Identity') and (v == 0):
495 fitModel = outputMaps[detectorMapName]
496 origXPoly = origModel['XPoly']['Coefficients']
497 origYPoly = origModel['YPoly']['Coefficients']
498 fitXPoly = fitModel[:len(origXPoly)]
499 fitYPoly = fitModel[len(origXPoly):]
500 absDiffX = abs(fitXPoly - origXPoly)
501 absDiffY = abs(fitYPoly - origYPoly)
502 # Check that input detector model matches fit
503 np.testing.assert_array_less(absDiffX, 1e-7)
504 np.testing.assert_array_less(absDiffY, 1e-7)
507def setup_module(module):
508 lsst.utils.tests.init()
511if __name__ == "__main__": 511 ↛ 512line 511 didn't jump to line 512, because the condition on line 511 was never true
512 lsst.utils.tests.init()
513 unittest.main()