Coverage for tests/fgcmcalTestBase.py : 9%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""General fgcmcal testing class.
25This class is used as the basis for individual obs package tests using
26data from testdata_jointcal.
27"""
29import os
30import shutil
31import numpy as np
32import glob
34import lsst.daf.persistence as dafPersist
35import lsst.geom as geom
36import lsst.log
37from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, LoadIndexedReferenceObjectsConfig
38from astropy import units
40import lsst.fgcmcal as fgcmcal
43class FgcmcalTestBase(object):
44 """
45 Base class for fgcmcal tests, to genericize some test running and setup.
47 Derive from this first, then from TestCase.
48 """
50 def setUp_base(self, inputDir=None, testDir=None, logLevel=None, otherArgs=[]):
51 """
52 Call from your child class's setUp() to get variables built.
54 Parameters
55 ----------
56 inputDir: `str`, optional
57 Input directory
58 testDir: `str`, optional
59 Test directory
60 logLevel: `str`, optional
61 Override loglevel for command-line tasks
62 otherArgs: `list`, default=[]
63 List of additional arguments to send to command-line tasks
64 """
66 self.inputDir = inputDir
67 self.testDir = testDir
68 self.logLevel = logLevel
69 self.otherArgs = otherArgs
71 self.config = None
72 self.configfiles = []
74 lsst.log.setLevel("daf.persistence.butler", lsst.log.FATAL)
75 lsst.log.setLevel("CameraMapper", lsst.log.FATAL)
77 if self.logLevel is not None:
78 self.otherArgs.extend(['--loglevel', 'fgcmcal=%s'%self.logLevel])
80 def _testFgcmMakeLut(self, nBand, i0Std, i0Recon, i10Std, i10Recon):
81 """
82 Test running of FgcmMakeLutTask
84 Parameters
85 ----------
86 nBand: `int`
87 Number of bands tested
88 i0Std: `np.array', size nBand
89 Values of i0Std to compare to
90 i10Std: `np.array`, size nBand
91 Values of i10Std to compare to
92 i0Recon: `np.array`, size nBand
93 Values of reconstructed i0 to compare to
94 i10Recon: `np.array`, size nBand
95 Values of reconsntructed i10 to compare to
97 Raises
98 ------
99 Exceptions on test failures
100 """
102 args = [self.inputDir, '--output', self.testDir,
103 '--doraise']
104 if len(self.configfiles) > 0:
105 args.extend(['--configfile', *self.configfiles])
106 args.extend(self.otherArgs)
108 result = fgcmcal.FgcmMakeLutTask.parseAndRun(args=args, config=self.config)
109 self._checkResult(result)
111 butler = dafPersist.butler.Butler(self.testDir)
112 tempTask = fgcmcal.FgcmFitCycleTask()
113 lutCat = butler.get('fgcmLookUpTable')
114 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat,
115 dict(tempTask.config.filterMap))
117 # Check that we got the requested number of bands...
118 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES']))
120 self.assertFloatsAlmostEqual(i0Std, lutStd[0]['I0STD'], msg='I0Std', rtol=1e-5)
121 self.assertFloatsAlmostEqual(i10Std, lutStd[0]['I10STD'], msg='I10Std', rtol=1e-5)
123 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32),
124 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
125 np.zeros(nBand) + lutStd[0]['O3STD'],
126 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
127 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
128 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
129 np.zeros(nBand, dtype=np.int32),
130 np.zeros(nBand) + lutStd[0]['PMBSTD'])
131 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
132 np.zeros(nBand) + lutStd[0]['O3STD'],
133 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
134 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
135 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
136 np.zeros(nBand) + lutStd[0]['PMBSTD'],
137 indices)
139 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5)
141 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
142 np.zeros(nBand) + lutStd[0]['O3STD'],
143 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
144 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
145 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
146 np.zeros(nBand) + lutStd[0]['PMBSTD'],
147 indices)
149 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5)
151 def _testFgcmBuildStars(self, visits, nStar, nObs):
152 """
153 Test running of FgcmBuildStarsTask
155 Parameters
156 ----------
157 visits: `list`
158 List of visits to calibrate
159 nStar: `int`
160 Number of stars expected
161 nObs: `int`
162 Number of observations of stars expected
164 Raises
165 ------
166 Exceptions on test failures
167 """
169 args = [self.inputDir, '--output', self.testDir,
170 '--id', 'visit='+'^'.join([str(visit) for visit in visits]),
171 '--doraise']
172 if len(self.configfiles) > 0:
173 args.extend(['--configfile', *self.configfiles])
174 args.extend(self.otherArgs)
176 result = fgcmcal.FgcmBuildStarsTask.parseAndRun(args=args, config=self.config)
177 self._checkResult(result)
179 butler = dafPersist.butler.Butler(self.testDir)
181 visitCat = butler.get('fgcmVisitCatalog')
182 self.assertEqual(len(visits), len(visitCat))
184 starIds = butler.get('fgcmStarIds')
185 self.assertEqual(nStar, len(starIds))
187 starObs = butler.get('fgcmStarObservations')
188 self.assertEqual(nObs, len(starObs))
190 def _testFgcmFitCycle(self, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=False):
191 """
192 Test running of FgcmFitCycleTask
194 Parameters
195 ----------
196 nZp: `int`
197 Number of zeropoints created by the task
198 nGoodZp: `int`
199 Number of good (photometric) zeropoints created
200 nOkZp: `int`
201 Number of constrained zeropoints (photometric or not)
202 nBadZp: `int`
203 Number of unconstrained (bad) zeropoints
204 nStdStars: `int`
205 Number of standard stars produced
206 nPlots: `int`
207 Number of plots produced
208 skipChecks: `bool`, optional
209 Skip number checks, when running less-than-final cycle.
210 Default is False.
211 """
213 args = [self.inputDir, '--output', self.testDir,
214 '--doraise']
215 if len(self.configfiles) > 0:
216 args.extend(['--configfile', *self.configfiles])
217 args.extend(self.otherArgs)
219 # Move into the test directory so the plots will get cleaned in tearDown
220 # In the future, with Gen3, we will probably have a better way of managing
221 # non-data output such as plots.
222 cwd = os.getcwd()
223 os.chdir(self.testDir)
225 result = fgcmcal.FgcmFitCycleTask.parseAndRun(args=args, config=self.config)
226 self._checkResult(result)
228 # Move back to the previous directory
229 os.chdir(cwd)
231 if skipChecks:
232 return
234 # Check that the expected number of plots are there.
235 plots = glob.glob(os.path.join(self.testDir, self.config.outfileBase +
236 '_cycle%02d_plots/' % (self.config.cycleNumber) +
237 '*.png'))
238 self.assertEqual(len(plots), nPlots)
240 butler = dafPersist.butler.Butler(self.testDir)
242 zps = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
244 # Check the numbers of zeropoints in all, good, okay, and bad
245 self.assertEqual(len(zps), nZp)
247 gd, = np.where(zps['fgcmFlag'] == 1)
248 self.assertEqual(len(gd), nGoodZp)
250 ok, = np.where(zps['fgcmFlag'] < 16)
251 self.assertEqual(len(ok), nOkZp)
253 bd, = np.where(zps['fgcmFlag'] >= 16)
254 self.assertEqual(len(bd), nBadZp)
256 # Check that there are no illegal values with the ok zeropoints
257 test, = np.where(zps['fgcmZpt'][gd] < -9000.0)
258 self.assertEqual(len(test), 0)
260 stds = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
262 self.assertEqual(len(stds), nStdStars)
264 def _testFgcmOutputProducts(self, visitDataRefName, ccdDataRefName, filterMapping,
265 zpOffsets, testVisit, testCcd, testFilter, testBandIndex):
266 """
267 Test running of FgcmOutputProductsTask
269 Parameters
270 ----------
271 visitDataRefName: `str`
272 Name of column in dataRef to get the visit
273 ccdDataRefName: `str`
274 Name of column in dataRef to get the ccd
275 filterMapping: `dict`
276 Mapping of filterName to dataRef filter names
277 zpOffsets: `np.array`
278 Zeropoint offsets expected
279 testVisit: `int`
280 Visit id to check for round-trip computations
281 testCcd: `int`
282 Ccd id to check for round-trip computations
283 testFilter: `str`
284 Filtername for testVisit/testCcd
285 testBandIndex: `int`
286 Band index for testVisit/testCcd
287 """
289 args = [self.inputDir, '--output', self.testDir,
290 '--doraise']
291 if len(self.configfiles) > 0:
292 args.extend(['--configfile', *self.configfiles])
293 args.extend(self.otherArgs)
295 result = fgcmcal.FgcmOutputProductsTask.parseAndRun(args=args, config=self.config,
296 doReturnResults=True)
297 self._checkResult(result)
299 # Extract the offsets from the results
300 offsets = result.resultList[0].results.offsets
302 # The tolerance here has been loosened to account for different
303 # results on different platforms.
304 # TODO: Tighten tolerances with fixes in DM-25114
305 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-5)
307 butler = dafPersist.butler.Butler(self.testDir)
309 # Test the reference catalog stars
311 # Read in the raw stars...
312 rawStars = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber)
314 # Read in the new reference catalog...
315 config = LoadIndexedReferenceObjectsConfig()
316 config.ref_dataset_name = 'fgcm_stars'
317 task = LoadIndexedReferenceObjectsTask(butler, config=config)
319 # Read in a giant radius to get them all
320 refStruct = task.loadSkyCircle(rawStars[0].getCoord(), 5.0*geom.degrees,
321 filterName='r')
323 # Make sure all the stars are there
324 self.assertEqual(len(rawStars), len(refStruct.refCat))
326 # And make sure the numbers are consistent
327 test, = np.where(rawStars['id'][0] == refStruct.refCat['id'])
329 # Perform math on numpy arrays to maintain datatypes
330 mags = rawStars['mag_std_noabs'][:, 0].astype(np.float64) + offsets[0]
331 fluxes = (mags*units.ABmag).to_value(units.nJy)
332 fluxErrs = (np.log(10.)/2.5)*fluxes*rawStars['magErr_std'][:, 0].astype(np.float64)
333 # Only check the first one
334 self.assertFloatsAlmostEqual(fluxes[0], refStruct.refCat['r_flux'][test[0]])
335 self.assertFloatsAlmostEqual(fluxErrs[0], refStruct.refCat['r_fluxErr'][test[0]])
337 # Test the psf candidate counting, ratio should be between 0.0 and 1.0
338 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
339 refStruct.refCat['r_nTotal'].astype(np.float64))
340 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
341 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
343 # Test the fgcm_photoCalib output
345 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber)
346 selected = (zptCat['fgcmFlag'] < 16)
348 # Read in all the calibrations, these should all be there
349 # This test is simply to ensure that all the photoCalib files exist
350 for rec in zptCat[selected]:
351 testCal = butler.get('fgcm_photoCalib',
352 dataId={visitDataRefName: int(rec['visit']),
353 ccdDataRefName: int(rec['ccd']),
354 'filter': filterMapping[rec['filtername']]})
355 self.assertIsNotNone(testCal)
357 # We do round-trip value checking on just the final one (chosen arbitrarily)
358 testCal = butler.get('fgcm_photoCalib',
359 dataId={visitDataRefName: int(testVisit),
360 ccdDataRefName: int(testCcd),
361 'filter': filterMapping[testFilter]})
362 self.assertIsNotNone(testCal)
364 src = butler.get('src', dataId={visitDataRefName: int(testVisit),
365 ccdDataRefName: int(testCcd)})
367 # Only test sources with positive flux
368 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0)
370 # We need to apply the calibration offset to the fgcmzpt (which is internal
371 # and doesn't know about that yet)
372 testZpInd, = np.where((zptCat['visit'] == testVisit) &
373 (zptCat['ccd'] == testCcd))
374 fgcmZpt = zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex]
375 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd])
377 if self.config.doComposeWcsJacobian:
378 # The raw zeropoint needs to be modified to know about the wcs jacobian
379 camera = butler.get('camera')
380 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera)
381 center = approxPixelAreaFields[testCcd].getBBox().getCenter()
382 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center)
383 fgcmZpt += -2.5*np.log10(pixAreaCorr)
385 # This is the magnitude through the mean calibration
386 photoCalMeanCalMags = np.zeros(gdSrc.sum())
387 # This is the magnitude through the full focal-plane variable mags
388 photoCalMags = np.zeros_like(photoCalMeanCalMags)
389 # This is the magnitude with the FGCM (central-ccd) zeropoint
390 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
392 for i, rec in enumerate(src[gdSrc]):
393 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'])
394 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'],
395 rec.getCentroid())
396 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
398 # These should be very close but some tiny differences because the fgcm value
399 # is defined at the center of the bbox, and the photoCal is the mean over the box
400 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
401 zptMeanCalMags, rtol=1e-6)
402 # These should be roughly equal, but not precisely because of the focal-plane
403 # variation. However, this is a useful sanity check for something going totally
404 # wrong.
405 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
406 photoCalMags, rtol=1e-2)
408 # And the photoCal error is just the zeropoint gray error
409 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(),
410 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr)
412 # Test the transmission output
414 visitCatalog = butler.get('fgcmVisitCatalog')
415 lutCat = butler.get('fgcmLookUpTable')
417 testTrans = butler.get('transmission_atmosphere_fgcm',
418 dataId={visitDataRefName: visitCatalog[0]['visit']})
419 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0),
420 wavelengths=lutCat[0]['atmLambda'])
422 # The test fit is performed with the atmosphere parameters frozen
423 # (freezeStdAtmosphere = True). Thus the only difference between
424 # these output atmospheres and the standard is the different
425 # airmass. Furthermore, this is a very rough comparison because
426 # the look-up table is computed with very coarse sampling for faster
427 # testing.
429 # To account for overall throughput changes, we scale by the median ratio,
430 # we only care about the shape
431 ratio = np.median(testResp/lutCat[0]['atmStdTrans'])
432 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04)
434 # The second should be close to the first, but there is the airmass
435 # difference so they aren't identical.
436 testTrans2 = butler.get('transmission_atmosphere_fgcm',
437 dataId={visitDataRefName: visitCatalog[1]['visit']})
438 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0),
439 wavelengths=lutCat[0]['atmLambda'])
440 # As above, we scale by the ratio to compare the shape of the curve.
441 ratio = np.median(testResp/testResp2)
442 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04)
444 def _testFgcmCalibrateTract(self, visits, tract,
445 rawRepeatability, filterNCalibMap):
446 """
447 Test running of FgcmCalibrateTractTask
449 Parameters
450 ----------
451 visits: `list`
452 List of visits to calibrate
453 tract: `int`
454 Tract number
455 rawRepeatability: `np.array`
456 Expected raw repeatability after convergence.
457 Length should be number of bands.
458 filterNCalibMap: `dict`
459 Mapping from filter name to number of photoCalibs created.
460 """
462 args = [self.inputDir, '--output', self.testDir,
463 '--id', 'visit='+'^'.join([str(visit) for visit in visits]),
464 'tract=%d' % (tract),
465 '--doraise']
466 if len(self.configfiles) > 0:
467 args.extend(['--configfile', *self.configfiles])
468 args.extend(self.otherArgs)
470 # Move into the test directory so the plots will get cleaned in tearDown
471 # In the future, with Gen3, we will probably have a better way of managing
472 # non-data output such as plots.
473 cwd = os.getcwd()
474 os.chdir(self.testDir)
476 result = fgcmcal.FgcmCalibrateTractTask.parseAndRun(args=args, config=self.config,
477 doReturnResults=True)
478 self._checkResult(result)
480 # Move back to the previous directory
481 os.chdir(cwd)
483 # Check that the converged repeatability is what we expect
484 # The tolerance here has been loosened to account for different
485 # results on different platforms.
486 # TODO: Tighten tolerances with fixes in DM-25114
487 repeatability = result.resultList[0].results.repeatability
488 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=3e-3)
490 butler = dafPersist.butler.Butler(self.testDir)
492 # Check that the number of photoCalib objects in each filter are what we expect
493 for filterName in filterNCalibMap.keys():
494 subset = butler.subset('fgcm_tract_photoCalib', tract=tract, filter=filterName)
495 tot = 0
496 for dataRef in subset:
497 if butler.datasetExists('fgcm_tract_photoCalib', dataId=dataRef.dataId):
498 tot += 1
499 self.assertEqual(tot, filterNCalibMap[filterName])
501 # Check that every visit got a transmission
502 visits = butler.queryMetadata('fgcm_tract_photoCalib', ('visit'), tract=tract)
503 for visit in visits:
504 self.assertTrue(butler.datasetExists('transmission_atmosphere_fgcm_tract',
505 tract=tract, visit=visit))
507 # Check that we got the reference catalog output.
508 # This will raise an exception if the catalog is not there.
509 config = LoadIndexedReferenceObjectsConfig()
510 config.ref_dataset_name = 'fgcm_stars_%d' % (tract)
511 task = LoadIndexedReferenceObjectsTask(butler, config=config)
513 coord = geom.SpherePoint(337.656174*geom.degrees, 0.823595*geom.degrees)
515 refStruct = task.loadSkyCircle(coord, 5.0*geom.degrees, filterName='r')
517 # Test the psf candidate counting, ratio should be between 0.0 and 1.0
518 candRatio = (refStruct.refCat['r_nPsfCandidate'].astype(np.float64) /
519 refStruct.refCat['r_nTotal'].astype(np.float64))
520 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
521 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
523 # Test that temporary files aren't stored
524 self.assertFalse(butler.datasetExists('fgcmVisitCatalog'))
525 self.assertFalse(butler.datasetExists('fgcmStarObservations'))
526 self.assertFalse(butler.datasetExists('fgcmStarIndices'))
527 self.assertFalse(butler.datasetExists('fgcmReferenceStars'))
529 def _checkResult(self, result):
530 """
531 Check the result output from the task
533 Parameters
534 ----------
535 result: `pipeBase.struct`
536 Result structure output from a task
538 Raises
539 ------
540 Exceptions on test failures
541 """
543 self.assertNotEqual(result.resultList, [], 'resultList should not be empty')
544 self.assertEqual(result.resultList[0].exitStatus, 0)
546 def tearDown(self):
547 """
548 Tear down and clear directories
549 """
551 if getattr(self, 'config', None) is not None:
552 del self.config
553 if os.path.exists(self.testDir):
554 shutil.rmtree(self.testDir, True)