Coverage for tests/fgcmcalTestBase.py: 10%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""General fgcmcal testing class.
25This class is used as the basis for individual obs package tests using
26data from testdata_jointcal for Gen3 repos.
27"""
29import os
30import shutil
31import numpy as np
32import glob
33import esutil
35import click.testing
36import lsst.ctrl.mpexec.cli.pipetask
38import lsst.daf.butler as dafButler
39import lsst.obs.base as obsBase
40import lsst.geom as geom
41import lsst.log
43import lsst.fgcmcal as fgcmcal
45ROOT = os.path.abspath(os.path.dirname(__file__))
48class FgcmcalTestBase(object):
49 """Base class for gen3 fgcmcal tests, to genericize some test running and setup.
51 Derive from this first, then from TestCase.
52 """
53 @classmethod
54 def _importRepository(cls, instrument, exportPath, exportFile):
55 """Import a test repository into self.testDir
57 Parameters
58 ----------
59 instrument : `str`
60 Full string name for the instrument.
61 exportPath : `str`
62 Path to location of repository to export.
63 exportFile : `str`
64 Filename of export data.
65 """
66 cls.repo = os.path.join(cls.testDir, 'testrepo')
68 print('Importing %s into %s' % (exportFile, cls.testDir))
70 # Make the repo and retrieve a writeable Butler
71 _ = dafButler.Butler.makeRepo(cls.repo)
72 butler = dafButler.Butler(cls.repo, writeable=True)
73 # Register the instrument
74 instrInstance = obsBase.utils.getInstrument(instrument)
75 instrInstance.register(butler.registry)
76 # Import the exportFile
77 butler.import_(directory=exportPath, filename=exportFile,
78 transfer='symlink',
79 skip_dimensions={'instrument', 'detector', 'physical_filter'})
81 def _runPipeline(self, repo, pipelineFile, queryString=None,
82 inputCollections=None, outputCollection=None,
83 configFiles=None, configOptions=None,
84 registerDatasetTypes=False):
85 """Run a pipeline via pipetask.
87 Parameters
88 ----------
89 repo : `str`
90 Gen3 repository yaml file.
91 pipelineFile : `str`
92 Pipeline definition file.
93 queryString : `str`, optional
94 String to use for "-d" data query.
95 inputCollections : `str`, optional
96 String to use for "-i" input collections (comma delimited).
97 outputCollection : `str`, optional
98 String to use for "-o" output collection.
99 configFiles : `list` [`str`], optional
100 List of config files to use (with "-C").
101 configOptions : `list` [`str`], optional
102 List of individual config options to use (with "-c").
103 registerDatasetTypes : `bool`, optional
104 Set "--register-dataset-types".
106 Returns
107 -------
108 exit_code : `int`
109 Exit code for pipetask run.
111 Raises
112 ------
113 RuntimeError : Raised if the "pipetask" call fails.
114 """
115 pipelineArgs = ["run",
116 "-b", repo,
117 "-p", pipelineFile]
119 if queryString is not None:
120 pipelineArgs.extend(["-d", queryString])
121 if inputCollections is not None:
122 pipelineArgs.extend(["-i", inputCollections])
123 if outputCollection is not None:
124 pipelineArgs.extend(["-o", outputCollection])
125 if configFiles is not None:
126 for configFile in configFiles:
127 pipelineArgs.extend(["-C", configFile])
128 if configOptions is not None:
129 for configOption in configOptions:
130 pipelineArgs.extend(["-c", configOption])
131 if registerDatasetTypes:
132 pipelineArgs.extend(["--register-dataset-types"])
134 # CliRunner is an unsafe workaround for DM-26239
135 runner = click.testing.CliRunner()
136 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs)
137 if results.exception:
138 raise RuntimeError("Pipeline %s failed." % (pipelineFile)) from results.exception
139 return results.exit_code
141 def _testFgcmMakeLut(self, instName, testName, nBand, i0Std, i0Recon, i10Std, i10Recon):
142 """Test running of FgcmMakeLutTask
144 Parameters
145 ----------
146 instName : `str`
147 Short name of the instrument
148 testName : `str`
149 Base name of the test collection
150 nBand : `int`
151 Number of bands tested
152 i0Std : `np.ndarray'
153 Values of i0Std to compare to
154 i10Std : `np.ndarray`
155 Values of i10Std to compare to
156 i0Recon : `np.ndarray`
157 Values of reconstructed i0 to compare to
158 i10Recon : `np.ndarray`
159 Values of reconsntructed i10 to compare to
160 """
161 instCamel = instName.title()
163 configFile = 'fgcmMakeLut:' + os.path.join(ROOT,
164 'config',
165 'fgcmMakeLut%s.py' % (instCamel))
166 outputCollection = f'{instName}/{testName}/lut'
168 self._runPipeline(self.repo,
169 os.path.join(ROOT,
170 'pipelines',
171 'fgcmMakeLut%s.yaml' % (instCamel)),
172 configFiles=[configFile],
173 inputCollections='%s/calib,%s/testdata' % (instName, instName),
174 outputCollection=outputCollection,
175 registerDatasetTypes=True)
177 # Check output values
178 butler = dafButler.Butler(self.repo)
179 lutCat = butler.get('fgcmLookUpTable',
180 collections=[outputCollection],
181 instrument=instName)
182 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat, {})
184 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES']))
186 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32),
187 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
188 np.zeros(nBand) + lutStd[0]['O3STD'],
189 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
190 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
191 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
192 np.zeros(nBand, dtype=np.int32),
193 np.zeros(nBand) + lutStd[0]['PMBSTD'])
194 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
195 np.zeros(nBand) + lutStd[0]['O3STD'],
196 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
197 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
198 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
199 np.zeros(nBand) + lutStd[0]['PMBSTD'],
200 indices)
202 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5)
204 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
205 np.zeros(nBand) + lutStd[0]['O3STD'],
206 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
207 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
208 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
209 np.zeros(nBand) + lutStd[0]['PMBSTD'],
210 indices)
212 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5)
214 def _testFgcmBuildStarsTable(self, instName, testName, queryString, visits, nStar, nObs):
215 """Test running of FgcmBuildStarsTableTask
217 Parameters
218 ----------
219 instName : `str`
220 Short name of the instrument
221 testName : `str`
222 Base name of the test collection
223 queryString : `str`
224 Query to send to the pipetask.
225 visits : `list`
226 List of visits to calibrate
227 nStar : `int`
228 Number of stars expected
229 nObs : `int`
230 Number of observations of stars expected
231 """
232 instCamel = instName.title()
234 configFile = 'fgcmBuildStarsTable:' + os.path.join(ROOT,
235 'config',
236 'fgcmBuildStarsTable%s.py' % (instCamel))
237 outputCollection = f'{instName}/{testName}/buildstars'
239 self._runPipeline(self.repo,
240 os.path.join(ROOT,
241 'pipelines',
242 'fgcmBuildStarsTable%s.yaml' % (instCamel)),
243 configFiles=[configFile],
244 inputCollections=f'{instName}/{testName}/lut,refcats/gen2',
245 outputCollection=outputCollection,
246 configOptions=['fgcmBuildStarsTable:ccdDataRefName=detector'],
247 queryString=queryString,
248 registerDatasetTypes=True)
250 butler = dafButler.Butler(self.repo)
252 visitCat = butler.get('fgcmVisitCatalog', collections=[outputCollection],
253 instrument=instName)
254 self.assertEqual(len(visits), len(visitCat))
256 starIds = butler.get('fgcmStarIds', collections=[outputCollection],
257 instrument=instName)
258 self.assertEqual(len(starIds), nStar)
260 starObs = butler.get('fgcmStarObservations', collections=[outputCollection],
261 instrument=instName)
262 self.assertEqual(len(starObs), nObs)
264 def _testFgcmFitCycle(self, instName, testName, cycleNumber,
265 nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots,
266 skipChecks=False, extraConfig=None):
267 """Test running of FgcmFitCycleTask
269 Parameters
270 ----------
271 instName : `str`
272 Short name of the instrument
273 testName : `str`
274 Base name of the test collection
275 cycleNumber : `int`
276 Fit cycle number.
277 nZp : `int`
278 Number of zeropoints created by the task
279 nGoodZp : `int`
280 Number of good (photometric) zeropoints created
281 nOkZp : `int`
282 Number of constrained zeropoints (photometric or not)
283 nBadZp : `int`
284 Number of unconstrained (bad) zeropoints
285 nStdStars : `int`
286 Number of standard stars produced
287 nPlots : `int`
288 Number of plots produced
289 skipChecks : `bool`, optional
290 Skip number checks, when running less-than-final cycle.
291 extraConfig : `str`, optional
292 Name of an extra config file to apply.
293 """
294 instCamel = instName.title()
296 configFiles = ['fgcmFitCycle:' + os.path.join(ROOT,
297 'config',
298 'fgcmFitCycle%s.py' % (instCamel))]
299 if extraConfig is not None:
300 configFiles.append('fgcmFitCycle:' + extraConfig)
302 outputCollection = f'{instName}/{testName}/fit'
304 if cycleNumber == 0:
305 inputCollections = f'{instName}/{testName}/buildstars'
306 else:
307 # We are reusing the outputCollection so we can't specify the input
308 inputCollections = None
310 cwd = os.getcwd()
311 runDir = os.path.join(self.testDir, testName)
312 os.makedirs(runDir, exist_ok=True)
313 os.chdir(runDir)
315 configOptions = ['fgcmFitCycle:cycleNumber=%d' % (cycleNumber),
316 'fgcmFitCycle:connections.previousCycleNumber=%d' %
317 (cycleNumber - 1),
318 'fgcmFitCycle:connections.cycleNumber=%d' %
319 (cycleNumber)]
321 self._runPipeline(self.repo,
322 os.path.join(ROOT,
323 'pipelines',
324 'fgcmFitCycle%s.yaml' % (instCamel)),
325 configFiles=configFiles,
326 inputCollections=inputCollections,
327 outputCollection=outputCollection,
328 configOptions=configOptions,
329 registerDatasetTypes=True)
331 os.chdir(cwd)
333 if skipChecks:
334 return
336 butler = dafButler.Butler(self.repo)
338 config = butler.get('fgcmFitCycle_config', collections=[outputCollection])
340 # Check that the expected number of plots are there.
341 plots = glob.glob(os.path.join(runDir, config.outfileBase
342 + '_cycle%02d_plots/' % (cycleNumber)
343 + '*.png'))
344 self.assertEqual(len(plots), nPlots)
346 zps = butler.get('fgcmZeropoints%d' % (cycleNumber),
347 collections=[outputCollection],
348 instrument=instName)
349 self.assertEqual(len(zps), nZp)
351 gd, = np.where(zps['fgcmFlag'] == 1)
352 self.assertEqual(len(gd), nGoodZp)
354 ok, = np.where(zps['fgcmFlag'] < 16)
355 self.assertEqual(len(ok), nOkZp)
357 bd, = np.where(zps['fgcmFlag'] >= 16)
358 self.assertEqual(len(bd), nBadZp)
360 # Check that there are no illegal values with the ok zeropoints
361 test, = np.where(zps['fgcmZpt'][gd] < -9000.0)
362 self.assertEqual(len(test), 0)
364 stds = butler.get('fgcmStandardStars%d' % (cycleNumber),
365 collections=[outputCollection],
366 instrument=instName)
368 self.assertEqual(len(stds), nStdStars)
370 def _testFgcmOutputProducts(self, instName, testName,
371 zpOffsets, testVisit, testCcd, testFilter, testBandIndex):
372 """Test running of FgcmOutputProductsTask.
374 Parameters
375 ----------
376 instName : `str`
377 Short name of the instrument
378 testName : `str`
379 Base name of the test collection
380 zpOffsets : `np.ndarray`
381 Zeropoint offsets expected
382 testVisit : `int`
383 Visit id to check for round-trip computations
384 testCcd : `int`
385 Ccd id to check for round-trip computations
386 testFilter : `str`
387 Filtername for testVisit/testCcd
388 testBandIndex : `int`
389 Band index for testVisit/testCcd
390 """
391 instCamel = instName.title()
393 configFile = 'fgcmOutputProducts:' + os.path.join(ROOT,
394 'config',
395 'fgcmOutputProducts%s.py' % (instCamel))
396 inputCollection = f'{instName}/{testName}/fit'
397 outputCollection = f'{instName}/{testName}/fit/output'
399 self._runPipeline(self.repo,
400 os.path.join(ROOT,
401 'pipelines',
402 'fgcmOutputProducts%s.yaml' % (instCamel)),
403 configFiles=[configFile],
404 inputCollections=inputCollection,
405 outputCollection=outputCollection,
406 configOptions=['fgcmOutputProducts:doRefcatOutput=False'],
407 registerDatasetTypes=True)
409 butler = dafButler.Butler(self.repo)
410 offsetCat = butler.get('fgcmReferenceCalibrationOffsets',
411 collections=[outputCollection], instrument=instName)
412 offsets = offsetCat['offset'][:]
413 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
415 config = butler.get('fgcmOutputProducts_config',
416 collections=[outputCollection], instrument=instName)
418 rawStars = butler.get('fgcmStandardStars' + config.connections.cycleNumber,
419 collections=[inputCollection], instrument=instName)
421 candRatio = (rawStars['npsfcand'][:, 0].astype(np.float64)
422 / rawStars['ntotal'][:, 0].astype(np.float64))
423 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
424 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
426 # Test the fgcm_photoCalib output
427 zptCat = butler.get('fgcmZeropoints' + config.connections.cycleNumber,
428 collections=[inputCollection], instrument=instName)
430 good = (zptCat['fgcmFlag'] < 16)
431 bad = (zptCat['fgcmFlag'] >= 16)
433 # Read in all the calibrations, these should all be there
434 # This test is simply to ensure that all the photoCalib files exist
435 visits = np.unique(zptCat['visit'][good])
436 photoCalibDict = {}
437 for visit in visits:
438 expCat = butler.get('fgcmPhotoCalibCatalog',
439 visit=visit,
440 collections=[outputCollection], instrument=instName)
441 for row in expCat:
442 if row['visit'] == visit:
443 photoCalibDict[(visit, row['id'])] = row.getPhotoCalib()
445 # Check that all of the good photocalibs are there.
446 for rec in zptCat[good]:
447 self.assertTrue((rec['visit'], rec['detector']) in photoCalibDict)
449 # Check that none of the bad photocalibs are there.
450 for rec in zptCat[bad]:
451 self.assertFalse((rec['visit'], rec['detector']) in photoCalibDict)
453 # We do round-trip value checking on just the final one (chosen arbitrarily)
454 testCal = photoCalibDict[(testVisit, testCcd)]
456 src = butler.get('src', visit=int(testVisit), detector=int(testCcd),
457 collections=[outputCollection], instrument=instName)
459 # Only test sources with positive flux
460 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0)
462 # We need to apply the calibration offset to the fgcmzpt (which is internal
463 # and doesn't know about that yet)
464 testZpInd, = np.where((zptCat['visit'] == testVisit)
465 & (zptCat['detector'] == testCcd))
466 fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex]
467 + zptCat['fgcmDeltaChrom'][testZpInd])
468 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd])
470 if config.doComposeWcsJacobian:
471 # The raw zeropoint needs to be modified to know about the wcs jacobian
472 refs = butler.registry.queryDatasets('camera', dimensions=['instrument'],
473 collections=...)
474 camera = butler.getDirect(list(refs)[0])
475 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera)
476 center = approxPixelAreaFields[testCcd].getBBox().getCenter()
477 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center)
478 fgcmZpt += -2.5*np.log10(pixAreaCorr)
480 # This is the magnitude through the mean calibration
481 photoCalMeanCalMags = np.zeros(gdSrc.sum())
482 # This is the magnitude through the full focal-plane variable mags
483 photoCalMags = np.zeros_like(photoCalMeanCalMags)
484 # This is the magnitude with the FGCM (central-ccd) zeropoint
485 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
487 for i, rec in enumerate(src[gdSrc]):
488 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'])
489 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'],
490 rec.getCentroid())
491 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
493 # These should be very close but some tiny differences because the fgcm value
494 # is defined at the center of the bbox, and the photoCal is the mean over the box
495 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
496 zptMeanCalMags, rtol=1e-6)
497 # These should be roughly equal, but not precisely because of the focal-plane
498 # variation. However, this is a useful sanity check for something going totally
499 # wrong.
500 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
501 photoCalMags, rtol=1e-2)
503 # The next test compares the "FGCM standard magnitudes" (which are output
504 # from the fgcm code itself) to the "calibrated magnitudes" that are
505 # obtained from running photoCalib.calibrateCatalog() on the original
506 # src catalogs. This summary comparison ensures that using photoCalibs
507 # yields the same results as what FGCM is computing internally.
508 # Note that we additionally need to take into account the post-processing
509 # offsets used in the tests.
511 # For decent statistics, we are matching all the sources from one visit
512 # (multiple ccds)
513 whereClause = f"instrument='{instName:s}' and visit={testVisit:d}"
514 srcRefs = butler.registry.queryDatasets('src', dimensions=['visit'],
515 collections='%s/testdata' % (instName),
516 where=whereClause,
517 findFirst=True)
518 photoCals = []
519 for srcRef in srcRefs:
520 photoCals.append(photoCalibDict[(testVisit, srcRef.dataId['detector'])])
522 matchMag, matchDelta = self._getMatchedVisitCat(butler, srcRefs, photoCals,
523 rawStars, testBandIndex, offsets)
525 st = np.argsort(matchMag)
526 # Compare the brightest 25% of stars. No matter the setting of
527 # deltaMagBkgOffsetPercentile, we want to ensure that these stars
528 # match on average.
529 brightest, = np.where(matchMag < matchMag[st[int(0.25*st.size)]])
530 self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]), 0.0, atol=0.002)
532 # And the photoCal error is just the zeropoint gray error
533 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(),
534 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr)
536 # Test the transmission output
537 visitCatalog = butler.get('fgcmVisitCatalog', collections=[inputCollection],
538 instrument=instName)
539 lutCat = butler.get('fgcmLookUpTable', collections=[inputCollection],
540 instrument=instName)
542 testTrans = butler.get('transmission_atmosphere_fgcm',
543 visit=visitCatalog[0]['visit'],
544 collections=[outputCollection], instrument=instName)
545 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0),
546 wavelengths=lutCat[0]['atmLambda'])
548 # The test fit is performed with the atmosphere parameters frozen
549 # (freezeStdAtmosphere = True). Thus the only difference between
550 # these output atmospheres and the standard is the different
551 # airmass. Furthermore, this is a very rough comparison because
552 # the look-up table is computed with very coarse sampling for faster
553 # testing.
555 # To account for overall throughput changes, we scale by the median ratio,
556 # we only care about the shape
557 ratio = np.median(testResp/lutCat[0]['atmStdTrans'])
558 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04)
560 # The second should be close to the first, but there is the airmass
561 # difference so they aren't identical.
562 testTrans2 = butler.get('transmission_atmosphere_fgcm',
563 visit=visitCatalog[1]['visit'],
564 collections=[outputCollection], instrument=instName)
565 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0),
566 wavelengths=lutCat[0]['atmLambda'])
568 # As above, we scale by the ratio to compare the shape of the curve.
569 ratio = np.median(testResp/testResp2)
570 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04)
572 def _testFgcmMultiFit(self, instName, testName, queryString, visits, zpOffsets):
573 """Test running the full pipeline with multiple fit cycles.
575 Parameters
576 ----------
577 instName : `str`
578 Short name of the instrument
579 testName : `str`
580 Base name of the test collection
581 queryString : `str`
582 Query to send to the pipetask.
583 visits : `list`
584 List of visits to calibrate
585 zpOffsets : `np.ndarray`
586 Zeropoint offsets expected
587 """
588 instCamel = instName.title()
590 configFiles = ['fgcmBuildStarsTable:' + os.path.join(ROOT,
591 'config',
592 f'fgcmBuildStarsTable{instCamel}.py'),
593 'fgcmFitCycle:' + os.path.join(ROOT,
594 'config',
595 f'fgcmFitCycle{instCamel}.py'),
596 'fgcmOutputProducts:' + os.path.join(ROOT,
597 'config',
598 f'fgcmOutputProducts{instCamel}.py')]
599 outputCollection = f'{instName}/{testName}/unified'
601 cwd = os.getcwd()
602 runDir = os.path.join(self.testDir, testName)
603 os.makedirs(runDir)
604 os.chdir(runDir)
606 self._runPipeline(self.repo,
607 os.path.join(ROOT,
608 'pipelines',
609 f'fgcmFullPipeline{instCamel}.yaml'),
610 configFiles=configFiles,
611 inputCollections=f'{instName}/{testName}/lut,refcats/gen2',
612 outputCollection=outputCollection,
613 configOptions=['fgcmBuildStarsTable:ccdDataRefName=detector'],
614 queryString=queryString,
615 registerDatasetTypes=True)
617 os.chdir(cwd)
619 butler = dafButler.Butler(self.repo)
621 offsetCat = butler.get('fgcmReferenceCalibrationOffsets',
622 collections=[outputCollection], instrument=instName)
623 offsets = offsetCat['offset'][:]
624 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
626 def _getMatchedVisitCat(self, butler, srcRefs, photoCals,
627 rawStars, bandIndex, offsets):
628 """
629 Get a list of matched magnitudes and deltas from calibrated src catalogs.
631 Parameters
632 ----------
633 butler : `lsst.daf.butler.Butler`
634 srcRefs : `list`
635 dataRefs of source catalogs
636 photoCalibRefs : `list`
637 dataRefs of photoCalib files, matched to srcRefs.
638 photoCals : `list`
639 photoCalib objects, matched to srcRefs.
640 rawStars : `lsst.afw.table.SourceCatalog`
641 Fgcm standard stars
642 bandIndex : `int`
643 Index of the band for the source catalogs
644 offsets : `np.ndarray`
645 Testing calibration offsets to apply to rawStars
647 Returns
648 -------
649 matchMag : `np.ndarray`
650 Array of matched magnitudes
651 matchDelta : `np.ndarray`
652 Array of matched deltas between src and standard stars.
653 """
654 matcher = esutil.htm.Matcher(11, np.rad2deg(rawStars['coord_ra']),
655 np.rad2deg(rawStars['coord_dec']))
657 matchDelta = None
658 # for dataRef in dataRefs:
659 for srcRef, photoCal in zip(srcRefs, photoCals):
660 src = butler.getDirect(srcRef)
661 src = photoCal.calibrateCatalog(src)
663 gdSrc, = np.where(np.nan_to_num(src['slot_CalibFlux_flux']) > 0.0)
665 matches = matcher.match(np.rad2deg(src['coord_ra'][gdSrc]),
666 np.rad2deg(src['coord_dec'][gdSrc]),
667 1./3600., maxmatch=1)
669 srcMag = src['slot_CalibFlux_mag'][gdSrc][matches[0]]
670 # Apply offset here to the catalog mag
671 catMag = rawStars['mag_std_noabs'][matches[1]][:, bandIndex] + offsets[bandIndex]
672 delta = srcMag - catMag
673 if matchDelta is None:
674 matchDelta = delta
675 matchMag = catMag
676 else:
677 matchDelta = np.append(matchDelta, delta)
678 matchMag = np.append(matchMag, catMag)
680 return matchMag, matchDelta
682 def _testFgcmCalibrateTract(self, instName, testName, visits, tract, skymapName,
683 rawRepeatability, filterNCalibMap):
684 """Test running of FgcmCalibrateTractTask
686 Parameters
687 ----------
688 instName : `str`
689 Short name of the instrument
690 testName : `str`
691 Base name of the test collection
692 visits : `list`
693 List of visits to calibrate
694 tract : `int`
695 Tract number
696 skymapName : `str`
697 Name of the sky map
698 rawRepeatability : `np.array`
699 Expected raw repeatability after convergence.
700 Length should be number of bands.
701 filterNCalibMap : `dict`
702 Mapping from filter name to number of photoCalibs created.
703 """
704 instCamel = instName.title()
706 configFile = os.path.join(ROOT,
707 'config',
708 'fgcmCalibrateTractTable%s.py' % (instCamel))
710 configFiles = ['fgcmCalibrateTractTable:' + configFile]
711 outputCollection = f'{instName}/{testName}/tract'
713 inputCollections = f'{instName}/{testName}/lut,refcats/gen2'
714 configOption = 'fgcmCalibrateTractTable:fgcmOutputProducts.doRefcatOutput=False'
716 queryString = f"tract={tract:d} and skymap='{skymapName:s}'"
718 self._runPipeline(self.repo,
719 os.path.join(ROOT,
720 'pipelines',
721 f'fgcmCalibrateTractTable{instCamel:s}.yaml'),
722 queryString=queryString,
723 configFiles=configFiles,
724 inputCollections=inputCollections,
725 outputCollection=outputCollection,
726 configOptions=[configOption],
727 registerDatasetTypes=True)
729 butler = dafButler.Butler(self.repo)
731 whereClause = f"instrument='{instName:s}' and tract={tract:d} and skymap='{skymapName:s}'"
733 repRefs = butler.registry.queryDatasets('fgcmRawRepeatability',
734 dimensions=['tract'],
735 collections=outputCollection,
736 where=whereClause)
738 repeatabilityCat = butler.getDirect(list(repRefs)[0])
739 repeatability = repeatabilityCat['rawRepeatability'][:]
740 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6)
742 # Check that the number of photoCalib objects in each filter are what we expect
743 for filterName in filterNCalibMap.keys():
744 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and "
745 f"physical_filter='{filterName:s}' and skymap='{skymapName:s}'")
747 refs = butler.registry.queryDatasets('fgcmPhotoCalibTractCatalog',
748 dimensions=['tract', 'physical_filter'],
749 collections=outputCollection,
750 where=whereClause)
752 count = 0
753 for ref in set(refs):
754 expCat = butler.getDirect(ref)
755 test, = np.where((expCat['visit'] > 0) & (expCat['id'] >= 0))
756 count += test.size
758 self.assertEqual(count, filterNCalibMap[filterName])
760 # Check that every visit got a transmission
761 for visit in visits:
762 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and "
763 f"visit={visit:d} and skymap='{skymapName:s}'")
764 refs = butler.registry.queryDatasets('transmission_atmosphere_fgcm_tract',
765 dimensions=['tract', 'visit'],
766 collections=outputCollection,
767 where=whereClause)
768 self.assertEqual(len(set(refs)), 1)
770 @classmethod
771 def tearDownClass(cls):
772 """Tear down and clear directories
773 """
774 if os.path.exists(cls.testDir):
775 shutil.rmtree(cls.testDir, True)