Coverage for tests/fgcmcalTestBase.py: 10%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# See COPYRIGHT file at the top of the source tree.
2#
3# This file is part of fgcmcal.
4#
5# Developed for the LSST Data Management System.
6# This product includes software developed by the LSST Project
7# (https://www.lsst.org).
8# See the COPYRIGHT file at the top-level directory of this distribution
9# for details of code ownership.
10#
11# This program is free software: you can redistribute it and/or modify
12# it under the terms of the GNU General Public License as published by
13# the Free Software Foundation, either version 3 of the License, or
14# (at your option) any later version.
15#
16# This program is distributed in the hope that it will be useful,
17# but WITHOUT ANY WARRANTY; without even the implied warranty of
18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19# GNU General Public License for more details.
20#
21# You should have received a copy of the GNU General Public License
22# along with this program. If not, see <https://www.gnu.org/licenses/>.
23"""General fgcmcal testing class.
25This class is used as the basis for individual obs package tests using
26data from testdata_jointcal for Gen3 repos.
27"""
29import os
30import shutil
31import numpy as np
32import glob
33import esutil
35import lsst.daf.butler as dafButler
36import lsst.pipe.base as pipeBase
37import lsst.geom as geom
38from lsst.pipe.base import Pipeline
39from lsst.ctrl.mpexec import SimplePipelineExecutor
41import lsst.fgcmcal as fgcmcal
43ROOT = os.path.abspath(os.path.dirname(__file__))
46class FgcmcalTestBase(object):
47 """Base class for gen3 fgcmcal tests, to genericize some test running and setup.
49 Derive from this first, then from TestCase.
50 """
51 @classmethod
52 def _importRepository(cls, instrument, exportPath, exportFile):
53 """Import a test repository into self.testDir
55 Parameters
56 ----------
57 instrument : `str`
58 Full string name for the instrument.
59 exportPath : `str`
60 Path to location of repository to export.
61 exportFile : `str`
62 Filename of export data.
63 """
64 cls.repo = os.path.join(cls.testDir, 'testrepo')
66 # Make the repo and retrieve a writeable Butler
67 _ = dafButler.Butler.makeRepo(cls.repo)
68 butler = dafButler.Butler(cls.repo, writeable=True)
69 # Register the instrument
70 instrInstance = pipeBase.Instrument.from_string(instrument)
71 instrInstance.register(butler.registry)
72 # Import the exportFile
73 butler.import_(directory=exportPath, filename=exportFile,
74 transfer='symlink',
75 skip_dimensions={'instrument', 'detector', 'physical_filter'})
77 def _runPipeline(self, repo, pipelineFile, queryString='',
78 inputCollections=None, outputCollection=None,
79 configFiles={}, configOptions={},
80 registerDatasetTypes=False):
81 """Run a pipeline via pipetask.
83 Parameters
84 ----------
85 repo : `str`
86 Gen3 repository yaml file.
87 pipelineFile : `str`
88 Pipeline definition file.
89 queryString : `str`, optional
90 Where query that defines the data to use.
91 inputCollections : `list` [`str`], optional
92 Input collections list.
93 outputCollection : `str`, optional
94 Output collection name.
95 configFiles : `dict` [`list`], optional
96 Dictionary of config files. The key of the ``configFiles``
97 dict is the relevant task label. The value of ``configFiles``
98 is a list of config files to apply (in order) to that task.
99 configOptions : `dict` [`dict`], optional
100 Dictionary of individual config options. The key of the
101 ``configOptions`` dict is the relevant task label. The value
102 of ``configOptions`` is another dict that contains config
103 key/value overrides to apply.
104 configOptions : `list` [`str`], optional
105 List of individual config options to use. Each string will
106 be of the form ``taskName:configField=value``.
107 registerDatasetTypes : `bool`, optional
108 Register new dataset types?
110 Returns
111 -------
112 exit_code : `int`
113 Exit code for pipetask run.
115 Raises
116 ------
117 RuntimeError : Raised if the "pipetask" call fails.
118 """
119 butler = SimplePipelineExecutor.prep_butler(repo,
120 inputs=inputCollections,
121 output=outputCollection)
123 pipeline = Pipeline.fromFile(pipelineFile)
124 for taskName, fileList in configFiles.items():
125 for fileName in fileList:
126 pipeline.addConfigFile(taskName, fileName)
127 for taskName, configDict in configOptions.items():
128 for option, value in configDict.items():
129 pipeline.addConfigOverride(taskName, option, value)
131 executor = SimplePipelineExecutor.from_pipeline(pipeline,
132 where=queryString,
133 root=repo,
134 butler=butler)
135 quanta = executor.run(register_dataset_types=registerDatasetTypes)
137 return len(quanta)
139 def _testFgcmMakeLut(self, instName, testName, nBand, i0Std, i0Recon, i10Std, i10Recon):
140 """Test running of FgcmMakeLutTask
142 Parameters
143 ----------
144 instName : `str`
145 Short name of the instrument
146 testName : `str`
147 Base name of the test collection
148 nBand : `int`
149 Number of bands tested
150 i0Std : `np.ndarray'
151 Values of i0Std to compare to
152 i10Std : `np.ndarray`
153 Values of i10Std to compare to
154 i0Recon : `np.ndarray`
155 Values of reconstructed i0 to compare to
156 i10Recon : `np.ndarray`
157 Values of reconsntructed i10 to compare to
158 """
159 instCamel = instName.title()
161 configFiles = {'fgcmMakeLut': [os.path.join(ROOT,
162 'config',
163 f'fgcmMakeLut{instCamel}.py')]}
164 outputCollection = f'{instName}/{testName}/lut'
166 self._runPipeline(self.repo,
167 os.path.join(ROOT,
168 'pipelines',
169 f'fgcmMakeLut{instCamel}.yaml'),
170 configFiles=configFiles,
171 inputCollections=[f'{instName}/calib', f'{instName}/testdata'],
172 outputCollection=outputCollection,
173 registerDatasetTypes=True)
175 # Check output values
176 butler = dafButler.Butler(self.repo)
177 lutCat = butler.get('fgcmLookUpTable',
178 collections=[outputCollection],
179 instrument=instName)
180 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat, {})
182 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES']))
184 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32),
185 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
186 np.zeros(nBand) + lutStd[0]['O3STD'],
187 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
188 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
189 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
190 np.zeros(nBand, dtype=np.int32),
191 np.zeros(nBand) + lutStd[0]['PMBSTD'])
192 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
193 np.zeros(nBand) + lutStd[0]['O3STD'],
194 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
195 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
196 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
197 np.zeros(nBand) + lutStd[0]['PMBSTD'],
198 indices)
200 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5)
202 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']),
203 np.zeros(nBand) + lutStd[0]['O3STD'],
204 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']),
205 np.zeros(nBand) + lutStd[0]['ALPHASTD'],
206 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])),
207 np.zeros(nBand) + lutStd[0]['PMBSTD'],
208 indices)
210 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5)
212 def _testFgcmBuildStarsTable(self, instName, testName, queryString, visits, nStar, nObs):
213 """Test running of FgcmBuildStarsTableTask
215 Parameters
216 ----------
217 instName : `str`
218 Short name of the instrument
219 testName : `str`
220 Base name of the test collection
221 queryString : `str`
222 Query to send to the pipetask.
223 visits : `list`
224 List of visits to calibrate
225 nStar : `int`
226 Number of stars expected
227 nObs : `int`
228 Number of observations of stars expected
229 """
230 instCamel = instName.title()
232 configFiles = {'fgcmBuildStarsTable': [os.path.join(ROOT,
233 'config',
234 f'fgcmBuildStarsTable{instCamel}.py')]}
235 outputCollection = f'{instName}/{testName}/buildstars'
237 self._runPipeline(self.repo,
238 os.path.join(ROOT,
239 'pipelines',
240 'fgcmBuildStarsTable%s.yaml' % (instCamel)),
241 configFiles=configFiles,
242 inputCollections=[f'{instName}/{testName}/lut',
243 'refcats/gen2'],
244 outputCollection=outputCollection,
245 configOptions={'fgcmBuildStarsTable':
246 {'ccdDataRefName': 'detector'}},
247 queryString=queryString,
248 registerDatasetTypes=True)
250 butler = dafButler.Butler(self.repo)
252 visitCat = butler.get('fgcmVisitCatalog', collections=[outputCollection],
253 instrument=instName)
254 self.assertEqual(len(visits), len(visitCat))
256 starIds = butler.get('fgcmStarIds', collections=[outputCollection],
257 instrument=instName)
258 self.assertEqual(len(starIds), nStar)
260 starObs = butler.get('fgcmStarObservations', collections=[outputCollection],
261 instrument=instName)
262 self.assertEqual(len(starObs), nObs)
264 def _testFgcmFitCycle(self, instName, testName, cycleNumber,
265 nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots,
266 skipChecks=False, extraConfig=None):
267 """Test running of FgcmFitCycleTask
269 Parameters
270 ----------
271 instName : `str`
272 Short name of the instrument
273 testName : `str`
274 Base name of the test collection
275 cycleNumber : `int`
276 Fit cycle number.
277 nZp : `int`
278 Number of zeropoints created by the task
279 nGoodZp : `int`
280 Number of good (photometric) zeropoints created
281 nOkZp : `int`
282 Number of constrained zeropoints (photometric or not)
283 nBadZp : `int`
284 Number of unconstrained (bad) zeropoints
285 nStdStars : `int`
286 Number of standard stars produced
287 nPlots : `int`
288 Number of plots produced
289 skipChecks : `bool`, optional
290 Skip number checks, when running less-than-final cycle.
291 extraConfig : `str`, optional
292 Name of an extra config file to apply.
293 """
294 instCamel = instName.title()
296 configFiles = {'fgcmFitCycle': [os.path.join(ROOT,
297 'config',
298 f'fgcmFitCycle{instCamel}.py')]}
299 if extraConfig is not None:
300 configFiles['fgcmFitCycle'].append(extraConfig)
302 outputCollection = f'{instName}/{testName}/fit'
304 if cycleNumber == 0:
305 inputCollections = [f'{instName}/{testName}/buildstars']
306 else:
307 # In these tests we are running the fit cycle task multiple
308 # times into the same output collection. This code allows
309 # us to find the correct chained input collections to use
310 # so that we can both read from previous runs in the output
311 # collection and write to a new run in the output collection.
312 # Note that this behavior is handled automatically by the
313 # pipetask command-line interface, but not by the python
314 # API.
315 butler = dafButler.Butler(self.repo)
316 inputCollections = list(butler.registry.getCollectionChain(outputCollection))
318 cwd = os.getcwd()
319 runDir = os.path.join(self.testDir, testName)
320 os.makedirs(runDir, exist_ok=True)
321 os.chdir(runDir)
323 configOptions = {'fgcmFitCycle':
324 {'cycleNumber': f'{cycleNumber}',
325 'connections.previousCycleNumber': f'{cycleNumber - 1}',
326 'connections.cycleNumber': f'{cycleNumber}'}}
327 self._runPipeline(self.repo,
328 os.path.join(ROOT,
329 'pipelines',
330 f'fgcmFitCycle{instCamel}.yaml'),
331 configFiles=configFiles,
332 inputCollections=inputCollections,
333 outputCollection=outputCollection,
334 configOptions=configOptions,
335 registerDatasetTypes=True)
337 os.chdir(cwd)
339 if skipChecks:
340 return
342 butler = dafButler.Butler(self.repo)
344 config = butler.get('fgcmFitCycle_config', collections=[outputCollection])
346 # Check that the expected number of plots are there.
347 plots = glob.glob(os.path.join(runDir, config.outfileBase
348 + '_cycle%02d_plots/' % (cycleNumber)
349 + '*.png'))
350 self.assertEqual(len(plots), nPlots)
352 zps = butler.get('fgcmZeropoints%d' % (cycleNumber),
353 collections=[outputCollection],
354 instrument=instName)
355 self.assertEqual(len(zps), nZp)
357 gd, = np.where(zps['fgcmFlag'] == 1)
358 self.assertEqual(len(gd), nGoodZp)
360 ok, = np.where(zps['fgcmFlag'] < 16)
361 self.assertEqual(len(ok), nOkZp)
363 bd, = np.where(zps['fgcmFlag'] >= 16)
364 self.assertEqual(len(bd), nBadZp)
366 # Check that there are no illegal values with the ok zeropoints
367 test, = np.where(zps['fgcmZpt'][gd] < -9000.0)
368 self.assertEqual(len(test), 0)
370 stds = butler.get('fgcmStandardStars%d' % (cycleNumber),
371 collections=[outputCollection],
372 instrument=instName)
374 self.assertEqual(len(stds), nStdStars)
376 def _testFgcmOutputProducts(self, instName, testName,
377 zpOffsets, testVisit, testCcd, testFilter, testBandIndex):
378 """Test running of FgcmOutputProductsTask.
380 Parameters
381 ----------
382 instName : `str`
383 Short name of the instrument
384 testName : `str`
385 Base name of the test collection
386 zpOffsets : `np.ndarray`
387 Zeropoint offsets expected
388 testVisit : `int`
389 Visit id to check for round-trip computations
390 testCcd : `int`
391 Ccd id to check for round-trip computations
392 testFilter : `str`
393 Filtername for testVisit/testCcd
394 testBandIndex : `int`
395 Band index for testVisit/testCcd
396 """
397 instCamel = instName.title()
399 configFiles = {'fgcmOutputProducts': [os.path.join(ROOT,
400 'config',
401 f'fgcmOutputProducts{instCamel}.py')]}
402 inputCollection = f'{instName}/{testName}/fit'
403 outputCollection = f'{instName}/{testName}/fit/output'
405 self._runPipeline(self.repo,
406 os.path.join(ROOT,
407 'pipelines',
408 'fgcmOutputProducts%s.yaml' % (instCamel)),
409 configFiles=configFiles,
410 inputCollections=[inputCollection],
411 outputCollection=outputCollection,
412 configOptions={'fgcmOutputProducts':
413 {'doRefcatOutput': 'False'}},
414 registerDatasetTypes=True)
416 butler = dafButler.Butler(self.repo)
417 offsetCat = butler.get('fgcmReferenceCalibrationOffsets',
418 collections=[outputCollection], instrument=instName)
419 offsets = offsetCat['offset'][:]
420 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
422 config = butler.get('fgcmOutputProducts_config',
423 collections=[outputCollection], instrument=instName)
425 rawStars = butler.get('fgcmStandardStars' + config.connections.cycleNumber,
426 collections=[inputCollection], instrument=instName)
428 candRatio = (rawStars['npsfcand'][:, 0].astype(np.float64)
429 / rawStars['ntotal'][:, 0].astype(np.float64))
430 self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
431 self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
433 # Test the fgcm_photoCalib output
434 zptCat = butler.get('fgcmZeropoints' + config.connections.cycleNumber,
435 collections=[inputCollection], instrument=instName)
437 good = (zptCat['fgcmFlag'] < 16)
438 bad = (zptCat['fgcmFlag'] >= 16)
440 # Read in all the calibrations, these should all be there
441 # This test is simply to ensure that all the photoCalib files exist
442 visits = np.unique(zptCat['visit'][good])
443 photoCalibDict = {}
444 for visit in visits:
445 expCat = butler.get('fgcmPhotoCalibCatalog',
446 visit=visit,
447 collections=[outputCollection], instrument=instName)
448 for row in expCat:
449 if row['visit'] == visit:
450 photoCalibDict[(visit, row['id'])] = row.getPhotoCalib()
452 # Check that all of the good photocalibs are there.
453 for rec in zptCat[good]:
454 self.assertTrue((rec['visit'], rec['detector']) in photoCalibDict)
456 # Check that none of the bad photocalibs are there.
457 for rec in zptCat[bad]:
458 self.assertFalse((rec['visit'], rec['detector']) in photoCalibDict)
460 # We do round-trip value checking on just the final one (chosen arbitrarily)
461 testCal = photoCalibDict[(testVisit, testCcd)]
463 src = butler.get('src', visit=int(testVisit), detector=int(testCcd),
464 collections=[outputCollection], instrument=instName)
466 # Only test sources with positive flux
467 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0)
469 # We need to apply the calibration offset to the fgcmzpt (which is internal
470 # and doesn't know about that yet)
471 testZpInd, = np.where((zptCat['visit'] == testVisit)
472 & (zptCat['detector'] == testCcd))
473 fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex]
474 + zptCat['fgcmDeltaChrom'][testZpInd])
475 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd])
477 if config.doComposeWcsJacobian:
478 # The raw zeropoint needs to be modified to know about the wcs jacobian
479 refs = butler.registry.queryDatasets('camera', dimensions=['instrument'],
480 collections=...)
481 camera = butler.getDirect(list(refs)[0])
482 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera)
483 center = approxPixelAreaFields[testCcd].getBBox().getCenter()
484 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center)
485 fgcmZpt += -2.5*np.log10(pixAreaCorr)
487 # This is the magnitude through the mean calibration
488 photoCalMeanCalMags = np.zeros(gdSrc.sum())
489 # This is the magnitude through the full focal-plane variable mags
490 photoCalMags = np.zeros_like(photoCalMeanCalMags)
491 # This is the magnitude with the FGCM (central-ccd) zeropoint
492 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags)
494 for i, rec in enumerate(src[gdSrc]):
495 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'])
496 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'],
497 rec.getCentroid())
498 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
500 # These should be very close but some tiny differences because the fgcm value
501 # is defined at the center of the bbox, and the photoCal is the mean over the box
502 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
503 zptMeanCalMags, rtol=1e-6)
504 # These should be roughly equal, but not precisely because of the focal-plane
505 # variation. However, this is a useful sanity check for something going totally
506 # wrong.
507 self.assertFloatsAlmostEqual(photoCalMeanCalMags,
508 photoCalMags, rtol=1e-2)
510 # The next test compares the "FGCM standard magnitudes" (which are output
511 # from the fgcm code itself) to the "calibrated magnitudes" that are
512 # obtained from running photoCalib.calibrateCatalog() on the original
513 # src catalogs. This summary comparison ensures that using photoCalibs
514 # yields the same results as what FGCM is computing internally.
515 # Note that we additionally need to take into account the post-processing
516 # offsets used in the tests.
518 # For decent statistics, we are matching all the sources from one visit
519 # (multiple ccds)
520 whereClause = f"instrument='{instName:s}' and visit={testVisit:d}"
521 srcRefs = butler.registry.queryDatasets('src', dimensions=['visit'],
522 collections='%s/testdata' % (instName),
523 where=whereClause,
524 findFirst=True)
525 photoCals = []
526 for srcRef in srcRefs:
527 photoCals.append(photoCalibDict[(testVisit, srcRef.dataId['detector'])])
529 matchMag, matchDelta = self._getMatchedVisitCat(butler, srcRefs, photoCals,
530 rawStars, testBandIndex, offsets)
532 st = np.argsort(matchMag)
533 # Compare the brightest 25% of stars. No matter the setting of
534 # deltaMagBkgOffsetPercentile, we want to ensure that these stars
535 # match on average.
536 brightest, = np.where(matchMag < matchMag[st[int(0.25*st.size)]])
537 self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]), 0.0, atol=0.002)
539 # And the photoCal error is just the zeropoint gray error
540 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(),
541 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr)
543 # Test the transmission output
544 visitCatalog = butler.get('fgcmVisitCatalog', collections=[inputCollection],
545 instrument=instName)
546 lutCat = butler.get('fgcmLookUpTable', collections=[inputCollection],
547 instrument=instName)
549 testTrans = butler.get('transmission_atmosphere_fgcm',
550 visit=visitCatalog[0]['visit'],
551 collections=[outputCollection], instrument=instName)
552 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0),
553 wavelengths=lutCat[0]['atmLambda'])
555 # The test fit is performed with the atmosphere parameters frozen
556 # (freezeStdAtmosphere = True). Thus the only difference between
557 # these output atmospheres and the standard is the different
558 # airmass. Furthermore, this is a very rough comparison because
559 # the look-up table is computed with very coarse sampling for faster
560 # testing.
562 # To account for overall throughput changes, we scale by the median ratio,
563 # we only care about the shape
564 ratio = np.median(testResp/lutCat[0]['atmStdTrans'])
565 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04)
567 # The second should be close to the first, but there is the airmass
568 # difference so they aren't identical.
569 testTrans2 = butler.get('transmission_atmosphere_fgcm',
570 visit=visitCatalog[1]['visit'],
571 collections=[outputCollection], instrument=instName)
572 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0),
573 wavelengths=lutCat[0]['atmLambda'])
575 # As above, we scale by the ratio to compare the shape of the curve.
576 ratio = np.median(testResp/testResp2)
577 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04)
579 def _testFgcmMultiFit(self, instName, testName, queryString, visits, zpOffsets):
580 """Test running the full pipeline with multiple fit cycles.
582 Parameters
583 ----------
584 instName : `str`
585 Short name of the instrument
586 testName : `str`
587 Base name of the test collection
588 queryString : `str`
589 Query to send to the pipetask.
590 visits : `list`
591 List of visits to calibrate
592 zpOffsets : `np.ndarray`
593 Zeropoint offsets expected
594 """
595 instCamel = instName.title()
597 configFiles = {'fgcmBuildStarsTable': [os.path.join(ROOT,
598 'config',
599 f'fgcmBuildStarsTable{instCamel}.py')],
600 'fgcmFitCycle': [os.path.join(ROOT,
601 'config',
602 f'fgcmFitCycle{instCamel}.py')],
603 'fgcmOutputProducts': [os.path.join(ROOT,
604 'config',
605 f'fgcmOutputProducts{instCamel}.py')]}
606 outputCollection = f'{instName}/{testName}/unified'
608 cwd = os.getcwd()
609 runDir = os.path.join(self.testDir, testName)
610 os.makedirs(runDir)
611 os.chdir(runDir)
613 self._runPipeline(self.repo,
614 os.path.join(ROOT,
615 'pipelines',
616 f'fgcmFullPipeline{instCamel}.yaml'),
617 configFiles=configFiles,
618 inputCollections=[f'{instName}/{testName}/lut',
619 'refcats/gen2'],
620 outputCollection=outputCollection,
621 configOptions={'fgcmBuildStarsTable':
622 {'ccdDataRefName': 'detector'}},
623 queryString=queryString,
624 registerDatasetTypes=True)
626 os.chdir(cwd)
628 butler = dafButler.Butler(self.repo)
630 offsetCat = butler.get('fgcmReferenceCalibrationOffsets',
631 collections=[outputCollection], instrument=instName)
632 offsets = offsetCat['offset'][:]
633 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6)
635 def _getMatchedVisitCat(self, butler, srcRefs, photoCals,
636 rawStars, bandIndex, offsets):
637 """
638 Get a list of matched magnitudes and deltas from calibrated src catalogs.
640 Parameters
641 ----------
642 butler : `lsst.daf.butler.Butler`
643 srcRefs : `list`
644 dataRefs of source catalogs
645 photoCalibRefs : `list`
646 dataRefs of photoCalib files, matched to srcRefs.
647 photoCals : `list`
648 photoCalib objects, matched to srcRefs.
649 rawStars : `lsst.afw.table.SourceCatalog`
650 Fgcm standard stars
651 bandIndex : `int`
652 Index of the band for the source catalogs
653 offsets : `np.ndarray`
654 Testing calibration offsets to apply to rawStars
656 Returns
657 -------
658 matchMag : `np.ndarray`
659 Array of matched magnitudes
660 matchDelta : `np.ndarray`
661 Array of matched deltas between src and standard stars.
662 """
663 matcher = esutil.htm.Matcher(11, np.rad2deg(rawStars['coord_ra']),
664 np.rad2deg(rawStars['coord_dec']))
666 matchDelta = None
667 # for dataRef in dataRefs:
668 for srcRef, photoCal in zip(srcRefs, photoCals):
669 src = butler.getDirect(srcRef)
670 src = photoCal.calibrateCatalog(src)
672 gdSrc, = np.where(np.nan_to_num(src['slot_CalibFlux_flux']) > 0.0)
674 matches = matcher.match(np.rad2deg(src['coord_ra'][gdSrc]),
675 np.rad2deg(src['coord_dec'][gdSrc]),
676 1./3600., maxmatch=1)
678 srcMag = src['slot_CalibFlux_mag'][gdSrc][matches[0]]
679 # Apply offset here to the catalog mag
680 catMag = rawStars['mag_std_noabs'][matches[1]][:, bandIndex] + offsets[bandIndex]
681 delta = srcMag - catMag
682 if matchDelta is None:
683 matchDelta = delta
684 matchMag = catMag
685 else:
686 matchDelta = np.append(matchDelta, delta)
687 matchMag = np.append(matchMag, catMag)
689 return matchMag, matchDelta
691 def _testFgcmCalibrateTract(self, instName, testName, visits, tract, skymapName,
692 rawRepeatability, filterNCalibMap):
693 """Test running of FgcmCalibrateTractTask
695 Parameters
696 ----------
697 instName : `str`
698 Short name of the instrument
699 testName : `str`
700 Base name of the test collection
701 visits : `list`
702 List of visits to calibrate
703 tract : `int`
704 Tract number
705 skymapName : `str`
706 Name of the sky map
707 rawRepeatability : `np.array`
708 Expected raw repeatability after convergence.
709 Length should be number of bands.
710 filterNCalibMap : `dict`
711 Mapping from filter name to number of photoCalibs created.
712 """
713 instCamel = instName.title()
715 configFiles = {'fgcmCalibrateTractTable':
716 [os.path.join(ROOT,
717 'config',
718 f'fgcmCalibrateTractTable{instCamel}.py')]}
720 outputCollection = f'{instName}/{testName}/tract'
722 inputCollections = [f'{instName}/{testName}/lut',
723 'refcats/gen2']
724 configOptions = {'fgcmCalibrateTractTable':
725 {'fgcmOutputProducts.doRefcatOutput': 'False'}}
727 queryString = f"tract={tract:d} and skymap='{skymapName:s}'"
729 self._runPipeline(self.repo,
730 os.path.join(ROOT,
731 'pipelines',
732 f'fgcmCalibrateTractTable{instCamel:s}.yaml'),
733 queryString=queryString,
734 configFiles=configFiles,
735 inputCollections=inputCollections,
736 outputCollection=outputCollection,
737 configOptions=configOptions,
738 registerDatasetTypes=True)
740 butler = dafButler.Butler(self.repo)
742 whereClause = f"instrument='{instName:s}' and tract={tract:d} and skymap='{skymapName:s}'"
744 repRefs = butler.registry.queryDatasets('fgcmRawRepeatability',
745 dimensions=['tract'],
746 collections=outputCollection,
747 where=whereClause)
749 repeatabilityCat = butler.getDirect(list(repRefs)[0])
750 repeatability = repeatabilityCat['rawRepeatability'][:]
751 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6)
753 # Check that the number of photoCalib objects in each filter are what we expect
754 for filterName in filterNCalibMap.keys():
755 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and "
756 f"physical_filter='{filterName:s}' and skymap='{skymapName:s}'")
758 refs = butler.registry.queryDatasets('fgcmPhotoCalibTractCatalog',
759 dimensions=['tract', 'physical_filter'],
760 collections=outputCollection,
761 where=whereClause)
763 count = 0
764 for ref in set(refs):
765 expCat = butler.getDirect(ref)
766 test, = np.where((expCat['visit'] > 0) & (expCat['id'] >= 0))
767 count += test.size
769 self.assertEqual(count, filterNCalibMap[filterName])
771 # Check that every visit got a transmission
772 for visit in visits:
773 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and "
774 f"visit={visit:d} and skymap='{skymapName:s}'")
775 refs = butler.registry.queryDatasets('transmission_atmosphere_fgcm_tract',
776 dimensions=['tract', 'visit'],
777 collections=outputCollection,
778 where=whereClause)
779 self.assertEqual(len(set(refs)), 1)
781 @classmethod
782 def tearDownClass(cls):
783 """Tear down and clear directories
784 """
785 if os.path.exists(cls.testDir):
786 shutil.rmtree(cls.testDir, True)