Coverage for tests/fgcmcalTestBase.py: 10%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

228 statements  

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""General fgcmcal testing class. 

24 

25This class is used as the basis for individual obs package tests using 

26data from testdata_jointcal for Gen3 repos. 

27""" 

28 

29import os 

30import shutil 

31import numpy as np 

32import glob 

33import esutil 

34 

35import lsst.daf.butler as dafButler 

36import lsst.obs.base as obsBase 

37import lsst.geom as geom 

38from lsst.pipe.base import Pipeline 

39from lsst.ctrl.mpexec import SimplePipelineExecutor 

40 

41import lsst.fgcmcal as fgcmcal 

42 

43ROOT = os.path.abspath(os.path.dirname(__file__)) 

44 

45 

46class FgcmcalTestBase(object): 

47 """Base class for gen3 fgcmcal tests, to genericize some test running and setup. 

48 

49 Derive from this first, then from TestCase. 

50 """ 

51 @classmethod 

52 def _importRepository(cls, instrument, exportPath, exportFile): 

53 """Import a test repository into self.testDir 

54 

55 Parameters 

56 ---------- 

57 instrument : `str` 

58 Full string name for the instrument. 

59 exportPath : `str` 

60 Path to location of repository to export. 

61 exportFile : `str` 

62 Filename of export data. 

63 """ 

64 cls.repo = os.path.join(cls.testDir, 'testrepo') 

65 

66 # Make the repo and retrieve a writeable Butler 

67 _ = dafButler.Butler.makeRepo(cls.repo) 

68 butler = dafButler.Butler(cls.repo, writeable=True) 

69 # Register the instrument 

70 instrInstance = obsBase.utils.getInstrument(instrument) 

71 instrInstance.register(butler.registry) 

72 # Import the exportFile 

73 butler.import_(directory=exportPath, filename=exportFile, 

74 transfer='symlink', 

75 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

76 

77 def _runPipeline(self, repo, pipelineFile, queryString='', 

78 inputCollections=None, outputCollection=None, 

79 configFiles={}, configOptions={}, 

80 registerDatasetTypes=False): 

81 """Run a pipeline via pipetask. 

82 

83 Parameters 

84 ---------- 

85 repo : `str` 

86 Gen3 repository yaml file. 

87 pipelineFile : `str` 

88 Pipeline definition file. 

89 queryString : `str`, optional 

90 Where query that defines the data to use. 

91 inputCollections : `list` [`str`], optional 

92 Input collections list. 

93 outputCollection : `str`, optional 

94 Output collection name. 

95 configFiles : `dict` [`list`], optional 

96 Dictionary of config files. The key of the ``configFiles`` 

97 dict is the relevant task label. The value of ``configFiles`` 

98 is a list of config files to apply (in order) to that task. 

99 configOptions : `dict` [`dict`], optional 

100 Dictionary of individual config options. The key of the 

101 ``configOptions`` dict is the relevant task label. The value 

102 of ``configOptions`` is another dict that contains config 

103 key/value overrides to apply. 

104 configOptions : `list` [`str`], optional 

105 List of individual config options to use. Each string will 

106 be of the form ``taskName:configField=value``. 

107 registerDatasetTypes : `bool`, optional 

108 Register new dataset types? 

109 

110 Returns 

111 ------- 

112 exit_code : `int` 

113 Exit code for pipetask run. 

114 

115 Raises 

116 ------ 

117 RuntimeError : Raised if the "pipetask" call fails. 

118 """ 

119 butler = SimplePipelineExecutor.prep_butler(repo, 

120 inputs=inputCollections, 

121 output=outputCollection) 

122 

123 pipeline = Pipeline.fromFile(pipelineFile) 

124 for taskName, fileList in configFiles.items(): 

125 for fileName in fileList: 

126 pipeline.addConfigFile(taskName, fileName) 

127 for taskName, configDict in configOptions.items(): 

128 for option, value in configDict.items(): 

129 pipeline.addConfigOverride(taskName, option, value) 

130 

131 executor = SimplePipelineExecutor.from_pipeline(pipeline, 

132 where=queryString, 

133 root=repo, 

134 butler=butler) 

135 quanta = executor.run(register_dataset_types=registerDatasetTypes) 

136 

137 return len(quanta) 

138 

139 def _testFgcmMakeLut(self, instName, testName, nBand, i0Std, i0Recon, i10Std, i10Recon): 

140 """Test running of FgcmMakeLutTask 

141 

142 Parameters 

143 ---------- 

144 instName : `str` 

145 Short name of the instrument 

146 testName : `str` 

147 Base name of the test collection 

148 nBand : `int` 

149 Number of bands tested 

150 i0Std : `np.ndarray' 

151 Values of i0Std to compare to 

152 i10Std : `np.ndarray` 

153 Values of i10Std to compare to 

154 i0Recon : `np.ndarray` 

155 Values of reconstructed i0 to compare to 

156 i10Recon : `np.ndarray` 

157 Values of reconsntructed i10 to compare to 

158 """ 

159 instCamel = instName.title() 

160 

161 configFiles = {'fgcmMakeLut': [os.path.join(ROOT, 

162 'config', 

163 f'fgcmMakeLut{instCamel}.py')]} 

164 outputCollection = f'{instName}/{testName}/lut' 

165 

166 self._runPipeline(self.repo, 

167 os.path.join(ROOT, 

168 'pipelines', 

169 f'fgcmMakeLut{instCamel}.yaml'), 

170 configFiles=configFiles, 

171 inputCollections=[f'{instName}/calib', f'{instName}/testdata'], 

172 outputCollection=outputCollection, 

173 registerDatasetTypes=True) 

174 

175 # Check output values 

176 butler = dafButler.Butler(self.repo) 

177 lutCat = butler.get('fgcmLookUpTable', 

178 collections=[outputCollection], 

179 instrument=instName) 

180 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat, {}) 

181 

182 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES'])) 

183 

184 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32), 

185 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

186 np.zeros(nBand) + lutStd[0]['O3STD'], 

187 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

188 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

189 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

190 np.zeros(nBand, dtype=np.int32), 

191 np.zeros(nBand) + lutStd[0]['PMBSTD']) 

192 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

193 np.zeros(nBand) + lutStd[0]['O3STD'], 

194 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

195 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

196 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

197 np.zeros(nBand) + lutStd[0]['PMBSTD'], 

198 indices) 

199 

200 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5) 

201 

202 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

203 np.zeros(nBand) + lutStd[0]['O3STD'], 

204 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

205 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

206 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

207 np.zeros(nBand) + lutStd[0]['PMBSTD'], 

208 indices) 

209 

210 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5) 

211 

212 def _testFgcmBuildStarsTable(self, instName, testName, queryString, visits, nStar, nObs): 

213 """Test running of FgcmBuildStarsTableTask 

214 

215 Parameters 

216 ---------- 

217 instName : `str` 

218 Short name of the instrument 

219 testName : `str` 

220 Base name of the test collection 

221 queryString : `str` 

222 Query to send to the pipetask. 

223 visits : `list` 

224 List of visits to calibrate 

225 nStar : `int` 

226 Number of stars expected 

227 nObs : `int` 

228 Number of observations of stars expected 

229 """ 

230 instCamel = instName.title() 

231 

232 configFiles = {'fgcmBuildStarsTable': [os.path.join(ROOT, 

233 'config', 

234 f'fgcmBuildStarsTable{instCamel}.py')]} 

235 outputCollection = f'{instName}/{testName}/buildstars' 

236 

237 self._runPipeline(self.repo, 

238 os.path.join(ROOT, 

239 'pipelines', 

240 'fgcmBuildStarsTable%s.yaml' % (instCamel)), 

241 configFiles=configFiles, 

242 inputCollections=[f'{instName}/{testName}/lut', 

243 'refcats/gen2'], 

244 outputCollection=outputCollection, 

245 configOptions={'fgcmBuildStarsTable': 

246 {'ccdDataRefName': 'detector'}}, 

247 queryString=queryString, 

248 registerDatasetTypes=True) 

249 

250 butler = dafButler.Butler(self.repo) 

251 

252 visitCat = butler.get('fgcmVisitCatalog', collections=[outputCollection], 

253 instrument=instName) 

254 self.assertEqual(len(visits), len(visitCat)) 

255 

256 starIds = butler.get('fgcmStarIds', collections=[outputCollection], 

257 instrument=instName) 

258 self.assertEqual(len(starIds), nStar) 

259 

260 starObs = butler.get('fgcmStarObservations', collections=[outputCollection], 

261 instrument=instName) 

262 self.assertEqual(len(starObs), nObs) 

263 

264 def _testFgcmFitCycle(self, instName, testName, cycleNumber, 

265 nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

266 skipChecks=False, extraConfig=None): 

267 """Test running of FgcmFitCycleTask 

268 

269 Parameters 

270 ---------- 

271 instName : `str` 

272 Short name of the instrument 

273 testName : `str` 

274 Base name of the test collection 

275 cycleNumber : `int` 

276 Fit cycle number. 

277 nZp : `int` 

278 Number of zeropoints created by the task 

279 nGoodZp : `int` 

280 Number of good (photometric) zeropoints created 

281 nOkZp : `int` 

282 Number of constrained zeropoints (photometric or not) 

283 nBadZp : `int` 

284 Number of unconstrained (bad) zeropoints 

285 nStdStars : `int` 

286 Number of standard stars produced 

287 nPlots : `int` 

288 Number of plots produced 

289 skipChecks : `bool`, optional 

290 Skip number checks, when running less-than-final cycle. 

291 extraConfig : `str`, optional 

292 Name of an extra config file to apply. 

293 """ 

294 instCamel = instName.title() 

295 

296 configFiles = {'fgcmFitCycle': [os.path.join(ROOT, 

297 'config', 

298 f'fgcmFitCycle{instCamel}.py')]} 

299 if extraConfig is not None: 

300 configFiles['fgcmFitCycle'].append(extraConfig) 

301 

302 outputCollection = f'{instName}/{testName}/fit' 

303 

304 if cycleNumber == 0: 

305 inputCollections = [f'{instName}/{testName}/buildstars'] 

306 else: 

307 # In these tests we are running the fit cycle task multiple 

308 # times into the same output collection. This code allows 

309 # us to find the correct chained input collections to use 

310 # so that we can both read from previous runs in the output 

311 # collection and write to a new run in the output collection. 

312 # Note that this behavior is handled automatically by the 

313 # pipetask command-line interface, but not by the python 

314 # API. 

315 butler = dafButler.Butler(self.repo) 

316 inputCollections = list(butler.registry.getCollectionChain(outputCollection)) 

317 

318 cwd = os.getcwd() 

319 runDir = os.path.join(self.testDir, testName) 

320 os.makedirs(runDir, exist_ok=True) 

321 os.chdir(runDir) 

322 

323 configOptions = {'fgcmFitCycle': 

324 {'cycleNumber': f'{cycleNumber}', 

325 'connections.previousCycleNumber': f'{cycleNumber - 1}', 

326 'connections.cycleNumber': f'{cycleNumber}'}} 

327 self._runPipeline(self.repo, 

328 os.path.join(ROOT, 

329 'pipelines', 

330 f'fgcmFitCycle{instCamel}.yaml'), 

331 configFiles=configFiles, 

332 inputCollections=inputCollections, 

333 outputCollection=outputCollection, 

334 configOptions=configOptions, 

335 registerDatasetTypes=True) 

336 

337 os.chdir(cwd) 

338 

339 if skipChecks: 

340 return 

341 

342 butler = dafButler.Butler(self.repo) 

343 

344 config = butler.get('fgcmFitCycle_config', collections=[outputCollection]) 

345 

346 # Check that the expected number of plots are there. 

347 plots = glob.glob(os.path.join(runDir, config.outfileBase 

348 + '_cycle%02d_plots/' % (cycleNumber) 

349 + '*.png')) 

350 self.assertEqual(len(plots), nPlots) 

351 

352 zps = butler.get('fgcmZeropoints%d' % (cycleNumber), 

353 collections=[outputCollection], 

354 instrument=instName) 

355 self.assertEqual(len(zps), nZp) 

356 

357 gd, = np.where(zps['fgcmFlag'] == 1) 

358 self.assertEqual(len(gd), nGoodZp) 

359 

360 ok, = np.where(zps['fgcmFlag'] < 16) 

361 self.assertEqual(len(ok), nOkZp) 

362 

363 bd, = np.where(zps['fgcmFlag'] >= 16) 

364 self.assertEqual(len(bd), nBadZp) 

365 

366 # Check that there are no illegal values with the ok zeropoints 

367 test, = np.where(zps['fgcmZpt'][gd] < -9000.0) 

368 self.assertEqual(len(test), 0) 

369 

370 stds = butler.get('fgcmStandardStars%d' % (cycleNumber), 

371 collections=[outputCollection], 

372 instrument=instName) 

373 

374 self.assertEqual(len(stds), nStdStars) 

375 

376 def _testFgcmOutputProducts(self, instName, testName, 

377 zpOffsets, testVisit, testCcd, testFilter, testBandIndex): 

378 """Test running of FgcmOutputProductsTask. 

379 

380 Parameters 

381 ---------- 

382 instName : `str` 

383 Short name of the instrument 

384 testName : `str` 

385 Base name of the test collection 

386 zpOffsets : `np.ndarray` 

387 Zeropoint offsets expected 

388 testVisit : `int` 

389 Visit id to check for round-trip computations 

390 testCcd : `int` 

391 Ccd id to check for round-trip computations 

392 testFilter : `str` 

393 Filtername for testVisit/testCcd 

394 testBandIndex : `int` 

395 Band index for testVisit/testCcd 

396 """ 

397 instCamel = instName.title() 

398 

399 configFiles = {'fgcmOutputProducts': [os.path.join(ROOT, 

400 'config', 

401 f'fgcmOutputProducts{instCamel}.py')]} 

402 inputCollection = f'{instName}/{testName}/fit' 

403 outputCollection = f'{instName}/{testName}/fit/output' 

404 

405 self._runPipeline(self.repo, 

406 os.path.join(ROOT, 

407 'pipelines', 

408 'fgcmOutputProducts%s.yaml' % (instCamel)), 

409 configFiles=configFiles, 

410 inputCollections=[inputCollection], 

411 outputCollection=outputCollection, 

412 configOptions={'fgcmOutputProducts': 

413 {'doRefcatOutput': 'False'}}, 

414 registerDatasetTypes=True) 

415 

416 butler = dafButler.Butler(self.repo) 

417 offsetCat = butler.get('fgcmReferenceCalibrationOffsets', 

418 collections=[outputCollection], instrument=instName) 

419 offsets = offsetCat['offset'][:] 

420 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6) 

421 

422 config = butler.get('fgcmOutputProducts_config', 

423 collections=[outputCollection], instrument=instName) 

424 

425 rawStars = butler.get('fgcmStandardStars' + config.connections.cycleNumber, 

426 collections=[inputCollection], instrument=instName) 

427 

428 candRatio = (rawStars['npsfcand'][:, 0].astype(np.float64) 

429 / rawStars['ntotal'][:, 0].astype(np.float64)) 

430 self.assertFloatsAlmostEqual(candRatio.min(), 0.0) 

431 self.assertFloatsAlmostEqual(candRatio.max(), 1.0) 

432 

433 # Test the fgcm_photoCalib output 

434 zptCat = butler.get('fgcmZeropoints' + config.connections.cycleNumber, 

435 collections=[inputCollection], instrument=instName) 

436 

437 good = (zptCat['fgcmFlag'] < 16) 

438 bad = (zptCat['fgcmFlag'] >= 16) 

439 

440 # Read in all the calibrations, these should all be there 

441 # This test is simply to ensure that all the photoCalib files exist 

442 visits = np.unique(zptCat['visit'][good]) 

443 photoCalibDict = {} 

444 for visit in visits: 

445 expCat = butler.get('fgcmPhotoCalibCatalog', 

446 visit=visit, 

447 collections=[outputCollection], instrument=instName) 

448 for row in expCat: 

449 if row['visit'] == visit: 

450 photoCalibDict[(visit, row['id'])] = row.getPhotoCalib() 

451 

452 # Check that all of the good photocalibs are there. 

453 for rec in zptCat[good]: 

454 self.assertTrue((rec['visit'], rec['detector']) in photoCalibDict) 

455 

456 # Check that none of the bad photocalibs are there. 

457 for rec in zptCat[bad]: 

458 self.assertFalse((rec['visit'], rec['detector']) in photoCalibDict) 

459 

460 # We do round-trip value checking on just the final one (chosen arbitrarily) 

461 testCal = photoCalibDict[(testVisit, testCcd)] 

462 

463 src = butler.get('src', visit=int(testVisit), detector=int(testCcd), 

464 collections=[outputCollection], instrument=instName) 

465 

466 # Only test sources with positive flux 

467 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0) 

468 

469 # We need to apply the calibration offset to the fgcmzpt (which is internal 

470 # and doesn't know about that yet) 

471 testZpInd, = np.where((zptCat['visit'] == testVisit) 

472 & (zptCat['detector'] == testCcd)) 

473 fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex] 

474 + zptCat['fgcmDeltaChrom'][testZpInd]) 

475 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd]) 

476 

477 if config.doComposeWcsJacobian: 

478 # The raw zeropoint needs to be modified to know about the wcs jacobian 

479 refs = butler.registry.queryDatasets('camera', dimensions=['instrument'], 

480 collections=...) 

481 camera = butler.getDirect(list(refs)[0]) 

482 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera) 

483 center = approxPixelAreaFields[testCcd].getBBox().getCenter() 

484 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center) 

485 fgcmZpt += -2.5*np.log10(pixAreaCorr) 

486 

487 # This is the magnitude through the mean calibration 

488 photoCalMeanCalMags = np.zeros(gdSrc.sum()) 

489 # This is the magnitude through the full focal-plane variable mags 

490 photoCalMags = np.zeros_like(photoCalMeanCalMags) 

491 # This is the magnitude with the FGCM (central-ccd) zeropoint 

492 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags) 

493 

494 for i, rec in enumerate(src[gdSrc]): 

495 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux']) 

496 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'], 

497 rec.getCentroid()) 

498 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux']) 

499 

500 # These should be very close but some tiny differences because the fgcm value 

501 # is defined at the center of the bbox, and the photoCal is the mean over the box 

502 self.assertFloatsAlmostEqual(photoCalMeanCalMags, 

503 zptMeanCalMags, rtol=1e-6) 

504 # These should be roughly equal, but not precisely because of the focal-plane 

505 # variation. However, this is a useful sanity check for something going totally 

506 # wrong. 

507 self.assertFloatsAlmostEqual(photoCalMeanCalMags, 

508 photoCalMags, rtol=1e-2) 

509 

510 # The next test compares the "FGCM standard magnitudes" (which are output 

511 # from the fgcm code itself) to the "calibrated magnitudes" that are 

512 # obtained from running photoCalib.calibrateCatalog() on the original 

513 # src catalogs. This summary comparison ensures that using photoCalibs 

514 # yields the same results as what FGCM is computing internally. 

515 # Note that we additionally need to take into account the post-processing 

516 # offsets used in the tests. 

517 

518 # For decent statistics, we are matching all the sources from one visit 

519 # (multiple ccds) 

520 whereClause = f"instrument='{instName:s}' and visit={testVisit:d}" 

521 srcRefs = butler.registry.queryDatasets('src', dimensions=['visit'], 

522 collections='%s/testdata' % (instName), 

523 where=whereClause, 

524 findFirst=True) 

525 photoCals = [] 

526 for srcRef in srcRefs: 

527 photoCals.append(photoCalibDict[(testVisit, srcRef.dataId['detector'])]) 

528 

529 matchMag, matchDelta = self._getMatchedVisitCat(butler, srcRefs, photoCals, 

530 rawStars, testBandIndex, offsets) 

531 

532 st = np.argsort(matchMag) 

533 # Compare the brightest 25% of stars. No matter the setting of 

534 # deltaMagBkgOffsetPercentile, we want to ensure that these stars 

535 # match on average. 

536 brightest, = np.where(matchMag < matchMag[st[int(0.25*st.size)]]) 

537 self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]), 0.0, atol=0.002) 

538 

539 # And the photoCal error is just the zeropoint gray error 

540 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(), 

541 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr) 

542 

543 # Test the transmission output 

544 visitCatalog = butler.get('fgcmVisitCatalog', collections=[inputCollection], 

545 instrument=instName) 

546 lutCat = butler.get('fgcmLookUpTable', collections=[inputCollection], 

547 instrument=instName) 

548 

549 testTrans = butler.get('transmission_atmosphere_fgcm', 

550 visit=visitCatalog[0]['visit'], 

551 collections=[outputCollection], instrument=instName) 

552 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0), 

553 wavelengths=lutCat[0]['atmLambda']) 

554 

555 # The test fit is performed with the atmosphere parameters frozen 

556 # (freezeStdAtmosphere = True). Thus the only difference between 

557 # these output atmospheres and the standard is the different 

558 # airmass. Furthermore, this is a very rough comparison because 

559 # the look-up table is computed with very coarse sampling for faster 

560 # testing. 

561 

562 # To account for overall throughput changes, we scale by the median ratio, 

563 # we only care about the shape 

564 ratio = np.median(testResp/lutCat[0]['atmStdTrans']) 

565 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04) 

566 

567 # The second should be close to the first, but there is the airmass 

568 # difference so they aren't identical. 

569 testTrans2 = butler.get('transmission_atmosphere_fgcm', 

570 visit=visitCatalog[1]['visit'], 

571 collections=[outputCollection], instrument=instName) 

572 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0), 

573 wavelengths=lutCat[0]['atmLambda']) 

574 

575 # As above, we scale by the ratio to compare the shape of the curve. 

576 ratio = np.median(testResp/testResp2) 

577 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04) 

578 

579 def _testFgcmMultiFit(self, instName, testName, queryString, visits, zpOffsets): 

580 """Test running the full pipeline with multiple fit cycles. 

581 

582 Parameters 

583 ---------- 

584 instName : `str` 

585 Short name of the instrument 

586 testName : `str` 

587 Base name of the test collection 

588 queryString : `str` 

589 Query to send to the pipetask. 

590 visits : `list` 

591 List of visits to calibrate 

592 zpOffsets : `np.ndarray` 

593 Zeropoint offsets expected 

594 """ 

595 instCamel = instName.title() 

596 

597 configFiles = {'fgcmBuildStarsTable': [os.path.join(ROOT, 

598 'config', 

599 f'fgcmBuildStarsTable{instCamel}.py')], 

600 'fgcmFitCycle': [os.path.join(ROOT, 

601 'config', 

602 f'fgcmFitCycle{instCamel}.py')], 

603 'fgcmOutputProducts': [os.path.join(ROOT, 

604 'config', 

605 f'fgcmOutputProducts{instCamel}.py')]} 

606 outputCollection = f'{instName}/{testName}/unified' 

607 

608 cwd = os.getcwd() 

609 runDir = os.path.join(self.testDir, testName) 

610 os.makedirs(runDir) 

611 os.chdir(runDir) 

612 

613 self._runPipeline(self.repo, 

614 os.path.join(ROOT, 

615 'pipelines', 

616 f'fgcmFullPipeline{instCamel}.yaml'), 

617 configFiles=configFiles, 

618 inputCollections=[f'{instName}/{testName}/lut', 

619 'refcats/gen2'], 

620 outputCollection=outputCollection, 

621 configOptions={'fgcmBuildStarsTable': 

622 {'ccdDataRefName': 'detector'}}, 

623 queryString=queryString, 

624 registerDatasetTypes=True) 

625 

626 os.chdir(cwd) 

627 

628 butler = dafButler.Butler(self.repo) 

629 

630 offsetCat = butler.get('fgcmReferenceCalibrationOffsets', 

631 collections=[outputCollection], instrument=instName) 

632 offsets = offsetCat['offset'][:] 

633 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6) 

634 

635 def _getMatchedVisitCat(self, butler, srcRefs, photoCals, 

636 rawStars, bandIndex, offsets): 

637 """ 

638 Get a list of matched magnitudes and deltas from calibrated src catalogs. 

639 

640 Parameters 

641 ---------- 

642 butler : `lsst.daf.butler.Butler` 

643 srcRefs : `list` 

644 dataRefs of source catalogs 

645 photoCalibRefs : `list` 

646 dataRefs of photoCalib files, matched to srcRefs. 

647 photoCals : `list` 

648 photoCalib objects, matched to srcRefs. 

649 rawStars : `lsst.afw.table.SourceCatalog` 

650 Fgcm standard stars 

651 bandIndex : `int` 

652 Index of the band for the source catalogs 

653 offsets : `np.ndarray` 

654 Testing calibration offsets to apply to rawStars 

655 

656 Returns 

657 ------- 

658 matchMag : `np.ndarray` 

659 Array of matched magnitudes 

660 matchDelta : `np.ndarray` 

661 Array of matched deltas between src and standard stars. 

662 """ 

663 matcher = esutil.htm.Matcher(11, np.rad2deg(rawStars['coord_ra']), 

664 np.rad2deg(rawStars['coord_dec'])) 

665 

666 matchDelta = None 

667 # for dataRef in dataRefs: 

668 for srcRef, photoCal in zip(srcRefs, photoCals): 

669 src = butler.getDirect(srcRef) 

670 src = photoCal.calibrateCatalog(src) 

671 

672 gdSrc, = np.where(np.nan_to_num(src['slot_CalibFlux_flux']) > 0.0) 

673 

674 matches = matcher.match(np.rad2deg(src['coord_ra'][gdSrc]), 

675 np.rad2deg(src['coord_dec'][gdSrc]), 

676 1./3600., maxmatch=1) 

677 

678 srcMag = src['slot_CalibFlux_mag'][gdSrc][matches[0]] 

679 # Apply offset here to the catalog mag 

680 catMag = rawStars['mag_std_noabs'][matches[1]][:, bandIndex] + offsets[bandIndex] 

681 delta = srcMag - catMag 

682 if matchDelta is None: 

683 matchDelta = delta 

684 matchMag = catMag 

685 else: 

686 matchDelta = np.append(matchDelta, delta) 

687 matchMag = np.append(matchMag, catMag) 

688 

689 return matchMag, matchDelta 

690 

691 def _testFgcmCalibrateTract(self, instName, testName, visits, tract, skymapName, 

692 rawRepeatability, filterNCalibMap): 

693 """Test running of FgcmCalibrateTractTask 

694 

695 Parameters 

696 ---------- 

697 instName : `str` 

698 Short name of the instrument 

699 testName : `str` 

700 Base name of the test collection 

701 visits : `list` 

702 List of visits to calibrate 

703 tract : `int` 

704 Tract number 

705 skymapName : `str` 

706 Name of the sky map 

707 rawRepeatability : `np.array` 

708 Expected raw repeatability after convergence. 

709 Length should be number of bands. 

710 filterNCalibMap : `dict` 

711 Mapping from filter name to number of photoCalibs created. 

712 """ 

713 instCamel = instName.title() 

714 

715 configFiles = {'fgcmCalibrateTractTable': 

716 [os.path.join(ROOT, 

717 'config', 

718 f'fgcmCalibrateTractTable{instCamel}.py')]} 

719 

720 outputCollection = f'{instName}/{testName}/tract' 

721 

722 inputCollections = [f'{instName}/{testName}/lut', 

723 'refcats/gen2'] 

724 configOptions = {'fgcmCalibrateTractTable': 

725 {'fgcmOutputProducts.doRefcatOutput': 'False'}} 

726 

727 queryString = f"tract={tract:d} and skymap='{skymapName:s}'" 

728 

729 self._runPipeline(self.repo, 

730 os.path.join(ROOT, 

731 'pipelines', 

732 f'fgcmCalibrateTractTable{instCamel:s}.yaml'), 

733 queryString=queryString, 

734 configFiles=configFiles, 

735 inputCollections=inputCollections, 

736 outputCollection=outputCollection, 

737 configOptions=configOptions, 

738 registerDatasetTypes=True) 

739 

740 butler = dafButler.Butler(self.repo) 

741 

742 whereClause = f"instrument='{instName:s}' and tract={tract:d} and skymap='{skymapName:s}'" 

743 

744 repRefs = butler.registry.queryDatasets('fgcmRawRepeatability', 

745 dimensions=['tract'], 

746 collections=outputCollection, 

747 where=whereClause) 

748 

749 repeatabilityCat = butler.getDirect(list(repRefs)[0]) 

750 repeatability = repeatabilityCat['rawRepeatability'][:] 

751 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6) 

752 

753 # Check that the number of photoCalib objects in each filter are what we expect 

754 for filterName in filterNCalibMap.keys(): 

755 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and " 

756 f"physical_filter='{filterName:s}' and skymap='{skymapName:s}'") 

757 

758 refs = butler.registry.queryDatasets('fgcmPhotoCalibTractCatalog', 

759 dimensions=['tract', 'physical_filter'], 

760 collections=outputCollection, 

761 where=whereClause) 

762 

763 count = 0 

764 for ref in set(refs): 

765 expCat = butler.getDirect(ref) 

766 test, = np.where((expCat['visit'] > 0) & (expCat['id'] >= 0)) 

767 count += test.size 

768 

769 self.assertEqual(count, filterNCalibMap[filterName]) 

770 

771 # Check that every visit got a transmission 

772 for visit in visits: 

773 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and " 

774 f"visit={visit:d} and skymap='{skymapName:s}'") 

775 refs = butler.registry.queryDatasets('transmission_atmosphere_fgcm_tract', 

776 dimensions=['tract', 'visit'], 

777 collections=outputCollection, 

778 where=whereClause) 

779 self.assertEqual(len(set(refs)), 1) 

780 

781 @classmethod 

782 def tearDownClass(cls): 

783 """Tear down and clear directories 

784 """ 

785 if os.path.exists(cls.testDir): 

786 shutil.rmtree(cls.testDir, True)