Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""General fgcmcal testing class. 

24 

25This class is used as the basis for individual obs package tests using 

26data from testdata_jointcal for Gen3 repos. 

27""" 

28 

29import os 

30import shutil 

31import numpy as np 

32import glob 

33import esutil 

34 

35import click.testing 

36import lsst.ctrl.mpexec.cli.pipetask 

37 

38import lsst.daf.butler as dafButler 

39import lsst.obs.base as obsBase 

40import lsst.geom as geom 

41import lsst.log 

42 

43import lsst.fgcmcal as fgcmcal 

44 

45ROOT = os.path.abspath(os.path.dirname(__file__)) 

46 

47 

48class FgcmcalTestBase(object): 

49 """Base class for gen3 fgcmcal tests, to genericize some test running and setup. 

50 

51 Derive from this first, then from TestCase. 

52 """ 

53 @classmethod 

54 def _importRepository(cls, instrument, exportPath, exportFile): 

55 """Import a test repository into self.testDir 

56 

57 Parameters 

58 ---------- 

59 instrument : `str` 

60 Full string name for the instrument. 

61 exportPath : `str` 

62 Path to location of repository to export. 

63 exportFile : `str` 

64 Filename of export data. 

65 """ 

66 cls.repo = os.path.join(cls.testDir, 'testrepo') 

67 

68 print('Importing %s into %s' % (exportFile, cls.testDir)) 

69 

70 # Make the repo and retrieve a writeable Butler 

71 _ = dafButler.Butler.makeRepo(cls.repo) 

72 butler = dafButler.Butler(cls.repo, writeable=True) 

73 # Register the instrument 

74 instrInstance = obsBase.utils.getInstrument(instrument) 

75 instrInstance.register(butler.registry) 

76 # Import the exportFile 

77 butler.import_(directory=exportPath, filename=exportFile, 

78 transfer='symlink', 

79 skip_dimensions={'instrument', 'detector', 'physical_filter'}) 

80 

81 def _runPipeline(self, repo, pipelineFile, queryString=None, 

82 inputCollections=None, outputCollection=None, 

83 configFiles=None, configOptions=None, 

84 registerDatasetTypes=False): 

85 """Run a pipeline via pipetask. 

86 

87 Parameters 

88 ---------- 

89 repo : `str` 

90 Gen3 repository yaml file. 

91 pipelineFile : `str` 

92 Pipeline definition file. 

93 queryString : `str`, optional 

94 String to use for "-d" data query. 

95 inputCollections : `str`, optional 

96 String to use for "-i" input collections (comma delimited). 

97 outputCollection : `str`, optional 

98 String to use for "-o" output collection. 

99 configFiles : `list` [`str`], optional 

100 List of config files to use (with "-C"). 

101 configOptions : `list` [`str`], optional 

102 List of individual config options to use (with "-c"). 

103 registerDatasetTypes : `bool`, optional 

104 Set "--register-dataset-types". 

105 

106 Returns 

107 ------- 

108 exit_code : `int` 

109 Exit code for pipetask run. 

110 

111 Raises 

112 ------ 

113 RuntimeError : Raised if the "pipetask" call fails. 

114 """ 

115 pipelineArgs = ["run", 

116 "-b", repo, 

117 "-p", pipelineFile] 

118 

119 if queryString is not None: 

120 pipelineArgs.extend(["-d", queryString]) 

121 if inputCollections is not None: 

122 pipelineArgs.extend(["-i", inputCollections]) 

123 if outputCollection is not None: 

124 pipelineArgs.extend(["-o", outputCollection]) 

125 if configFiles is not None: 

126 for configFile in configFiles: 

127 pipelineArgs.extend(["-C", configFile]) 

128 if configOptions is not None: 

129 for configOption in configOptions: 

130 pipelineArgs.extend(["-c", configOption]) 

131 if registerDatasetTypes: 

132 pipelineArgs.extend(["--register-dataset-types"]) 

133 

134 # CliRunner is an unsafe workaround for DM-26239 

135 runner = click.testing.CliRunner() 

136 results = runner.invoke(lsst.ctrl.mpexec.cli.pipetask.cli, pipelineArgs) 

137 if results.exception: 

138 raise RuntimeError("Pipeline %s failed." % (pipelineFile)) from results.exception 

139 return results.exit_code 

140 

141 def _testFgcmMakeLut(self, instName, testName, nBand, i0Std, i0Recon, i10Std, i10Recon): 

142 """Test running of FgcmMakeLutTask 

143 

144 Parameters 

145 ---------- 

146 instName : `str` 

147 Short name of the instrument 

148 testName : `str` 

149 Base name of the test collection 

150 nBand : `int` 

151 Number of bands tested 

152 i0Std : `np.ndarray' 

153 Values of i0Std to compare to 

154 i10Std : `np.ndarray` 

155 Values of i10Std to compare to 

156 i0Recon : `np.ndarray` 

157 Values of reconstructed i0 to compare to 

158 i10Recon : `np.ndarray` 

159 Values of reconsntructed i10 to compare to 

160 """ 

161 instCamel = instName.title() 

162 

163 configFile = 'fgcmMakeLut:' + os.path.join(ROOT, 

164 'config', 

165 'fgcmMakeLut%s.py' % (instCamel)) 

166 outputCollection = f'{instName}/{testName}/lut' 

167 

168 self._runPipeline(self.repo, 

169 os.path.join(ROOT, 

170 'pipelines', 

171 'fgcmMakeLut%s.yaml' % (instCamel)), 

172 configFiles=[configFile], 

173 inputCollections='%s/calib,%s/testdata' % (instName, instName), 

174 outputCollection=outputCollection, 

175 registerDatasetTypes=True) 

176 

177 # Check output values 

178 butler = dafButler.Butler(self.repo) 

179 lutCat = butler.get('fgcmLookUpTable', 

180 collections=[outputCollection], 

181 instrument=instName) 

182 fgcmLut, lutIndexVals, lutStd = fgcmcal.utilities.translateFgcmLut(lutCat, {}) 

183 

184 self.assertEqual(nBand, len(lutIndexVals[0]['FILTERNAMES'])) 

185 

186 indices = fgcmLut.getIndices(np.arange(nBand, dtype=np.int32), 

187 np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

188 np.zeros(nBand) + lutStd[0]['O3STD'], 

189 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

190 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

191 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

192 np.zeros(nBand, dtype=np.int32), 

193 np.zeros(nBand) + lutStd[0]['PMBSTD']) 

194 i0 = fgcmLut.computeI0(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

195 np.zeros(nBand) + lutStd[0]['O3STD'], 

196 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

197 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

198 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

199 np.zeros(nBand) + lutStd[0]['PMBSTD'], 

200 indices) 

201 

202 self.assertFloatsAlmostEqual(i0Recon, i0, msg='i0Recon', rtol=1e-5) 

203 

204 i1 = fgcmLut.computeI1(np.zeros(nBand) + np.log(lutStd[0]['PWVSTD']), 

205 np.zeros(nBand) + lutStd[0]['O3STD'], 

206 np.zeros(nBand) + np.log(lutStd[0]['TAUSTD']), 

207 np.zeros(nBand) + lutStd[0]['ALPHASTD'], 

208 np.zeros(nBand) + 1./np.cos(np.radians(lutStd[0]['ZENITHSTD'])), 

209 np.zeros(nBand) + lutStd[0]['PMBSTD'], 

210 indices) 

211 

212 self.assertFloatsAlmostEqual(i10Recon, i1/i0, msg='i10Recon', rtol=1e-5) 

213 

214 def _testFgcmBuildStarsTable(self, instName, testName, queryString, visits, nStar, nObs): 

215 """Test running of FgcmBuildStarsTableTask 

216 

217 Parameters 

218 ---------- 

219 instName : `str` 

220 Short name of the instrument 

221 testName : `str` 

222 Base name of the test collection 

223 queryString : `str` 

224 Query to send to the pipetask. 

225 visits : `list` 

226 List of visits to calibrate 

227 nStar : `int` 

228 Number of stars expected 

229 nObs : `int` 

230 Number of observations of stars expected 

231 """ 

232 instCamel = instName.title() 

233 

234 configFile = 'fgcmBuildStarsTable:' + os.path.join(ROOT, 

235 'config', 

236 'fgcmBuildStarsTable%s.py' % (instCamel)) 

237 outputCollection = f'{instName}/{testName}/buildstars' 

238 

239 self._runPipeline(self.repo, 

240 os.path.join(ROOT, 

241 'pipelines', 

242 'fgcmBuildStarsTable%s.yaml' % (instCamel)), 

243 configFiles=[configFile], 

244 inputCollections=f'{instName}/{testName}/lut,refcats/gen2', 

245 outputCollection=outputCollection, 

246 configOptions=['fgcmBuildStarsTable:ccdDataRefName=detector'], 

247 queryString=queryString, 

248 registerDatasetTypes=True) 

249 

250 butler = dafButler.Butler(self.repo) 

251 

252 visitCat = butler.get('fgcmVisitCatalog', collections=[outputCollection], 

253 instrument=instName) 

254 self.assertEqual(len(visits), len(visitCat)) 

255 

256 starIds = butler.get('fgcmStarIds', collections=[outputCollection], 

257 instrument=instName) 

258 self.assertEqual(len(starIds), nStar) 

259 

260 starObs = butler.get('fgcmStarObservations', collections=[outputCollection], 

261 instrument=instName) 

262 self.assertEqual(len(starObs), nObs) 

263 

264 def _testFgcmFitCycle(self, instName, testName, cycleNumber, 

265 nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, 

266 skipChecks=False, extraConfig=None): 

267 """Test running of FgcmFitCycleTask 

268 

269 Parameters 

270 ---------- 

271 instName : `str` 

272 Short name of the instrument 

273 testName : `str` 

274 Base name of the test collection 

275 cycleNumber : `int` 

276 Fit cycle number. 

277 nZp : `int` 

278 Number of zeropoints created by the task 

279 nGoodZp : `int` 

280 Number of good (photometric) zeropoints created 

281 nOkZp : `int` 

282 Number of constrained zeropoints (photometric or not) 

283 nBadZp : `int` 

284 Number of unconstrained (bad) zeropoints 

285 nStdStars : `int` 

286 Number of standard stars produced 

287 nPlots : `int` 

288 Number of plots produced 

289 skipChecks : `bool`, optional 

290 Skip number checks, when running less-than-final cycle. 

291 extraConfig : `str`, optional 

292 Name of an extra config file to apply. 

293 """ 

294 instCamel = instName.title() 

295 

296 configFiles = ['fgcmFitCycle:' + os.path.join(ROOT, 

297 'config', 

298 'fgcmFitCycle%s.py' % (instCamel))] 

299 if extraConfig is not None: 

300 configFiles.append('fgcmFitCycle:' + extraConfig) 

301 

302 outputCollection = f'{instName}/{testName}/fit' 

303 

304 if cycleNumber == 0: 

305 inputCollections = f'{instName}/{testName}/buildstars' 

306 else: 

307 # We are reusing the outputCollection so we can't specify the input 

308 inputCollections = None 

309 

310 cwd = os.getcwd() 

311 runDir = os.path.join(self.testDir, testName) 

312 os.makedirs(runDir, exist_ok=True) 

313 os.chdir(runDir) 

314 

315 configOptions = ['fgcmFitCycle:cycleNumber=%d' % (cycleNumber), 

316 'fgcmFitCycle:connections.previousCycleNumber=%d' % 

317 (cycleNumber - 1), 

318 'fgcmFitCycle:connections.cycleNumber=%d' % 

319 (cycleNumber)] 

320 

321 self._runPipeline(self.repo, 

322 os.path.join(ROOT, 

323 'pipelines', 

324 'fgcmFitCycle%s.yaml' % (instCamel)), 

325 configFiles=configFiles, 

326 inputCollections=inputCollections, 

327 outputCollection=outputCollection, 

328 configOptions=configOptions, 

329 registerDatasetTypes=True) 

330 

331 os.chdir(cwd) 

332 

333 if skipChecks: 

334 return 

335 

336 butler = dafButler.Butler(self.repo) 

337 

338 config = butler.get('fgcmFitCycle_config', collections=[outputCollection]) 

339 

340 # Check that the expected number of plots are there. 

341 plots = glob.glob(os.path.join(runDir, config.outfileBase 

342 + '_cycle%02d_plots/' % (cycleNumber) 

343 + '*.png')) 

344 self.assertEqual(len(plots), nPlots) 

345 

346 zps = butler.get('fgcmZeropoints%d' % (cycleNumber), 

347 collections=[outputCollection], 

348 instrument=instName) 

349 self.assertEqual(len(zps), nZp) 

350 

351 gd, = np.where(zps['fgcmFlag'] == 1) 

352 self.assertEqual(len(gd), nGoodZp) 

353 

354 ok, = np.where(zps['fgcmFlag'] < 16) 

355 self.assertEqual(len(ok), nOkZp) 

356 

357 bd, = np.where(zps['fgcmFlag'] >= 16) 

358 self.assertEqual(len(bd), nBadZp) 

359 

360 # Check that there are no illegal values with the ok zeropoints 

361 test, = np.where(zps['fgcmZpt'][gd] < -9000.0) 

362 self.assertEqual(len(test), 0) 

363 

364 stds = butler.get('fgcmStandardStars%d' % (cycleNumber), 

365 collections=[outputCollection], 

366 instrument=instName) 

367 

368 self.assertEqual(len(stds), nStdStars) 

369 

370 def _testFgcmOutputProducts(self, instName, testName, 

371 zpOffsets, testVisit, testCcd, testFilter, testBandIndex): 

372 """Test running of FgcmOutputProductsTask. 

373 

374 Parameters 

375 ---------- 

376 instName : `str` 

377 Short name of the instrument 

378 testName : `str` 

379 Base name of the test collection 

380 zpOffsets : `np.ndarray` 

381 Zeropoint offsets expected 

382 testVisit : `int` 

383 Visit id to check for round-trip computations 

384 testCcd : `int` 

385 Ccd id to check for round-trip computations 

386 testFilter : `str` 

387 Filtername for testVisit/testCcd 

388 testBandIndex : `int` 

389 Band index for testVisit/testCcd 

390 """ 

391 instCamel = instName.title() 

392 

393 configFile = 'fgcmOutputProducts:' + os.path.join(ROOT, 

394 'config', 

395 'fgcmOutputProducts%s.py' % (instCamel)) 

396 inputCollection = f'{instName}/{testName}/fit' 

397 outputCollection = f'{instName}/{testName}/fit/output' 

398 

399 self._runPipeline(self.repo, 

400 os.path.join(ROOT, 

401 'pipelines', 

402 'fgcmOutputProducts%s.yaml' % (instCamel)), 

403 configFiles=[configFile], 

404 inputCollections=inputCollection, 

405 outputCollection=outputCollection, 

406 configOptions=['fgcmOutputProducts:doRefcatOutput=False'], 

407 registerDatasetTypes=True) 

408 

409 butler = dafButler.Butler(self.repo) 

410 offsetCat = butler.get('fgcmReferenceCalibrationOffsets', 

411 collections=[outputCollection], instrument=instName) 

412 offsets = offsetCat['offset'][:] 

413 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6) 

414 

415 config = butler.get('fgcmOutputProducts_config', 

416 collections=[outputCollection], instrument=instName) 

417 

418 rawStars = butler.get('fgcmStandardStars' + config.connections.cycleNumber, 

419 collections=[inputCollection], instrument=instName) 

420 

421 candRatio = (rawStars['npsfcand'][:, 0].astype(np.float64) 

422 / rawStars['ntotal'][:, 0].astype(np.float64)) 

423 self.assertFloatsAlmostEqual(candRatio.min(), 0.0) 

424 self.assertFloatsAlmostEqual(candRatio.max(), 1.0) 

425 

426 # Test the fgcm_photoCalib output 

427 zptCat = butler.get('fgcmZeropoints' + config.connections.cycleNumber, 

428 collections=[inputCollection], instrument=instName) 

429 selected = (zptCat['fgcmFlag'] < 16) 

430 

431 # Read in all the calibrations, these should all be there 

432 # This test is simply to ensure that all the photoCalib files exist 

433 visits = np.unique(zptCat['visit']) 

434 photoCalibDict = {} 

435 for visit in visits: 

436 expCat = butler.get('fgcmPhotoCalibCatalog', 

437 visit=visit, 

438 collections=[outputCollection], instrument=instName) 

439 for row in expCat: 

440 if row['visit'] == visit: 

441 photoCalibDict[(visit, row['id'])] = row.getPhotoCalib() 

442 

443 for rec in zptCat[selected]: 

444 self.assertTrue((rec['visit'], rec['detector']) in photoCalibDict) 

445 

446 # We do round-trip value checking on just the final one (chosen arbitrarily) 

447 testCal = photoCalibDict[(testVisit, testCcd)] 

448 

449 src = butler.get('src', visit=int(testVisit), detector=int(testCcd), 

450 collections=[outputCollection], instrument=instName) 

451 

452 # Only test sources with positive flux 

453 gdSrc = (src['slot_CalibFlux_instFlux'] > 0.0) 

454 

455 # We need to apply the calibration offset to the fgcmzpt (which is internal 

456 # and doesn't know about that yet) 

457 testZpInd, = np.where((zptCat['visit'] == testVisit) 

458 & (zptCat['detector'] == testCcd)) 

459 fgcmZpt = (zptCat['fgcmZpt'][testZpInd] + offsets[testBandIndex] 

460 + zptCat['fgcmDeltaChrom'][testZpInd]) 

461 fgcmZptGrayErr = np.sqrt(zptCat['fgcmZptVar'][testZpInd]) 

462 

463 if config.doComposeWcsJacobian: 

464 # The raw zeropoint needs to be modified to know about the wcs jacobian 

465 refs = butler.registry.queryDatasets('camera', dimensions=['instrument'], 

466 collections=...) 

467 camera = butler.getDirect(list(refs)[0]) 

468 approxPixelAreaFields = fgcmcal.utilities.computeApproxPixelAreaFields(camera) 

469 center = approxPixelAreaFields[testCcd].getBBox().getCenter() 

470 pixAreaCorr = approxPixelAreaFields[testCcd].evaluate(center) 

471 fgcmZpt += -2.5*np.log10(pixAreaCorr) 

472 

473 # This is the magnitude through the mean calibration 

474 photoCalMeanCalMags = np.zeros(gdSrc.sum()) 

475 # This is the magnitude through the full focal-plane variable mags 

476 photoCalMags = np.zeros_like(photoCalMeanCalMags) 

477 # This is the magnitude with the FGCM (central-ccd) zeropoint 

478 zptMeanCalMags = np.zeros_like(photoCalMeanCalMags) 

479 

480 for i, rec in enumerate(src[gdSrc]): 

481 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux']) 

482 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'], 

483 rec.getCentroid()) 

484 zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux']) 

485 

486 # These should be very close but some tiny differences because the fgcm value 

487 # is defined at the center of the bbox, and the photoCal is the mean over the box 

488 self.assertFloatsAlmostEqual(photoCalMeanCalMags, 

489 zptMeanCalMags, rtol=1e-6) 

490 # These should be roughly equal, but not precisely because of the focal-plane 

491 # variation. However, this is a useful sanity check for something going totally 

492 # wrong. 

493 self.assertFloatsAlmostEqual(photoCalMeanCalMags, 

494 photoCalMags, rtol=1e-2) 

495 

496 # The next test compares the "FGCM standard magnitudes" (which are output 

497 # from the fgcm code itself) to the "calibrated magnitudes" that are 

498 # obtained from running photoCalib.calibrateCatalog() on the original 

499 # src catalogs. This summary comparison ensures that using photoCalibs 

500 # yields the same results as what FGCM is computing internally. 

501 # Note that we additionally need to take into account the post-processing 

502 # offsets used in the tests. 

503 

504 # For decent statistics, we are matching all the sources from one visit 

505 # (multiple ccds) 

506 whereClause = f"instrument='{instName:s}' and visit={testVisit:d}" 

507 srcRefs = butler.registry.queryDatasets('src', dimensions=['visit'], 

508 collections='%s/testdata' % (instName), 

509 where=whereClause, 

510 findFirst=True) 

511 photoCals = [] 

512 for srcRef in srcRefs: 

513 photoCals.append(photoCalibDict[(testVisit, srcRef.dataId['detector'])]) 

514 

515 matchMag, matchDelta = self._getMatchedVisitCat(butler, srcRefs, photoCals, 

516 rawStars, testBandIndex, offsets) 

517 

518 st = np.argsort(matchMag) 

519 # Compare the brightest 25% of stars. No matter the setting of 

520 # deltaMagBkgOffsetPercentile, we want to ensure that these stars 

521 # match on average. 

522 brightest, = np.where(matchMag < matchMag[st[int(0.25*st.size)]]) 

523 self.assertFloatsAlmostEqual(np.median(matchDelta[brightest]), 0.0, atol=0.002) 

524 

525 # And the photoCal error is just the zeropoint gray error 

526 self.assertFloatsAlmostEqual(testCal.getCalibrationErr(), 

527 (np.log(10.0)/2.5)*testCal.getCalibrationMean()*fgcmZptGrayErr) 

528 

529 # Test the transmission output 

530 visitCatalog = butler.get('fgcmVisitCatalog', collections=[inputCollection], 

531 instrument=instName) 

532 lutCat = butler.get('fgcmLookUpTable', collections=[inputCollection], 

533 instrument=instName) 

534 

535 testTrans = butler.get('transmission_atmosphere_fgcm', 

536 visit=visitCatalog[0]['visit'], 

537 collections=[outputCollection], instrument=instName) 

538 testResp = testTrans.sampleAt(position=geom.Point2D(0, 0), 

539 wavelengths=lutCat[0]['atmLambda']) 

540 

541 # The test fit is performed with the atmosphere parameters frozen 

542 # (freezeStdAtmosphere = True). Thus the only difference between 

543 # these output atmospheres and the standard is the different 

544 # airmass. Furthermore, this is a very rough comparison because 

545 # the look-up table is computed with very coarse sampling for faster 

546 # testing. 

547 

548 # To account for overall throughput changes, we scale by the median ratio, 

549 # we only care about the shape 

550 ratio = np.median(testResp/lutCat[0]['atmStdTrans']) 

551 self.assertFloatsAlmostEqual(testResp/ratio, lutCat[0]['atmStdTrans'], atol=0.04) 

552 

553 # The second should be close to the first, but there is the airmass 

554 # difference so they aren't identical. 

555 testTrans2 = butler.get('transmission_atmosphere_fgcm', 

556 visit=visitCatalog[1]['visit'], 

557 collections=[outputCollection], instrument=instName) 

558 testResp2 = testTrans2.sampleAt(position=geom.Point2D(0, 0), 

559 wavelengths=lutCat[0]['atmLambda']) 

560 

561 # As above, we scale by the ratio to compare the shape of the curve. 

562 ratio = np.median(testResp/testResp2) 

563 self.assertFloatsAlmostEqual(testResp/ratio, testResp2, atol=0.04) 

564 

565 def _testFgcmMultiFit(self, instName, testName, queryString, visits, zpOffsets): 

566 """Test running the full pipeline with multiple fit cycles. 

567 

568 Parameters 

569 ---------- 

570 instName : `str` 

571 Short name of the instrument 

572 testName : `str` 

573 Base name of the test collection 

574 queryString : `str` 

575 Query to send to the pipetask. 

576 visits : `list` 

577 List of visits to calibrate 

578 zpOffsets : `np.ndarray` 

579 Zeropoint offsets expected 

580 """ 

581 instCamel = instName.title() 

582 

583 configFiles = ['fgcmBuildStarsTable:' + os.path.join(ROOT, 

584 'config', 

585 f'fgcmBuildStarsTable{instCamel}.py'), 

586 'fgcmFitCycle:' + os.path.join(ROOT, 

587 'config', 

588 f'fgcmFitCycle{instCamel}.py'), 

589 'fgcmOutputProducts:' + os.path.join(ROOT, 

590 'config', 

591 f'fgcmOutputProducts{instCamel}.py')] 

592 outputCollection = f'{instName}/{testName}/unified' 

593 

594 cwd = os.getcwd() 

595 runDir = os.path.join(self.testDir, testName) 

596 os.makedirs(runDir) 

597 os.chdir(runDir) 

598 

599 self._runPipeline(self.repo, 

600 os.path.join(ROOT, 

601 'pipelines', 

602 f'fgcmFullPipeline{instCamel}.yaml'), 

603 configFiles=configFiles, 

604 inputCollections=f'{instName}/{testName}/lut,refcats/gen2', 

605 outputCollection=outputCollection, 

606 configOptions=['fgcmBuildStarsTable:ccdDataRefName=detector'], 

607 queryString=queryString, 

608 registerDatasetTypes=True) 

609 

610 os.chdir(cwd) 

611 

612 butler = dafButler.Butler(self.repo) 

613 

614 offsetCat = butler.get('fgcmReferenceCalibrationOffsets', 

615 collections=[outputCollection], instrument=instName) 

616 offsets = offsetCat['offset'][:] 

617 self.assertFloatsAlmostEqual(offsets, zpOffsets, atol=1e-6) 

618 

619 def _getMatchedVisitCat(self, butler, srcRefs, photoCals, 

620 rawStars, bandIndex, offsets): 

621 """ 

622 Get a list of matched magnitudes and deltas from calibrated src catalogs. 

623 

624 Parameters 

625 ---------- 

626 butler : `lsst.daf.butler.Butler` 

627 srcRefs : `list` 

628 dataRefs of source catalogs 

629 photoCalibRefs : `list` 

630 dataRefs of photoCalib files, matched to srcRefs. 

631 photoCals : `list` 

632 photoCalib objects, matched to srcRefs. 

633 rawStars : `lsst.afw.table.SourceCatalog` 

634 Fgcm standard stars 

635 bandIndex : `int` 

636 Index of the band for the source catalogs 

637 offsets : `np.ndarray` 

638 Testing calibration offsets to apply to rawStars 

639 

640 Returns 

641 ------- 

642 matchMag : `np.ndarray` 

643 Array of matched magnitudes 

644 matchDelta : `np.ndarray` 

645 Array of matched deltas between src and standard stars. 

646 """ 

647 matcher = esutil.htm.Matcher(11, np.rad2deg(rawStars['coord_ra']), 

648 np.rad2deg(rawStars['coord_dec'])) 

649 

650 matchDelta = None 

651 # for dataRef in dataRefs: 

652 for srcRef, photoCal in zip(srcRefs, photoCals): 

653 src = butler.getDirect(srcRef) 

654 src = photoCal.calibrateCatalog(src) 

655 

656 gdSrc, = np.where(np.nan_to_num(src['slot_CalibFlux_flux']) > 0.0) 

657 

658 matches = matcher.match(np.rad2deg(src['coord_ra'][gdSrc]), 

659 np.rad2deg(src['coord_dec'][gdSrc]), 

660 1./3600., maxmatch=1) 

661 

662 srcMag = src['slot_CalibFlux_mag'][gdSrc][matches[0]] 

663 # Apply offset here to the catalog mag 

664 catMag = rawStars['mag_std_noabs'][matches[1]][:, bandIndex] + offsets[bandIndex] 

665 delta = srcMag - catMag 

666 if matchDelta is None: 

667 matchDelta = delta 

668 matchMag = catMag 

669 else: 

670 matchDelta = np.append(matchDelta, delta) 

671 matchMag = np.append(matchMag, catMag) 

672 

673 return matchMag, matchDelta 

674 

675 def _testFgcmCalibrateTract(self, instName, testName, visits, tract, skymapName, 

676 rawRepeatability, filterNCalibMap): 

677 """Test running of FgcmCalibrateTractTask 

678 

679 Parameters 

680 ---------- 

681 instName : `str` 

682 Short name of the instrument 

683 testName : `str` 

684 Base name of the test collection 

685 visits : `list` 

686 List of visits to calibrate 

687 tract : `int` 

688 Tract number 

689 skymapName : `str` 

690 Name of the sky map 

691 rawRepeatability : `np.array` 

692 Expected raw repeatability after convergence. 

693 Length should be number of bands. 

694 filterNCalibMap : `dict` 

695 Mapping from filter name to number of photoCalibs created. 

696 """ 

697 instCamel = instName.title() 

698 

699 configFile = os.path.join(ROOT, 

700 'config', 

701 'fgcmCalibrateTractTable%s.py' % (instCamel)) 

702 

703 configFiles = ['fgcmCalibrateTractTable:' + configFile] 

704 outputCollection = f'{instName}/{testName}/tract' 

705 

706 inputCollections = f'{instName}/{testName}/lut,refcats/gen2' 

707 configOption = 'fgcmCalibrateTractTable:fgcmOutputProducts.doRefcatOutput=False' 

708 

709 queryString = f"tract={tract:d} and skymap='{skymapName:s}'" 

710 

711 self._runPipeline(self.repo, 

712 os.path.join(ROOT, 

713 'pipelines', 

714 f'fgcmCalibrateTractTable{instCamel:s}.yaml'), 

715 queryString=queryString, 

716 configFiles=configFiles, 

717 inputCollections=inputCollections, 

718 outputCollection=outputCollection, 

719 configOptions=[configOption], 

720 registerDatasetTypes=True) 

721 

722 butler = dafButler.Butler(self.repo) 

723 

724 whereClause = f"instrument='{instName:s}' and tract={tract:d} and skymap='{skymapName:s}'" 

725 

726 repRefs = butler.registry.queryDatasets('fgcmRawRepeatability', 

727 dimensions=['tract'], 

728 collections=outputCollection, 

729 where=whereClause) 

730 

731 repeatabilityCat = butler.getDirect(list(repRefs)[0]) 

732 repeatability = repeatabilityCat['rawRepeatability'][:] 

733 self.assertFloatsAlmostEqual(repeatability, rawRepeatability, atol=4e-6) 

734 

735 # Check that the number of photoCalib objects in each filter are what we expect 

736 for filterName in filterNCalibMap.keys(): 

737 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and " 

738 f"physical_filter='{filterName:s}' and skymap='{skymapName:s}'") 

739 

740 refs = butler.registry.queryDatasets('fgcmPhotoCalibTractCatalog', 

741 dimensions=['tract', 'physical_filter'], 

742 collections=outputCollection, 

743 where=whereClause) 

744 

745 count = 0 

746 for ref in set(refs): 

747 expCat = butler.getDirect(ref) 

748 test, = np.where((expCat['visit'] > 0) & (expCat['id'] >= 0)) 

749 count += test.size 

750 

751 self.assertEqual(count, filterNCalibMap[filterName]) 

752 

753 # Check that every visit got a transmission 

754 for visit in visits: 

755 whereClause = (f"instrument='{instName:s}' and tract={tract:d} and " 

756 f"visit={visit:d} and skymap='{skymapName:s}'") 

757 refs = butler.registry.queryDatasets('transmission_atmosphere_fgcm_tract', 

758 dimensions=['tract', 'visit'], 

759 collections=outputCollection, 

760 where=whereClause) 

761 self.assertEqual(len(set(refs)), 1) 

762 

763 @classmethod 

764 def tearDownClass(cls): 

765 """Tear down and clear directories 

766 """ 

767 if os.path.exists(cls.testDir): 

768 shutil.rmtree(cls.testDir, True)