Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34 

35import sys 

36import traceback 

37import copy 

38 

39import numpy as np 

40import healpy as hp 

41import esutil 

42from astropy import units 

43 

44import lsst.pex.config as pexConfig 

45import lsst.pipe.base as pipeBase 

46from lsst.afw.image import TransmissionCurve 

47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

48from lsst.pipe.tasks.photoCal import PhotoCalTask 

49import lsst.geom 

50import lsst.afw.image as afwImage 

51import lsst.afw.math as afwMath 

52import lsst.afw.table as afwTable 

53from lsst.meas.algorithms import IndexerRegistry 

54from lsst.meas.algorithms import DatasetConfig 

55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

56 

57from .utilities import computeApproxPixelAreaFields 

58 

59import fgcm 

60 

61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

62 

63 

64class FgcmOutputProductsConfig(pexConfig.Config): 

65 """Config for FgcmOutputProductsTask""" 

66 

67 cycleNumber = pexConfig.Field( 

68 doc="Final fit cycle from FGCM fit", 

69 dtype=int, 

70 default=None, 

71 ) 

72 

73 # The following fields refer to calibrating from a reference 

74 # catalog, but in the future this might need to be expanded 

75 doReferenceCalibration = pexConfig.Field( 

76 doc=("Transfer 'absolute' calibration from reference catalog? " 

77 "This afterburner step is unnecessary if reference stars " 

78 "were used in the full fit in FgcmFitCycleTask."), 

79 dtype=bool, 

80 default=False, 

81 ) 

82 doRefcatOutput = pexConfig.Field( 

83 doc="Output standard stars in reference catalog format", 

84 dtype=bool, 

85 default=True, 

86 ) 

87 doAtmosphereOutput = pexConfig.Field( 

88 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

89 dtype=bool, 

90 default=True, 

91 ) 

92 doZeropointOutput = pexConfig.Field( 

93 doc="Output zeropoints in fgcm_photoCalib format", 

94 dtype=bool, 

95 default=True, 

96 ) 

97 doComposeWcsJacobian = pexConfig.Field( 

98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

99 dtype=bool, 

100 default=True, 

101 ) 

102 doApplyMeanChromaticCorrection = pexConfig.Field( 

103 doc="Apply the mean chromatic correction to the zeropoints?", 

104 dtype=bool, 

105 default=True, 

106 ) 

107 refObjLoader = pexConfig.ConfigurableField( 

108 target=LoadIndexedReferenceObjectsTask, 

109 doc="reference object loader for 'absolute' photometric calibration", 

110 ) 

111 photoCal = pexConfig.ConfigurableField( 

112 target=PhotoCalTask, 

113 doc="task to perform 'absolute' calibration", 

114 ) 

115 referencePixelizationNside = pexConfig.Field( 

116 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

117 dtype=int, 

118 default=64, 

119 ) 

120 referencePixelizationMinStars = pexConfig.Field( 

121 doc=("Minimum number of stars per healpix pixel to select for comparison" 

122 "to the specified reference catalog"), 

123 dtype=int, 

124 default=200, 

125 ) 

126 referenceMinMatch = pexConfig.Field( 

127 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

128 dtype=int, 

129 default=50, 

130 ) 

131 referencePixelizationNPixels = pexConfig.Field( 

132 doc=("Number of healpix pixels to sample to do comparison. " 

133 "Doing too many will take a long time and not yield any more " 

134 "precise results because the final number is the median offset " 

135 "(per band) from the set of pixels."), 

136 dtype=int, 

137 default=100, 

138 ) 

139 datasetConfig = pexConfig.ConfigField( 

140 dtype=DatasetConfig, 

141 doc="Configuration for writing/reading ingested catalog", 

142 ) 

143 

144 def setDefaults(self): 

145 pexConfig.Config.setDefaults(self) 

146 

147 # In order to transfer the "absolute" calibration from a reference 

148 # catalog to the relatively calibrated FGCM standard stars (one number 

149 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

150 # pixels. These basic settings ensure that only well-measured, good stars 

151 # from the source and reference catalogs are used for the matching. 

152 

153 # applyColorTerms needs to be False if doReferenceCalibration is False, 

154 # as is the new default after DM-16702 

155 self.photoCal.applyColorTerms = False 

156 self.photoCal.fluxField = 'instFlux' 

157 self.photoCal.magErrFloor = 0.003 

158 self.photoCal.match.referenceSelection.doSignalToNoise = True 

159 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

160 self.photoCal.match.sourceSelection.doSignalToNoise = True 

161 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

162 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

163 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

164 self.photoCal.match.sourceSelection.doFlags = True 

165 self.photoCal.match.sourceSelection.flags.good = [] 

166 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

167 self.photoCal.match.sourceSelection.doUnresolved = False 

168 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

169 self.datasetConfig.format_version = 1 

170 

171 

172class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

173 """Subclass of TaskRunner for fgcmOutputProductsTask 

174 

175 fgcmOutputProductsTask.run() takes one argument, the butler, and 

176 does not run on any data in the repository. 

177 This runner does not use any parallelization. 

178 """ 

179 

180 @staticmethod 

181 def getTargetList(parsedCmd): 

182 """ 

183 Return a list with one element, the butler. 

184 """ 

185 return [parsedCmd.butler] 

186 

187 def __call__(self, butler): 

188 """ 

189 Parameters 

190 ---------- 

191 butler: `lsst.daf.persistence.Butler` 

192 

193 Returns 

194 ------- 

195 exitStatus: `list` with `pipeBase.Struct` 

196 exitStatus (0: success; 1: failure) 

197 if self.doReturnResults also 

198 results (`np.array` with absolute zeropoint offsets) 

199 """ 

200 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

201 

202 exitStatus = 0 

203 if self.doRaise: 

204 results = task.runDataRef(butler) 

205 else: 

206 try: 

207 results = task.runDataRef(butler) 

208 except Exception as e: 

209 exitStatus = 1 

210 task.log.fatal("Failed: %s" % e) 

211 if not isinstance(e, pipeBase.TaskError): 

212 traceback.print_exc(file=sys.stderr) 

213 

214 task.writeMetadata(butler) 

215 

216 if self.doReturnResults: 

217 # The results here are the zeropoint offsets for each band 

218 return [pipeBase.Struct(exitStatus=exitStatus, 

219 results=results)] 

220 else: 

221 return [pipeBase.Struct(exitStatus=exitStatus)] 

222 

223 def run(self, parsedCmd): 

224 """ 

225 Run the task, with no multiprocessing 

226 

227 Parameters 

228 ---------- 

229 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

230 """ 

231 

232 resultList = [] 

233 

234 if self.precall(parsedCmd): 

235 targetList = self.getTargetList(parsedCmd) 

236 # make sure that we only get 1 

237 resultList = self(targetList[0]) 

238 

239 return resultList 

240 

241 

242class FgcmOutputProductsTask(pipeBase.CmdLineTask): 

243 """ 

244 Output products from FGCM global calibration. 

245 """ 

246 

247 ConfigClass = FgcmOutputProductsConfig 

248 RunnerClass = FgcmOutputProductsRunner 

249 _DefaultName = "fgcmOutputProducts" 

250 

251 def __init__(self, butler=None, **kwargs): 

252 """ 

253 Instantiate an fgcmOutputProductsTask. 

254 

255 Parameters 

256 ---------- 

257 butler : `lsst.daf.persistence.Butler` 

258 """ 

259 

260 pipeBase.CmdLineTask.__init__(self, **kwargs) 

261 

262 if self.config.doReferenceCalibration: 

263 # We need the ref obj loader to get the flux field 

264 self.makeSubtask("refObjLoader", butler=butler) 

265 

266 if self.config.doRefcatOutput: 

267 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

268 self.config.datasetConfig.indexer.active) 

269 

270 # no saving of metadata for now 

271 def _getMetadataName(self): 

272 return None 

273 

274 @pipeBase.timeMethod 

275 def runDataRef(self, butler): 

276 """ 

277 Make FGCM output products for use in the stack 

278 

279 Parameters 

280 ---------- 

281 butler: `lsst.daf.persistence.Butler` 

282 cycleNumber: `int` 

283 Final fit cycle number, override config. 

284 

285 Returns 

286 ------- 

287 offsets: `lsst.pipe.base.Struct` 

288 A structure with array of zeropoint offsets 

289 

290 Raises 

291 ------ 

292 RuntimeError: 

293 Raised if any one of the following is true: 

294 

295 - butler cannot find "fgcmBuildStars_config" or 

296 "fgcmBuildStarsTable_config". 

297 - butler cannot find "fgcmFitCycle_config". 

298 - "fgcmFitCycle_config" does not refer to 

299 `self.config.cycleNumber`. 

300 - butler cannot find "fgcmAtmosphereParameters" and 

301 `self.config.doAtmosphereOutput` is `True`. 

302 - butler cannot find "fgcmStandardStars" and 

303 `self.config.doReferenceCalibration` is `True` or 

304 `self.config.doRefcatOutput` is `True`. 

305 - butler cannot find "fgcmZeropoints" and 

306 `self.config.doZeropointOutput` is `True`. 

307 """ 

308 

309 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

310 # the visit and ccd dataset tags 

311 if not butler.datasetExists('fgcmBuildStarsTable_config') and \ 

312 not butler.datasetExists('fgcmBuildStars_config'): 

313 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, " 

314 "which is prereq for fgcmOutputProducts") 

315 

316 if butler.datasetExists('fgcmBuildStarsTable_config'): 

317 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config') 

318 else: 

319 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

320 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

321 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

322 self.filterMap = fgcmBuildStarsConfig.filterMap 

323 

324 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

325 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

326 "in fgcmBuildStarsTask.") 

327 

328 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

329 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

330 

331 # Make sure that the fit config exists, to retrieve bands and other info 

332 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber): 

333 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) + 

334 "which is required for fgcmOutputProducts.") 

335 

336 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber) 

337 self.configBands = fitCycleConfig.bands 

338 

339 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration: 

340 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

341 "fitCycleConfig.doReferenceCalibration") 

342 

343 # And make sure that the atmosphere was output properly 

344 if (self.config.doAtmosphereOutput and 

345 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

346 raise RuntimeError("Atmosphere parameters are missing for cycle %d." % 

347 (self.config.cycleNumber)) 

348 

349 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and 

350 (not butler.datasetExists('fgcmStandardStars', 

351 fgcmcycle=self.config.cycleNumber))): 

352 raise RuntimeError("Standard stars are missing for cycle %d." % 

353 (self.config.cycleNumber)) 

354 

355 if (self.config.doZeropointOutput and 

356 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

357 raise RuntimeError("Zeropoints are missing for cycle %d." % 

358 (self.config.cycleNumber)) 

359 

360 # And make sure this is the last cycle 

361 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1): 

362 raise RuntimeError("The task fgcmOutputProducts should only be run" 

363 "on the final fit cycle products") 

364 

365 if self.config.doReferenceCalibration or self.config.doRefcatOutput: 

366 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber) 

367 md = stdCat.getMetadata() 

368 self.bands = md.getArray('BANDS') 

369 else: 

370 stdCat = None 

371 self.bands = self.configBands 

372 

373 if self.config.doReferenceCalibration: 

374 offsets = self._computeReferenceOffsets(butler, stdCat) 

375 else: 

376 offsets = np.zeros(len(self.bands)) 

377 

378 # Output the standard stars in stack format 

379 if self.config.doRefcatOutput: 

380 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig) 

381 

382 del stdCat 

383 

384 # Output the gray zeropoints 

385 if self.config.doZeropointOutput: 

386 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber) 

387 visitCat = butler.get('fgcmVisitCatalog') 

388 

389 self._outputZeropoints(butler, zptCat, visitCat, offsets) 

390 

391 # Output the atmospheres 

392 if self.config.doAtmosphereOutput: 

393 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber) 

394 self._outputAtmospheres(butler, atmCat) 

395 

396 # We return the zp offsets 

397 return pipeBase.Struct(offsets=offsets) 

398 

399 def generateTractOutputProducts(self, butler, tract, 

400 visitCat, zptCat, atmCat, stdCat, 

401 fgcmBuildStarsConfig, fgcmFitCycleConfig): 

402 """ 

403 Generate the output products for a given tract, as specified in the config. 

404 

405 This method is here to have an alternate entry-point for 

406 FgcmCalibrateTract. 

407 

408 Parameters 

409 ---------- 

410 butler: `lsst.daf.persistence.Butler` 

411 tract: `int` 

412 Tract number 

413 visitCat: `lsst.afw.table.BaseCatalog` 

414 FGCM visitCat from `FgcmBuildStarsTask` 

415 zptCat: `lsst.afw.table.BaseCatalog` 

416 FGCM zeropoint catalog from `FgcmFitCycleTask` 

417 atmCat: `lsst.afw.table.BaseCatalog` 

418 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

419 stdCat: `lsst.afw.table.SimpleCatalog` 

420 FGCM standard star catalog from `FgcmFitCycleTask` 

421 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig` 

422 Configuration object from `FgcmBuildStarsTask` 

423 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig` 

424 Configuration object from `FgcmFitCycleTask` 

425 """ 

426 

427 self.configBands = fgcmFitCycleConfig.bands 

428 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

429 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

430 self.filterMap = fgcmBuildStarsConfig.filterMap 

431 

432 if stdCat is not None: 

433 md = stdCat.getMetadata() 

434 self.bands = md.getArray('BANDS') 

435 else: 

436 self.bands = self.configBands 

437 

438 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration: 

439 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

440 "fitCycleConfig.doReferenceCalibration") 

441 

442 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

443 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

444 "in fgcmBuildStarsTask.") 

445 

446 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

447 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

448 

449 if self.config.doReferenceCalibration: 

450 offsets = self._computeReferenceOffsets(butler, stdCat) 

451 else: 

452 offsets = np.zeros(len(self.bands)) 

453 

454 if self.config.doRefcatOutput: 

455 # Create a special config that has the tract number in it 

456 datasetConfig = copy.copy(self.config.datasetConfig) 

457 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

458 tract) 

459 self._outputStandardStars(butler, stdCat, offsets, datasetConfig) 

460 

461 if self.config.doZeropointOutput: 

462 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract) 

463 

464 if self.config.doAtmosphereOutput: 

465 self._outputAtmospheres(butler, atmCat, tract=tract) 

466 

467 return pipeBase.Struct(offsets=offsets) 

468 

469 def _computeReferenceOffsets(self, butler, stdCat): 

470 """ 

471 Compute offsets relative to a reference catalog. 

472 

473 This method splits the star catalog into healpix pixels 

474 and computes the calibration transfer for a sample of 

475 these pixels to approximate the 'absolute' calibration 

476 values (on for each band) to apply to transfer the 

477 absolute scale. 

478 

479 Parameters 

480 ---------- 

481 butler: `lsst.daf.persistence.Butler` 

482 stdCat: `lsst.afw.table.SimpleCatalog` 

483 FGCM standard stars 

484 

485 Returns 

486 ------- 

487 offsets: `numpy.array` of floats 

488 Per band zeropoint offsets 

489 """ 

490 

491 # Only use stars that are observed in all the bands that were actually used 

492 # This will ensure that we use the same healpix pixels for the absolute 

493 # calibration of each band. 

494 minObs = stdCat['ngood'].min(axis=1) 

495 

496 goodStars = (minObs >= 1) 

497 stdCat = stdCat[goodStars] 

498 

499 self.log.info("Found %d stars with at least 1 good observation in each band" % 

500 (len(stdCat))) 

501 

502 # We have to make a table for each pixel with flux/fluxErr 

503 # This is a temporary table generated for input to the photoCal task. 

504 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

505 # have had chromatic corrections applied to get to the standard system 

506 # specified by the atmosphere/instrumental parameters), nor are they 

507 # in Jansky (since they don't have a proper absolute calibration: the overall 

508 # zeropoint is estimated from the telescope size, etc.) 

509 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

510 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

511 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

512 doc="instrumental flux (counts)") 

513 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

514 doc="instrumental flux error (counts)") 

515 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

516 type='Flag', 

517 doc="bad flag") 

518 

519 # Split up the stars 

520 # Note that there is an assumption here that the ra/dec coords stored 

521 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

522 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

523 theta = np.pi/2. - stdCat['coord_dec'] 

524 phi = stdCat['coord_ra'] 

525 

526 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

527 h, rev = esutil.stat.histogram(ipring, rev=True) 

528 

529 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

530 

531 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

532 (gdpix.size, 

533 self.config.referencePixelizationNside, 

534 self.config.referencePixelizationMinStars)) 

535 

536 if gdpix.size < self.config.referencePixelizationNPixels: 

537 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

538 (gdpix.size, self.config.referencePixelizationNPixels)) 

539 else: 

540 # Sample out the pixels we want to use 

541 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

542 

543 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

544 ('nstar', 'i4', len(self.bands)), 

545 ('nmatch', 'i4', len(self.bands)), 

546 ('zp', 'f4', len(self.bands)), 

547 ('zpErr', 'f4', len(self.bands))]) 

548 results['hpix'] = ipring[rev[rev[gdpix]]] 

549 

550 # We need a boolean index to deal with catalogs... 

551 selected = np.zeros(len(stdCat), dtype=np.bool) 

552 

553 refFluxFields = [None]*len(self.bands) 

554 

555 for p, pix in enumerate(gdpix): 

556 i1a = rev[rev[pix]: rev[pix + 1]] 

557 

558 # the stdCat afwTable can only be indexed with boolean arrays, 

559 # and not numpy index arrays (see DM-16497). This little trick 

560 # converts the index array into a boolean array 

561 selected[:] = False 

562 selected[i1a] = True 

563 

564 for b, band in enumerate(self.bands): 

565 

566 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat, 

567 selected, refFluxFields) 

568 results['nstar'][p, b] = len(i1a) 

569 results['nmatch'][p, b] = len(struct.arrays.refMag) 

570 results['zp'][p, b] = struct.zp 

571 results['zpErr'][p, b] = struct.sigma 

572 

573 # And compute the summary statistics 

574 offsets = np.zeros(len(self.bands)) 

575 

576 for b, band in enumerate(self.bands): 

577 # make configurable 

578 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch) 

579 offsets[b] = np.median(results['zp'][ok, b]) 

580 # use median absolute deviation to estimate Normal sigma 

581 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

582 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b])) 

583 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" % 

584 (band, offsets[b], madSigma)) 

585 

586 return offsets 

587 

588 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

589 b, band, stdCat, selected, refFluxFields): 

590 """ 

591 Compute the zeropoint offset between the fgcm stdCat and the reference 

592 stars for one pixel in one band 

593 

594 Parameters 

595 ---------- 

596 sourceMapper: `lsst.afw.table.SchemaMapper` 

597 Mapper to go from stdCat to calibratable catalog 

598 badStarKey: `lsst.afw.table.Key` 

599 Key for the field with bad stars 

600 b: `int` 

601 Index of the band in the star catalog 

602 band: `str` 

603 Name of band for reference catalog 

604 stdCat: `lsst.afw.table.SimpleCatalog` 

605 FGCM standard stars 

606 selected: `numpy.array(dtype=np.bool)` 

607 Boolean array of which stars are in the pixel 

608 refFluxFields: `list` 

609 List of names of flux fields for reference catalog 

610 """ 

611 

612 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

613 sourceCat.reserve(selected.sum()) 

614 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

615 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5)) 

616 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] * 

617 sourceCat['instFlux']) 

618 # Make sure we only use stars that have valid measurements 

619 # (This is perhaps redundant with requirements above that the 

620 # stars be observed in all bands, but it can't hurt) 

621 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0) 

622 for rec in sourceCat[badStar]: 

623 rec.set(badStarKey, True) 

624 

625 exposure = afwImage.ExposureF() 

626 exposure.setFilter(afwImage.Filter(band)) 

627 

628 if refFluxFields[b] is None: 

629 # Need to find the flux field in the reference catalog 

630 # to work around limitations of DirectMatch in PhotoCal 

631 ctr = stdCat[0].getCoord() 

632 rad = 0.05*lsst.geom.degrees 

633 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band) 

634 refFluxFields[b] = refDataTest.fluxField 

635 

636 # Make a copy of the config so that we can modify it 

637 calConfig = copy.copy(self.config.photoCal.value) 

638 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b] 

639 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err' 

640 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

641 config=calConfig, 

642 schema=sourceCat.getSchema()) 

643 

644 struct = calTask.run(exposure, sourceCat) 

645 

646 return struct 

647 

648 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig): 

649 """ 

650 Output standard stars in indexed reference catalog format. 

651 

652 Parameters 

653 ---------- 

654 butler: `lsst.daf.persistence.Butler` 

655 stdCat: `lsst.afw.table.SimpleCatalog` 

656 FGCM standard star catalog from fgcmFitCycleTask 

657 offsets: `numpy.array` of floats 

658 Per band zeropoint offsets 

659 datasetConfig: `lsst.meas.algorithms.DatasetConfig` 

660 Config for reference dataset 

661 """ 

662 

663 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

664 

665 # We determine the conversion from the native units (typically radians) to 

666 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

667 # numpy arrays rather than Angles, which would we approximately 600x slower. 

668 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

669 # (as Angles) for input 

670 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

671 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv, 

672 stdCat['coord_dec']*conv)) 

673 

674 formattedCat = self._formatCatalog(stdCat, offsets) 

675 

676 # Write the master schema 

677 dataId = self.indexer.makeDataId('master_schema', 

678 datasetConfig.ref_dataset_name) 

679 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

680 addRefCatMetadata(masterCat) 

681 butler.put(masterCat, 'ref_cat', dataId=dataId) 

682 

683 # Break up the pixels using a histogram 

684 h, rev = esutil.stat.histogram(indices, rev=True) 

685 gd, = np.where(h > 0) 

686 selected = np.zeros(len(formattedCat), dtype=np.bool) 

687 for i in gd: 

688 i1a = rev[rev[i]: rev[i + 1]] 

689 

690 # the formattedCat afwTable can only be indexed with boolean arrays, 

691 # and not numpy index arrays (see DM-16497). This little trick 

692 # converts the index array into a boolean array 

693 selected[:] = False 

694 selected[i1a] = True 

695 

696 # Write the individual pixel 

697 dataId = self.indexer.makeDataId(indices[i1a[0]], 

698 datasetConfig.ref_dataset_name) 

699 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

700 

701 # And save the dataset configuration 

702 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

703 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

704 

705 self.log.info("Done outputting standard stars.") 

706 

707 def _formatCatalog(self, fgcmStarCat, offsets): 

708 """ 

709 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

710 

711 Parameters 

712 ---------- 

713 fgcmStarCat: `lsst.afw.Table.SimpleCatalog` 

714 SimpleCatalog as output by fgcmcal 

715 offsets: `list` with len(self.bands) entries 

716 Zeropoint offsets to apply 

717 

718 Returns 

719 ------- 

720 formattedCat: `lsst.afw.table.SimpleCatalog` 

721 SimpleCatalog suitable for using as a reference catalog 

722 """ 

723 

724 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

725 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands, 

726 addCentroid=False, 

727 addIsResolved=True, 

728 coordErrDim=0) 

729 sourceMapper.addMinimalSchema(minSchema) 

730 for band in self.bands: 

731 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

732 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

733 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

734 

735 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

736 formattedCat.reserve(len(fgcmStarCat)) 

737 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

738 

739 # Note that we don't have to set `resolved` because the default is False 

740 

741 for b, band in enumerate(self.bands): 

742 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

743 # We want fluxes in nJy from calibrated AB magnitudes 

744 # (after applying offset). Updated after RFC-549 and RFC-575. 

745 flux = (mag*units.ABmag).to_value(units.nJy) 

746 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

747 

748 formattedCat['%s_flux' % (band)][:] = flux 

749 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

750 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

751 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

752 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

753 

754 addRefCatMetadata(formattedCat) 

755 

756 return formattedCat 

757 

758 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None): 

759 """ 

760 Output the zeropoints in fgcm_photoCalib format. 

761 

762 Parameters 

763 ---------- 

764 butler: `lsst.daf.persistence.Butler` 

765 zptCat: `lsst.afw.table.BaseCatalog` 

766 FGCM zeropoint catalog from `FgcmFitCycleTask` 

767 visitCat: `lsst.afw.table.BaseCatalog` 

768 FGCM visitCat from `FgcmBuildStarsTask` 

769 offsets: `numpy.array` 

770 Float array of absolute calibration offsets, one for each filter 

771 tract: `int`, optional 

772 Tract number to output. Default is None (global calibration) 

773 """ 

774 

775 if tract is None: 

776 datasetType = 'fgcm_photoCalib' 

777 else: 

778 datasetType = 'fgcm_tract_photoCalib' 

779 

780 self.log.info("Outputting %s objects" % (datasetType)) 

781 

782 # Select visit/ccds where we have a calibration 

783 # This includes ccds where we were able to interpolate from neighboring 

784 # ccds. 

785 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

786 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD'] 

787 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) & 

788 (zptCat['fgcmZptVar'] > 0.0)) 

789 

790 # We also select the "best" calibrations, avoiding interpolation. These 

791 # are only used for mapping filternames 

792 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) & 

793 (zptCat['fgcmZptVar'] > 0.0)) 

794 

795 # Log warnings for any visit which has no valid zeropoints 

796 badVisits = np.unique(zptCat['visit'][~selected]) 

797 goodVisits = np.unique(zptCat['visit'][selected]) 

798 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

799 for allBadVisit in allBadVisits: 

800 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}') 

801 

802 # Get the mapping from filtername to dataId filter name, empirically 

803 filterMapping = {} 

804 nFound = 0 

805 for rec in zptCat[selected_best]: 

806 if rec['filtername'] in filterMapping: 

807 continue 

808 dataId = {self.visitDataRefName: int(rec['visit']), 

809 self.ccdDataRefName: int(rec['ccd'])} 

810 dataRef = butler.dataRef('raw', dataId=dataId) 

811 filterMapping[rec['filtername']] = dataRef.dataId['filter'] 

812 nFound += 1 

813 if nFound == len(self.filterMap): 

814 break 

815 

816 # Get a mapping from filtername to the offsets 

817 offsetMapping = {} 

818 for f in self.filterMap: 

819 # Not every filter in the map will necesarily have a band. 

820 if self.filterMap[f] in self.bands: 

821 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])] 

822 

823 # Get a mapping from "ccd" to the ccd index used for the scaling 

824 camera = butler.get('camera') 

825 ccdMapping = {} 

826 for ccdIndex, detector in enumerate(camera): 

827 ccdMapping[detector.getId()] = ccdIndex 

828 

829 # And a mapping to get the flat-field scaling values 

830 scalingMapping = {} 

831 for rec in visitCat: 

832 scalingMapping[rec['visit']] = rec['scaling'] 

833 

834 if self.config.doComposeWcsJacobian: 

835 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

836 

837 for rec in zptCat[selected]: 

838 

839 # Retrieve overall scaling 

840 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]] 

841 

842 # The postCalibrationOffset describe any zeropoint offsets 

843 # to apply after the fgcm calibration. The first part comes 

844 # from the reference catalog match (used in testing). The 

845 # second part comes from the mean chromatic correction 

846 # (if configured). 

847 postCalibrationOffset = offsetMapping[rec['filtername']] 

848 if self.config.doApplyMeanChromaticCorrection: 

849 postCalibrationOffset += rec['fgcmDeltaChrom'] 

850 

851 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

852 rec['fgcmfZptChebXyMax']) 

853 # Convert from FGCM AB to nJy 

854 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

855 rec['fgcmfZptChebXyMax'], 

856 offset=postCalibrationOffset, 

857 scaling=scaling) 

858 

859 if self.config.doComposeWcsJacobian: 

860 

861 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']], 

862 fgcmSuperStarField, 

863 fgcmZptField]) 

864 else: 

865 # The photoCalib is just the product of the fgcmSuperStarField and the 

866 # fgcmZptField 

867 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

868 

869 # The "mean" calibration will be set to the center of the ccd for reference 

870 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

871 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

872 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

873 calibrationErr=calibErr, 

874 calibration=fgcmField, 

875 isConstant=False) 

876 

877 if tract is None: 

878 butler.put(photoCalib, datasetType, 

879 dataId={self.visitDataRefName: int(rec['visit']), 

880 self.ccdDataRefName: int(rec['ccd']), 

881 'filter': filterMapping[rec['filtername']]}) 

882 else: 

883 butler.put(photoCalib, datasetType, 

884 dataId={self.visitDataRefName: int(rec['visit']), 

885 self.ccdDataRefName: int(rec['ccd']), 

886 'filter': filterMapping[rec['filtername']], 

887 'tract': tract}) 

888 

889 self.log.info("Done outputting %s objects" % (datasetType)) 

890 

891 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

892 """ 

893 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

894 and scaling. 

895 

896 Parameters 

897 ---------- 

898 coefficients: `numpy.array` 

899 Flattened array of chebyshev coefficients 

900 xyMax: `list` of length 2 

901 Maximum x and y of the chebyshev bounding box 

902 offset: `float`, optional 

903 Absolute calibration offset. Default is 0.0 

904 scaling: `float`, optional 

905 Flat scaling value from fgcmBuildStars. Default is 1.0 

906 

907 Returns 

908 ------- 

909 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

910 """ 

911 

912 orderPlus1 = int(np.sqrt(coefficients.size)) 

913 pars = np.zeros((orderPlus1, orderPlus1)) 

914 

915 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

916 lsst.geom.Point2I(*xyMax)) 

917 

918 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) * 

919 (10.**(offset/-2.5))*scaling) 

920 

921 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

922 

923 return boundedField 

924 

925 def _outputAtmospheres(self, butler, atmCat, tract=None): 

926 """ 

927 Output the atmospheres. 

928 

929 Parameters 

930 ---------- 

931 butler: `lsst.daf.persistence.Butler` 

932 atmCat: `lsst.afw.table.BaseCatalog` 

933 FGCM atmosphere parameter catalog from fgcmFitCycleTask 

934 tract: `int`, optional 

935 Tract number to output. Default is None (global calibration) 

936 """ 

937 

938 self.log.info("Outputting atmosphere transmissions") 

939 

940 # First, we need to grab the look-up table and key info 

941 lutCat = butler.get('fgcmLookUpTable') 

942 

943 atmosphereTableName = lutCat[0]['tablename'] 

944 elevation = lutCat[0]['elevation'] 

945 atmLambda = lutCat[0]['atmLambda'] 

946 lutCat = None 

947 

948 # Make the atmosphere table if possible 

949 try: 

950 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

951 atmTable.loadTable() 

952 except IOError: 

953 atmTable = None 

954 

955 if atmTable is None: 

956 # Try to use MODTRAN instead 

957 try: 

958 modGen = fgcm.ModtranGenerator(elevation) 

959 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

960 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

961 except (ValueError, IOError) as e: 

962 raise RuntimeError("FGCM look-up-table generated with modtran, " 

963 "but modtran not configured to run.") from e 

964 

965 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

966 

967 for i, visit in enumerate(atmCat['visit']): 

968 if atmTable is not None: 

969 # Interpolate the atmosphere table 

970 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

971 pwv=atmCat[i]['pwv'], 

972 o3=atmCat[i]['o3'], 

973 tau=atmCat[i]['tau'], 

974 alpha=atmCat[i]['alpha'], 

975 zenith=zenith[i], 

976 ctranslamstd=[atmCat[i]['cTrans'], 

977 atmCat[i]['lamStd']]) 

978 else: 

979 # Run modtran 

980 modAtm = modGen(pmb=atmCat[i]['pmb'], 

981 pwv=atmCat[i]['pwv'], 

982 o3=atmCat[i]['o3'], 

983 tau=atmCat[i]['tau'], 

984 alpha=atmCat[i]['alpha'], 

985 zenith=zenith[i], 

986 lambdaRange=lambdaRange, 

987 lambdaStep=lambdaStep, 

988 ctranslamstd=[atmCat[i]['cTrans'], 

989 atmCat[i]['lamStd']]) 

990 atmVals = modAtm['COMBINED'] 

991 

992 # Now need to create something to persist... 

993 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

994 wavelengths=atmLambda, 

995 throughputAtMin=atmVals[0], 

996 throughputAtMax=atmVals[-1]) 

997 

998 if tract is None: 

999 butler.put(curve, "transmission_atmosphere_fgcm", 

1000 dataId={self.visitDataRefName: visit}) 

1001 else: 

1002 butler.put(curve, "transmission_atmosphere_fgcm_tract", 

1003 dataId={self.visitDataRefName: visit, 

1004 'tract': tract}) 

1005 

1006 self.log.info("Done outputting atmosphere transmissions")