Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34 

35import sys 

36import traceback 

37import copy 

38 

39import numpy as np 

40import healpy as hp 

41import esutil 

42from astropy import units 

43 

44import lsst.pex.config as pexConfig 

45import lsst.pipe.base as pipeBase 

46from lsst.afw.image import TransmissionCurve 

47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

48from lsst.pipe.tasks.photoCal import PhotoCalTask 

49import lsst.geom 

50import lsst.afw.image as afwImage 

51import lsst.afw.math as afwMath 

52import lsst.afw.table as afwTable 

53from lsst.meas.algorithms import IndexerRegistry 

54from lsst.meas.algorithms import DatasetConfig 

55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

56 

57from .utilities import computeApproxPixelAreaFields 

58 

59import fgcm 

60 

61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

62 

63 

64class FgcmOutputProductsConfig(pexConfig.Config): 

65 """Config for FgcmOutputProductsTask""" 

66 

67 cycleNumber = pexConfig.Field( 

68 doc="Final fit cycle from FGCM fit", 

69 dtype=int, 

70 default=None, 

71 ) 

72 

73 # The following fields refer to calibrating from a reference 

74 # catalog, but in the future this might need to be expanded 

75 doReferenceCalibration = pexConfig.Field( 

76 doc=("Transfer 'absolute' calibration from reference catalog? " 

77 "This afterburner step is unnecessary if reference stars " 

78 "were used in the full fit in FgcmFitCycleTask."), 

79 dtype=bool, 

80 default=False, 

81 ) 

82 doRefcatOutput = pexConfig.Field( 

83 doc="Output standard stars in reference catalog format", 

84 dtype=bool, 

85 default=True, 

86 ) 

87 doAtmosphereOutput = pexConfig.Field( 

88 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

89 dtype=bool, 

90 default=True, 

91 ) 

92 doZeropointOutput = pexConfig.Field( 

93 doc="Output zeropoints in fgcm_photoCalib format", 

94 dtype=bool, 

95 default=True, 

96 ) 

97 doComposeWcsJacobian = pexConfig.Field( 

98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

99 dtype=bool, 

100 default=True, 

101 ) 

102 refObjLoader = pexConfig.ConfigurableField( 

103 target=LoadIndexedReferenceObjectsTask, 

104 doc="reference object loader for 'absolute' photometric calibration", 

105 ) 

106 photoCal = pexConfig.ConfigurableField( 

107 target=PhotoCalTask, 

108 doc="task to perform 'absolute' calibration", 

109 ) 

110 referencePixelizationNside = pexConfig.Field( 

111 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

112 dtype=int, 

113 default=64, 

114 ) 

115 referencePixelizationMinStars = pexConfig.Field( 

116 doc=("Minimum number of stars per healpix pixel to select for comparison" 

117 "to the specified reference catalog"), 

118 dtype=int, 

119 default=200, 

120 ) 

121 referenceMinMatch = pexConfig.Field( 

122 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

123 dtype=int, 

124 default=50, 

125 ) 

126 referencePixelizationNPixels = pexConfig.Field( 

127 doc=("Number of healpix pixels to sample to do comparison. " 

128 "Doing too many will take a long time and not yield any more " 

129 "precise results because the final number is the median offset " 

130 "(per band) from the set of pixels."), 

131 dtype=int, 

132 default=100, 

133 ) 

134 datasetConfig = pexConfig.ConfigField( 

135 dtype=DatasetConfig, 

136 doc="Configuration for writing/reading ingested catalog", 

137 ) 

138 

139 def setDefaults(self): 

140 pexConfig.Config.setDefaults(self) 

141 

142 # In order to transfer the "absolute" calibration from a reference 

143 # catalog to the relatively calibrated FGCM standard stars (one number 

144 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

145 # pixels. These basic settings ensure that only well-measured, good stars 

146 # from the source and reference catalogs are used for the matching. 

147 

148 # applyColorTerms needs to be False if doReferenceCalibration is False, 

149 # as is the new default after DM-16702 

150 self.photoCal.applyColorTerms = False 

151 self.photoCal.fluxField = 'instFlux' 

152 self.photoCal.magErrFloor = 0.003 

153 self.photoCal.match.referenceSelection.doSignalToNoise = True 

154 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

155 self.photoCal.match.sourceSelection.doSignalToNoise = True 

156 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

157 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

158 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

159 self.photoCal.match.sourceSelection.doFlags = True 

160 self.photoCal.match.sourceSelection.flags.good = [] 

161 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

162 self.photoCal.match.sourceSelection.doUnresolved = False 

163 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

164 self.datasetConfig.format_version = 1 

165 

166 

167class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

168 """Subclass of TaskRunner for fgcmOutputProductsTask 

169 

170 fgcmOutputProductsTask.run() takes one argument, the butler, and 

171 does not run on any data in the repository. 

172 This runner does not use any parallelization. 

173 """ 

174 

175 @staticmethod 

176 def getTargetList(parsedCmd): 

177 """ 

178 Return a list with one element, the butler. 

179 """ 

180 return [parsedCmd.butler] 

181 

182 def __call__(self, butler): 

183 """ 

184 Parameters 

185 ---------- 

186 butler: `lsst.daf.persistence.Butler` 

187 

188 Returns 

189 ------- 

190 exitStatus: `list` with `pipeBase.Struct` 

191 exitStatus (0: success; 1: failure) 

192 if self.doReturnResults also 

193 results (`np.array` with absolute zeropoint offsets) 

194 """ 

195 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

196 

197 exitStatus = 0 

198 if self.doRaise: 

199 results = task.runDataRef(butler) 

200 else: 

201 try: 

202 results = task.runDataRef(butler) 

203 except Exception as e: 

204 exitStatus = 1 

205 task.log.fatal("Failed: %s" % e) 

206 if not isinstance(e, pipeBase.TaskError): 

207 traceback.print_exc(file=sys.stderr) 

208 

209 task.writeMetadata(butler) 

210 

211 if self.doReturnResults: 

212 # The results here are the zeropoint offsets for each band 

213 return [pipeBase.Struct(exitStatus=exitStatus, 

214 results=results)] 

215 else: 

216 return [pipeBase.Struct(exitStatus=exitStatus)] 

217 

218 def run(self, parsedCmd): 

219 """ 

220 Run the task, with no multiprocessing 

221 

222 Parameters 

223 ---------- 

224 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

225 """ 

226 

227 resultList = [] 

228 

229 if self.precall(parsedCmd): 

230 targetList = self.getTargetList(parsedCmd) 

231 # make sure that we only get 1 

232 resultList = self(targetList[0]) 

233 

234 return resultList 

235 

236 

237class FgcmOutputProductsTask(pipeBase.CmdLineTask): 

238 """ 

239 Output products from FGCM global calibration. 

240 """ 

241 

242 ConfigClass = FgcmOutputProductsConfig 

243 RunnerClass = FgcmOutputProductsRunner 

244 _DefaultName = "fgcmOutputProducts" 

245 

246 def __init__(self, butler=None, **kwargs): 

247 """ 

248 Instantiate an fgcmOutputProductsTask. 

249 

250 Parameters 

251 ---------- 

252 butler : `lsst.daf.persistence.Butler` 

253 """ 

254 

255 pipeBase.CmdLineTask.__init__(self, **kwargs) 

256 

257 if self.config.doReferenceCalibration: 

258 # We need the ref obj loader to get the flux field 

259 self.makeSubtask("refObjLoader", butler=butler) 

260 

261 if self.config.doRefcatOutput: 

262 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

263 self.config.datasetConfig.indexer.active) 

264 

265 # no saving of metadata for now 

266 def _getMetadataName(self): 

267 return None 

268 

269 @pipeBase.timeMethod 

270 def runDataRef(self, butler): 

271 """ 

272 Make FGCM output products for use in the stack 

273 

274 Parameters 

275 ---------- 

276 butler: `lsst.daf.persistence.Butler` 

277 cycleNumber: `int` 

278 Final fit cycle number, override config. 

279 

280 Returns 

281 ------- 

282 offsets: `lsst.pipe.base.Struct` 

283 A structure with array of zeropoint offsets 

284 

285 Raises 

286 ------ 

287 RuntimeError: Raised if butler cannot find fgcmBuildStars_config, or 

288 fgcmFitCycle_config, or fgcmAtmosphereParameters (and 

289 `self.config.doAtmosphereOutput` is true), or fgcmStandardStars (and 

290 `self.config.doReferenceCalibration or `self.config.doRefcatOutput` 

291 is true), or fgcmZeropoints (and self.config.doZeropointOutput is true). 

292 Also will raise if the fgcmFitCycle_config does not refer to the 

293 final fit cycle. 

294 """ 

295 

296 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

297 # the visit and ccd dataset tags 

298 if not butler.datasetExists('fgcmBuildStars_config'): 

299 raise RuntimeError("Cannot find fgcmBuildStars_config, which is prereq for fgcmOutputProducts") 

300 

301 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

302 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

303 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

304 self.filterMap = fgcmBuildStarsConfig.filterMap 

305 

306 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

307 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

308 "in fgcmBuildStarsTask.") 

309 

310 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

311 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

312 

313 # Make sure that the fit config exists, to retrieve bands and other info 

314 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber): 

315 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) + 

316 "which is required for fgcmOutputProducts.") 

317 

318 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber) 

319 self.configBands = fitCycleConfig.bands 

320 

321 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration: 

322 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

323 "fitCycleConfig.doReferenceCalibration") 

324 

325 # And make sure that the atmosphere was output properly 

326 if (self.config.doAtmosphereOutput and 

327 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

328 raise RuntimeError("Atmosphere parameters are missing for cycle %d." % 

329 (self.config.cycleNumber)) 

330 

331 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and 

332 (not butler.datasetExists('fgcmStandardStars', 

333 fgcmcycle=self.config.cycleNumber))): 

334 raise RuntimeError("Standard stars are missing for cycle %d." % 

335 (self.config.cycleNumber)) 

336 

337 if (self.config.doZeropointOutput and 

338 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

339 raise RuntimeError("Zeropoints are missing for cycle %d." % 

340 (self.config.cycleNumber)) 

341 

342 # And make sure this is the last cycle 

343 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1): 

344 raise RuntimeError("The task fgcmOutputProducts should only be run" 

345 "on the final fit cycle products") 

346 

347 if self.config.doReferenceCalibration or self.config.doRefcatOutput: 

348 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber) 

349 md = stdCat.getMetadata() 

350 self.bands = md.getArray('BANDS') 

351 else: 

352 stdCat = None 

353 self.bands = self.configBands 

354 

355 if self.config.doReferenceCalibration: 

356 offsets = self._computeReferenceOffsets(butler, stdCat) 

357 else: 

358 offsets = np.zeros(len(self.bands)) 

359 

360 # Output the standard stars in stack format 

361 if self.config.doRefcatOutput: 

362 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig) 

363 

364 del stdCat 

365 

366 # Output the gray zeropoints 

367 if self.config.doZeropointOutput: 

368 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber) 

369 visitCat = butler.get('fgcmVisitCatalog') 

370 

371 self._outputZeropoints(butler, zptCat, visitCat, offsets) 

372 

373 # Output the atmospheres 

374 if self.config.doAtmosphereOutput: 

375 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber) 

376 self._outputAtmospheres(butler, atmCat) 

377 

378 # We return the zp offsets 

379 return pipeBase.Struct(offsets=offsets) 

380 

381 def generateTractOutputProducts(self, butler, tract, 

382 visitCat, zptCat, atmCat, stdCat, 

383 fgcmBuildStarsConfig, fgcmFitCycleConfig): 

384 """ 

385 Generate the output products for a given tract, as specified in the config. 

386 

387 This method is here to have an alternate entry-point for 

388 FgcmCalibrateTract. 

389 

390 Parameters 

391 ---------- 

392 butler: `lsst.daf.persistence.Butler` 

393 tract: `int` 

394 Tract number 

395 visitCat: `lsst.afw.table.BaseCatalog` 

396 FGCM visitCat from `FgcmBuildStarsTask` 

397 zptCat: `lsst.afw.table.BaseCatalog` 

398 FGCM zeropoint catalog from `FgcmFitCycleTask` 

399 atmCat: `lsst.afw.table.BaseCatalog` 

400 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

401 stdCat: `lsst.afw.table.SimpleCatalog` 

402 FGCM standard star catalog from `FgcmFitCycleTask` 

403 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig` 

404 Configuration object from `FgcmBuildStarsTask` 

405 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig` 

406 Configuration object from `FgcmFitCycleTask` 

407 """ 

408 

409 self.configBands = fgcmFitCycleConfig.bands 

410 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

411 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

412 self.filterMap = fgcmBuildStarsConfig.filterMap 

413 

414 if stdCat is not None: 

415 md = stdCat.getMetadata() 

416 self.bands = md.getArray('BANDS') 

417 else: 

418 self.bands = self.configBands 

419 

420 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration: 

421 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

422 "fitCycleConfig.doReferenceCalibration") 

423 

424 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

425 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

426 "in fgcmBuildStarsTask.") 

427 

428 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

429 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

430 

431 if self.config.doReferenceCalibration: 

432 offsets = self._computeReferenceOffsets(butler, stdCat) 

433 else: 

434 offsets = np.zeros(len(self.bands)) 

435 

436 if self.config.doRefcatOutput: 

437 # Create a special config that has the tract number in it 

438 datasetConfig = copy.copy(self.config.datasetConfig) 

439 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

440 tract) 

441 self._outputStandardStars(butler, stdCat, offsets, datasetConfig) 

442 

443 if self.config.doZeropointOutput: 

444 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract) 

445 

446 if self.config.doAtmosphereOutput: 

447 self._outputAtmospheres(butler, atmCat, tract=tract) 

448 

449 return pipeBase.Struct(offsets=offsets) 

450 

451 def _computeReferenceOffsets(self, butler, stdCat): 

452 """ 

453 Compute offsets relative to a reference catalog. 

454 

455 This method splits the star catalog into healpix pixels 

456 and computes the calibration transfer for a sample of 

457 these pixels to approximate the 'absolute' calibration 

458 values (on for each band) to apply to transfer the 

459 absolute scale. 

460 

461 Parameters 

462 ---------- 

463 butler: `lsst.daf.persistence.Butler` 

464 stdCat: `lsst.afw.table.SimpleCatalog` 

465 FGCM standard stars 

466 

467 Returns 

468 ------- 

469 offsets: `numpy.array` of floats 

470 Per band zeropoint offsets 

471 """ 

472 

473 # Only use stars that are observed in all the bands that were actually used 

474 # This will ensure that we use the same healpix pixels for the absolute 

475 # calibration of each band. 

476 minObs = stdCat['ngood'].min(axis=1) 

477 

478 goodStars = (minObs >= 1) 

479 stdCat = stdCat[goodStars] 

480 

481 self.log.info("Found %d stars with at least 1 good observation in each band" % 

482 (len(stdCat))) 

483 

484 # We have to make a table for each pixel with flux/fluxErr 

485 # This is a temporary table generated for input to the photoCal task. 

486 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

487 # have had chromatic corrections applied to get to the standard system 

488 # specified by the atmosphere/instrumental parameters), nor are they 

489 # in Jansky (since they don't have a proper absolute calibration: the overall 

490 # zeropoint is estimated from the telescope size, etc.) 

491 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

492 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

493 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

494 doc="instrumental flux (counts)") 

495 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

496 doc="instrumental flux error (counts)") 

497 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

498 type='Flag', 

499 doc="bad flag") 

500 

501 # Split up the stars 

502 # Note that there is an assumption here that the ra/dec coords stored 

503 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

504 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

505 theta = np.pi/2. - stdCat['coord_dec'] 

506 phi = stdCat['coord_ra'] 

507 

508 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

509 h, rev = esutil.stat.histogram(ipring, rev=True) 

510 

511 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

512 

513 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

514 (gdpix.size, 

515 self.config.referencePixelizationNside, 

516 self.config.referencePixelizationMinStars)) 

517 

518 if gdpix.size < self.config.referencePixelizationNPixels: 

519 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

520 (gdpix.size, self.config.referencePixelizationNPixels)) 

521 else: 

522 # Sample out the pixels we want to use 

523 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

524 

525 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

526 ('nstar', 'i4', len(self.bands)), 

527 ('nmatch', 'i4', len(self.bands)), 

528 ('zp', 'f4', len(self.bands)), 

529 ('zpErr', 'f4', len(self.bands))]) 

530 results['hpix'] = ipring[rev[rev[gdpix]]] 

531 

532 # We need a boolean index to deal with catalogs... 

533 selected = np.zeros(len(stdCat), dtype=np.bool) 

534 

535 refFluxFields = [None]*len(self.bands) 

536 

537 for p, pix in enumerate(gdpix): 

538 i1a = rev[rev[pix]: rev[pix + 1]] 

539 

540 # the stdCat afwTable can only be indexed with boolean arrays, 

541 # and not numpy index arrays (see DM-16497). This little trick 

542 # converts the index array into a boolean array 

543 selected[:] = False 

544 selected[i1a] = True 

545 

546 for b, band in enumerate(self.bands): 

547 

548 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat, 

549 selected, refFluxFields) 

550 results['nstar'][p, b] = len(i1a) 

551 results['nmatch'][p, b] = len(struct.arrays.refMag) 

552 results['zp'][p, b] = struct.zp 

553 results['zpErr'][p, b] = struct.sigma 

554 

555 # And compute the summary statistics 

556 offsets = np.zeros(len(self.bands)) 

557 

558 for b, band in enumerate(self.bands): 

559 # make configurable 

560 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch) 

561 offsets[b] = np.median(results['zp'][ok, b]) 

562 # use median absolute deviation to estimate Normal sigma 

563 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

564 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b])) 

565 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" % 

566 (band, offsets[b], madSigma)) 

567 

568 return offsets 

569 

570 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

571 b, band, stdCat, selected, refFluxFields): 

572 """ 

573 Compute the zeropoint offset between the fgcm stdCat and the reference 

574 stars for one pixel in one band 

575 

576 Parameters 

577 ---------- 

578 sourceMapper: `lsst.afw.table.SchemaMapper` 

579 Mapper to go from stdCat to calibratable catalog 

580 badStarKey: `lsst.afw.table.Key` 

581 Key for the field with bad stars 

582 b: `int` 

583 Index of the band in the star catalog 

584 band: `str` 

585 Name of band for reference catalog 

586 stdCat: `lsst.afw.table.SimpleCatalog` 

587 FGCM standard stars 

588 selected: `numpy.array(dtype=np.bool)` 

589 Boolean array of which stars are in the pixel 

590 refFluxFields: `list` 

591 List of names of flux fields for reference catalog 

592 """ 

593 

594 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

595 sourceCat.reserve(selected.sum()) 

596 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

597 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5)) 

598 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] * 

599 sourceCat['instFlux']) 

600 # Make sure we only use stars that have valid measurements 

601 # (This is perhaps redundant with requirements above that the 

602 # stars be observed in all bands, but it can't hurt) 

603 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0) 

604 for rec in sourceCat[badStar]: 

605 rec.set(badStarKey, True) 

606 

607 exposure = afwImage.ExposureF() 

608 exposure.setFilter(afwImage.Filter(band)) 

609 

610 if refFluxFields[b] is None: 

611 # Need to find the flux field in the reference catalog 

612 # to work around limitations of DirectMatch in PhotoCal 

613 ctr = stdCat[0].getCoord() 

614 rad = 0.05*lsst.geom.degrees 

615 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band) 

616 refFluxFields[b] = refDataTest.fluxField 

617 

618 # Make a copy of the config so that we can modify it 

619 calConfig = copy.copy(self.config.photoCal.value) 

620 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b] 

621 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err' 

622 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

623 config=calConfig, 

624 schema=sourceCat.getSchema()) 

625 

626 struct = calTask.run(exposure, sourceCat) 

627 

628 return struct 

629 

630 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig): 

631 """ 

632 Output standard stars in indexed reference catalog format. 

633 

634 Parameters 

635 ---------- 

636 butler: `lsst.daf.persistence.Butler` 

637 stdCat: `lsst.afw.table.SimpleCatalog` 

638 FGCM standard star catalog from fgcmFitCycleTask 

639 offsets: `numpy.array` of floats 

640 Per band zeropoint offsets 

641 datasetConfig: `lsst.meas.algorithms.DatasetConfig` 

642 Config for reference dataset 

643 """ 

644 

645 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

646 

647 # We determine the conversion from the native units (typically radians) to 

648 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

649 # numpy arrays rather than Angles, which would we approximately 600x slower. 

650 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

651 # (as Angles) for input 

652 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

653 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv, 

654 stdCat['coord_dec']*conv)) 

655 

656 formattedCat = self._formatCatalog(stdCat, offsets) 

657 

658 # Write the master schema 

659 dataId = self.indexer.makeDataId('master_schema', 

660 datasetConfig.ref_dataset_name) 

661 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

662 addRefCatMetadata(masterCat) 

663 butler.put(masterCat, 'ref_cat', dataId=dataId) 

664 

665 # Break up the pixels using a histogram 

666 h, rev = esutil.stat.histogram(indices, rev=True) 

667 gd, = np.where(h > 0) 

668 selected = np.zeros(len(formattedCat), dtype=np.bool) 

669 for i in gd: 

670 i1a = rev[rev[i]: rev[i + 1]] 

671 

672 # the formattedCat afwTable can only be indexed with boolean arrays, 

673 # and not numpy index arrays (see DM-16497). This little trick 

674 # converts the index array into a boolean array 

675 selected[:] = False 

676 selected[i1a] = True 

677 

678 # Write the individual pixel 

679 dataId = self.indexer.makeDataId(indices[i1a[0]], 

680 datasetConfig.ref_dataset_name) 

681 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

682 

683 # And save the dataset configuration 

684 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

685 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

686 

687 self.log.info("Done outputting standard stars.") 

688 

689 def _formatCatalog(self, fgcmStarCat, offsets): 

690 """ 

691 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

692 

693 Parameters 

694 ---------- 

695 fgcmStarCat: `lsst.afw.Table.SimpleCatalog` 

696 SimpleCatalog as output by fgcmcal 

697 offsets: `list` with len(self.bands) entries 

698 Zeropoint offsets to apply 

699 

700 Returns 

701 ------- 

702 formattedCat: `lsst.afw.table.SimpleCatalog` 

703 SimpleCatalog suitable for using as a reference catalog 

704 """ 

705 

706 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

707 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands, 

708 addCentroid=False, 

709 addIsResolved=True, 

710 coordErrDim=0) 

711 sourceMapper.addMinimalSchema(minSchema) 

712 for band in self.bands: 

713 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

714 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

715 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

716 

717 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

718 formattedCat.reserve(len(fgcmStarCat)) 

719 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

720 

721 # Note that we don't have to set `resolved` because the default is False 

722 

723 for b, band in enumerate(self.bands): 

724 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

725 # We want fluxes in nJy from calibrated AB magnitudes 

726 # (after applying offset). Updated after RFC-549 and RFC-575. 

727 flux = (mag*units.ABmag).to_value(units.nJy) 

728 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

729 

730 formattedCat['%s_flux' % (band)][:] = flux 

731 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

732 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

733 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

734 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

735 

736 addRefCatMetadata(formattedCat) 

737 

738 return formattedCat 

739 

740 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None): 

741 """ 

742 Output the zeropoints in fgcm_photoCalib format. 

743 

744 Parameters 

745 ---------- 

746 butler: `lsst.daf.persistence.Butler` 

747 zptCat: `lsst.afw.table.BaseCatalog` 

748 FGCM zeropoint catalog from `FgcmFitCycleTask` 

749 visitCat: `lsst.afw.table.BaseCatalog` 

750 FGCM visitCat from `FgcmBuildStarsTask` 

751 offsets: `numpy.array` 

752 Float array of absolute calibration offsets, one for each filter 

753 tract: `int`, optional 

754 Tract number to output. Default is None (global calibration) 

755 """ 

756 

757 if tract is None: 

758 datasetType = 'fgcm_photoCalib' 

759 else: 

760 datasetType = 'fgcm_tract_photoCalib' 

761 

762 self.log.info("Outputting %s objects" % (datasetType)) 

763 

764 # Select visit/ccds where we have a calibration 

765 # This includes ccds where we were able to interpolate from neighboring 

766 # ccds. 

767 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

768 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD'] 

769 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) & 

770 (zptCat['fgcmZptVar'] > 0.0)) 

771 

772 # We also select the "best" calibrations, avoiding interpolation. These 

773 # are only used for mapping filternames 

774 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) & 

775 (zptCat['fgcmZptVar'] > 0.0)) 

776 

777 # Log warnings for any visit which has no valid zeropoints 

778 badVisits = np.unique(zptCat['visit'][~selected]) 

779 goodVisits = np.unique(zptCat['visit'][selected]) 

780 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

781 for allBadVisit in allBadVisits: 

782 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}') 

783 

784 # Get the mapping from filtername to dataId filter name, empirically 

785 filterMapping = {} 

786 nFound = 0 

787 for rec in zptCat[selected_best]: 

788 if rec['filtername'] in filterMapping: 

789 continue 

790 dataId = {self.visitDataRefName: int(rec['visit']), 

791 self.ccdDataRefName: int(rec['ccd'])} 

792 dataRef = butler.dataRef('raw', dataId=dataId) 

793 filterMapping[rec['filtername']] = dataRef.dataId['filter'] 

794 nFound += 1 

795 if nFound == len(self.filterMap): 

796 break 

797 

798 # Get a mapping from filtername to the offsets 

799 offsetMapping = {} 

800 for f in self.filterMap: 

801 # Not every filter in the map will necesarily have a band. 

802 if self.filterMap[f] in self.bands: 

803 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])] 

804 

805 # Get a mapping from "ccd" to the ccd index used for the scaling 

806 camera = butler.get('camera') 

807 ccdMapping = {} 

808 for ccdIndex, detector in enumerate(camera): 

809 ccdMapping[detector.getId()] = ccdIndex 

810 

811 # And a mapping to get the flat-field scaling values 

812 scalingMapping = {} 

813 for rec in visitCat: 

814 scalingMapping[rec['visit']] = rec['scaling'] 

815 

816 if self.config.doComposeWcsJacobian: 

817 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

818 

819 for rec in zptCat[selected]: 

820 

821 # Retrieve overall scaling 

822 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]] 

823 

824 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

825 rec['fgcmfZptChebXyMax']) 

826 # Convert from FGCM AB to nJy 

827 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

828 rec['fgcmfZptChebXyMax'], 

829 offset=offsetMapping[rec['filtername']], 

830 scaling=scaling) 

831 

832 if self.config.doComposeWcsJacobian: 

833 

834 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']], 

835 fgcmSuperStarField, 

836 fgcmZptField]) 

837 else: 

838 # The photoCalib is just the product of the fgcmSuperStarField and the 

839 # fgcmZptField 

840 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

841 

842 # The "mean" calibration will be set to the center of the ccd for reference 

843 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

844 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

845 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

846 calibrationErr=calibErr, 

847 calibration=fgcmField, 

848 isConstant=False) 

849 

850 if tract is None: 

851 butler.put(photoCalib, datasetType, 

852 dataId={self.visitDataRefName: int(rec['visit']), 

853 self.ccdDataRefName: int(rec['ccd']), 

854 'filter': filterMapping[rec['filtername']]}) 

855 else: 

856 butler.put(photoCalib, datasetType, 

857 dataId={self.visitDataRefName: int(rec['visit']), 

858 self.ccdDataRefName: int(rec['ccd']), 

859 'filter': filterMapping[rec['filtername']], 

860 'tract': tract}) 

861 

862 self.log.info("Done outputting %s objects" % (datasetType)) 

863 

864 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

865 """ 

866 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

867 and scaling. 

868 

869 Parameters 

870 ---------- 

871 coefficients: `numpy.array` 

872 Flattened array of chebyshev coefficients 

873 xyMax: `list` of length 2 

874 Maximum x and y of the chebyshev bounding box 

875 offset: `float`, optional 

876 Absolute calibration offset. Default is 0.0 

877 scaling: `float`, optional 

878 Flat scaling value from fgcmBuildStars. Default is 1.0 

879 

880 Returns 

881 ------- 

882 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

883 """ 

884 

885 orderPlus1 = int(np.sqrt(coefficients.size)) 

886 pars = np.zeros((orderPlus1, orderPlus1)) 

887 

888 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

889 lsst.geom.Point2I(*xyMax)) 

890 

891 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) * 

892 (10.**(offset/-2.5))*scaling) 

893 

894 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

895 

896 return boundedField 

897 

898 def _outputAtmospheres(self, butler, atmCat, tract=None): 

899 """ 

900 Output the atmospheres. 

901 

902 Parameters 

903 ---------- 

904 butler: `lsst.daf.persistence.Butler` 

905 atmCat: `lsst.afw.table.BaseCatalog` 

906 FGCM atmosphere parameter catalog from fgcmFitCycleTask 

907 tract: `int`, optional 

908 Tract number to output. Default is None (global calibration) 

909 """ 

910 

911 self.log.info("Outputting atmosphere transmissions") 

912 

913 # First, we need to grab the look-up table and key info 

914 lutCat = butler.get('fgcmLookUpTable') 

915 

916 atmosphereTableName = lutCat[0]['tablename'] 

917 elevation = lutCat[0]['elevation'] 

918 atmLambda = lutCat[0]['atmLambda'] 

919 lutCat = None 

920 

921 # Make the atmosphere table if possible 

922 try: 

923 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

924 atmTable.loadTable() 

925 except IOError: 

926 atmTable = None 

927 

928 if atmTable is None: 

929 # Try to use MODTRAN instead 

930 try: 

931 modGen = fgcm.ModtranGenerator(elevation) 

932 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

933 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

934 except (ValueError, IOError) as e: 

935 raise RuntimeError("FGCM look-up-table generated with modtran, " 

936 "but modtran not configured to run.") from e 

937 

938 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

939 

940 for i, visit in enumerate(atmCat['visit']): 

941 if atmTable is not None: 

942 # Interpolate the atmosphere table 

943 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

944 pwv=atmCat[i]['pwv'], 

945 o3=atmCat[i]['o3'], 

946 tau=atmCat[i]['tau'], 

947 alpha=atmCat[i]['alpha'], 

948 zenith=zenith[i], 

949 ctranslamstd=[atmCat[i]['cTrans'], 

950 atmCat[i]['lamStd']]) 

951 else: 

952 # Run modtran 

953 modAtm = modGen(pmb=atmCat[i]['pmb'], 

954 pwv=atmCat[i]['pwv'], 

955 o3=atmCat[i]['o3'], 

956 tau=atmCat[i]['tau'], 

957 alpha=atmCat[i]['alpha'], 

958 zenith=zenith[i], 

959 lambdaRange=lambdaRange, 

960 lambdaStep=lambdaStep, 

961 ctranslamstd=[atmCat[i]['cTrans'], 

962 atmCat[i]['lamStd']]) 

963 atmVals = modAtm['COMBINED'] 

964 

965 # Now need to create something to persist... 

966 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

967 wavelengths=atmLambda, 

968 throughputAtMin=atmVals[0], 

969 throughputAtMax=atmVals[-1]) 

970 

971 if tract is None: 

972 butler.put(curve, "transmission_atmosphere_fgcm", 

973 dataId={self.visitDataRefName: visit}) 

974 else: 

975 butler.put(curve, "transmission_atmosphere_fgcm_tract", 

976 dataId={self.visitDataRefName: visit, 

977 'tract': tract}) 

978 

979 self.log.info("Done outputting atmosphere transmissions")