Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Make the final fgcmcal output products. 

24 

25This task takes the final output from fgcmFitCycle and produces the following 

26outputs for use in the DM stack: the FGCM standard stars in a reference 

27catalog format; the model atmospheres in "transmission_atmosphere_fgcm" 

28format; and the zeropoints in "fgcm_photoCalib" format. Optionally, the 

29task can transfer the 'absolute' calibration from a reference catalog 

30to put the fgcm standard stars in units of Jansky. This is accomplished 

31by matching stars in a sample of healpix pixels, and applying the median 

32offset per band. 

33""" 

34 

35import sys 

36import traceback 

37import copy 

38 

39import numpy as np 

40import healpy as hp 

41import esutil 

42from astropy import units 

43 

44import lsst.pex.config as pexConfig 

45import lsst.pipe.base as pipeBase 

46from lsst.afw.image import TransmissionCurve 

47from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask 

48from lsst.pipe.tasks.photoCal import PhotoCalTask 

49import lsst.geom 

50import lsst.afw.image as afwImage 

51import lsst.afw.math as afwMath 

52import lsst.afw.table as afwTable 

53from lsst.meas.algorithms import IndexerRegistry 

54from lsst.meas.algorithms import DatasetConfig 

55from lsst.meas.algorithms.ingestIndexReferenceTask import addRefCatMetadata 

56 

57from .utilities import computeApproxPixelAreaFields 

58 

59import fgcm 

60 

61__all__ = ['FgcmOutputProductsConfig', 'FgcmOutputProductsTask', 'FgcmOutputProductsRunner'] 

62 

63 

64class FgcmOutputProductsConfig(pexConfig.Config): 

65 """Config for FgcmOutputProductsTask""" 

66 

67 cycleNumber = pexConfig.Field( 

68 doc="Final fit cycle from FGCM fit", 

69 dtype=int, 

70 default=None, 

71 ) 

72 

73 # The following fields refer to calibrating from a reference 

74 # catalog, but in the future this might need to be expanded 

75 doReferenceCalibration = pexConfig.Field( 

76 doc=("Transfer 'absolute' calibration from reference catalog? " 

77 "This afterburner step is unnecessary if reference stars " 

78 "were used in the full fit in FgcmFitCycleTask."), 

79 dtype=bool, 

80 default=False, 

81 ) 

82 doRefcatOutput = pexConfig.Field( 

83 doc="Output standard stars in reference catalog format", 

84 dtype=bool, 

85 default=True, 

86 ) 

87 doAtmosphereOutput = pexConfig.Field( 

88 doc="Output atmospheres in transmission_atmosphere_fgcm format", 

89 dtype=bool, 

90 default=True, 

91 ) 

92 doZeropointOutput = pexConfig.Field( 

93 doc="Output zeropoints in fgcm_photoCalib format", 

94 dtype=bool, 

95 default=True, 

96 ) 

97 doComposeWcsJacobian = pexConfig.Field( 

98 doc="Compose Jacobian of WCS with fgcm calibration for output photoCalib?", 

99 dtype=bool, 

100 default=True, 

101 ) 

102 refObjLoader = pexConfig.ConfigurableField( 

103 target=LoadIndexedReferenceObjectsTask, 

104 doc="reference object loader for 'absolute' photometric calibration", 

105 ) 

106 photoCal = pexConfig.ConfigurableField( 

107 target=PhotoCalTask, 

108 doc="task to perform 'absolute' calibration", 

109 ) 

110 referencePixelizationNside = pexConfig.Field( 

111 doc="Healpix nside to pixelize catalog to compare to reference catalog", 

112 dtype=int, 

113 default=64, 

114 ) 

115 referencePixelizationMinStars = pexConfig.Field( 

116 doc=("Minimum number of stars per healpix pixel to select for comparison" 

117 "to the specified reference catalog"), 

118 dtype=int, 

119 default=200, 

120 ) 

121 referenceMinMatch = pexConfig.Field( 

122 doc="Minimum number of stars matched to reference catalog to be used in statistics", 

123 dtype=int, 

124 default=50, 

125 ) 

126 referencePixelizationNPixels = pexConfig.Field( 

127 doc=("Number of healpix pixels to sample to do comparison. " 

128 "Doing too many will take a long time and not yield any more " 

129 "precise results because the final number is the median offset " 

130 "(per band) from the set of pixels."), 

131 dtype=int, 

132 default=100, 

133 ) 

134 datasetConfig = pexConfig.ConfigField( 

135 dtype=DatasetConfig, 

136 doc="Configuration for writing/reading ingested catalog", 

137 ) 

138 

139 def setDefaults(self): 

140 pexConfig.Config.setDefaults(self) 

141 

142 # In order to transfer the "absolute" calibration from a reference 

143 # catalog to the relatively calibrated FGCM standard stars (one number 

144 # per band), we use the PhotoCalTask to match stars in a sample of healpix 

145 # pixels. These basic settings ensure that only well-measured, good stars 

146 # from the source and reference catalogs are used for the matching. 

147 

148 # applyColorTerms needs to be False if doReferenceCalibration is False, 

149 # as is the new default after DM-16702 

150 self.photoCal.applyColorTerms = False 

151 self.photoCal.fluxField = 'instFlux' 

152 self.photoCal.magErrFloor = 0.003 

153 self.photoCal.match.referenceSelection.doSignalToNoise = True 

154 self.photoCal.match.referenceSelection.signalToNoise.minimum = 10.0 

155 self.photoCal.match.sourceSelection.doSignalToNoise = True 

156 self.photoCal.match.sourceSelection.signalToNoise.minimum = 10.0 

157 self.photoCal.match.sourceSelection.signalToNoise.fluxField = 'instFlux' 

158 self.photoCal.match.sourceSelection.signalToNoise.errField = 'instFluxErr' 

159 self.photoCal.match.sourceSelection.doFlags = True 

160 self.photoCal.match.sourceSelection.flags.good = [] 

161 self.photoCal.match.sourceSelection.flags.bad = ['flag_badStar'] 

162 self.photoCal.match.sourceSelection.doUnresolved = False 

163 self.datasetConfig.ref_dataset_name = 'fgcm_stars' 

164 self.datasetConfig.format_version = 1 

165 

166 

167class FgcmOutputProductsRunner(pipeBase.ButlerInitializedTaskRunner): 

168 """Subclass of TaskRunner for fgcmOutputProductsTask 

169 

170 fgcmOutputProductsTask.run() takes one argument, the butler, and 

171 does not run on any data in the repository. 

172 This runner does not use any parallelization. 

173 """ 

174 

175 @staticmethod 

176 def getTargetList(parsedCmd): 

177 """ 

178 Return a list with one element, the butler. 

179 """ 

180 return [parsedCmd.butler] 

181 

182 def __call__(self, butler): 

183 """ 

184 Parameters 

185 ---------- 

186 butler: `lsst.daf.persistence.Butler` 

187 

188 Returns 

189 ------- 

190 exitStatus: `list` with `pipeBase.Struct` 

191 exitStatus (0: success; 1: failure) 

192 if self.doReturnResults also 

193 results (`np.array` with absolute zeropoint offsets) 

194 """ 

195 task = self.TaskClass(butler=butler, config=self.config, log=self.log) 

196 

197 exitStatus = 0 

198 if self.doRaise: 

199 results = task.runDataRef(butler) 

200 else: 

201 try: 

202 results = task.runDataRef(butler) 

203 except Exception as e: 

204 exitStatus = 1 

205 task.log.fatal("Failed: %s" % e) 

206 if not isinstance(e, pipeBase.TaskError): 

207 traceback.print_exc(file=sys.stderr) 

208 

209 task.writeMetadata(butler) 

210 

211 if self.doReturnResults: 

212 # The results here are the zeropoint offsets for each band 

213 return [pipeBase.Struct(exitStatus=exitStatus, 

214 results=results)] 

215 else: 

216 return [pipeBase.Struct(exitStatus=exitStatus)] 

217 

218 def run(self, parsedCmd): 

219 """ 

220 Run the task, with no multiprocessing 

221 

222 Parameters 

223 ---------- 

224 parsedCmd: `lsst.pipe.base.ArgumentParser` parsed command line 

225 """ 

226 

227 resultList = [] 

228 

229 if self.precall(parsedCmd): 

230 targetList = self.getTargetList(parsedCmd) 

231 # make sure that we only get 1 

232 resultList = self(targetList[0]) 

233 

234 return resultList 

235 

236 

237class FgcmOutputProductsTask(pipeBase.CmdLineTask): 

238 """ 

239 Output products from FGCM global calibration. 

240 """ 

241 

242 ConfigClass = FgcmOutputProductsConfig 

243 RunnerClass = FgcmOutputProductsRunner 

244 _DefaultName = "fgcmOutputProducts" 

245 

246 def __init__(self, butler=None, **kwargs): 

247 """ 

248 Instantiate an fgcmOutputProductsTask. 

249 

250 Parameters 

251 ---------- 

252 butler : `lsst.daf.persistence.Butler` 

253 """ 

254 

255 pipeBase.CmdLineTask.__init__(self, **kwargs) 

256 

257 if self.config.doReferenceCalibration: 

258 # We need the ref obj loader to get the flux field 

259 self.makeSubtask("refObjLoader", butler=butler) 

260 

261 if self.config.doRefcatOutput: 

262 self.indexer = IndexerRegistry[self.config.datasetConfig.indexer.name]( 

263 self.config.datasetConfig.indexer.active) 

264 

265 # no saving of metadata for now 

266 def _getMetadataName(self): 

267 return None 

268 

269 @pipeBase.timeMethod 

270 def runDataRef(self, butler): 

271 """ 

272 Make FGCM output products for use in the stack 

273 

274 Parameters 

275 ---------- 

276 butler: `lsst.daf.persistence.Butler` 

277 cycleNumber: `int` 

278 Final fit cycle number, override config. 

279 

280 Returns 

281 ------- 

282 offsets: `lsst.pipe.base.Struct` 

283 A structure with array of zeropoint offsets 

284 

285 Raises 

286 ------ 

287 RuntimeError: 

288 Raised if any one of the following is true: 

289 

290 - butler cannot find "fgcmBuildStars_config" or 

291 "fgcmBuildStarsTable_config". 

292 - butler cannot find "fgcmFitCycle_config". 

293 - "fgcmFitCycle_config" does not refer to 

294 `self.config.cycleNumber`. 

295 - butler cannot find "fgcmAtmosphereParameters" and 

296 `self.config.doAtmosphereOutput` is `True`. 

297 - butler cannot find "fgcmStandardStars" and 

298 `self.config.doReferenceCalibration` is `True` or 

299 `self.config.doRefcatOutput` is `True`. 

300 - butler cannot find "fgcmZeropoints" and 

301 `self.config.doZeropointOutput` is `True`. 

302 """ 

303 

304 # Check to make sure that the fgcmBuildStars config exists, to retrieve 

305 # the visit and ccd dataset tags 

306 if not butler.datasetExists('fgcmBuildStarsTable_config') and \ 

307 not butler.datasetExists('fgcmBuildStars_config'): 

308 raise RuntimeError("Cannot find fgcmBuildStarsTable_config or fgcmBuildStars_config, " 

309 "which is prereq for fgcmOutputProducts") 

310 

311 if butler.datasetExists('fgcmBuildStarsTable_config'): 

312 fgcmBuildStarsConfig = butler.get('fgcmBuildStarsTable_config') 

313 else: 

314 fgcmBuildStarsConfig = butler.get('fgcmBuildStars_config') 

315 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

316 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

317 self.filterMap = fgcmBuildStarsConfig.filterMap 

318 

319 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

320 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

321 "in fgcmBuildStarsTask.") 

322 

323 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

324 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

325 

326 # Make sure that the fit config exists, to retrieve bands and other info 

327 if not butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber): 

328 raise RuntimeError("Cannot find fgcmFitCycle_config from cycle %d " % (self.config.cycleNumber) + 

329 "which is required for fgcmOutputProducts.") 

330 

331 fitCycleConfig = butler.get('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber) 

332 self.configBands = fitCycleConfig.bands 

333 

334 if self.config.doReferenceCalibration and fitCycleConfig.doReferenceCalibration: 

335 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

336 "fitCycleConfig.doReferenceCalibration") 

337 

338 # And make sure that the atmosphere was output properly 

339 if (self.config.doAtmosphereOutput and 

340 not butler.datasetExists('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber)): 

341 raise RuntimeError("Atmosphere parameters are missing for cycle %d." % 

342 (self.config.cycleNumber)) 

343 

344 if ((self.config.doReferenceCalibration or self.config.doRefcatOutput) and 

345 (not butler.datasetExists('fgcmStandardStars', 

346 fgcmcycle=self.config.cycleNumber))): 

347 raise RuntimeError("Standard stars are missing for cycle %d." % 

348 (self.config.cycleNumber)) 

349 

350 if (self.config.doZeropointOutput and 

351 (not butler.datasetExists('fgcmZeropoints', fgcmcycle=self.config.cycleNumber))): 

352 raise RuntimeError("Zeropoints are missing for cycle %d." % 

353 (self.config.cycleNumber)) 

354 

355 # And make sure this is the last cycle 

356 if butler.datasetExists('fgcmFitCycle_config', fgcmcycle=self.config.cycleNumber + 1): 

357 raise RuntimeError("The task fgcmOutputProducts should only be run" 

358 "on the final fit cycle products") 

359 

360 if self.config.doReferenceCalibration or self.config.doRefcatOutput: 

361 stdCat = butler.get('fgcmStandardStars', fgcmcycle=self.config.cycleNumber) 

362 md = stdCat.getMetadata() 

363 self.bands = md.getArray('BANDS') 

364 else: 

365 stdCat = None 

366 self.bands = self.configBands 

367 

368 if self.config.doReferenceCalibration: 

369 offsets = self._computeReferenceOffsets(butler, stdCat) 

370 else: 

371 offsets = np.zeros(len(self.bands)) 

372 

373 # Output the standard stars in stack format 

374 if self.config.doRefcatOutput: 

375 self._outputStandardStars(butler, stdCat, offsets, self.config.datasetConfig) 

376 

377 del stdCat 

378 

379 # Output the gray zeropoints 

380 if self.config.doZeropointOutput: 

381 zptCat = butler.get('fgcmZeropoints', fgcmcycle=self.config.cycleNumber) 

382 visitCat = butler.get('fgcmVisitCatalog') 

383 

384 self._outputZeropoints(butler, zptCat, visitCat, offsets) 

385 

386 # Output the atmospheres 

387 if self.config.doAtmosphereOutput: 

388 atmCat = butler.get('fgcmAtmosphereParameters', fgcmcycle=self.config.cycleNumber) 

389 self._outputAtmospheres(butler, atmCat) 

390 

391 # We return the zp offsets 

392 return pipeBase.Struct(offsets=offsets) 

393 

394 def generateTractOutputProducts(self, butler, tract, 

395 visitCat, zptCat, atmCat, stdCat, 

396 fgcmBuildStarsConfig, fgcmFitCycleConfig): 

397 """ 

398 Generate the output products for a given tract, as specified in the config. 

399 

400 This method is here to have an alternate entry-point for 

401 FgcmCalibrateTract. 

402 

403 Parameters 

404 ---------- 

405 butler: `lsst.daf.persistence.Butler` 

406 tract: `int` 

407 Tract number 

408 visitCat: `lsst.afw.table.BaseCatalog` 

409 FGCM visitCat from `FgcmBuildStarsTask` 

410 zptCat: `lsst.afw.table.BaseCatalog` 

411 FGCM zeropoint catalog from `FgcmFitCycleTask` 

412 atmCat: `lsst.afw.table.BaseCatalog` 

413 FGCM atmosphere parameter catalog from `FgcmFitCycleTask` 

414 stdCat: `lsst.afw.table.SimpleCatalog` 

415 FGCM standard star catalog from `FgcmFitCycleTask` 

416 fgcmBuildStarsConfig: `lsst.fgcmcal.FgcmBuildStarsConfig` 

417 Configuration object from `FgcmBuildStarsTask` 

418 fgcmFitCycleConfig: `lsst.fgcmcal.FgcmFitCycleConfig` 

419 Configuration object from `FgcmFitCycleTask` 

420 """ 

421 

422 self.configBands = fgcmFitCycleConfig.bands 

423 self.visitDataRefName = fgcmBuildStarsConfig.visitDataRefName 

424 self.ccdDataRefName = fgcmBuildStarsConfig.ccdDataRefName 

425 self.filterMap = fgcmBuildStarsConfig.filterMap 

426 

427 if stdCat is not None: 

428 md = stdCat.getMetadata() 

429 self.bands = md.getArray('BANDS') 

430 else: 

431 self.bands = self.configBands 

432 

433 if self.config.doReferenceCalibration and fgcmFitCycleConfig.doReferenceCalibration: 

434 self.log.warn("doReferenceCalibration is set, and is possibly redundant with " 

435 "fitCycleConfig.doReferenceCalibration") 

436 

437 if self.config.doComposeWcsJacobian and not fgcmBuildStarsConfig.doApplyWcsJacobian: 

438 raise RuntimeError("Cannot compose the WCS jacobian if it hasn't been applied " 

439 "in fgcmBuildStarsTask.") 

440 

441 if not self.config.doComposeWcsJacobian and fgcmBuildStarsConfig.doApplyWcsJacobian: 

442 self.log.warn("Jacobian was applied in build-stars but doComposeWcsJacobian is not set.") 

443 

444 if self.config.doReferenceCalibration: 

445 offsets = self._computeReferenceOffsets(butler, stdCat) 

446 else: 

447 offsets = np.zeros(len(self.bands)) 

448 

449 if self.config.doRefcatOutput: 

450 # Create a special config that has the tract number in it 

451 datasetConfig = copy.copy(self.config.datasetConfig) 

452 datasetConfig.ref_dataset_name = '%s_%d' % (self.config.datasetConfig.ref_dataset_name, 

453 tract) 

454 self._outputStandardStars(butler, stdCat, offsets, datasetConfig) 

455 

456 if self.config.doZeropointOutput: 

457 self._outputZeropoints(butler, zptCat, visitCat, offsets, tract=tract) 

458 

459 if self.config.doAtmosphereOutput: 

460 self._outputAtmospheres(butler, atmCat, tract=tract) 

461 

462 return pipeBase.Struct(offsets=offsets) 

463 

464 def _computeReferenceOffsets(self, butler, stdCat): 

465 """ 

466 Compute offsets relative to a reference catalog. 

467 

468 This method splits the star catalog into healpix pixels 

469 and computes the calibration transfer for a sample of 

470 these pixels to approximate the 'absolute' calibration 

471 values (on for each band) to apply to transfer the 

472 absolute scale. 

473 

474 Parameters 

475 ---------- 

476 butler: `lsst.daf.persistence.Butler` 

477 stdCat: `lsst.afw.table.SimpleCatalog` 

478 FGCM standard stars 

479 

480 Returns 

481 ------- 

482 offsets: `numpy.array` of floats 

483 Per band zeropoint offsets 

484 """ 

485 

486 # Only use stars that are observed in all the bands that were actually used 

487 # This will ensure that we use the same healpix pixels for the absolute 

488 # calibration of each band. 

489 minObs = stdCat['ngood'].min(axis=1) 

490 

491 goodStars = (minObs >= 1) 

492 stdCat = stdCat[goodStars] 

493 

494 self.log.info("Found %d stars with at least 1 good observation in each band" % 

495 (len(stdCat))) 

496 

497 # We have to make a table for each pixel with flux/fluxErr 

498 # This is a temporary table generated for input to the photoCal task. 

499 # These fluxes are not instFlux (they are top-of-the-atmosphere approximate and 

500 # have had chromatic corrections applied to get to the standard system 

501 # specified by the atmosphere/instrumental parameters), nor are they 

502 # in Jansky (since they don't have a proper absolute calibration: the overall 

503 # zeropoint is estimated from the telescope size, etc.) 

504 sourceMapper = afwTable.SchemaMapper(stdCat.schema) 

505 sourceMapper.addMinimalSchema(afwTable.SimpleTable.makeMinimalSchema()) 

506 sourceMapper.editOutputSchema().addField('instFlux', type=np.float64, 

507 doc="instrumental flux (counts)") 

508 sourceMapper.editOutputSchema().addField('instFluxErr', type=np.float64, 

509 doc="instrumental flux error (counts)") 

510 badStarKey = sourceMapper.editOutputSchema().addField('flag_badStar', 

511 type='Flag', 

512 doc="bad flag") 

513 

514 # Split up the stars 

515 # Note that there is an assumption here that the ra/dec coords stored 

516 # on-disk are in radians, and therefore that starObs['coord_ra'] / 

517 # starObs['coord_dec'] return radians when used as an array of numpy float64s. 

518 theta = np.pi/2. - stdCat['coord_dec'] 

519 phi = stdCat['coord_ra'] 

520 

521 ipring = hp.ang2pix(self.config.referencePixelizationNside, theta, phi) 

522 h, rev = esutil.stat.histogram(ipring, rev=True) 

523 

524 gdpix, = np.where(h >= self.config.referencePixelizationMinStars) 

525 

526 self.log.info("Found %d pixels (nside=%d) with at least %d good stars" % 

527 (gdpix.size, 

528 self.config.referencePixelizationNside, 

529 self.config.referencePixelizationMinStars)) 

530 

531 if gdpix.size < self.config.referencePixelizationNPixels: 

532 self.log.warn("Found fewer good pixels (%d) than preferred in configuration (%d)" % 

533 (gdpix.size, self.config.referencePixelizationNPixels)) 

534 else: 

535 # Sample out the pixels we want to use 

536 gdpix = np.random.choice(gdpix, size=self.config.referencePixelizationNPixels, replace=False) 

537 

538 results = np.zeros(gdpix.size, dtype=[('hpix', 'i4'), 

539 ('nstar', 'i4', len(self.bands)), 

540 ('nmatch', 'i4', len(self.bands)), 

541 ('zp', 'f4', len(self.bands)), 

542 ('zpErr', 'f4', len(self.bands))]) 

543 results['hpix'] = ipring[rev[rev[gdpix]]] 

544 

545 # We need a boolean index to deal with catalogs... 

546 selected = np.zeros(len(stdCat), dtype=np.bool) 

547 

548 refFluxFields = [None]*len(self.bands) 

549 

550 for p, pix in enumerate(gdpix): 

551 i1a = rev[rev[pix]: rev[pix + 1]] 

552 

553 # the stdCat afwTable can only be indexed with boolean arrays, 

554 # and not numpy index arrays (see DM-16497). This little trick 

555 # converts the index array into a boolean array 

556 selected[:] = False 

557 selected[i1a] = True 

558 

559 for b, band in enumerate(self.bands): 

560 

561 struct = self._computeOffsetOneBand(sourceMapper, badStarKey, b, band, stdCat, 

562 selected, refFluxFields) 

563 results['nstar'][p, b] = len(i1a) 

564 results['nmatch'][p, b] = len(struct.arrays.refMag) 

565 results['zp'][p, b] = struct.zp 

566 results['zpErr'][p, b] = struct.sigma 

567 

568 # And compute the summary statistics 

569 offsets = np.zeros(len(self.bands)) 

570 

571 for b, band in enumerate(self.bands): 

572 # make configurable 

573 ok, = np.where(results['nmatch'][:, b] >= self.config.referenceMinMatch) 

574 offsets[b] = np.median(results['zp'][ok, b]) 

575 # use median absolute deviation to estimate Normal sigma 

576 # see https://en.wikipedia.org/wiki/Median_absolute_deviation 

577 madSigma = 1.4826*np.median(np.abs(results['zp'][ok, b] - offsets[b])) 

578 self.log.info("Reference catalog offset for %s band: %.12f +/- %.12f" % 

579 (band, offsets[b], madSigma)) 

580 

581 return offsets 

582 

583 def _computeOffsetOneBand(self, sourceMapper, badStarKey, 

584 b, band, stdCat, selected, refFluxFields): 

585 """ 

586 Compute the zeropoint offset between the fgcm stdCat and the reference 

587 stars for one pixel in one band 

588 

589 Parameters 

590 ---------- 

591 sourceMapper: `lsst.afw.table.SchemaMapper` 

592 Mapper to go from stdCat to calibratable catalog 

593 badStarKey: `lsst.afw.table.Key` 

594 Key for the field with bad stars 

595 b: `int` 

596 Index of the band in the star catalog 

597 band: `str` 

598 Name of band for reference catalog 

599 stdCat: `lsst.afw.table.SimpleCatalog` 

600 FGCM standard stars 

601 selected: `numpy.array(dtype=np.bool)` 

602 Boolean array of which stars are in the pixel 

603 refFluxFields: `list` 

604 List of names of flux fields for reference catalog 

605 """ 

606 

607 sourceCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

608 sourceCat.reserve(selected.sum()) 

609 sourceCat.extend(stdCat[selected], mapper=sourceMapper) 

610 sourceCat['instFlux'] = 10.**(stdCat['mag_std_noabs'][selected, b]/(-2.5)) 

611 sourceCat['instFluxErr'] = (np.log(10.)/2.5)*(stdCat['magErr_std'][selected, b] * 

612 sourceCat['instFlux']) 

613 # Make sure we only use stars that have valid measurements 

614 # (This is perhaps redundant with requirements above that the 

615 # stars be observed in all bands, but it can't hurt) 

616 badStar = (stdCat['mag_std_noabs'][selected, b] > 90.0) 

617 for rec in sourceCat[badStar]: 

618 rec.set(badStarKey, True) 

619 

620 exposure = afwImage.ExposureF() 

621 exposure.setFilter(afwImage.Filter(band)) 

622 

623 if refFluxFields[b] is None: 

624 # Need to find the flux field in the reference catalog 

625 # to work around limitations of DirectMatch in PhotoCal 

626 ctr = stdCat[0].getCoord() 

627 rad = 0.05*lsst.geom.degrees 

628 refDataTest = self.refObjLoader.loadSkyCircle(ctr, rad, band) 

629 refFluxFields[b] = refDataTest.fluxField 

630 

631 # Make a copy of the config so that we can modify it 

632 calConfig = copy.copy(self.config.photoCal.value) 

633 calConfig.match.referenceSelection.signalToNoise.fluxField = refFluxFields[b] 

634 calConfig.match.referenceSelection.signalToNoise.errField = refFluxFields[b] + 'Err' 

635 calTask = self.config.photoCal.target(refObjLoader=self.refObjLoader, 

636 config=calConfig, 

637 schema=sourceCat.getSchema()) 

638 

639 struct = calTask.run(exposure, sourceCat) 

640 

641 return struct 

642 

643 def _outputStandardStars(self, butler, stdCat, offsets, datasetConfig): 

644 """ 

645 Output standard stars in indexed reference catalog format. 

646 

647 Parameters 

648 ---------- 

649 butler: `lsst.daf.persistence.Butler` 

650 stdCat: `lsst.afw.table.SimpleCatalog` 

651 FGCM standard star catalog from fgcmFitCycleTask 

652 offsets: `numpy.array` of floats 

653 Per band zeropoint offsets 

654 datasetConfig: `lsst.meas.algorithms.DatasetConfig` 

655 Config for reference dataset 

656 """ 

657 

658 self.log.info("Outputting standard stars to %s" % (datasetConfig.ref_dataset_name)) 

659 

660 # We determine the conversion from the native units (typically radians) to 

661 # degrees for the first star. This allows us to treat coord_ra/coord_dec as 

662 # numpy arrays rather than Angles, which would we approximately 600x slower. 

663 # TODO: Fix this after DM-16524 (HtmIndexer.indexPoints should take coords 

664 # (as Angles) for input 

665 conv = stdCat[0]['coord_ra'].asDegrees()/float(stdCat[0]['coord_ra']) 

666 indices = np.array(self.indexer.indexPoints(stdCat['coord_ra']*conv, 

667 stdCat['coord_dec']*conv)) 

668 

669 formattedCat = self._formatCatalog(stdCat, offsets) 

670 

671 # Write the master schema 

672 dataId = self.indexer.makeDataId('master_schema', 

673 datasetConfig.ref_dataset_name) 

674 masterCat = afwTable.SimpleCatalog(formattedCat.schema) 

675 addRefCatMetadata(masterCat) 

676 butler.put(masterCat, 'ref_cat', dataId=dataId) 

677 

678 # Break up the pixels using a histogram 

679 h, rev = esutil.stat.histogram(indices, rev=True) 

680 gd, = np.where(h > 0) 

681 selected = np.zeros(len(formattedCat), dtype=np.bool) 

682 for i in gd: 

683 i1a = rev[rev[i]: rev[i + 1]] 

684 

685 # the formattedCat afwTable can only be indexed with boolean arrays, 

686 # and not numpy index arrays (see DM-16497). This little trick 

687 # converts the index array into a boolean array 

688 selected[:] = False 

689 selected[i1a] = True 

690 

691 # Write the individual pixel 

692 dataId = self.indexer.makeDataId(indices[i1a[0]], 

693 datasetConfig.ref_dataset_name) 

694 butler.put(formattedCat[selected], 'ref_cat', dataId=dataId) 

695 

696 # And save the dataset configuration 

697 dataId = self.indexer.makeDataId(None, datasetConfig.ref_dataset_name) 

698 butler.put(datasetConfig, 'ref_cat_config', dataId=dataId) 

699 

700 self.log.info("Done outputting standard stars.") 

701 

702 def _formatCatalog(self, fgcmStarCat, offsets): 

703 """ 

704 Turn an FGCM-formatted star catalog, applying zeropoint offsets. 

705 

706 Parameters 

707 ---------- 

708 fgcmStarCat: `lsst.afw.Table.SimpleCatalog` 

709 SimpleCatalog as output by fgcmcal 

710 offsets: `list` with len(self.bands) entries 

711 Zeropoint offsets to apply 

712 

713 Returns 

714 ------- 

715 formattedCat: `lsst.afw.table.SimpleCatalog` 

716 SimpleCatalog suitable for using as a reference catalog 

717 """ 

718 

719 sourceMapper = afwTable.SchemaMapper(fgcmStarCat.schema) 

720 minSchema = LoadIndexedReferenceObjectsTask.makeMinimalSchema(self.bands, 

721 addCentroid=False, 

722 addIsResolved=True, 

723 coordErrDim=0) 

724 sourceMapper.addMinimalSchema(minSchema) 

725 for band in self.bands: 

726 sourceMapper.editOutputSchema().addField('%s_nGood' % (band), type=np.int32) 

727 sourceMapper.editOutputSchema().addField('%s_nTotal' % (band), type=np.int32) 

728 sourceMapper.editOutputSchema().addField('%s_nPsfCandidate' % (band), type=np.int32) 

729 

730 formattedCat = afwTable.SimpleCatalog(sourceMapper.getOutputSchema()) 

731 formattedCat.reserve(len(fgcmStarCat)) 

732 formattedCat.extend(fgcmStarCat, mapper=sourceMapper) 

733 

734 # Note that we don't have to set `resolved` because the default is False 

735 

736 for b, band in enumerate(self.bands): 

737 mag = fgcmStarCat['mag_std_noabs'][:, b].astype(np.float64) + offsets[b] 

738 # We want fluxes in nJy from calibrated AB magnitudes 

739 # (after applying offset). Updated after RFC-549 and RFC-575. 

740 flux = (mag*units.ABmag).to_value(units.nJy) 

741 fluxErr = (np.log(10.)/2.5)*flux*fgcmStarCat['magErr_std'][:, b].astype(np.float64) 

742 

743 formattedCat['%s_flux' % (band)][:] = flux 

744 formattedCat['%s_fluxErr' % (band)][:] = fluxErr 

745 formattedCat['%s_nGood' % (band)][:] = fgcmStarCat['ngood'][:, b] 

746 formattedCat['%s_nTotal' % (band)][:] = fgcmStarCat['ntotal'][:, b] 

747 formattedCat['%s_nPsfCandidate' % (band)][:] = fgcmStarCat['npsfcand'][:, b] 

748 

749 addRefCatMetadata(formattedCat) 

750 

751 return formattedCat 

752 

753 def _outputZeropoints(self, butler, zptCat, visitCat, offsets, tract=None): 

754 """ 

755 Output the zeropoints in fgcm_photoCalib format. 

756 

757 Parameters 

758 ---------- 

759 butler: `lsst.daf.persistence.Butler` 

760 zptCat: `lsst.afw.table.BaseCatalog` 

761 FGCM zeropoint catalog from `FgcmFitCycleTask` 

762 visitCat: `lsst.afw.table.BaseCatalog` 

763 FGCM visitCat from `FgcmBuildStarsTask` 

764 offsets: `numpy.array` 

765 Float array of absolute calibration offsets, one for each filter 

766 tract: `int`, optional 

767 Tract number to output. Default is None (global calibration) 

768 """ 

769 

770 if tract is None: 

771 datasetType = 'fgcm_photoCalib' 

772 else: 

773 datasetType = 'fgcm_tract_photoCalib' 

774 

775 self.log.info("Outputting %s objects" % (datasetType)) 

776 

777 # Select visit/ccds where we have a calibration 

778 # This includes ccds where we were able to interpolate from neighboring 

779 # ccds. 

780 cannot_compute = fgcm.fgcmUtilities.zpFlagDict['CANNOT_COMPUTE_ZEROPOINT'] 

781 too_few_stars = fgcm.fgcmUtilities.zpFlagDict['TOO_FEW_STARS_ON_CCD'] 

782 selected = (((zptCat['fgcmFlag'] & cannot_compute) == 0) & 

783 (zptCat['fgcmZptVar'] > 0.0)) 

784 

785 # We also select the "best" calibrations, avoiding interpolation. These 

786 # are only used for mapping filternames 

787 selected_best = (((zptCat['fgcmFlag'] & (cannot_compute | too_few_stars)) == 0) & 

788 (zptCat['fgcmZptVar'] > 0.0)) 

789 

790 # Log warnings for any visit which has no valid zeropoints 

791 badVisits = np.unique(zptCat['visit'][~selected]) 

792 goodVisits = np.unique(zptCat['visit'][selected]) 

793 allBadVisits = badVisits[~np.isin(badVisits, goodVisits)] 

794 for allBadVisit in allBadVisits: 

795 self.log.warn(f'No suitable photoCalib for {self.visitDataRefName} {allBadVisit}') 

796 

797 # Get the mapping from filtername to dataId filter name, empirically 

798 filterMapping = {} 

799 nFound = 0 

800 for rec in zptCat[selected_best]: 

801 if rec['filtername'] in filterMapping: 

802 continue 

803 dataId = {self.visitDataRefName: int(rec['visit']), 

804 self.ccdDataRefName: int(rec['ccd'])} 

805 dataRef = butler.dataRef('raw', dataId=dataId) 

806 filterMapping[rec['filtername']] = dataRef.dataId['filter'] 

807 nFound += 1 

808 if nFound == len(self.filterMap): 

809 break 

810 

811 # Get a mapping from filtername to the offsets 

812 offsetMapping = {} 

813 for f in self.filterMap: 

814 # Not every filter in the map will necesarily have a band. 

815 if self.filterMap[f] in self.bands: 

816 offsetMapping[f] = offsets[self.bands.index(self.filterMap[f])] 

817 

818 # Get a mapping from "ccd" to the ccd index used for the scaling 

819 camera = butler.get('camera') 

820 ccdMapping = {} 

821 for ccdIndex, detector in enumerate(camera): 

822 ccdMapping[detector.getId()] = ccdIndex 

823 

824 # And a mapping to get the flat-field scaling values 

825 scalingMapping = {} 

826 for rec in visitCat: 

827 scalingMapping[rec['visit']] = rec['scaling'] 

828 

829 if self.config.doComposeWcsJacobian: 

830 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

831 

832 for rec in zptCat[selected]: 

833 

834 # Retrieve overall scaling 

835 scaling = scalingMapping[rec['visit']][ccdMapping[rec['ccd']]] 

836 

837 fgcmSuperStarField = self._getChebyshevBoundedField(rec['fgcmfZptSstarCheb'], 

838 rec['fgcmfZptChebXyMax']) 

839 # Convert from FGCM AB to nJy 

840 fgcmZptField = self._getChebyshevBoundedField((rec['fgcmfZptCheb']*units.AB).to_value(units.nJy), 

841 rec['fgcmfZptChebXyMax'], 

842 offset=offsetMapping[rec['filtername']], 

843 scaling=scaling) 

844 

845 if self.config.doComposeWcsJacobian: 

846 

847 fgcmField = afwMath.ProductBoundedField([approxPixelAreaFields[rec['ccd']], 

848 fgcmSuperStarField, 

849 fgcmZptField]) 

850 else: 

851 # The photoCalib is just the product of the fgcmSuperStarField and the 

852 # fgcmZptField 

853 fgcmField = afwMath.ProductBoundedField([fgcmSuperStarField, fgcmZptField]) 

854 

855 # The "mean" calibration will be set to the center of the ccd for reference 

856 calibCenter = fgcmField.evaluate(fgcmField.getBBox().getCenter()) 

857 calibErr = (np.log(10.0)/2.5)*calibCenter*np.sqrt(rec['fgcmZptVar']) 

858 photoCalib = afwImage.PhotoCalib(calibrationMean=calibCenter, 

859 calibrationErr=calibErr, 

860 calibration=fgcmField, 

861 isConstant=False) 

862 

863 if tract is None: 

864 butler.put(photoCalib, datasetType, 

865 dataId={self.visitDataRefName: int(rec['visit']), 

866 self.ccdDataRefName: int(rec['ccd']), 

867 'filter': filterMapping[rec['filtername']]}) 

868 else: 

869 butler.put(photoCalib, datasetType, 

870 dataId={self.visitDataRefName: int(rec['visit']), 

871 self.ccdDataRefName: int(rec['ccd']), 

872 'filter': filterMapping[rec['filtername']], 

873 'tract': tract}) 

874 

875 self.log.info("Done outputting %s objects" % (datasetType)) 

876 

877 def _getChebyshevBoundedField(self, coefficients, xyMax, offset=0.0, scaling=1.0): 

878 """ 

879 Make a ChebyshevBoundedField from fgcm coefficients, with optional offset 

880 and scaling. 

881 

882 Parameters 

883 ---------- 

884 coefficients: `numpy.array` 

885 Flattened array of chebyshev coefficients 

886 xyMax: `list` of length 2 

887 Maximum x and y of the chebyshev bounding box 

888 offset: `float`, optional 

889 Absolute calibration offset. Default is 0.0 

890 scaling: `float`, optional 

891 Flat scaling value from fgcmBuildStars. Default is 1.0 

892 

893 Returns 

894 ------- 

895 boundedField: `lsst.afw.math.ChebyshevBoundedField` 

896 """ 

897 

898 orderPlus1 = int(np.sqrt(coefficients.size)) 

899 pars = np.zeros((orderPlus1, orderPlus1)) 

900 

901 bbox = lsst.geom.Box2I(lsst.geom.Point2I(0.0, 0.0), 

902 lsst.geom.Point2I(*xyMax)) 

903 

904 pars[:, :] = (coefficients.reshape(orderPlus1, orderPlus1) * 

905 (10.**(offset/-2.5))*scaling) 

906 

907 boundedField = afwMath.ChebyshevBoundedField(bbox, pars) 

908 

909 return boundedField 

910 

911 def _outputAtmospheres(self, butler, atmCat, tract=None): 

912 """ 

913 Output the atmospheres. 

914 

915 Parameters 

916 ---------- 

917 butler: `lsst.daf.persistence.Butler` 

918 atmCat: `lsst.afw.table.BaseCatalog` 

919 FGCM atmosphere parameter catalog from fgcmFitCycleTask 

920 tract: `int`, optional 

921 Tract number to output. Default is None (global calibration) 

922 """ 

923 

924 self.log.info("Outputting atmosphere transmissions") 

925 

926 # First, we need to grab the look-up table and key info 

927 lutCat = butler.get('fgcmLookUpTable') 

928 

929 atmosphereTableName = lutCat[0]['tablename'] 

930 elevation = lutCat[0]['elevation'] 

931 atmLambda = lutCat[0]['atmLambda'] 

932 lutCat = None 

933 

934 # Make the atmosphere table if possible 

935 try: 

936 atmTable = fgcm.FgcmAtmosphereTable.initWithTableName(atmosphereTableName) 

937 atmTable.loadTable() 

938 except IOError: 

939 atmTable = None 

940 

941 if atmTable is None: 

942 # Try to use MODTRAN instead 

943 try: 

944 modGen = fgcm.ModtranGenerator(elevation) 

945 lambdaRange = np.array([atmLambda[0], atmLambda[-1]])/10. 

946 lambdaStep = (atmLambda[1] - atmLambda[0])/10. 

947 except (ValueError, IOError) as e: 

948 raise RuntimeError("FGCM look-up-table generated with modtran, " 

949 "but modtran not configured to run.") from e 

950 

951 zenith = np.degrees(np.arccos(1./atmCat['secZenith'])) 

952 

953 for i, visit in enumerate(atmCat['visit']): 

954 if atmTable is not None: 

955 # Interpolate the atmosphere table 

956 atmVals = atmTable.interpolateAtmosphere(pmb=atmCat[i]['pmb'], 

957 pwv=atmCat[i]['pwv'], 

958 o3=atmCat[i]['o3'], 

959 tau=atmCat[i]['tau'], 

960 alpha=atmCat[i]['alpha'], 

961 zenith=zenith[i], 

962 ctranslamstd=[atmCat[i]['cTrans'], 

963 atmCat[i]['lamStd']]) 

964 else: 

965 # Run modtran 

966 modAtm = modGen(pmb=atmCat[i]['pmb'], 

967 pwv=atmCat[i]['pwv'], 

968 o3=atmCat[i]['o3'], 

969 tau=atmCat[i]['tau'], 

970 alpha=atmCat[i]['alpha'], 

971 zenith=zenith[i], 

972 lambdaRange=lambdaRange, 

973 lambdaStep=lambdaStep, 

974 ctranslamstd=[atmCat[i]['cTrans'], 

975 atmCat[i]['lamStd']]) 

976 atmVals = modAtm['COMBINED'] 

977 

978 # Now need to create something to persist... 

979 curve = TransmissionCurve.makeSpatiallyConstant(throughput=atmVals, 

980 wavelengths=atmLambda, 

981 throughputAtMin=atmVals[0], 

982 throughputAtMax=atmVals[-1]) 

983 

984 if tract is None: 

985 butler.put(curve, "transmission_atmosphere_fgcm", 

986 dataId={self.visitDataRefName: visit}) 

987 else: 

988 butler.put(curve, "transmission_atmosphere_fgcm_tract", 

989 dataId={self.visitDataRefName: visit, 

990 'tract': tract}) 

991 

992 self.log.info("Done outputting atmosphere transmissions")