Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35import collections 

36 

37import lsst.daf.persistence as dafPersist 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40from lsst.pipe.base import connectionTypes 

41import lsst.afw.table as afwTable 

42from lsst.meas.algorithms import ReferenceObjectLoader 

43 

44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsRunner, FgcmBuildStarsBaseTask 

45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromDataRef 

46from .utilities import lookupStaticCalibrations 

47 

48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

49 

50 

51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

52 dimensions=("instrument",), 

53 defaultTemplates={}): 

54 camera = connectionTypes.PrerequisiteInput( 

55 doc="Camera instrument", 

56 name="camera", 

57 storageClass="Camera", 

58 dimensions=("instrument",), 

59 lookupFunction=lookupStaticCalibrations, 

60 isCalibration=True, 

61 ) 

62 

63 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

65 "chromatic corrections."), 

66 name="fgcmLookUpTable", 

67 storageClass="Catalog", 

68 dimensions=("instrument",), 

69 deferLoad=True, 

70 ) 

71 

72 sourceSchema = connectionTypes.PrerequisiteInput( 

73 doc="Schema for source catalogs", 

74 name="src_schema", 

75 storageClass="SourceCatalog", 

76 deferLoad=True, 

77 ) 

78 

79 refCat = connectionTypes.PrerequisiteInput( 

80 doc="Reference catalog to use for photometric calibration", 

81 name="cal_ref_cat", 

82 storageClass="SimpleCatalog", 

83 dimensions=("skypix",), 

84 deferLoad=True, 

85 multiple=True, 

86 ) 

87 

88 sourceTable_visit = connectionTypes.Input( 

89 doc="Source table in parquet format, per visit", 

90 name="sourceTable_visit", 

91 storageClass="DataFrame", 

92 dimensions=("instrument", "visit"), 

93 deferLoad=True, 

94 multiple=True, 

95 ) 

96 

97 calexp = connectionTypes.Input( 

98 doc="Calibrated exposures used for psf and metadata", 

99 name="calexp", 

100 storageClass="ExposureF", 

101 dimensions=("instrument", "visit", "detector"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 background = connectionTypes.Input( 

107 doc="Calexp background model", 

108 name="calexpBackground", 

109 storageClass="Background", 

110 dimensions=("instrument", "visit", "detector"), 

111 deferLoad=True, 

112 multiple=True, 

113 ) 

114 

115 fgcmVisitCatalog = connectionTypes.Output( 

116 doc="Catalog of visit information for fgcm", 

117 name="fgcmVisitCatalog", 

118 storageClass="Catalog", 

119 dimensions=("instrument",), 

120 ) 

121 

122 fgcmStarObservations = connectionTypes.Output( 

123 doc="Catalog of star observations for fgcm", 

124 name="fgcmStarObservations", 

125 storageClass="Catalog", 

126 dimensions=("instrument",), 

127 ) 

128 

129 fgcmStarIds = connectionTypes.Output( 

130 doc="Catalog of fgcm calibration star IDs", 

131 name="fgcmStarIds", 

132 storageClass="Catalog", 

133 dimensions=("instrument",), 

134 ) 

135 

136 fgcmStarIndices = connectionTypes.Output( 

137 doc="Catalog of fgcm calibration star indices", 

138 name="fgcmStarIndices", 

139 storageClass="Catalog", 

140 dimensions=("instrument",), 

141 ) 

142 

143 fgcmReferenceStars = connectionTypes.Output( 

144 doc="Catalog of fgcm-matched reference stars", 

145 name="fgcmReferenceStars", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 ) 

149 

150 def __init__(self, *, config=None): 

151 super().__init__(config=config) 

152 

153 if not config.doReferenceMatches: 

154 self.prerequisiteInputs.remove("refCat") 

155 self.prerequisiteInputs.remove("fgcmLookUpTable") 

156 

157 if not config.doModelErrorsWithBackground: 

158 self.inputs.remove("background") 

159 

160 if not config.doReferenceMatches: 

161 self.outputs.remove("fgcmReferenceStars") 

162 

163 

164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

165 pipelineConnections=FgcmBuildStarsTableConnections): 

166 """Config for FgcmBuildStarsTableTask""" 

167 

168 referenceCCD = pexConfig.Field( 

169 doc="Reference CCD for checking PSF and background", 

170 dtype=int, 

171 default=40, 

172 ) 

173 

174 def setDefaults(self): 

175 super().setDefaults() 

176 

177 # The names here correspond to the post-transformed 

178 # sourceTable_visit catalogs, which differ from the raw src 

179 # catalogs. Therefore, all field and flag names cannot 

180 # be derived from the base config class. 

181 self.instFluxField = 'ApFlux_12_0_instFlux' 

182 self.localBackgroundFluxField = 'LocalBackground_instFlux' 

183 self.apertureInnerInstFluxField = 'ApFlux_12_0_instFlux' 

184 self.apertureOuterInstFluxField = 'ApFlux_17_0_instFlux' 

185 self.psfCandidateName = 'Calib_psf_candidate' 

186 

187 sourceSelector = self.sourceSelector["science"] 

188 

189 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

190 

191 sourceSelector.flags.bad = ['PixelFlags_edge', 

192 'PixelFlags_interpolatedCenter', 

193 'PixelFlags_saturatedCenter', 

194 'PixelFlags_crCenter', 

195 'PixelFlags_bad', 

196 'PixelFlags_interpolated', 

197 'PixelFlags_saturated', 

198 'Centroid_flag', 

199 fluxFlagName] 

200 

201 if self.doSubtractLocalBackground: 

202 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

203 sourceSelector.flags.bad.append(localBackgroundFlagName) 

204 

205 sourceSelector.signalToNoise.fluxField = self.instFluxField 

206 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

207 

208 sourceSelector.isolated.parentName = 'parentSourceId' 

209 sourceSelector.isolated.nChildName = 'Deblend_nChild' 

210 

211 sourceSelector.unresolved.name = 'extendedness' 

212 

213 

214class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

215 """ 

216 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

217 """ 

218 ConfigClass = FgcmBuildStarsTableConfig 

219 RunnerClass = FgcmBuildStarsRunner 

220 _DefaultName = "fgcmBuildStarsTable" 

221 

222 canMultiprocess = False 

223 

224 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

225 inputRefDict = butlerQC.get(inputRefs) 

226 

227 dataRefs = inputRefDict['sourceTable_visit'] 

228 

229 self.log.info("Running with %d sourceTable_visit dataRefs", (len(dataRefs))) 

230 

231 if self.config.doReferenceMatches: 

232 # Get the LUT dataRef 

233 lutDataRef = inputRefDict['fgcmLookUpTable'] 

234 

235 # Prepare the refCat loader 

236 refConfig = self.config.fgcmLoadReferenceCatalog.refObjLoader 

237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

238 for ref in inputRefs.refCat], 

239 refCats=butlerQC.get(inputRefs.refCat), 

240 config=refConfig, 

241 log=self.log) 

242 self.makeSubtask('fgcmLoadReferenceCatalog', refObjLoader=refObjLoader) 

243 else: 

244 lutDataRef = None 

245 

246 # Compute aperture radius if necessary. This is useful to do now before 

247 # any heave lifting has happened (fail early). 

248 calibFluxApertureRadius = None 

249 if self.config.doSubtractLocalBackground: 

250 try: 

251 calibFluxApertureRadius = computeApertureRadiusFromDataRef(dataRefs[0], 

252 self.config.instFluxField) 

253 except RuntimeError as e: 

254 raise RuntimeError("Could not determine aperture radius from %s. " 

255 "Cannot use doSubtractLocalBackground." % 

256 (self.config.instFluxField)) from e 

257 

258 calexpRefs = inputRefDict['calexp'] 

259 calexpDataRefDict = {(calexpRef.dataId.byName()['visit'], 

260 calexpRef.dataId.byName()['detector']): calexpRef for 

261 calexpRef in calexpRefs} 

262 

263 camera = inputRefDict['camera'] 

264 groupedDataRefs = self._findAndGroupDataRefs(camera, dataRefs, 

265 calexpDataRefDict=calexpDataRefDict) 

266 

267 if self.config.doModelErrorsWithBackground: 

268 bkgRefs = inputRefDict['background'] 

269 bkgDataRefDict = {(bkgRef.dataId.byName()['visit'], 

270 bkgRef.dataId.byName()['detector']): bkgRef for 

271 bkgRef in bkgRefs} 

272 else: 

273 bkgDataRefDict = None 

274 

275 # Gen3 does not currently allow "checkpoint" saving of datasets, 

276 # so we need to have this all in one go. 

277 visitCat = self.fgcmMakeVisitCatalog(camera, groupedDataRefs, 

278 bkgDataRefDict=bkgDataRefDict, 

279 visitCatDataRef=None, 

280 inVisitCat=None) 

281 

282 rad = calibFluxApertureRadius 

283 sourceSchemaDataRef = inputRefDict['sourceSchema'] 

284 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedDataRefs, 

285 visitCat, 

286 sourceSchemaDataRef, 

287 camera, 

288 calibFluxApertureRadius=rad, 

289 starObsDataRef=None, 

290 visitCatDataRef=None, 

291 inStarObsCat=None) 

292 

293 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

294 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

295 

296 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

297 fgcmStarObservationCat, 

298 lutDataRef=lutDataRef) 

299 

300 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

301 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

302 if fgcmRefCat is not None: 

303 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

304 

305 @classmethod 

306 def _makeArgumentParser(cls): 

307 """Create an argument parser""" 

308 parser = pipeBase.ArgumentParser(name=cls._DefaultName) 

309 parser.add_id_argument("--id", "sourceTable_visit", help="Data ID, e.g. --id visit=6789") 

310 

311 return parser 

312 

313 def _findAndGroupDataRefs(self, camera, dataRefs, butler=None, calexpDataRefDict=None): 

314 if (butler is None and calexpDataRefDict is None) or \ 

315 (butler is not None and calexpDataRefDict is not None): 

316 raise RuntimeError("Must either set butler (Gen2) or dataRefDict (Gen3)") 

317 

318 self.log.info("Grouping dataRefs by %s", (self.config.visitDataRefName)) 

319 

320 ccdIds = [] 

321 for detector in camera: 

322 ccdIds.append(detector.getId()) 

323 # Insert our preferred referenceCCD first: 

324 # It is fine that this is listed twice, because we only need 

325 # the first calexp that is found. 

326 ccdIds.insert(0, self.config.referenceCCD) 

327 

328 # The visitTable building code expects a dictionary of groupedDataRefs 

329 # keyed by visit, the first element as the "primary" calexp dataRef. 

330 # We then append the sourceTable_visit dataRef at the end for the 

331 # code which does the data reading (fgcmMakeAllStarObservations). 

332 

333 groupedDataRefs = collections.defaultdict(list) 

334 for dataRef in dataRefs: 

335 visit = dataRef.dataId[self.config.visitDataRefName] 

336 

337 # Find an existing calexp (we need for psf and metadata) 

338 # and make the relevant dataRef 

339 for ccdId in ccdIds: 

340 if butler is not None: 

341 # Gen2 Mode 

342 try: 

343 calexpRef = butler.dataRef('calexp', dataId={self.config.visitDataRefName: visit, 

344 self.config.ccdDataRefName: ccdId}) 

345 except RuntimeError: 

346 # Not found 

347 continue 

348 else: 

349 # Gen3 mode 

350 calexpRef = calexpDataRefDict.get((visit, ccdId)) 

351 if calexpRef is None: 

352 continue 

353 

354 # It was found. Add and quit out, since we only 

355 # need one calexp per visit. 

356 groupedDataRefs[visit].append(calexpRef) 

357 break 

358 

359 # And append this dataRef 

360 groupedDataRefs[visit].append(dataRef) 

361 

362 # This should be sorted by visit (the key) 

363 return dict(sorted(groupedDataRefs.items())) 

364 

365 def fgcmMakeAllStarObservations(self, groupedDataRefs, visitCat, 

366 sourceSchemaDataRef, 

367 camera, 

368 calibFluxApertureRadius=None, 

369 visitCatDataRef=None, 

370 starObsDataRef=None, 

371 inStarObsCat=None): 

372 startTime = time.time() 

373 

374 # If both dataRefs are None, then we assume the caller does not 

375 # want to store checkpoint files. If both are set, we will 

376 # do checkpoint files. And if only one is set, this is potentially 

377 # unintentional and we will warn. 

378 if (visitCatDataRef is not None and starObsDataRef is None 

379 or visitCatDataRef is None and starObsDataRef is not None): 

380 self.log.warn("Only one of visitCatDataRef and starObsDataRef are set, so " 

381 "no checkpoint files will be persisted.") 

382 

383 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

384 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

385 

386 # To get the correct output schema, we use similar code as fgcmBuildStarsTask 

387 # We are not actually using this mapper, except to grab the outputSchema 

388 sourceSchema = sourceSchemaDataRef.get().schema 

389 sourceMapper = self._makeSourceMapper(sourceSchema) 

390 outputSchema = sourceMapper.getOutputSchema() 

391 

392 # Construct mapping from ccd number to index 

393 ccdMapping = {} 

394 for ccdIndex, detector in enumerate(camera): 

395 ccdMapping[detector.getId()] = ccdIndex 

396 

397 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

398 

399 if inStarObsCat is not None: 

400 fullCatalog = inStarObsCat 

401 comp1 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_KEYS) 

402 comp2 = fullCatalog.schema.compare(outputSchema, outputSchema.EQUAL_NAMES) 

403 if not comp1 or not comp2: 

404 raise RuntimeError("Existing fgcmStarObservations file found with mismatched schema.") 

405 else: 

406 fullCatalog = afwTable.BaseCatalog(outputSchema) 

407 

408 visitKey = outputSchema['visit'].asKey() 

409 ccdKey = outputSchema['ccd'].asKey() 

410 instMagKey = outputSchema['instMag'].asKey() 

411 instMagErrKey = outputSchema['instMagErr'].asKey() 

412 

413 # Prepare local background if desired 

414 if self.config.doSubtractLocalBackground: 

415 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

416 

417 # Determine which columns we need from the sourceTable_visit catalogs 

418 columns = self._get_sourceTable_visit_columns() 

419 

420 k = 2.5/np.log(10.) 

421 

422 for counter, visit in enumerate(visitCat): 

423 # Check if these sources have already been read and stored in the checkpoint file 

424 if visit['sources_read']: 

425 continue 

426 

427 expTime = visit['exptime'] 

428 

429 dataRef = groupedDataRefs[visit['visit']][-1] 

430 

431 if isinstance(dataRef, dafPersist.ButlerDataRef): 

432 srcTable = dataRef.get() 

433 df = srcTable.toDataFrame(columns) 

434 else: 

435 df = dataRef.get(parameters={'columns': columns}) 

436 

437 goodSrc = self.sourceSelector.selectSources(df) 

438 

439 # Need to add a selection based on the local background correction 

440 # if necessary 

441 if self.config.doSubtractLocalBackground: 

442 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

443 use, = np.where((goodSrc.selected) 

444 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

445 else: 

446 use, = np.where(goodSrc.selected) 

447 

448 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

449 tempCat.resize(use.size) 

450 

451 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

452 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

453 tempCat['x'][:] = df['x'].values[use] 

454 tempCat['y'][:] = df['y'].values[use] 

455 # These "visit" and "ccd" names in the parquet tables are 

456 # hard-coded. 

457 tempCat[visitKey][:] = df['visit'].values[use] 

458 tempCat[ccdKey][:] = df['ccd'].values[use] 

459 tempCat['psf_candidate'] = df['Calib_psf_candidate'].values[use] 

460 

461 if self.config.doSubtractLocalBackground: 

462 # At the moment we only adjust the flux and not the flux 

463 # error by the background because the error on 

464 # base_LocalBackground_instFlux is the rms error in the 

465 # background annulus, not the error on the mean in the 

466 # background estimate (which is much smaller, by sqrt(n) 

467 # pixels used to estimate the background, which we do not 

468 # have access to in this task). In the default settings, 

469 # the annulus is sufficiently large such that these 

470 # additional errors are are negligibly small (much less 

471 # than a mmag in quadrature). 

472 

473 # This is the difference between the mag with local background correction 

474 # and the mag without local background correction. 

475 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

476 - localBackground[use]) - 

477 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

478 else: 

479 tempCat['deltaMagBkg'][:] = 0.0 

480 

481 # Need to loop over ccds here 

482 for detector in camera: 

483 ccdId = detector.getId() 

484 # used index for all observations with a given ccd 

485 use2 = (tempCat[ccdKey] == ccdId) 

486 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

487 tempCat['y'][use2]) 

488 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

489 * visit['scaling'][ccdMapping[ccdId]]) 

490 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

491 

492 # Compute instMagErr from instFluxErr/instFlux, any scaling 

493 # will cancel out. 

494 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

495 / df[self.config.instFluxField].values[use]) 

496 

497 # Apply the jacobian if configured 

498 if self.config.doApplyWcsJacobian: 

499 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

500 

501 fullCatalog.extend(tempCat) 

502 

503 # Now do the aperture information 

504 with np.warnings.catch_warnings(): 

505 # Ignore warnings, we will filter infinites and nans below 

506 np.warnings.simplefilter("ignore") 

507 

508 instMagIn = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

509 instMagErrIn = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

510 / df[self.config.apertureInnerInstFluxField].values[use]) 

511 instMagOut = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

512 instMagErrOut = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

513 / df[self.config.apertureOuterInstFluxField].values[use]) 

514 

515 ok = (np.isfinite(instMagIn) & np.isfinite(instMagErrIn) 

516 & np.isfinite(instMagOut) & np.isfinite(instMagErrOut)) 

517 

518 visit['deltaAper'] = np.median(instMagIn[ok] - instMagOut[ok]) 

519 visit['sources_read'] = True 

520 

521 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

522 use.size, visit['visit'], visit['deltaAper']) 

523 

524 if ((counter % self.config.nVisitsPerCheckpoint) == 0 

525 and starObsDataRef is not None and visitCatDataRef is not None): 

526 # We need to persist both the stars and the visit catalog which gets 

527 # additional metadata from each visit. 

528 starObsDataRef.put(fullCatalog) 

529 visitCatDataRef.put(visitCat) 

530 

531 self.log.info("Found all good star observations in %.2f s" % 

532 (time.time() - startTime)) 

533 

534 return fullCatalog 

535 

536 def _get_sourceTable_visit_columns(self): 

537 """ 

538 Get the sourceTable_visit columns from the config. 

539 

540 Returns 

541 ------- 

542 columns : `list` 

543 List of columns to read from sourceTable_visit 

544 """ 

545 # These "visit" and "ccd" names in the parquet tables are hard-coded. 

546 columns = ['visit', 'ccd', 

547 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

548 self.config.instFluxField, self.config.instFluxField + 'Err', 

549 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

550 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

551 if self.sourceSelector.config.doFlags: 

552 columns.extend(self.sourceSelector.config.flags.bad) 

553 if self.sourceSelector.config.doUnresolved: 

554 columns.append(self.sourceSelector.config.unresolved.name) 

555 if self.sourceSelector.config.doIsolated: 

556 columns.append(self.sourceSelector.config.isolated.parentName) 

557 columns.append(self.sourceSelector.config.isolated.nChildName) 

558 if self.config.doSubtractLocalBackground: 

559 columns.append(self.config.localBackgroundFluxField) 

560 

561 return columns