Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 18%

188 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-01 02:26 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35import collections 

36 

37import lsst.pex.config as pexConfig 

38import lsst.pipe.base as pipeBase 

39from lsst.pipe.base import connectionTypes 

40import lsst.afw.table as afwTable 

41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

42 

43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

45from .utilities import lookupStaticCalibrations 

46 

47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

48 

49 

50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument",), 

52 defaultTemplates={}): 

53 camera = connectionTypes.PrerequisiteInput( 

54 doc="Camera instrument", 

55 name="camera", 

56 storageClass="Camera", 

57 dimensions=("instrument",), 

58 lookupFunction=lookupStaticCalibrations, 

59 isCalibration=True, 

60 ) 

61 

62 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

64 "chromatic corrections."), 

65 name="fgcmLookUpTable", 

66 storageClass="Catalog", 

67 dimensions=("instrument",), 

68 deferLoad=True, 

69 ) 

70 

71 sourceSchema = connectionTypes.InitInput( 

72 doc="Schema for source catalogs", 

73 name="src_schema", 

74 storageClass="SourceCatalog", 

75 ) 

76 

77 refCat = connectionTypes.PrerequisiteInput( 

78 doc="Reference catalog to use for photometric calibration", 

79 name="cal_ref_cat", 

80 storageClass="SimpleCatalog", 

81 dimensions=("skypix",), 

82 deferLoad=True, 

83 multiple=True, 

84 ) 

85 

86 sourceTable_visit = connectionTypes.Input( 

87 doc="Source table in parquet format, per visit", 

88 name="sourceTable_visit", 

89 storageClass="DataFrame", 

90 dimensions=("instrument", "visit"), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 visitSummary = connectionTypes.Input( 

96 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

97 "detector id for the id and must be sorted for fast lookups of a " 

98 "detector."), 

99 name="visitSummary", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 background = connectionTypes.Input( 

107 doc="Calexp background model", 

108 name="calexpBackground", 

109 storageClass="Background", 

110 dimensions=("instrument", "visit", "detector"), 

111 deferLoad=True, 

112 multiple=True, 

113 ) 

114 

115 fgcmVisitCatalog = connectionTypes.Output( 

116 doc="Catalog of visit information for fgcm", 

117 name="fgcmVisitCatalog", 

118 storageClass="Catalog", 

119 dimensions=("instrument",), 

120 ) 

121 

122 fgcmStarObservations = connectionTypes.Output( 

123 doc="Catalog of star observations for fgcm", 

124 name="fgcmStarObservations", 

125 storageClass="Catalog", 

126 dimensions=("instrument",), 

127 ) 

128 

129 fgcmStarIds = connectionTypes.Output( 

130 doc="Catalog of fgcm calibration star IDs", 

131 name="fgcmStarIds", 

132 storageClass="Catalog", 

133 dimensions=("instrument",), 

134 ) 

135 

136 fgcmStarIndices = connectionTypes.Output( 

137 doc="Catalog of fgcm calibration star indices", 

138 name="fgcmStarIndices", 

139 storageClass="Catalog", 

140 dimensions=("instrument",), 

141 ) 

142 

143 fgcmReferenceStars = connectionTypes.Output( 

144 doc="Catalog of fgcm-matched reference stars", 

145 name="fgcmReferenceStars", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 ) 

149 

150 def __init__(self, *, config=None): 

151 super().__init__(config=config) 

152 

153 if not config.doReferenceMatches: 

154 self.prerequisiteInputs.remove("refCat") 

155 self.prerequisiteInputs.remove("fgcmLookUpTable") 

156 

157 if not config.doModelErrorsWithBackground: 

158 self.inputs.remove("background") 

159 

160 if not config.doReferenceMatches: 

161 self.outputs.remove("fgcmReferenceStars") 

162 

163 

164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

165 pipelineConnections=FgcmBuildStarsTableConnections): 

166 """Config for FgcmBuildStarsTableTask""" 

167 

168 referenceCCD = pexConfig.Field( 

169 doc="Reference CCD for checking PSF and background", 

170 dtype=int, 

171 default=40, 

172 ) 

173 

174 def setDefaults(self): 

175 super().setDefaults() 

176 

177 # The names here correspond to the post-transformed 

178 # sourceTable_visit catalogs, which differ from the raw src 

179 # catalogs. Therefore, all field and flag names cannot 

180 # be derived from the base config class. 

181 self.instFluxField = 'apFlux_12_0_instFlux' 

182 self.localBackgroundFluxField = 'localBackground_instFlux' 

183 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

184 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

185 self.psfCandidateName = 'calib_psf_candidate' 

186 

187 sourceSelector = self.sourceSelector["science"] 

188 

189 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

190 

191 sourceSelector.flags.bad = ['pixelFlags_edge', 

192 'pixelFlags_interpolatedCenter', 

193 'pixelFlags_saturatedCenter', 

194 'pixelFlags_crCenter', 

195 'pixelFlags_bad', 

196 'pixelFlags_interpolated', 

197 'pixelFlags_saturated', 

198 'centroid_flag', 

199 fluxFlagName] 

200 

201 if self.doSubtractLocalBackground: 

202 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

203 sourceSelector.flags.bad.append(localBackgroundFlagName) 

204 

205 sourceSelector.signalToNoise.fluxField = self.instFluxField 

206 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

207 

208 sourceSelector.isolated.parentName = 'parentSourceId' 

209 sourceSelector.isolated.nChildName = 'deblend_nChild' 

210 

211 sourceSelector.unresolved.name = 'extendedness' 

212 

213 

214class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

215 """ 

216 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

217 """ 

218 ConfigClass = FgcmBuildStarsTableConfig 

219 _DefaultName = "fgcmBuildStarsTable" 

220 

221 canMultiprocess = False 

222 

223 def __init__(self, initInputs=None, **kwargs): 

224 super().__init__(initInputs=initInputs, **kwargs) 

225 if initInputs is not None: 

226 self.sourceSchema = initInputs["sourceSchema"].schema 

227 

228 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

229 inputRefDict = butlerQC.get(inputRefs) 

230 

231 sourceTableHandles = inputRefDict['sourceTable_visit'] 

232 

233 self.log.info("Running with %d sourceTable_visit handles", 

234 len(sourceTableHandles)) 

235 

236 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

237 sourceTableHandle in sourceTableHandles} 

238 

239 if self.config.doReferenceMatches: 

240 # Get the LUT handle 

241 lutHandle = inputRefDict['fgcmLookUpTable'] 

242 

243 # Prepare the reference catalog loader 

244 refConfig = LoadReferenceObjectsConfig() 

245 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

246 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

247 for ref in inputRefs.refCat], 

248 refCats=butlerQC.get(inputRefs.refCat), 

249 log=self.log, 

250 config=refConfig) 

251 self.makeSubtask('fgcmLoadReferenceCatalog', 

252 refObjLoader=refObjLoader, 

253 refCatName=self.config.connections.refCat) 

254 else: 

255 lutHandle = None 

256 

257 # Compute aperture radius if necessary. This is useful to do now before 

258 # any heave lifting has happened (fail early). 

259 calibFluxApertureRadius = None 

260 if self.config.doSubtractLocalBackground: 

261 try: 

262 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

263 except RuntimeError as e: 

264 raise RuntimeError("Could not determine aperture radius from %s. " 

265 "Cannot use doSubtractLocalBackground." % 

266 (self.config.instFluxField)) from e 

267 

268 visitSummaryHandles = inputRefDict['visitSummary'] 

269 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

270 visitSummaryHandle in visitSummaryHandles} 

271 

272 camera = inputRefDict['camera'] 

273 groupedHandles = self._groupHandles(sourceTableHandleDict, 

274 visitSummaryHandleDict) 

275 

276 if self.config.doModelErrorsWithBackground: 

277 bkgHandles = inputRefDict['background'] 

278 bkgHandleDict = {(bkgHandle.dataId.byName()['visit'], 

279 bkgHandle.dataId.byName()['detector']): bkgHandle for 

280 bkgHandle in bkgHandles} 

281 else: 

282 bkgHandleDict = None 

283 

284 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles, 

285 bkgHandleDict=bkgHandleDict) 

286 

287 rad = calibFluxApertureRadius 

288 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

289 visitCat, 

290 self.sourceSchema, 

291 camera, 

292 calibFluxApertureRadius=rad) 

293 

294 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

295 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

296 

297 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

298 fgcmStarObservationCat, 

299 lutHandle=lutHandle) 

300 

301 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

302 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

303 if fgcmRefCat is not None: 

304 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

305 

306 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

307 """Group sourceTable and visitSummary handles. 

308 

309 Parameters 

310 ---------- 

311 sourceTableHandleDict : `dict` [`int`, `str`] 

312 Dict of source tables, keyed by visit. 

313 visitSummaryHandleDict : `dict` [int, `str`] 

314 Dict of visit summary catalogs, keyed by visit. 

315 

316 Returns 

317 ------- 

318 groupedHandles : `dict` [`int`, `list`] 

319 Dictionary with sorted visit keys, and `list`s with 

320 `lsst.daf.butler.DeferredDataSetHandle`. The first 

321 item in the list will be the visitSummary ref, and 

322 the second will be the source table ref. 

323 """ 

324 groupedHandles = collections.defaultdict(list) 

325 visits = sorted(sourceTableHandleDict.keys()) 

326 

327 for visit in visits: 

328 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

329 sourceTableHandleDict[visit]] 

330 

331 return groupedHandles 

332 

333 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

334 sourceSchema, 

335 camera, 

336 calibFluxApertureRadius=None): 

337 startTime = time.time() 

338 

339 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

340 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

341 

342 # To get the correct output schema, we use the legacy code. 

343 # We are not actually using this mapper, except to grab the outputSchema 

344 sourceMapper = self._makeSourceMapper(sourceSchema) 

345 outputSchema = sourceMapper.getOutputSchema() 

346 

347 # Construct mapping from ccd number to index 

348 ccdMapping = {} 

349 for ccdIndex, detector in enumerate(camera): 

350 ccdMapping[detector.getId()] = ccdIndex 

351 

352 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

353 

354 fullCatalog = afwTable.BaseCatalog(outputSchema) 

355 

356 visitKey = outputSchema['visit'].asKey() 

357 ccdKey = outputSchema['ccd'].asKey() 

358 instMagKey = outputSchema['instMag'].asKey() 

359 instMagErrKey = outputSchema['instMagErr'].asKey() 

360 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

361 

362 # Prepare local background if desired 

363 if self.config.doSubtractLocalBackground: 

364 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

365 

366 columns = None 

367 

368 k = 2.5/np.log(10.) 

369 

370 for counter, visit in enumerate(visitCat): 

371 expTime = visit['exptime'] 

372 

373 handle = groupedHandles[visit['visit']][-1] 

374 

375 if columns is None: 

376 inColumns = handle.get(component='columns') 

377 columns, detColumn = self._get_sourceTable_visit_columns(inColumns) 

378 df = handle.get(parameters={'columns': columns}) 

379 

380 goodSrc = self.sourceSelector.selectSources(df) 

381 

382 # Need to add a selection based on the local background correction 

383 # if necessary 

384 if self.config.doSubtractLocalBackground: 

385 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

386 use, = np.where((goodSrc.selected) 

387 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

388 else: 

389 use, = np.where(goodSrc.selected) 

390 

391 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

392 tempCat.resize(use.size) 

393 

394 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

395 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

396 tempCat['x'][:] = df['x'].values[use] 

397 tempCat['y'][:] = df['y'].values[use] 

398 # The "visit" name in the parquet table is hard-coded. 

399 tempCat[visitKey][:] = df['visit'].values[use] 

400 tempCat[ccdKey][:] = df[detColumn].values[use] 

401 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

402 

403 with np.warnings.catch_warnings(): 

404 # Ignore warnings, we will filter infinites and nans below 

405 np.warnings.simplefilter("ignore") 

406 

407 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

408 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

409 / df[self.config.apertureInnerInstFluxField].values[use]) 

410 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

411 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

412 / df[self.config.apertureOuterInstFluxField].values[use]) 

413 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

414 # Set bad values to illegal values for fgcm. 

415 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

416 

417 if self.config.doSubtractLocalBackground: 

418 # At the moment we only adjust the flux and not the flux 

419 # error by the background because the error on 

420 # base_LocalBackground_instFlux is the rms error in the 

421 # background annulus, not the error on the mean in the 

422 # background estimate (which is much smaller, by sqrt(n) 

423 # pixels used to estimate the background, which we do not 

424 # have access to in this task). In the default settings, 

425 # the annulus is sufficiently large such that these 

426 # additional errors are are negligibly small (much less 

427 # than a mmag in quadrature). 

428 

429 # This is the difference between the mag with local background correction 

430 # and the mag without local background correction. 

431 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

432 - localBackground[use]) - 

433 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

434 else: 

435 tempCat['deltaMagBkg'][:] = 0.0 

436 

437 # Need to loop over ccds here 

438 for detector in camera: 

439 ccdId = detector.getId() 

440 # used index for all observations with a given ccd 

441 use2 = (tempCat[ccdKey] == ccdId) 

442 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

443 tempCat['y'][use2]) 

444 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

445 * visit['scaling'][ccdMapping[ccdId]]) 

446 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

447 

448 # Compute instMagErr from instFluxErr/instFlux, any scaling 

449 # will cancel out. 

450 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

451 / df[self.config.instFluxField].values[use]) 

452 

453 # Apply the jacobian if configured 

454 if self.config.doApplyWcsJacobian: 

455 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

456 

457 fullCatalog.extend(tempCat) 

458 

459 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

460 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

461 

462 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

463 visit['sources_read'] = True 

464 

465 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

466 use.size, visit['visit'], visit['deltaAper']) 

467 

468 self.log.info("Found all good star observations in %.2f s" % 

469 (time.time() - startTime)) 

470 

471 return fullCatalog 

472 

473 def _get_sourceTable_visit_columns(self, inColumns): 

474 """ 

475 Get the sourceTable_visit columns from the config. 

476 

477 Parameters 

478 ---------- 

479 inColumns : `list` 

480 List of columns available in the sourceTable_visit 

481 

482 Returns 

483 ------- 

484 columns : `list` 

485 List of columns to read from sourceTable_visit. 

486 detectorColumn : `str` 

487 Name of the detector column. 

488 """ 

489 if 'detector' in inColumns: 

490 # Default name for Gen3. 

491 detectorColumn = 'detector' 

492 else: 

493 # Default name for Gen2 conversions (including test data). 

494 detectorColumn = 'ccd' 

495 # Some names are hard-coded in the parquet table. 

496 columns = ['visit', detectorColumn, 

497 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

498 self.config.instFluxField, self.config.instFluxField + 'Err', 

499 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

500 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

501 if self.sourceSelector.config.doFlags: 

502 columns.extend(self.sourceSelector.config.flags.bad) 

503 if self.sourceSelector.config.doUnresolved: 

504 columns.append(self.sourceSelector.config.unresolved.name) 

505 if self.sourceSelector.config.doIsolated: 

506 columns.append(self.sourceSelector.config.isolated.parentName) 

507 columns.append(self.sourceSelector.config.isolated.nChildName) 

508 if self.config.doSubtractLocalBackground: 

509 columns.append(self.config.localBackgroundFluxField) 

510 

511 return columns, detectorColumn