Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 16%

187 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-28 03:59 -0800

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35import collections 

36 

37import lsst.pex.config as pexConfig 

38import lsst.pipe.base as pipeBase 

39from lsst.pipe.base import connectionTypes 

40import lsst.afw.table as afwTable 

41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

42 

43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

45from .utilities import lookupStaticCalibrations 

46 

47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

48 

49 

50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument",), 

52 defaultTemplates={}): 

53 camera = connectionTypes.PrerequisiteInput( 

54 doc="Camera instrument", 

55 name="camera", 

56 storageClass="Camera", 

57 dimensions=("instrument",), 

58 lookupFunction=lookupStaticCalibrations, 

59 isCalibration=True, 

60 ) 

61 

62 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

64 "chromatic corrections."), 

65 name="fgcmLookUpTable", 

66 storageClass="Catalog", 

67 dimensions=("instrument",), 

68 deferLoad=True, 

69 ) 

70 

71 sourceSchema = connectionTypes.InitInput( 

72 doc="Schema for source catalogs", 

73 name="src_schema", 

74 storageClass="SourceCatalog", 

75 ) 

76 

77 refCat = connectionTypes.PrerequisiteInput( 

78 doc="Reference catalog to use for photometric calibration", 

79 name="cal_ref_cat", 

80 storageClass="SimpleCatalog", 

81 dimensions=("skypix",), 

82 deferLoad=True, 

83 multiple=True, 

84 ) 

85 

86 sourceTable_visit = connectionTypes.Input( 

87 doc="Source table in parquet format, per visit", 

88 name="sourceTable_visit", 

89 storageClass="DataFrame", 

90 dimensions=("instrument", "visit"), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 visitSummary = connectionTypes.Input( 

96 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

97 "detector id for the id and must be sorted for fast lookups of a " 

98 "detector."), 

99 name="visitSummary", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 background = connectionTypes.Input( 

107 doc="Calexp background model", 

108 name="calexpBackground", 

109 storageClass="Background", 

110 dimensions=("instrument", "visit", "detector"), 

111 deferLoad=True, 

112 multiple=True, 

113 ) 

114 

115 fgcmVisitCatalog = connectionTypes.Output( 

116 doc="Catalog of visit information for fgcm", 

117 name="fgcmVisitCatalog", 

118 storageClass="Catalog", 

119 dimensions=("instrument",), 

120 ) 

121 

122 fgcmStarObservations = connectionTypes.Output( 

123 doc="Catalog of star observations for fgcm", 

124 name="fgcmStarObservations", 

125 storageClass="Catalog", 

126 dimensions=("instrument",), 

127 ) 

128 

129 fgcmStarIds = connectionTypes.Output( 

130 doc="Catalog of fgcm calibration star IDs", 

131 name="fgcmStarIds", 

132 storageClass="Catalog", 

133 dimensions=("instrument",), 

134 ) 

135 

136 fgcmStarIndices = connectionTypes.Output( 

137 doc="Catalog of fgcm calibration star indices", 

138 name="fgcmStarIndices", 

139 storageClass="Catalog", 

140 dimensions=("instrument",), 

141 ) 

142 

143 fgcmReferenceStars = connectionTypes.Output( 

144 doc="Catalog of fgcm-matched reference stars", 

145 name="fgcmReferenceStars", 

146 storageClass="Catalog", 

147 dimensions=("instrument",), 

148 ) 

149 

150 def __init__(self, *, config=None): 

151 super().__init__(config=config) 

152 

153 if not config.doReferenceMatches: 

154 self.prerequisiteInputs.remove("refCat") 

155 self.prerequisiteInputs.remove("fgcmLookUpTable") 

156 

157 if not config.doModelErrorsWithBackground: 

158 self.inputs.remove("background") 

159 

160 if not config.doReferenceMatches: 

161 self.outputs.remove("fgcmReferenceStars") 

162 

163 

164class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

165 pipelineConnections=FgcmBuildStarsTableConnections): 

166 """Config for FgcmBuildStarsTableTask""" 

167 

168 referenceCCD = pexConfig.Field( 

169 doc="Reference CCD for checking PSF and background", 

170 dtype=int, 

171 default=40, 

172 ) 

173 

174 def setDefaults(self): 

175 super().setDefaults() 

176 

177 # The names here correspond to the post-transformed 

178 # sourceTable_visit catalogs, which differ from the raw src 

179 # catalogs. Therefore, all field and flag names cannot 

180 # be derived from the base config class. 

181 self.instFluxField = 'apFlux_12_0_instFlux' 

182 self.localBackgroundFluxField = 'localBackground_instFlux' 

183 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

184 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

185 self.psfCandidateName = 'calib_psf_candidate' 

186 

187 sourceSelector = self.sourceSelector["science"] 

188 

189 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

190 

191 sourceSelector.flags.bad = ['pixelFlags_edge', 

192 'pixelFlags_interpolatedCenter', 

193 'pixelFlags_saturatedCenter', 

194 'pixelFlags_crCenter', 

195 'pixelFlags_bad', 

196 'pixelFlags_interpolated', 

197 'pixelFlags_saturated', 

198 'centroid_flag', 

199 fluxFlagName] 

200 

201 if self.doSubtractLocalBackground: 

202 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

203 sourceSelector.flags.bad.append(localBackgroundFlagName) 

204 

205 sourceSelector.signalToNoise.fluxField = self.instFluxField 

206 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

207 

208 sourceSelector.isolated.parentName = 'parentSourceId' 

209 sourceSelector.isolated.nChildName = 'deblend_nChild' 

210 

211 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

212 sourceSelector.requireFiniteRaDec.decColName = 'decl' 

213 

214 sourceSelector.unresolved.name = 'extendedness' 

215 

216 

217class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

218 """ 

219 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

220 """ 

221 ConfigClass = FgcmBuildStarsTableConfig 

222 _DefaultName = "fgcmBuildStarsTable" 

223 

224 canMultiprocess = False 

225 

226 def __init__(self, initInputs=None, **kwargs): 

227 super().__init__(initInputs=initInputs, **kwargs) 

228 if initInputs is not None: 

229 self.sourceSchema = initInputs["sourceSchema"].schema 

230 

231 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

232 inputRefDict = butlerQC.get(inputRefs) 

233 

234 sourceTableHandles = inputRefDict['sourceTable_visit'] 

235 

236 self.log.info("Running with %d sourceTable_visit handles", 

237 len(sourceTableHandles)) 

238 

239 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

240 sourceTableHandle in sourceTableHandles} 

241 

242 if self.config.doReferenceMatches: 

243 # Get the LUT handle 

244 lutHandle = inputRefDict['fgcmLookUpTable'] 

245 

246 # Prepare the reference catalog loader 

247 refConfig = LoadReferenceObjectsConfig() 

248 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

249 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

250 for ref in inputRefs.refCat], 

251 refCats=butlerQC.get(inputRefs.refCat), 

252 name=self.config.connections.refCat, 

253 log=self.log, 

254 config=refConfig) 

255 self.makeSubtask('fgcmLoadReferenceCatalog', 

256 refObjLoader=refObjLoader, 

257 refCatName=self.config.connections.refCat) 

258 else: 

259 lutHandle = None 

260 

261 # Compute aperture radius if necessary. This is useful to do now before 

262 # any heave lifting has happened (fail early). 

263 calibFluxApertureRadius = None 

264 if self.config.doSubtractLocalBackground: 

265 try: 

266 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

267 except RuntimeError as e: 

268 raise RuntimeError("Could not determine aperture radius from %s. " 

269 "Cannot use doSubtractLocalBackground." % 

270 (self.config.instFluxField)) from e 

271 

272 visitSummaryHandles = inputRefDict['visitSummary'] 

273 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

274 visitSummaryHandle in visitSummaryHandles} 

275 

276 camera = inputRefDict['camera'] 

277 groupedHandles = self._groupHandles(sourceTableHandleDict, 

278 visitSummaryHandleDict) 

279 

280 if self.config.doModelErrorsWithBackground: 

281 bkgHandles = inputRefDict['background'] 

282 bkgHandleDict = {(bkgHandle.dataId.byName()['visit'], 

283 bkgHandle.dataId.byName()['detector']): bkgHandle for 

284 bkgHandle in bkgHandles} 

285 else: 

286 bkgHandleDict = None 

287 

288 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles, 

289 bkgHandleDict=bkgHandleDict) 

290 

291 rad = calibFluxApertureRadius 

292 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

293 visitCat, 

294 self.sourceSchema, 

295 camera, 

296 calibFluxApertureRadius=rad) 

297 

298 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

299 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

300 

301 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

302 fgcmStarObservationCat, 

303 lutHandle=lutHandle) 

304 

305 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

306 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

307 if fgcmRefCat is not None: 

308 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

309 

310 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

311 """Group sourceTable and visitSummary handles. 

312 

313 Parameters 

314 ---------- 

315 sourceTableHandleDict : `dict` [`int`, `str`] 

316 Dict of source tables, keyed by visit. 

317 visitSummaryHandleDict : `dict` [int, `str`] 

318 Dict of visit summary catalogs, keyed by visit. 

319 

320 Returns 

321 ------- 

322 groupedHandles : `dict` [`int`, `list`] 

323 Dictionary with sorted visit keys, and `list`s with 

324 `lsst.daf.butler.DeferredDataSetHandle`. The first 

325 item in the list will be the visitSummary ref, and 

326 the second will be the source table ref. 

327 """ 

328 groupedHandles = collections.defaultdict(list) 

329 visits = sorted(sourceTableHandleDict.keys()) 

330 

331 for visit in visits: 

332 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

333 sourceTableHandleDict[visit]] 

334 

335 return groupedHandles 

336 

337 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

338 sourceSchema, 

339 camera, 

340 calibFluxApertureRadius=None): 

341 startTime = time.time() 

342 

343 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

344 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

345 

346 # To get the correct output schema, we use the legacy code. 

347 # We are not actually using this mapper, except to grab the outputSchema 

348 sourceMapper = self._makeSourceMapper(sourceSchema) 

349 outputSchema = sourceMapper.getOutputSchema() 

350 

351 # Construct mapping from ccd number to index 

352 ccdMapping = {} 

353 for ccdIndex, detector in enumerate(camera): 

354 ccdMapping[detector.getId()] = ccdIndex 

355 

356 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

357 

358 fullCatalog = afwTable.BaseCatalog(outputSchema) 

359 

360 visitKey = outputSchema['visit'].asKey() 

361 ccdKey = outputSchema['ccd'].asKey() 

362 instMagKey = outputSchema['instMag'].asKey() 

363 instMagErrKey = outputSchema['instMagErr'].asKey() 

364 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

365 

366 # Prepare local background if desired 

367 if self.config.doSubtractLocalBackground: 

368 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

369 

370 columns = None 

371 

372 k = 2.5/np.log(10.) 

373 

374 for counter, visit in enumerate(visitCat): 

375 expTime = visit['exptime'] 

376 

377 handle = groupedHandles[visit['visit']][-1] 

378 

379 if columns is None: 

380 inColumns = handle.get(component='columns') 

381 columns = self._get_sourceTable_visit_columns(inColumns) 

382 df = handle.get(parameters={'columns': columns}) 

383 

384 goodSrc = self.sourceSelector.selectSources(df) 

385 

386 # Need to add a selection based on the local background correction 

387 # if necessary 

388 if self.config.doSubtractLocalBackground: 

389 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

390 use, = np.where((goodSrc.selected) 

391 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

392 else: 

393 use, = np.where(goodSrc.selected) 

394 

395 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

396 tempCat.resize(use.size) 

397 

398 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

399 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

400 tempCat['x'][:] = df['x'].values[use] 

401 tempCat['y'][:] = df['y'].values[use] 

402 # The "visit" name in the parquet table is hard-coded. 

403 tempCat[visitKey][:] = df['visit'].values[use] 

404 tempCat[ccdKey][:] = df['detector'].values[use] 

405 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

406 

407 with np.warnings.catch_warnings(): 

408 # Ignore warnings, we will filter infinites and nans below 

409 np.warnings.simplefilter("ignore") 

410 

411 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

412 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

413 / df[self.config.apertureInnerInstFluxField].values[use]) 

414 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

415 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

416 / df[self.config.apertureOuterInstFluxField].values[use]) 

417 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

418 # Set bad values to illegal values for fgcm. 

419 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

420 

421 if self.config.doSubtractLocalBackground: 

422 # At the moment we only adjust the flux and not the flux 

423 # error by the background because the error on 

424 # base_LocalBackground_instFlux is the rms error in the 

425 # background annulus, not the error on the mean in the 

426 # background estimate (which is much smaller, by sqrt(n) 

427 # pixels used to estimate the background, which we do not 

428 # have access to in this task). In the default settings, 

429 # the annulus is sufficiently large such that these 

430 # additional errors are are negligibly small (much less 

431 # than a mmag in quadrature). 

432 

433 # This is the difference between the mag with local background correction 

434 # and the mag without local background correction. 

435 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

436 - localBackground[use]) - 

437 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

438 else: 

439 tempCat['deltaMagBkg'][:] = 0.0 

440 

441 # Need to loop over ccds here 

442 for detector in camera: 

443 ccdId = detector.getId() 

444 # used index for all observations with a given ccd 

445 use2 = (tempCat[ccdKey] == ccdId) 

446 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

447 tempCat['y'][use2]) 

448 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

449 * visit['scaling'][ccdMapping[ccdId]]) 

450 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

451 

452 # Compute instMagErr from instFluxErr/instFlux, any scaling 

453 # will cancel out. 

454 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

455 / df[self.config.instFluxField].values[use]) 

456 

457 # Apply the jacobian if configured 

458 if self.config.doApplyWcsJacobian: 

459 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

460 

461 fullCatalog.extend(tempCat) 

462 

463 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

464 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

465 

466 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

467 visit['sources_read'] = True 

468 

469 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

470 use.size, visit['visit'], visit['deltaAper']) 

471 

472 self.log.info("Found all good star observations in %.2f s" % 

473 (time.time() - startTime)) 

474 

475 return fullCatalog 

476 

477 def _get_sourceTable_visit_columns(self, inColumns): 

478 """ 

479 Get the sourceTable_visit columns from the config. 

480 

481 Parameters 

482 ---------- 

483 inColumns : `list` 

484 List of columns available in the sourceTable_visit 

485 

486 Returns 

487 ------- 

488 columns : `list` 

489 List of columns to read from sourceTable_visit. 

490 """ 

491 # Some names are hard-coded in the parquet table. 

492 columns = ['visit', 'detector', 

493 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

494 self.config.instFluxField, self.config.instFluxField + 'Err', 

495 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

496 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

497 if self.sourceSelector.config.doFlags: 

498 columns.extend(self.sourceSelector.config.flags.bad) 

499 if self.sourceSelector.config.doUnresolved: 

500 columns.append(self.sourceSelector.config.unresolved.name) 

501 if self.sourceSelector.config.doIsolated: 

502 columns.append(self.sourceSelector.config.isolated.parentName) 

503 columns.append(self.sourceSelector.config.isolated.nChildName) 

504 if self.config.doSubtractLocalBackground: 

505 columns.append(self.config.localBackgroundFluxField) 

506 

507 return columns