Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%

181 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-23 04:26 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33import warnings 

34 

35import numpy as np 

36import collections 

37 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40from lsst.pipe.base import connectionTypes 

41import lsst.afw.table as afwTable 

42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

43 

44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

46from .utilities import lookupStaticCalibrations 

47 

48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

49 

50 

51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

52 dimensions=("instrument",), 

53 defaultTemplates={}): 

54 camera = connectionTypes.PrerequisiteInput( 

55 doc="Camera instrument", 

56 name="camera", 

57 storageClass="Camera", 

58 dimensions=("instrument",), 

59 lookupFunction=lookupStaticCalibrations, 

60 isCalibration=True, 

61 ) 

62 

63 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

65 "chromatic corrections."), 

66 name="fgcmLookUpTable", 

67 storageClass="Catalog", 

68 dimensions=("instrument",), 

69 deferLoad=True, 

70 ) 

71 

72 sourceSchema = connectionTypes.InitInput( 

73 doc="Schema for source catalogs", 

74 name="src_schema", 

75 storageClass="SourceCatalog", 

76 ) 

77 

78 refCat = connectionTypes.PrerequisiteInput( 

79 doc="Reference catalog to use for photometric calibration", 

80 name="cal_ref_cat", 

81 storageClass="SimpleCatalog", 

82 dimensions=("skypix",), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 sourceTable_visit = connectionTypes.Input( 

88 doc="Source table in parquet format, per visit", 

89 name="sourceTable_visit", 

90 storageClass="DataFrame", 

91 dimensions=("instrument", "visit"), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 visitSummary = connectionTypes.Input( 

97 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

98 "detector id for the id and must be sorted for fast lookups of a " 

99 "detector."), 

100 name="visitSummary", 

101 storageClass="ExposureCatalog", 

102 dimensions=("instrument", "visit"), 

103 deferLoad=True, 

104 multiple=True, 

105 ) 

106 

107 fgcmVisitCatalog = connectionTypes.Output( 

108 doc="Catalog of visit information for fgcm", 

109 name="fgcmVisitCatalog", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 ) 

113 

114 fgcmStarObservations = connectionTypes.Output( 

115 doc="Catalog of star observations for fgcm", 

116 name="fgcmStarObservations", 

117 storageClass="Catalog", 

118 dimensions=("instrument",), 

119 ) 

120 

121 fgcmStarIds = connectionTypes.Output( 

122 doc="Catalog of fgcm calibration star IDs", 

123 name="fgcmStarIds", 

124 storageClass="Catalog", 

125 dimensions=("instrument",), 

126 ) 

127 

128 fgcmStarIndices = connectionTypes.Output( 

129 doc="Catalog of fgcm calibration star indices", 

130 name="fgcmStarIndices", 

131 storageClass="Catalog", 

132 dimensions=("instrument",), 

133 ) 

134 

135 fgcmReferenceStars = connectionTypes.Output( 

136 doc="Catalog of fgcm-matched reference stars", 

137 name="fgcmReferenceStars", 

138 storageClass="Catalog", 

139 dimensions=("instrument",), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doReferenceMatches: 

146 self.prerequisiteInputs.remove("refCat") 

147 self.prerequisiteInputs.remove("fgcmLookUpTable") 

148 

149 if not config.doReferenceMatches: 

150 self.outputs.remove("fgcmReferenceStars") 

151 

152 

153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

154 pipelineConnections=FgcmBuildStarsTableConnections): 

155 """Config for FgcmBuildStarsTableTask""" 

156 

157 referenceCCD = pexConfig.Field( 

158 doc="Reference CCD for checking PSF and background", 

159 dtype=int, 

160 default=40, 

161 ) 

162 

163 def setDefaults(self): 

164 super().setDefaults() 

165 

166 # The names here correspond to the post-transformed 

167 # sourceTable_visit catalogs, which differ from the raw src 

168 # catalogs. Therefore, all field and flag names cannot 

169 # be derived from the base config class. 

170 self.instFluxField = 'apFlux_12_0_instFlux' 

171 self.localBackgroundFluxField = 'localBackground_instFlux' 

172 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

173 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

174 self.psfCandidateName = 'calib_psf_candidate' 

175 

176 sourceSelector = self.sourceSelector["science"] 

177 

178 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

179 

180 sourceSelector.flags.bad = ['pixelFlags_edge', 

181 'pixelFlags_interpolatedCenter', 

182 'pixelFlags_saturatedCenter', 

183 'pixelFlags_crCenter', 

184 'pixelFlags_bad', 

185 'pixelFlags_interpolated', 

186 'pixelFlags_saturated', 

187 'centroid_flag', 

188 fluxFlagName] 

189 

190 if self.doSubtractLocalBackground: 

191 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

192 sourceSelector.flags.bad.append(localBackgroundFlagName) 

193 

194 sourceSelector.signalToNoise.fluxField = self.instFluxField 

195 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

196 

197 sourceSelector.isolated.parentName = 'parentSourceId' 

198 sourceSelector.isolated.nChildName = 'deblend_nChild' 

199 

200 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

201 sourceSelector.requireFiniteRaDec.decColName = 'decl' 

202 

203 sourceSelector.unresolved.name = 'extendedness' 

204 

205 

206class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

207 """ 

208 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

209 """ 

210 ConfigClass = FgcmBuildStarsTableConfig 

211 _DefaultName = "fgcmBuildStarsTable" 

212 

213 canMultiprocess = False 

214 

215 def __init__(self, initInputs=None, **kwargs): 

216 super().__init__(initInputs=initInputs, **kwargs) 

217 if initInputs is not None: 

218 self.sourceSchema = initInputs["sourceSchema"].schema 

219 

220 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

221 inputRefDict = butlerQC.get(inputRefs) 

222 

223 sourceTableHandles = inputRefDict['sourceTable_visit'] 

224 

225 self.log.info("Running with %d sourceTable_visit handles", 

226 len(sourceTableHandles)) 

227 

228 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

229 sourceTableHandle in sourceTableHandles} 

230 

231 if self.config.doReferenceMatches: 

232 # Get the LUT handle 

233 lutHandle = inputRefDict['fgcmLookUpTable'] 

234 

235 # Prepare the reference catalog loader 

236 refConfig = LoadReferenceObjectsConfig() 

237 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

238 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

239 for ref in inputRefs.refCat], 

240 refCats=butlerQC.get(inputRefs.refCat), 

241 name=self.config.connections.refCat, 

242 log=self.log, 

243 config=refConfig) 

244 self.makeSubtask('fgcmLoadReferenceCatalog', 

245 refObjLoader=refObjLoader, 

246 refCatName=self.config.connections.refCat) 

247 else: 

248 lutHandle = None 

249 

250 # Compute aperture radius if necessary. This is useful to do now before 

251 # any heave lifting has happened (fail early). 

252 calibFluxApertureRadius = None 

253 if self.config.doSubtractLocalBackground: 

254 try: 

255 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

256 except RuntimeError as e: 

257 raise RuntimeError("Could not determine aperture radius from %s. " 

258 "Cannot use doSubtractLocalBackground." % 

259 (self.config.instFluxField)) from e 

260 

261 visitSummaryHandles = inputRefDict['visitSummary'] 

262 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

263 visitSummaryHandle in visitSummaryHandles} 

264 

265 camera = inputRefDict['camera'] 

266 groupedHandles = self._groupHandles(sourceTableHandleDict, 

267 visitSummaryHandleDict) 

268 

269 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles) 

270 

271 rad = calibFluxApertureRadius 

272 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

273 visitCat, 

274 self.sourceSchema, 

275 camera, 

276 calibFluxApertureRadius=rad) 

277 

278 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

279 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

280 

281 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

282 fgcmStarObservationCat, 

283 lutHandle=lutHandle) 

284 

285 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

286 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

287 if fgcmRefCat is not None: 

288 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

289 

290 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

291 """Group sourceTable and visitSummary handles. 

292 

293 Parameters 

294 ---------- 

295 sourceTableHandleDict : `dict` [`int`, `str`] 

296 Dict of source tables, keyed by visit. 

297 visitSummaryHandleDict : `dict` [int, `str`] 

298 Dict of visit summary catalogs, keyed by visit. 

299 

300 Returns 

301 ------- 

302 groupedHandles : `dict` [`int`, `list`] 

303 Dictionary with sorted visit keys, and `list`s with 

304 `lsst.daf.butler.DeferredDataSetHandle`. The first 

305 item in the list will be the visitSummary ref, and 

306 the second will be the source table ref. 

307 """ 

308 groupedHandles = collections.defaultdict(list) 

309 visits = sorted(sourceTableHandleDict.keys()) 

310 

311 for visit in visits: 

312 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

313 sourceTableHandleDict[visit]] 

314 

315 return groupedHandles 

316 

317 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

318 sourceSchema, 

319 camera, 

320 calibFluxApertureRadius=None): 

321 startTime = time.time() 

322 

323 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

324 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

325 

326 # To get the correct output schema, we use the legacy code. 

327 # We are not actually using this mapper, except to grab the outputSchema 

328 sourceMapper = self._makeSourceMapper(sourceSchema) 

329 outputSchema = sourceMapper.getOutputSchema() 

330 

331 # Construct mapping from ccd number to index 

332 ccdMapping = {} 

333 for ccdIndex, detector in enumerate(camera): 

334 ccdMapping[detector.getId()] = ccdIndex 

335 

336 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

337 

338 fullCatalog = afwTable.BaseCatalog(outputSchema) 

339 

340 visitKey = outputSchema['visit'].asKey() 

341 ccdKey = outputSchema['ccd'].asKey() 

342 instMagKey = outputSchema['instMag'].asKey() 

343 instMagErrKey = outputSchema['instMagErr'].asKey() 

344 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

345 

346 # Prepare local background if desired 

347 if self.config.doSubtractLocalBackground: 

348 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

349 

350 columns = None 

351 

352 k = 2.5/np.log(10.) 

353 

354 for counter, visit in enumerate(visitCat): 

355 expTime = visit['exptime'] 

356 

357 handle = groupedHandles[visit['visit']][-1] 

358 

359 if columns is None: 

360 inColumns = handle.get(component='columns') 

361 columns = self._get_sourceTable_visit_columns(inColumns) 

362 df = handle.get(parameters={'columns': columns}) 

363 

364 goodSrc = self.sourceSelector.selectSources(df) 

365 

366 # Need to add a selection based on the local background correction 

367 # if necessary 

368 if self.config.doSubtractLocalBackground: 

369 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

370 use, = np.where((goodSrc.selected) 

371 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

372 else: 

373 use, = np.where(goodSrc.selected) 

374 

375 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

376 tempCat.resize(use.size) 

377 

378 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

379 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

380 tempCat['x'][:] = df['x'].values[use] 

381 tempCat['y'][:] = df['y'].values[use] 

382 # The "visit" name in the parquet table is hard-coded. 

383 tempCat[visitKey][:] = df['visit'].values[use] 

384 tempCat[ccdKey][:] = df['detector'].values[use] 

385 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

386 

387 with warnings.catch_warnings(): 

388 # Ignore warnings, we will filter infinites and nans below 

389 warnings.simplefilter("ignore") 

390 

391 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

392 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

393 / df[self.config.apertureInnerInstFluxField].values[use]) 

394 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

395 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

396 / df[self.config.apertureOuterInstFluxField].values[use]) 

397 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

398 # Set bad values to illegal values for fgcm. 

399 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

400 

401 if self.config.doSubtractLocalBackground: 

402 # At the moment we only adjust the flux and not the flux 

403 # error by the background because the error on 

404 # base_LocalBackground_instFlux is the rms error in the 

405 # background annulus, not the error on the mean in the 

406 # background estimate (which is much smaller, by sqrt(n) 

407 # pixels used to estimate the background, which we do not 

408 # have access to in this task). In the default settings, 

409 # the annulus is sufficiently large such that these 

410 # additional errors are are negligibly small (much less 

411 # than a mmag in quadrature). 

412 

413 # This is the difference between the mag with local background correction 

414 # and the mag without local background correction. 

415 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

416 - localBackground[use]) - 

417 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

418 else: 

419 tempCat['deltaMagBkg'][:] = 0.0 

420 

421 # Need to loop over ccds here 

422 for detector in camera: 

423 ccdId = detector.getId() 

424 # used index for all observations with a given ccd 

425 use2 = (tempCat[ccdKey] == ccdId) 

426 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

427 tempCat['y'][use2]) 

428 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

429 * visit['scaling'][ccdMapping[ccdId]]) 

430 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

431 

432 # Compute instMagErr from instFluxErr/instFlux, any scaling 

433 # will cancel out. 

434 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

435 / df[self.config.instFluxField].values[use]) 

436 

437 # Apply the jacobian if configured 

438 if self.config.doApplyWcsJacobian: 

439 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

440 

441 fullCatalog.extend(tempCat) 

442 

443 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

444 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

445 

446 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

447 visit['sources_read'] = True 

448 

449 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

450 use.size, visit['visit'], visit['deltaAper']) 

451 

452 self.log.info("Found all good star observations in %.2f s" % 

453 (time.time() - startTime)) 

454 

455 return fullCatalog 

456 

457 def _get_sourceTable_visit_columns(self, inColumns): 

458 """ 

459 Get the sourceTable_visit columns from the config. 

460 

461 Parameters 

462 ---------- 

463 inColumns : `list` 

464 List of columns available in the sourceTable_visit 

465 

466 Returns 

467 ------- 

468 columns : `list` 

469 List of columns to read from sourceTable_visit. 

470 """ 

471 # Some names are hard-coded in the parquet table. 

472 columns = ['visit', 'detector', 

473 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

474 self.config.instFluxField, self.config.instFluxField + 'Err', 

475 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

476 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

477 if self.sourceSelector.config.doFlags: 

478 columns.extend(self.sourceSelector.config.flags.bad) 

479 if self.sourceSelector.config.doUnresolved: 

480 columns.append(self.sourceSelector.config.unresolved.name) 

481 if self.sourceSelector.config.doIsolated: 

482 columns.append(self.sourceSelector.config.isolated.parentName) 

483 columns.append(self.sourceSelector.config.isolated.nChildName) 

484 if self.config.doSubtractLocalBackground: 

485 columns.append(self.config.localBackgroundFluxField) 

486 

487 return columns