Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%

180 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-29 11:20 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33 

34import numpy as np 

35import collections 

36 

37import lsst.pex.config as pexConfig 

38import lsst.pipe.base as pipeBase 

39from lsst.pipe.base import connectionTypes 

40import lsst.afw.table as afwTable 

41from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

42 

43from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

44from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

45from .utilities import lookupStaticCalibrations 

46 

47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

48 

49 

50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument",), 

52 defaultTemplates={}): 

53 camera = connectionTypes.PrerequisiteInput( 

54 doc="Camera instrument", 

55 name="camera", 

56 storageClass="Camera", 

57 dimensions=("instrument",), 

58 lookupFunction=lookupStaticCalibrations, 

59 isCalibration=True, 

60 ) 

61 

62 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

63 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

64 "chromatic corrections."), 

65 name="fgcmLookUpTable", 

66 storageClass="Catalog", 

67 dimensions=("instrument",), 

68 deferLoad=True, 

69 ) 

70 

71 sourceSchema = connectionTypes.InitInput( 

72 doc="Schema for source catalogs", 

73 name="src_schema", 

74 storageClass="SourceCatalog", 

75 ) 

76 

77 refCat = connectionTypes.PrerequisiteInput( 

78 doc="Reference catalog to use for photometric calibration", 

79 name="cal_ref_cat", 

80 storageClass="SimpleCatalog", 

81 dimensions=("skypix",), 

82 deferLoad=True, 

83 multiple=True, 

84 ) 

85 

86 sourceTable_visit = connectionTypes.Input( 

87 doc="Source table in parquet format, per visit", 

88 name="sourceTable_visit", 

89 storageClass="DataFrame", 

90 dimensions=("instrument", "visit"), 

91 deferLoad=True, 

92 multiple=True, 

93 ) 

94 

95 visitSummary = connectionTypes.Input( 

96 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

97 "detector id for the id and must be sorted for fast lookups of a " 

98 "detector."), 

99 name="visitSummary", 

100 storageClass="ExposureCatalog", 

101 dimensions=("instrument", "visit"), 

102 deferLoad=True, 

103 multiple=True, 

104 ) 

105 

106 fgcmVisitCatalog = connectionTypes.Output( 

107 doc="Catalog of visit information for fgcm", 

108 name="fgcmVisitCatalog", 

109 storageClass="Catalog", 

110 dimensions=("instrument",), 

111 ) 

112 

113 fgcmStarObservations = connectionTypes.Output( 

114 doc="Catalog of star observations for fgcm", 

115 name="fgcmStarObservations", 

116 storageClass="Catalog", 

117 dimensions=("instrument",), 

118 ) 

119 

120 fgcmStarIds = connectionTypes.Output( 

121 doc="Catalog of fgcm calibration star IDs", 

122 name="fgcmStarIds", 

123 storageClass="Catalog", 

124 dimensions=("instrument",), 

125 ) 

126 

127 fgcmStarIndices = connectionTypes.Output( 

128 doc="Catalog of fgcm calibration star indices", 

129 name="fgcmStarIndices", 

130 storageClass="Catalog", 

131 dimensions=("instrument",), 

132 ) 

133 

134 fgcmReferenceStars = connectionTypes.Output( 

135 doc="Catalog of fgcm-matched reference stars", 

136 name="fgcmReferenceStars", 

137 storageClass="Catalog", 

138 dimensions=("instrument",), 

139 ) 

140 

141 def __init__(self, *, config=None): 

142 super().__init__(config=config) 

143 

144 if not config.doReferenceMatches: 

145 self.prerequisiteInputs.remove("refCat") 

146 self.prerequisiteInputs.remove("fgcmLookUpTable") 

147 

148 if not config.doReferenceMatches: 

149 self.outputs.remove("fgcmReferenceStars") 

150 

151 

152class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

153 pipelineConnections=FgcmBuildStarsTableConnections): 

154 """Config for FgcmBuildStarsTableTask""" 

155 

156 referenceCCD = pexConfig.Field( 

157 doc="Reference CCD for checking PSF and background", 

158 dtype=int, 

159 default=40, 

160 ) 

161 

162 def setDefaults(self): 

163 super().setDefaults() 

164 

165 # The names here correspond to the post-transformed 

166 # sourceTable_visit catalogs, which differ from the raw src 

167 # catalogs. Therefore, all field and flag names cannot 

168 # be derived from the base config class. 

169 self.instFluxField = 'apFlux_12_0_instFlux' 

170 self.localBackgroundFluxField = 'localBackground_instFlux' 

171 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

172 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

173 self.psfCandidateName = 'calib_psf_candidate' 

174 

175 sourceSelector = self.sourceSelector["science"] 

176 

177 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

178 

179 sourceSelector.flags.bad = ['pixelFlags_edge', 

180 'pixelFlags_interpolatedCenter', 

181 'pixelFlags_saturatedCenter', 

182 'pixelFlags_crCenter', 

183 'pixelFlags_bad', 

184 'pixelFlags_interpolated', 

185 'pixelFlags_saturated', 

186 'centroid_flag', 

187 fluxFlagName] 

188 

189 if self.doSubtractLocalBackground: 

190 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

191 sourceSelector.flags.bad.append(localBackgroundFlagName) 

192 

193 sourceSelector.signalToNoise.fluxField = self.instFluxField 

194 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

195 

196 sourceSelector.isolated.parentName = 'parentSourceId' 

197 sourceSelector.isolated.nChildName = 'deblend_nChild' 

198 

199 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

200 sourceSelector.requireFiniteRaDec.decColName = 'decl' 

201 

202 sourceSelector.unresolved.name = 'extendedness' 

203 

204 

205class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

206 """ 

207 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

208 """ 

209 ConfigClass = FgcmBuildStarsTableConfig 

210 _DefaultName = "fgcmBuildStarsTable" 

211 

212 canMultiprocess = False 

213 

214 def __init__(self, initInputs=None, **kwargs): 

215 super().__init__(initInputs=initInputs, **kwargs) 

216 if initInputs is not None: 

217 self.sourceSchema = initInputs["sourceSchema"].schema 

218 

219 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

220 inputRefDict = butlerQC.get(inputRefs) 

221 

222 sourceTableHandles = inputRefDict['sourceTable_visit'] 

223 

224 self.log.info("Running with %d sourceTable_visit handles", 

225 len(sourceTableHandles)) 

226 

227 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

228 sourceTableHandle in sourceTableHandles} 

229 

230 if self.config.doReferenceMatches: 

231 # Get the LUT handle 

232 lutHandle = inputRefDict['fgcmLookUpTable'] 

233 

234 # Prepare the reference catalog loader 

235 refConfig = LoadReferenceObjectsConfig() 

236 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

237 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

238 for ref in inputRefs.refCat], 

239 refCats=butlerQC.get(inputRefs.refCat), 

240 name=self.config.connections.refCat, 

241 log=self.log, 

242 config=refConfig) 

243 self.makeSubtask('fgcmLoadReferenceCatalog', 

244 refObjLoader=refObjLoader, 

245 refCatName=self.config.connections.refCat) 

246 else: 

247 lutHandle = None 

248 

249 # Compute aperture radius if necessary. This is useful to do now before 

250 # any heave lifting has happened (fail early). 

251 calibFluxApertureRadius = None 

252 if self.config.doSubtractLocalBackground: 

253 try: 

254 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

255 except RuntimeError as e: 

256 raise RuntimeError("Could not determine aperture radius from %s. " 

257 "Cannot use doSubtractLocalBackground." % 

258 (self.config.instFluxField)) from e 

259 

260 visitSummaryHandles = inputRefDict['visitSummary'] 

261 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

262 visitSummaryHandle in visitSummaryHandles} 

263 

264 camera = inputRefDict['camera'] 

265 groupedHandles = self._groupHandles(sourceTableHandleDict, 

266 visitSummaryHandleDict) 

267 

268 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles) 

269 

270 rad = calibFluxApertureRadius 

271 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

272 visitCat, 

273 self.sourceSchema, 

274 camera, 

275 calibFluxApertureRadius=rad) 

276 

277 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

278 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

279 

280 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

281 fgcmStarObservationCat, 

282 lutHandle=lutHandle) 

283 

284 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

285 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

286 if fgcmRefCat is not None: 

287 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

288 

289 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

290 """Group sourceTable and visitSummary handles. 

291 

292 Parameters 

293 ---------- 

294 sourceTableHandleDict : `dict` [`int`, `str`] 

295 Dict of source tables, keyed by visit. 

296 visitSummaryHandleDict : `dict` [int, `str`] 

297 Dict of visit summary catalogs, keyed by visit. 

298 

299 Returns 

300 ------- 

301 groupedHandles : `dict` [`int`, `list`] 

302 Dictionary with sorted visit keys, and `list`s with 

303 `lsst.daf.butler.DeferredDataSetHandle`. The first 

304 item in the list will be the visitSummary ref, and 

305 the second will be the source table ref. 

306 """ 

307 groupedHandles = collections.defaultdict(list) 

308 visits = sorted(sourceTableHandleDict.keys()) 

309 

310 for visit in visits: 

311 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

312 sourceTableHandleDict[visit]] 

313 

314 return groupedHandles 

315 

316 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

317 sourceSchema, 

318 camera, 

319 calibFluxApertureRadius=None): 

320 startTime = time.time() 

321 

322 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

323 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

324 

325 # To get the correct output schema, we use the legacy code. 

326 # We are not actually using this mapper, except to grab the outputSchema 

327 sourceMapper = self._makeSourceMapper(sourceSchema) 

328 outputSchema = sourceMapper.getOutputSchema() 

329 

330 # Construct mapping from ccd number to index 

331 ccdMapping = {} 

332 for ccdIndex, detector in enumerate(camera): 

333 ccdMapping[detector.getId()] = ccdIndex 

334 

335 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

336 

337 fullCatalog = afwTable.BaseCatalog(outputSchema) 

338 

339 visitKey = outputSchema['visit'].asKey() 

340 ccdKey = outputSchema['ccd'].asKey() 

341 instMagKey = outputSchema['instMag'].asKey() 

342 instMagErrKey = outputSchema['instMagErr'].asKey() 

343 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

344 

345 # Prepare local background if desired 

346 if self.config.doSubtractLocalBackground: 

347 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

348 

349 columns = None 

350 

351 k = 2.5/np.log(10.) 

352 

353 for counter, visit in enumerate(visitCat): 

354 expTime = visit['exptime'] 

355 

356 handle = groupedHandles[visit['visit']][-1] 

357 

358 if columns is None: 

359 inColumns = handle.get(component='columns') 

360 columns = self._get_sourceTable_visit_columns(inColumns) 

361 df = handle.get(parameters={'columns': columns}) 

362 

363 goodSrc = self.sourceSelector.selectSources(df) 

364 

365 # Need to add a selection based on the local background correction 

366 # if necessary 

367 if self.config.doSubtractLocalBackground: 

368 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

369 use, = np.where((goodSrc.selected) 

370 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

371 else: 

372 use, = np.where(goodSrc.selected) 

373 

374 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

375 tempCat.resize(use.size) 

376 

377 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

378 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

379 tempCat['x'][:] = df['x'].values[use] 

380 tempCat['y'][:] = df['y'].values[use] 

381 # The "visit" name in the parquet table is hard-coded. 

382 tempCat[visitKey][:] = df['visit'].values[use] 

383 tempCat[ccdKey][:] = df['detector'].values[use] 

384 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

385 

386 with np.warnings.catch_warnings(): 

387 # Ignore warnings, we will filter infinites and nans below 

388 np.warnings.simplefilter("ignore") 

389 

390 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

391 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

392 / df[self.config.apertureInnerInstFluxField].values[use]) 

393 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

394 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

395 / df[self.config.apertureOuterInstFluxField].values[use]) 

396 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

397 # Set bad values to illegal values for fgcm. 

398 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

399 

400 if self.config.doSubtractLocalBackground: 

401 # At the moment we only adjust the flux and not the flux 

402 # error by the background because the error on 

403 # base_LocalBackground_instFlux is the rms error in the 

404 # background annulus, not the error on the mean in the 

405 # background estimate (which is much smaller, by sqrt(n) 

406 # pixels used to estimate the background, which we do not 

407 # have access to in this task). In the default settings, 

408 # the annulus is sufficiently large such that these 

409 # additional errors are are negligibly small (much less 

410 # than a mmag in quadrature). 

411 

412 # This is the difference between the mag with local background correction 

413 # and the mag without local background correction. 

414 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

415 - localBackground[use]) - 

416 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

417 else: 

418 tempCat['deltaMagBkg'][:] = 0.0 

419 

420 # Need to loop over ccds here 

421 for detector in camera: 

422 ccdId = detector.getId() 

423 # used index for all observations with a given ccd 

424 use2 = (tempCat[ccdKey] == ccdId) 

425 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

426 tempCat['y'][use2]) 

427 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

428 * visit['scaling'][ccdMapping[ccdId]]) 

429 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

430 

431 # Compute instMagErr from instFluxErr/instFlux, any scaling 

432 # will cancel out. 

433 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

434 / df[self.config.instFluxField].values[use]) 

435 

436 # Apply the jacobian if configured 

437 if self.config.doApplyWcsJacobian: 

438 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

439 

440 fullCatalog.extend(tempCat) 

441 

442 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

443 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

444 

445 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

446 visit['sources_read'] = True 

447 

448 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

449 use.size, visit['visit'], visit['deltaAper']) 

450 

451 self.log.info("Found all good star observations in %.2f s" % 

452 (time.time() - startTime)) 

453 

454 return fullCatalog 

455 

456 def _get_sourceTable_visit_columns(self, inColumns): 

457 """ 

458 Get the sourceTable_visit columns from the config. 

459 

460 Parameters 

461 ---------- 

462 inColumns : `list` 

463 List of columns available in the sourceTable_visit 

464 

465 Returns 

466 ------- 

467 columns : `list` 

468 List of columns to read from sourceTable_visit. 

469 """ 

470 # Some names are hard-coded in the parquet table. 

471 columns = ['visit', 'detector', 

472 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

473 self.config.instFluxField, self.config.instFluxField + 'Err', 

474 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

475 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

476 if self.sourceSelector.config.doFlags: 

477 columns.extend(self.sourceSelector.config.flags.bad) 

478 if self.sourceSelector.config.doUnresolved: 

479 columns.append(self.sourceSelector.config.unresolved.name) 

480 if self.sourceSelector.config.doIsolated: 

481 columns.append(self.sourceSelector.config.isolated.parentName) 

482 columns.append(self.sourceSelector.config.isolated.nChildName) 

483 if self.config.doSubtractLocalBackground: 

484 columns.append(self.config.localBackgroundFluxField) 

485 

486 return columns