Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%

184 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-02 03:27 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33import warnings 

34 

35import numpy as np 

36import collections 

37 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40from lsst.pipe.base import connectionTypes 

41import lsst.afw.table as afwTable 

42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

43 

44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

46from .utilities import lookupStaticCalibrations 

47 

48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

49 

50 

51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

52 dimensions=("instrument",), 

53 defaultTemplates={}): 

54 camera = connectionTypes.PrerequisiteInput( 

55 doc="Camera instrument", 

56 name="camera", 

57 storageClass="Camera", 

58 dimensions=("instrument",), 

59 lookupFunction=lookupStaticCalibrations, 

60 isCalibration=True, 

61 ) 

62 

63 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

65 "chromatic corrections."), 

66 name="fgcmLookUpTable", 

67 storageClass="Catalog", 

68 dimensions=("instrument",), 

69 deferLoad=True, 

70 ) 

71 

72 sourceSchema = connectionTypes.InitInput( 

73 doc="Schema for source catalogs", 

74 name="src_schema", 

75 storageClass="SourceCatalog", 

76 ) 

77 

78 refCat = connectionTypes.PrerequisiteInput( 

79 doc="Reference catalog to use for photometric calibration", 

80 name="cal_ref_cat", 

81 storageClass="SimpleCatalog", 

82 dimensions=("skypix",), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 sourceTable_visit = connectionTypes.Input( 

88 doc="Source table in parquet format, per visit", 

89 name="sourceTable_visit", 

90 storageClass="DataFrame", 

91 dimensions=("instrument", "visit"), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 visitSummary = connectionTypes.Input( 

97 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

98 "detector id for the id and must be sorted for fast lookups of a " 

99 "detector."), 

100 name="visitSummary", 

101 storageClass="ExposureCatalog", 

102 dimensions=("instrument", "visit"), 

103 deferLoad=True, 

104 multiple=True, 

105 ) 

106 

107 fgcmVisitCatalog = connectionTypes.Output( 

108 doc="Catalog of visit information for fgcm", 

109 name="fgcmVisitCatalog", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 ) 

113 

114 fgcmStarObservations = connectionTypes.Output( 

115 doc="Catalog of star observations for fgcm", 

116 name="fgcmStarObservations", 

117 storageClass="Catalog", 

118 dimensions=("instrument",), 

119 ) 

120 

121 fgcmStarIds = connectionTypes.Output( 

122 doc="Catalog of fgcm calibration star IDs", 

123 name="fgcmStarIds", 

124 storageClass="Catalog", 

125 dimensions=("instrument",), 

126 ) 

127 

128 fgcmStarIndices = connectionTypes.Output( 

129 doc="Catalog of fgcm calibration star indices", 

130 name="fgcmStarIndices", 

131 storageClass="Catalog", 

132 dimensions=("instrument",), 

133 ) 

134 

135 fgcmReferenceStars = connectionTypes.Output( 

136 doc="Catalog of fgcm-matched reference stars", 

137 name="fgcmReferenceStars", 

138 storageClass="Catalog", 

139 dimensions=("instrument",), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doReferenceMatches: 

146 self.prerequisiteInputs.remove("refCat") 

147 self.prerequisiteInputs.remove("fgcmLookUpTable") 

148 

149 if not config.doReferenceMatches: 

150 self.outputs.remove("fgcmReferenceStars") 

151 

152 

153class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

154 pipelineConnections=FgcmBuildStarsTableConnections): 

155 """Config for FgcmBuildStarsTableTask""" 

156 

157 referenceCCD = pexConfig.Field( 

158 doc="Reference CCD for checking PSF and background", 

159 dtype=int, 

160 default=40, 

161 ) 

162 

163 def setDefaults(self): 

164 super().setDefaults() 

165 

166 # The names here correspond to the post-transformed 

167 # sourceTable_visit catalogs, which differ from the raw src 

168 # catalogs. Therefore, all field and flag names cannot 

169 # be derived from the base config class. 

170 self.instFluxField = 'apFlux_12_0_instFlux' 

171 self.localBackgroundFluxField = 'localBackground_instFlux' 

172 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

173 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

174 self.psfCandidateName = 'calib_psf_candidate' 

175 

176 sourceSelector = self.sourceSelector["science"] 

177 

178 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

179 

180 sourceSelector.flags.bad = ['pixelFlags_edge', 

181 'pixelFlags_interpolatedCenter', 

182 'pixelFlags_saturatedCenter', 

183 'pixelFlags_crCenter', 

184 'pixelFlags_bad', 

185 'pixelFlags_interpolated', 

186 'pixelFlags_saturated', 

187 'centroid_flag', 

188 fluxFlagName] 

189 

190 if self.doSubtractLocalBackground: 

191 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

192 sourceSelector.flags.bad.append(localBackgroundFlagName) 

193 

194 sourceSelector.signalToNoise.fluxField = self.instFluxField 

195 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

196 

197 sourceSelector.isolated.parentName = 'parentSourceId' 

198 sourceSelector.isolated.nChildName = 'deblend_nChild' 

199 

200 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

201 sourceSelector.requireFiniteRaDec.decColName = 'decl' 

202 

203 sourceSelector.unresolved.name = 'extendedness' 

204 

205 sourceSelector.doRequirePrimary = True 

206 

207 

208class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

209 """ 

210 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

211 """ 

212 ConfigClass = FgcmBuildStarsTableConfig 

213 _DefaultName = "fgcmBuildStarsTable" 

214 

215 canMultiprocess = False 

216 

217 def __init__(self, initInputs=None, **kwargs): 

218 super().__init__(initInputs=initInputs, **kwargs) 

219 if initInputs is not None: 

220 self.sourceSchema = initInputs["sourceSchema"].schema 

221 

222 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

223 inputRefDict = butlerQC.get(inputRefs) 

224 

225 sourceTableHandles = inputRefDict['sourceTable_visit'] 

226 

227 self.log.info("Running with %d sourceTable_visit handles", 

228 len(sourceTableHandles)) 

229 

230 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

231 sourceTableHandle in sourceTableHandles} 

232 

233 if self.config.doReferenceMatches: 

234 # Get the LUT handle 

235 lutHandle = inputRefDict['fgcmLookUpTable'] 

236 

237 # Prepare the reference catalog loader 

238 refConfig = LoadReferenceObjectsConfig() 

239 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

240 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

241 for ref in inputRefs.refCat], 

242 refCats=butlerQC.get(inputRefs.refCat), 

243 name=self.config.connections.refCat, 

244 log=self.log, 

245 config=refConfig) 

246 self.makeSubtask('fgcmLoadReferenceCatalog', 

247 refObjLoader=refObjLoader, 

248 refCatName=self.config.connections.refCat) 

249 else: 

250 lutHandle = None 

251 

252 # Compute aperture radius if necessary. This is useful to do now before 

253 # any heave lifting has happened (fail early). 

254 calibFluxApertureRadius = None 

255 if self.config.doSubtractLocalBackground: 

256 try: 

257 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

258 except RuntimeError as e: 

259 raise RuntimeError("Could not determine aperture radius from %s. " 

260 "Cannot use doSubtractLocalBackground." % 

261 (self.config.instFluxField)) from e 

262 

263 visitSummaryHandles = inputRefDict['visitSummary'] 

264 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

265 visitSummaryHandle in visitSummaryHandles} 

266 

267 camera = inputRefDict['camera'] 

268 groupedHandles = self._groupHandles(sourceTableHandleDict, 

269 visitSummaryHandleDict) 

270 

271 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles) 

272 

273 rad = calibFluxApertureRadius 

274 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

275 visitCat, 

276 self.sourceSchema, 

277 camera, 

278 calibFluxApertureRadius=rad) 

279 

280 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

281 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

282 

283 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

284 fgcmStarObservationCat, 

285 lutHandle=lutHandle) 

286 

287 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

288 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

289 if fgcmRefCat is not None: 

290 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

291 

292 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

293 """Group sourceTable and visitSummary handles. 

294 

295 Parameters 

296 ---------- 

297 sourceTableHandleDict : `dict` [`int`, `str`] 

298 Dict of source tables, keyed by visit. 

299 visitSummaryHandleDict : `dict` [int, `str`] 

300 Dict of visit summary catalogs, keyed by visit. 

301 

302 Returns 

303 ------- 

304 groupedHandles : `dict` [`int`, `list`] 

305 Dictionary with sorted visit keys, and `list`s with 

306 `lsst.daf.butler.DeferredDataSetHandle`. The first 

307 item in the list will be the visitSummary ref, and 

308 the second will be the source table ref. 

309 """ 

310 groupedHandles = collections.defaultdict(list) 

311 visits = sorted(sourceTableHandleDict.keys()) 

312 

313 for visit in visits: 

314 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

315 sourceTableHandleDict[visit]] 

316 

317 return groupedHandles 

318 

319 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

320 sourceSchema, 

321 camera, 

322 calibFluxApertureRadius=None): 

323 startTime = time.time() 

324 

325 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

326 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

327 

328 # To get the correct output schema, we use the legacy code. 

329 # We are not actually using this mapper, except to grab the outputSchema 

330 sourceMapper = self._makeSourceMapper(sourceSchema) 

331 outputSchema = sourceMapper.getOutputSchema() 

332 

333 # Construct mapping from ccd number to index 

334 ccdMapping = {} 

335 for ccdIndex, detector in enumerate(camera): 

336 ccdMapping[detector.getId()] = ccdIndex 

337 

338 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

339 

340 fullCatalog = afwTable.BaseCatalog(outputSchema) 

341 

342 visitKey = outputSchema['visit'].asKey() 

343 ccdKey = outputSchema['ccd'].asKey() 

344 instMagKey = outputSchema['instMag'].asKey() 

345 instMagErrKey = outputSchema['instMagErr'].asKey() 

346 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

347 

348 # Prepare local background if desired 

349 if self.config.doSubtractLocalBackground: 

350 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

351 

352 columns = None 

353 

354 k = 2.5/np.log(10.) 

355 

356 for counter, visit in enumerate(visitCat): 

357 expTime = visit['exptime'] 

358 

359 handle = groupedHandles[visit['visit']][-1] 

360 

361 if columns is None: 

362 inColumns = handle.get(component='columns') 

363 columns = self._get_sourceTable_visit_columns(inColumns) 

364 df = handle.get(parameters={'columns': columns}) 

365 

366 goodSrc = self.sourceSelector.selectSources(df) 

367 

368 # Need to add a selection based on the local background correction 

369 # if necessary 

370 if self.config.doSubtractLocalBackground: 

371 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

372 use, = np.where((goodSrc.selected) 

373 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

374 else: 

375 use, = np.where(goodSrc.selected) 

376 

377 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

378 tempCat.resize(use.size) 

379 

380 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

381 tempCat['dec'][:] = np.deg2rad(df['decl'].values[use]) 

382 tempCat['x'][:] = df['x'].values[use] 

383 tempCat['y'][:] = df['y'].values[use] 

384 # The "visit" name in the parquet table is hard-coded. 

385 tempCat[visitKey][:] = df['visit'].values[use] 

386 tempCat[ccdKey][:] = df['detector'].values[use] 

387 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

388 

389 with warnings.catch_warnings(): 

390 # Ignore warnings, we will filter infinites and nans below 

391 warnings.simplefilter("ignore") 

392 

393 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

394 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

395 / df[self.config.apertureInnerInstFluxField].values[use]) 

396 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

397 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

398 / df[self.config.apertureOuterInstFluxField].values[use]) 

399 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

400 # Set bad values to illegal values for fgcm. 

401 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

402 

403 if self.config.doSubtractLocalBackground: 

404 # At the moment we only adjust the flux and not the flux 

405 # error by the background because the error on 

406 # base_LocalBackground_instFlux is the rms error in the 

407 # background annulus, not the error on the mean in the 

408 # background estimate (which is much smaller, by sqrt(n) 

409 # pixels used to estimate the background, which we do not 

410 # have access to in this task). In the default settings, 

411 # the annulus is sufficiently large such that these 

412 # additional errors are are negligibly small (much less 

413 # than a mmag in quadrature). 

414 

415 # This is the difference between the mag with local background correction 

416 # and the mag without local background correction. 

417 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

418 - localBackground[use]) - 

419 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

420 else: 

421 tempCat['deltaMagBkg'][:] = 0.0 

422 

423 # Need to loop over ccds here 

424 for detector in camera: 

425 ccdId = detector.getId() 

426 # used index for all observations with a given ccd 

427 use2 = (tempCat[ccdKey] == ccdId) 

428 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

429 tempCat['y'][use2]) 

430 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

431 * visit['scaling'][ccdMapping[ccdId]]) 

432 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

433 

434 # Compute instMagErr from instFluxErr/instFlux, any scaling 

435 # will cancel out. 

436 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

437 / df[self.config.instFluxField].values[use]) 

438 

439 # Apply the jacobian if configured 

440 if self.config.doApplyWcsJacobian: 

441 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

442 

443 fullCatalog.extend(tempCat) 

444 

445 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

446 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

447 

448 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

449 visit['sources_read'] = True 

450 

451 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

452 use.size, visit['visit'], visit['deltaAper']) 

453 

454 self.log.info("Found all good star observations in %.2f s" % 

455 (time.time() - startTime)) 

456 

457 return fullCatalog 

458 

459 def _get_sourceTable_visit_columns(self, inColumns): 

460 """ 

461 Get the sourceTable_visit columns from the config. 

462 

463 Parameters 

464 ---------- 

465 inColumns : `list` 

466 List of columns available in the sourceTable_visit 

467 

468 Returns 

469 ------- 

470 columns : `list` 

471 List of columns to read from sourceTable_visit. 

472 """ 

473 # Some names are hard-coded in the parquet table. 

474 columns = ['visit', 'detector', 

475 'ra', 'decl', 'x', 'y', self.config.psfCandidateName, 

476 self.config.instFluxField, self.config.instFluxField + 'Err', 

477 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

478 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

479 if self.sourceSelector.config.doFlags: 

480 columns.extend(self.sourceSelector.config.flags.bad) 

481 if self.sourceSelector.config.doUnresolved: 

482 columns.append(self.sourceSelector.config.unresolved.name) 

483 if self.sourceSelector.config.doIsolated: 

484 columns.append(self.sourceSelector.config.isolated.parentName) 

485 columns.append(self.sourceSelector.config.isolated.nChildName) 

486 if self.sourceSelector.config.doRequirePrimary: 

487 columns.append(self.sourceSelector.config.requirePrimary.primaryColName) 

488 if self.config.doSubtractLocalBackground: 

489 columns.append(self.config.localBackgroundFluxField) 

490 

491 return columns