Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 16%

185 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-11 03:54 -0700

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33import warnings 

34 

35import numpy as np 

36import collections 

37 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40from lsst.pipe.base import connectionTypes 

41import lsst.afw.table as afwTable 

42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

43 

44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

46 

47__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

48 

49 

50class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

51 dimensions=("instrument",), 

52 defaultTemplates={}): 

53 camera = connectionTypes.PrerequisiteInput( 

54 doc="Camera instrument", 

55 name="camera", 

56 storageClass="Camera", 

57 dimensions=("instrument",), 

58 isCalibration=True, 

59 ) 

60 

61 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

62 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

63 "chromatic corrections."), 

64 name="fgcmLookUpTable", 

65 storageClass="Catalog", 

66 dimensions=("instrument",), 

67 deferLoad=True, 

68 ) 

69 

70 sourceSchema = connectionTypes.InitInput( 

71 doc="Schema for source catalogs", 

72 name="src_schema", 

73 storageClass="SourceCatalog", 

74 ) 

75 

76 refCat = connectionTypes.PrerequisiteInput( 

77 doc="Reference catalog to use for photometric calibration", 

78 name="cal_ref_cat", 

79 storageClass="SimpleCatalog", 

80 dimensions=("skypix",), 

81 deferLoad=True, 

82 multiple=True, 

83 ) 

84 

85 sourceTable_visit = connectionTypes.Input( 

86 doc="Source table in parquet format, per visit", 

87 name="sourceTable_visit", 

88 storageClass="DataFrame", 

89 dimensions=("instrument", "visit"), 

90 deferLoad=True, 

91 multiple=True, 

92 ) 

93 

94 visitSummary = connectionTypes.Input( 

95 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

96 "detector id for the id and must be sorted for fast lookups of a " 

97 "detector."), 

98 name="visitSummary", 

99 storageClass="ExposureCatalog", 

100 dimensions=("instrument", "visit"), 

101 deferLoad=True, 

102 multiple=True, 

103 ) 

104 

105 fgcmVisitCatalog = connectionTypes.Output( 

106 doc="Catalog of visit information for fgcm", 

107 name="fgcmVisitCatalog", 

108 storageClass="Catalog", 

109 dimensions=("instrument",), 

110 ) 

111 

112 fgcmStarObservations = connectionTypes.Output( 

113 doc="Catalog of star observations for fgcm", 

114 name="fgcmStarObservations", 

115 storageClass="Catalog", 

116 dimensions=("instrument",), 

117 ) 

118 

119 fgcmStarIds = connectionTypes.Output( 

120 doc="Catalog of fgcm calibration star IDs", 

121 name="fgcmStarIds", 

122 storageClass="Catalog", 

123 dimensions=("instrument",), 

124 ) 

125 

126 fgcmStarIndices = connectionTypes.Output( 

127 doc="Catalog of fgcm calibration star indices", 

128 name="fgcmStarIndices", 

129 storageClass="Catalog", 

130 dimensions=("instrument",), 

131 ) 

132 

133 fgcmReferenceStars = connectionTypes.Output( 

134 doc="Catalog of fgcm-matched reference stars", 

135 name="fgcmReferenceStars", 

136 storageClass="Catalog", 

137 dimensions=("instrument",), 

138 ) 

139 

140 def __init__(self, *, config=None): 

141 super().__init__(config=config) 

142 

143 if not config.doReferenceMatches: 

144 self.prerequisiteInputs.remove("refCat") 

145 self.prerequisiteInputs.remove("fgcmLookUpTable") 

146 

147 if not config.doReferenceMatches: 

148 self.outputs.remove("fgcmReferenceStars") 

149 

150 def getSpatialBoundsConnections(self): 

151 return ("visitSummary",) 

152 

153 

154class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

155 pipelineConnections=FgcmBuildStarsTableConnections): 

156 """Config for FgcmBuildStarsTableTask""" 

157 

158 referenceCCD = pexConfig.Field( 

159 doc="Reference CCD for checking PSF and background", 

160 dtype=int, 

161 default=40, 

162 ) 

163 

164 def setDefaults(self): 

165 super().setDefaults() 

166 

167 # The names here correspond to the post-transformed 

168 # sourceTable_visit catalogs, which differ from the raw src 

169 # catalogs. Therefore, all field and flag names cannot 

170 # be derived from the base config class. 

171 self.instFluxField = 'apFlux_12_0_instFlux' 

172 self.localBackgroundFluxField = 'localBackground_instFlux' 

173 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

174 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

175 self.psfCandidateName = 'calib_psf_candidate' 

176 

177 sourceSelector = self.sourceSelector["science"] 

178 

179 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

180 

181 sourceSelector.flags.bad = ['pixelFlags_edge', 

182 'pixelFlags_interpolatedCenter', 

183 'pixelFlags_saturatedCenter', 

184 'pixelFlags_crCenter', 

185 'pixelFlags_bad', 

186 'pixelFlags_interpolated', 

187 'pixelFlags_saturated', 

188 'centroid_flag', 

189 fluxFlagName] 

190 

191 if self.doSubtractLocalBackground: 

192 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

193 sourceSelector.flags.bad.append(localBackgroundFlagName) 

194 

195 sourceSelector.signalToNoise.fluxField = self.instFluxField 

196 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

197 

198 sourceSelector.isolated.parentName = 'parentSourceId' 

199 sourceSelector.isolated.nChildName = 'deblend_nChild' 

200 

201 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

202 sourceSelector.requireFiniteRaDec.decColName = 'dec' 

203 

204 sourceSelector.unresolved.name = 'extendedness' 

205 

206 sourceSelector.doRequirePrimary = True 

207 

208 

209class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

210 """ 

211 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

212 """ 

213 ConfigClass = FgcmBuildStarsTableConfig 

214 _DefaultName = "fgcmBuildStarsTable" 

215 

216 canMultiprocess = False 

217 

218 def __init__(self, initInputs=None, **kwargs): 

219 super().__init__(initInputs=initInputs, **kwargs) 

220 if initInputs is not None: 

221 self.sourceSchema = initInputs["sourceSchema"].schema 

222 

223 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

224 inputRefDict = butlerQC.get(inputRefs) 

225 

226 sourceTableHandles = inputRefDict['sourceTable_visit'] 

227 

228 self.log.info("Running with %d sourceTable_visit handles", 

229 len(sourceTableHandles)) 

230 

231 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

232 sourceTableHandle in sourceTableHandles} 

233 

234 if self.config.doReferenceMatches: 

235 # Get the LUT handle 

236 lutHandle = inputRefDict['fgcmLookUpTable'] 

237 

238 # Prepare the reference catalog loader 

239 refConfig = LoadReferenceObjectsConfig() 

240 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

241 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

242 for ref in inputRefs.refCat], 

243 refCats=butlerQC.get(inputRefs.refCat), 

244 name=self.config.connections.refCat, 

245 log=self.log, 

246 config=refConfig) 

247 self.makeSubtask('fgcmLoadReferenceCatalog', 

248 refObjLoader=refObjLoader, 

249 refCatName=self.config.connections.refCat) 

250 else: 

251 lutHandle = None 

252 

253 # Compute aperture radius if necessary. This is useful to do now before 

254 # any heave lifting has happened (fail early). 

255 calibFluxApertureRadius = None 

256 if self.config.doSubtractLocalBackground: 

257 try: 

258 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

259 except RuntimeError as e: 

260 raise RuntimeError("Could not determine aperture radius from %s. " 

261 "Cannot use doSubtractLocalBackground." % 

262 (self.config.instFluxField)) from e 

263 

264 visitSummaryHandles = inputRefDict['visitSummary'] 

265 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

266 visitSummaryHandle in visitSummaryHandles} 

267 

268 camera = inputRefDict['camera'] 

269 groupedHandles = self._groupHandles(sourceTableHandleDict, 

270 visitSummaryHandleDict) 

271 

272 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles) 

273 

274 rad = calibFluxApertureRadius 

275 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

276 visitCat, 

277 self.sourceSchema, 

278 camera, 

279 calibFluxApertureRadius=rad) 

280 

281 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

282 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

283 

284 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

285 fgcmStarObservationCat, 

286 lutHandle=lutHandle) 

287 

288 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

289 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

290 if fgcmRefCat is not None: 

291 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

292 

293 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

294 """Group sourceTable and visitSummary handles. 

295 

296 Parameters 

297 ---------- 

298 sourceTableHandleDict : `dict` [`int`, `str`] 

299 Dict of source tables, keyed by visit. 

300 visitSummaryHandleDict : `dict` [int, `str`] 

301 Dict of visit summary catalogs, keyed by visit. 

302 

303 Returns 

304 ------- 

305 groupedHandles : `dict` [`int`, `list`] 

306 Dictionary with sorted visit keys, and `list`s with 

307 `lsst.daf.butler.DeferredDataSetHandle`. The first 

308 item in the list will be the visitSummary ref, and 

309 the second will be the source table ref. 

310 """ 

311 groupedHandles = collections.defaultdict(list) 

312 visits = sorted(sourceTableHandleDict.keys()) 

313 

314 for visit in visits: 

315 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

316 sourceTableHandleDict[visit]] 

317 

318 return groupedHandles 

319 

320 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

321 sourceSchema, 

322 camera, 

323 calibFluxApertureRadius=None): 

324 startTime = time.time() 

325 

326 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

327 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

328 

329 # To get the correct output schema, we use the legacy code. 

330 # We are not actually using this mapper, except to grab the outputSchema 

331 sourceMapper = self._makeSourceMapper(sourceSchema) 

332 outputSchema = sourceMapper.getOutputSchema() 

333 

334 # Construct mapping from ccd number to index 

335 ccdMapping = {} 

336 for ccdIndex, detector in enumerate(camera): 

337 ccdMapping[detector.getId()] = ccdIndex 

338 

339 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

340 

341 fullCatalog = afwTable.BaseCatalog(outputSchema) 

342 

343 visitKey = outputSchema['visit'].asKey() 

344 ccdKey = outputSchema['ccd'].asKey() 

345 instMagKey = outputSchema['instMag'].asKey() 

346 instMagErrKey = outputSchema['instMagErr'].asKey() 

347 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

348 

349 # Prepare local background if desired 

350 if self.config.doSubtractLocalBackground: 

351 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

352 

353 columns = None 

354 

355 k = 2.5/np.log(10.) 

356 

357 for counter, visit in enumerate(visitCat): 

358 expTime = visit['exptime'] 

359 

360 handle = groupedHandles[visit['visit']][-1] 

361 

362 if columns is None: 

363 inColumns = handle.get(component='columns') 

364 columns = self._get_sourceTable_visit_columns(inColumns) 

365 df = handle.get(parameters={'columns': columns}) 

366 

367 goodSrc = self.sourceSelector.selectSources(df) 

368 

369 # Need to add a selection based on the local background correction 

370 # if necessary 

371 if self.config.doSubtractLocalBackground: 

372 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

373 use, = np.where((goodSrc.selected) 

374 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

375 else: 

376 use, = np.where(goodSrc.selected) 

377 

378 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

379 tempCat.resize(use.size) 

380 

381 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

382 tempCat['dec'][:] = np.deg2rad(df['dec'].values[use]) 

383 tempCat['x'][:] = df['x'].values[use] 

384 tempCat['y'][:] = df['y'].values[use] 

385 # The "visit" name in the parquet table is hard-coded. 

386 tempCat[visitKey][:] = df['visit'].values[use] 

387 tempCat[ccdKey][:] = df['detector'].values[use] 

388 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

389 

390 with warnings.catch_warnings(): 

391 # Ignore warnings, we will filter infinites and nans below 

392 warnings.simplefilter("ignore") 

393 

394 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

395 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

396 / df[self.config.apertureInnerInstFluxField].values[use]) 

397 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

398 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

399 / df[self.config.apertureOuterInstFluxField].values[use]) 

400 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

401 # Set bad values to illegal values for fgcm. 

402 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

403 

404 if self.config.doSubtractLocalBackground: 

405 # At the moment we only adjust the flux and not the flux 

406 # error by the background because the error on 

407 # base_LocalBackground_instFlux is the rms error in the 

408 # background annulus, not the error on the mean in the 

409 # background estimate (which is much smaller, by sqrt(n) 

410 # pixels used to estimate the background, which we do not 

411 # have access to in this task). In the default settings, 

412 # the annulus is sufficiently large such that these 

413 # additional errors are are negligibly small (much less 

414 # than a mmag in quadrature). 

415 

416 # This is the difference between the mag with local background correction 

417 # and the mag without local background correction. 

418 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

419 - localBackground[use]) - 

420 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

421 else: 

422 tempCat['deltaMagBkg'][:] = 0.0 

423 

424 # Need to loop over ccds here 

425 for detector in camera: 

426 ccdId = detector.getId() 

427 # used index for all observations with a given ccd 

428 use2 = (tempCat[ccdKey] == ccdId) 

429 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

430 tempCat['y'][use2]) 

431 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

432 * visit['scaling'][ccdMapping[ccdId]]) 

433 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

434 

435 # Compute instMagErr from instFluxErr/instFlux, any scaling 

436 # will cancel out. 

437 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

438 / df[self.config.instFluxField].values[use]) 

439 

440 # Apply the jacobian if configured 

441 if self.config.doApplyWcsJacobian: 

442 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

443 

444 fullCatalog.extend(tempCat) 

445 

446 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

447 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

448 

449 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

450 visit['sources_read'] = True 

451 

452 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

453 use.size, visit['visit'], visit['deltaAper']) 

454 

455 self.log.info("Found all good star observations in %.2f s" % 

456 (time.time() - startTime)) 

457 

458 return fullCatalog 

459 

460 def _get_sourceTable_visit_columns(self, inColumns): 

461 """ 

462 Get the sourceTable_visit columns from the config. 

463 

464 Parameters 

465 ---------- 

466 inColumns : `list` 

467 List of columns available in the sourceTable_visit 

468 

469 Returns 

470 ------- 

471 columns : `list` 

472 List of columns to read from sourceTable_visit. 

473 """ 

474 # Some names are hard-coded in the parquet table. 

475 columns = ['visit', 'detector', 

476 'ra', 'dec', 'x', 'y', self.config.psfCandidateName, 

477 self.config.instFluxField, self.config.instFluxField + 'Err', 

478 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

479 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

480 if self.sourceSelector.config.doFlags: 

481 columns.extend(self.sourceSelector.config.flags.bad) 

482 if self.sourceSelector.config.doUnresolved: 

483 columns.append(self.sourceSelector.config.unresolved.name) 

484 if self.sourceSelector.config.doIsolated: 

485 columns.append(self.sourceSelector.config.isolated.parentName) 

486 columns.append(self.sourceSelector.config.isolated.nChildName) 

487 if self.sourceSelector.config.doRequirePrimary: 

488 columns.append(self.sourceSelector.config.requirePrimary.primaryColName) 

489 if self.config.doSubtractLocalBackground: 

490 columns.append(self.config.localBackgroundFluxField) 

491 

492 return columns