Coverage for python/lsst/fgcmcal/fgcmBuildStarsTable.py: 17%

186 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-25 12:04 +0000

1# See COPYRIGHT file at the top of the source tree. 

2# 

3# This file is part of fgcmcal. 

4# 

5# Developed for the LSST Data Management System. 

6# This product includes software developed by the LSST Project 

7# (https://www.lsst.org). 

8# See the COPYRIGHT file at the top-level directory of this distribution 

9# for details of code ownership. 

10# 

11# This program is free software: you can redistribute it and/or modify 

12# it under the terms of the GNU General Public License as published by 

13# the Free Software Foundation, either version 3 of the License, or 

14# (at your option) any later version. 

15# 

16# This program is distributed in the hope that it will be useful, 

17# but WITHOUT ANY WARRANTY; without even the implied warranty of 

18# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

19# GNU General Public License for more details. 

20# 

21# You should have received a copy of the GNU General Public License 

22# along with this program. If not, see <https://www.gnu.org/licenses/>. 

23"""Build star observations for input to FGCM using sourceTable_visit. 

24 

25This task finds all the visits and sourceTable_visits in a repository (or a 

26subset based on command line parameters) and extracts all the potential 

27calibration stars for input into fgcm. This task additionally uses fgcm to 

28match star observations into unique stars, and performs as much cleaning of the 

29input catalog as possible. 

30""" 

31 

32import time 

33import warnings 

34 

35import numpy as np 

36import collections 

37 

38import lsst.pex.config as pexConfig 

39import lsst.pipe.base as pipeBase 

40from lsst.pipe.base import connectionTypes 

41import lsst.afw.table as afwTable 

42from lsst.meas.algorithms import ReferenceObjectLoader, LoadReferenceObjectsConfig 

43 

44from .fgcmBuildStarsBase import FgcmBuildStarsConfigBase, FgcmBuildStarsBaseTask 

45from .utilities import computeApproxPixelAreaFields, computeApertureRadiusFromName 

46from .utilities import lookupStaticCalibrations 

47 

48__all__ = ['FgcmBuildStarsTableConfig', 'FgcmBuildStarsTableTask'] 

49 

50 

51class FgcmBuildStarsTableConnections(pipeBase.PipelineTaskConnections, 

52 dimensions=("instrument",), 

53 defaultTemplates={}): 

54 camera = connectionTypes.PrerequisiteInput( 

55 doc="Camera instrument", 

56 name="camera", 

57 storageClass="Camera", 

58 dimensions=("instrument",), 

59 lookupFunction=lookupStaticCalibrations, 

60 isCalibration=True, 

61 ) 

62 

63 fgcmLookUpTable = connectionTypes.PrerequisiteInput( 

64 doc=("Atmosphere + instrument look-up-table for FGCM throughput and " 

65 "chromatic corrections."), 

66 name="fgcmLookUpTable", 

67 storageClass="Catalog", 

68 dimensions=("instrument",), 

69 deferLoad=True, 

70 ) 

71 

72 sourceSchema = connectionTypes.InitInput( 

73 doc="Schema for source catalogs", 

74 name="src_schema", 

75 storageClass="SourceCatalog", 

76 ) 

77 

78 refCat = connectionTypes.PrerequisiteInput( 

79 doc="Reference catalog to use for photometric calibration", 

80 name="cal_ref_cat", 

81 storageClass="SimpleCatalog", 

82 dimensions=("skypix",), 

83 deferLoad=True, 

84 multiple=True, 

85 ) 

86 

87 sourceTable_visit = connectionTypes.Input( 

88 doc="Source table in parquet format, per visit", 

89 name="sourceTable_visit", 

90 storageClass="DataFrame", 

91 dimensions=("instrument", "visit"), 

92 deferLoad=True, 

93 multiple=True, 

94 ) 

95 

96 visitSummary = connectionTypes.Input( 

97 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

98 "detector id for the id and must be sorted for fast lookups of a " 

99 "detector."), 

100 name="visitSummary", 

101 storageClass="ExposureCatalog", 

102 dimensions=("instrument", "visit"), 

103 deferLoad=True, 

104 multiple=True, 

105 ) 

106 

107 fgcmVisitCatalog = connectionTypes.Output( 

108 doc="Catalog of visit information for fgcm", 

109 name="fgcmVisitCatalog", 

110 storageClass="Catalog", 

111 dimensions=("instrument",), 

112 ) 

113 

114 fgcmStarObservations = connectionTypes.Output( 

115 doc="Catalog of star observations for fgcm", 

116 name="fgcmStarObservations", 

117 storageClass="Catalog", 

118 dimensions=("instrument",), 

119 ) 

120 

121 fgcmStarIds = connectionTypes.Output( 

122 doc="Catalog of fgcm calibration star IDs", 

123 name="fgcmStarIds", 

124 storageClass="Catalog", 

125 dimensions=("instrument",), 

126 ) 

127 

128 fgcmStarIndices = connectionTypes.Output( 

129 doc="Catalog of fgcm calibration star indices", 

130 name="fgcmStarIndices", 

131 storageClass="Catalog", 

132 dimensions=("instrument",), 

133 ) 

134 

135 fgcmReferenceStars = connectionTypes.Output( 

136 doc="Catalog of fgcm-matched reference stars", 

137 name="fgcmReferenceStars", 

138 storageClass="Catalog", 

139 dimensions=("instrument",), 

140 ) 

141 

142 def __init__(self, *, config=None): 

143 super().__init__(config=config) 

144 

145 if not config.doReferenceMatches: 

146 self.prerequisiteInputs.remove("refCat") 

147 self.prerequisiteInputs.remove("fgcmLookUpTable") 

148 

149 if not config.doReferenceMatches: 

150 self.outputs.remove("fgcmReferenceStars") 

151 

152 def getSpatialBoundsConnections(self): 

153 return ("visitSummary",) 

154 

155 

156class FgcmBuildStarsTableConfig(FgcmBuildStarsConfigBase, pipeBase.PipelineTaskConfig, 

157 pipelineConnections=FgcmBuildStarsTableConnections): 

158 """Config for FgcmBuildStarsTableTask""" 

159 

160 referenceCCD = pexConfig.Field( 

161 doc="Reference CCD for checking PSF and background", 

162 dtype=int, 

163 default=40, 

164 ) 

165 

166 def setDefaults(self): 

167 super().setDefaults() 

168 

169 # The names here correspond to the post-transformed 

170 # sourceTable_visit catalogs, which differ from the raw src 

171 # catalogs. Therefore, all field and flag names cannot 

172 # be derived from the base config class. 

173 self.instFluxField = 'apFlux_12_0_instFlux' 

174 self.localBackgroundFluxField = 'localBackground_instFlux' 

175 self.apertureInnerInstFluxField = 'apFlux_12_0_instFlux' 

176 self.apertureOuterInstFluxField = 'apFlux_17_0_instFlux' 

177 self.psfCandidateName = 'calib_psf_candidate' 

178 

179 sourceSelector = self.sourceSelector["science"] 

180 

181 fluxFlagName = self.instFluxField[0: -len('instFlux')] + 'flag' 

182 

183 sourceSelector.flags.bad = ['pixelFlags_edge', 

184 'pixelFlags_interpolatedCenter', 

185 'pixelFlags_saturatedCenter', 

186 'pixelFlags_crCenter', 

187 'pixelFlags_bad', 

188 'pixelFlags_interpolated', 

189 'pixelFlags_saturated', 

190 'centroid_flag', 

191 fluxFlagName] 

192 

193 if self.doSubtractLocalBackground: 

194 localBackgroundFlagName = self.localBackgroundFluxField[0: -len('instFlux')] + 'flag' 

195 sourceSelector.flags.bad.append(localBackgroundFlagName) 

196 

197 sourceSelector.signalToNoise.fluxField = self.instFluxField 

198 sourceSelector.signalToNoise.errField = self.instFluxField + 'Err' 

199 

200 sourceSelector.isolated.parentName = 'parentSourceId' 

201 sourceSelector.isolated.nChildName = 'deblend_nChild' 

202 

203 sourceSelector.requireFiniteRaDec.raColName = 'ra' 

204 sourceSelector.requireFiniteRaDec.decColName = 'dec' 

205 

206 sourceSelector.unresolved.name = 'extendedness' 

207 

208 sourceSelector.doRequirePrimary = True 

209 

210 

211class FgcmBuildStarsTableTask(FgcmBuildStarsBaseTask): 

212 """ 

213 Build stars for the FGCM global calibration, using sourceTable_visit catalogs. 

214 """ 

215 ConfigClass = FgcmBuildStarsTableConfig 

216 _DefaultName = "fgcmBuildStarsTable" 

217 

218 canMultiprocess = False 

219 

220 def __init__(self, initInputs=None, **kwargs): 

221 super().__init__(initInputs=initInputs, **kwargs) 

222 if initInputs is not None: 

223 self.sourceSchema = initInputs["sourceSchema"].schema 

224 

225 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

226 inputRefDict = butlerQC.get(inputRefs) 

227 

228 sourceTableHandles = inputRefDict['sourceTable_visit'] 

229 

230 self.log.info("Running with %d sourceTable_visit handles", 

231 len(sourceTableHandles)) 

232 

233 sourceTableHandleDict = {sourceTableHandle.dataId['visit']: sourceTableHandle for 

234 sourceTableHandle in sourceTableHandles} 

235 

236 if self.config.doReferenceMatches: 

237 # Get the LUT handle 

238 lutHandle = inputRefDict['fgcmLookUpTable'] 

239 

240 # Prepare the reference catalog loader 

241 refConfig = LoadReferenceObjectsConfig() 

242 refConfig.filterMap = self.config.fgcmLoadReferenceCatalog.filterMap 

243 refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId 

244 for ref in inputRefs.refCat], 

245 refCats=butlerQC.get(inputRefs.refCat), 

246 name=self.config.connections.refCat, 

247 log=self.log, 

248 config=refConfig) 

249 self.makeSubtask('fgcmLoadReferenceCatalog', 

250 refObjLoader=refObjLoader, 

251 refCatName=self.config.connections.refCat) 

252 else: 

253 lutHandle = None 

254 

255 # Compute aperture radius if necessary. This is useful to do now before 

256 # any heave lifting has happened (fail early). 

257 calibFluxApertureRadius = None 

258 if self.config.doSubtractLocalBackground: 

259 try: 

260 calibFluxApertureRadius = computeApertureRadiusFromName(self.config.instFluxField) 

261 except RuntimeError as e: 

262 raise RuntimeError("Could not determine aperture radius from %s. " 

263 "Cannot use doSubtractLocalBackground." % 

264 (self.config.instFluxField)) from e 

265 

266 visitSummaryHandles = inputRefDict['visitSummary'] 

267 visitSummaryHandleDict = {visitSummaryHandle.dataId['visit']: visitSummaryHandle for 

268 visitSummaryHandle in visitSummaryHandles} 

269 

270 camera = inputRefDict['camera'] 

271 groupedHandles = self._groupHandles(sourceTableHandleDict, 

272 visitSummaryHandleDict) 

273 

274 visitCat = self.fgcmMakeVisitCatalog(camera, groupedHandles) 

275 

276 rad = calibFluxApertureRadius 

277 fgcmStarObservationCat = self.fgcmMakeAllStarObservations(groupedHandles, 

278 visitCat, 

279 self.sourceSchema, 

280 camera, 

281 calibFluxApertureRadius=rad) 

282 

283 butlerQC.put(visitCat, outputRefs.fgcmVisitCatalog) 

284 butlerQC.put(fgcmStarObservationCat, outputRefs.fgcmStarObservations) 

285 

286 fgcmStarIdCat, fgcmStarIndicesCat, fgcmRefCat = self.fgcmMatchStars(visitCat, 

287 fgcmStarObservationCat, 

288 lutHandle=lutHandle) 

289 

290 butlerQC.put(fgcmStarIdCat, outputRefs.fgcmStarIds) 

291 butlerQC.put(fgcmStarIndicesCat, outputRefs.fgcmStarIndices) 

292 if fgcmRefCat is not None: 

293 butlerQC.put(fgcmRefCat, outputRefs.fgcmReferenceStars) 

294 

295 def _groupHandles(self, sourceTableHandleDict, visitSummaryHandleDict): 

296 """Group sourceTable and visitSummary handles. 

297 

298 Parameters 

299 ---------- 

300 sourceTableHandleDict : `dict` [`int`, `str`] 

301 Dict of source tables, keyed by visit. 

302 visitSummaryHandleDict : `dict` [int, `str`] 

303 Dict of visit summary catalogs, keyed by visit. 

304 

305 Returns 

306 ------- 

307 groupedHandles : `dict` [`int`, `list`] 

308 Dictionary with sorted visit keys, and `list`s with 

309 `lsst.daf.butler.DeferredDataSetHandle`. The first 

310 item in the list will be the visitSummary ref, and 

311 the second will be the source table ref. 

312 """ 

313 groupedHandles = collections.defaultdict(list) 

314 visits = sorted(sourceTableHandleDict.keys()) 

315 

316 for visit in visits: 

317 groupedHandles[visit] = [visitSummaryHandleDict[visit], 

318 sourceTableHandleDict[visit]] 

319 

320 return groupedHandles 

321 

322 def fgcmMakeAllStarObservations(self, groupedHandles, visitCat, 

323 sourceSchema, 

324 camera, 

325 calibFluxApertureRadius=None): 

326 startTime = time.time() 

327 

328 if self.config.doSubtractLocalBackground and calibFluxApertureRadius is None: 

329 raise RuntimeError("Must set calibFluxApertureRadius if doSubtractLocalBackground is True.") 

330 

331 # To get the correct output schema, we use the legacy code. 

332 # We are not actually using this mapper, except to grab the outputSchema 

333 sourceMapper = self._makeSourceMapper(sourceSchema) 

334 outputSchema = sourceMapper.getOutputSchema() 

335 

336 # Construct mapping from ccd number to index 

337 ccdMapping = {} 

338 for ccdIndex, detector in enumerate(camera): 

339 ccdMapping[detector.getId()] = ccdIndex 

340 

341 approxPixelAreaFields = computeApproxPixelAreaFields(camera) 

342 

343 fullCatalog = afwTable.BaseCatalog(outputSchema) 

344 

345 visitKey = outputSchema['visit'].asKey() 

346 ccdKey = outputSchema['ccd'].asKey() 

347 instMagKey = outputSchema['instMag'].asKey() 

348 instMagErrKey = outputSchema['instMagErr'].asKey() 

349 deltaMagAperKey = outputSchema['deltaMagAper'].asKey() 

350 

351 # Prepare local background if desired 

352 if self.config.doSubtractLocalBackground: 

353 localBackgroundArea = np.pi*calibFluxApertureRadius**2. 

354 

355 columns = None 

356 

357 k = 2.5/np.log(10.) 

358 

359 for counter, visit in enumerate(visitCat): 

360 expTime = visit['exptime'] 

361 

362 handle = groupedHandles[visit['visit']][-1] 

363 

364 if columns is None: 

365 inColumns = handle.get(component='columns') 

366 columns = self._get_sourceTable_visit_columns(inColumns) 

367 df = handle.get(parameters={'columns': columns}) 

368 

369 goodSrc = self.sourceSelector.selectSources(df) 

370 

371 # Need to add a selection based on the local background correction 

372 # if necessary 

373 if self.config.doSubtractLocalBackground: 

374 localBackground = localBackgroundArea*df[self.config.localBackgroundFluxField].values 

375 use, = np.where((goodSrc.selected) 

376 & ((df[self.config.instFluxField].values - localBackground) > 0.0)) 

377 else: 

378 use, = np.where(goodSrc.selected) 

379 

380 tempCat = afwTable.BaseCatalog(fullCatalog.schema) 

381 tempCat.resize(use.size) 

382 

383 tempCat['ra'][:] = np.deg2rad(df['ra'].values[use]) 

384 tempCat['dec'][:] = np.deg2rad(df['dec'].values[use]) 

385 tempCat['x'][:] = df['x'].values[use] 

386 tempCat['y'][:] = df['y'].values[use] 

387 # The "visit" name in the parquet table is hard-coded. 

388 tempCat[visitKey][:] = df['visit'].values[use] 

389 tempCat[ccdKey][:] = df['detector'].values[use] 

390 tempCat['psf_candidate'] = df[self.config.psfCandidateName].values[use] 

391 

392 with warnings.catch_warnings(): 

393 # Ignore warnings, we will filter infinites and nans below 

394 warnings.simplefilter("ignore") 

395 

396 instMagInner = -2.5*np.log10(df[self.config.apertureInnerInstFluxField].values[use]) 

397 instMagErrInner = k*(df[self.config.apertureInnerInstFluxField + 'Err'].values[use] 

398 / df[self.config.apertureInnerInstFluxField].values[use]) 

399 instMagOuter = -2.5*np.log10(df[self.config.apertureOuterInstFluxField].values[use]) 

400 instMagErrOuter = k*(df[self.config.apertureOuterInstFluxField + 'Err'].values[use] 

401 / df[self.config.apertureOuterInstFluxField].values[use]) 

402 tempCat[deltaMagAperKey][:] = instMagInner - instMagOuter 

403 # Set bad values to illegal values for fgcm. 

404 tempCat[deltaMagAperKey][~np.isfinite(tempCat[deltaMagAperKey][:])] = 99.0 

405 

406 if self.config.doSubtractLocalBackground: 

407 # At the moment we only adjust the flux and not the flux 

408 # error by the background because the error on 

409 # base_LocalBackground_instFlux is the rms error in the 

410 # background annulus, not the error on the mean in the 

411 # background estimate (which is much smaller, by sqrt(n) 

412 # pixels used to estimate the background, which we do not 

413 # have access to in this task). In the default settings, 

414 # the annulus is sufficiently large such that these 

415 # additional errors are are negligibly small (much less 

416 # than a mmag in quadrature). 

417 

418 # This is the difference between the mag with local background correction 

419 # and the mag without local background correction. 

420 tempCat['deltaMagBkg'] = (-2.5*np.log10(df[self.config.instFluxField].values[use] 

421 - localBackground[use]) - 

422 -2.5*np.log10(df[self.config.instFluxField].values[use])) 

423 else: 

424 tempCat['deltaMagBkg'][:] = 0.0 

425 

426 # Need to loop over ccds here 

427 for detector in camera: 

428 ccdId = detector.getId() 

429 # used index for all observations with a given ccd 

430 use2 = (tempCat[ccdKey] == ccdId) 

431 tempCat['jacobian'][use2] = approxPixelAreaFields[ccdId].evaluate(tempCat['x'][use2], 

432 tempCat['y'][use2]) 

433 scaledInstFlux = (df[self.config.instFluxField].values[use[use2]] 

434 * visit['scaling'][ccdMapping[ccdId]]) 

435 tempCat[instMagKey][use2] = (-2.5*np.log10(scaledInstFlux) + 2.5*np.log10(expTime)) 

436 

437 # Compute instMagErr from instFluxErr/instFlux, any scaling 

438 # will cancel out. 

439 tempCat[instMagErrKey][:] = k*(df[self.config.instFluxField + 'Err'].values[use] 

440 / df[self.config.instFluxField].values[use]) 

441 

442 # Apply the jacobian if configured 

443 if self.config.doApplyWcsJacobian: 

444 tempCat[instMagKey][:] -= 2.5*np.log10(tempCat['jacobian'][:]) 

445 

446 fullCatalog.extend(tempCat) 

447 

448 deltaOk = (np.isfinite(instMagInner) & np.isfinite(instMagErrInner) 

449 & np.isfinite(instMagOuter) & np.isfinite(instMagErrOuter)) 

450 

451 visit['deltaAper'] = np.median(instMagInner[deltaOk] - instMagOuter[deltaOk]) 

452 visit['sources_read'] = True 

453 

454 self.log.info(" Found %d good stars in visit %d (deltaAper = %0.3f)", 

455 use.size, visit['visit'], visit['deltaAper']) 

456 

457 self.log.info("Found all good star observations in %.2f s" % 

458 (time.time() - startTime)) 

459 

460 return fullCatalog 

461 

462 def _get_sourceTable_visit_columns(self, inColumns): 

463 """ 

464 Get the sourceTable_visit columns from the config. 

465 

466 Parameters 

467 ---------- 

468 inColumns : `list` 

469 List of columns available in the sourceTable_visit 

470 

471 Returns 

472 ------- 

473 columns : `list` 

474 List of columns to read from sourceTable_visit. 

475 """ 

476 # Some names are hard-coded in the parquet table. 

477 columns = ['visit', 'detector', 

478 'ra', 'dec', 'x', 'y', self.config.psfCandidateName, 

479 self.config.instFluxField, self.config.instFluxField + 'Err', 

480 self.config.apertureInnerInstFluxField, self.config.apertureInnerInstFluxField + 'Err', 

481 self.config.apertureOuterInstFluxField, self.config.apertureOuterInstFluxField + 'Err'] 

482 if self.sourceSelector.config.doFlags: 

483 columns.extend(self.sourceSelector.config.flags.bad) 

484 if self.sourceSelector.config.doUnresolved: 

485 columns.append(self.sourceSelector.config.unresolved.name) 

486 if self.sourceSelector.config.doIsolated: 

487 columns.append(self.sourceSelector.config.isolated.parentName) 

488 columns.append(self.sourceSelector.config.isolated.nChildName) 

489 if self.sourceSelector.config.doRequirePrimary: 

490 columns.append(self.sourceSelector.config.requirePrimary.primaryColName) 

491 if self.config.doSubtractLocalBackground: 

492 columns.append(self.config.localBackgroundFluxField) 

493 

494 return columns