Coverage for python/lsst/pipe/tasks/postprocess.py: 28%

675 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 11:31 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["WriteObjectTableConfig", "WriteObjectTableTask", 

23 "WriteSourceTableConfig", "WriteSourceTableTask", 

24 "WriteRecalibratedSourceTableConfig", "WriteRecalibratedSourceTableTask", 

25 "PostprocessAnalysis", 

26 "TransformCatalogBaseConfig", "TransformCatalogBaseTask", 

27 "TransformObjectCatalogConfig", "TransformObjectCatalogTask", 

28 "ConsolidateObjectTableConfig", "ConsolidateObjectTableTask", 

29 "TransformSourceTableConfig", "TransformSourceTableTask", 

30 "ConsolidateVisitSummaryConfig", "ConsolidateVisitSummaryTask", 

31 "ConsolidateSourceTableConfig", "ConsolidateSourceTableTask", 

32 "MakeCcdVisitTableConfig", "MakeCcdVisitTableTask", 

33 "MakeVisitTableConfig", "MakeVisitTableTask", 

34 "WriteForcedSourceTableConfig", "WriteForcedSourceTableTask", 

35 "TransformForcedSourceTableConfig", "TransformForcedSourceTableTask", 

36 "ConsolidateTractConfig", "ConsolidateTractTask"] 

37 

38import functools 

39import pandas as pd 

40import logging 

41import numpy as np 

42import numbers 

43import os 

44import warnings 

45 

46from deprecated.sphinx import deprecated 

47 

48import lsst.geom 

49import lsst.pex.config as pexConfig 

50import lsst.pipe.base as pipeBase 

51import lsst.daf.base as dafBase 

52from lsst.utils.introspection import find_outside_stacklevel 

53from lsst.pipe.base import connectionTypes 

54import lsst.afw.table as afwTable 

55from lsst.afw.image import ExposureSummaryStats 

56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig 

57from lsst.skymap import BaseSkyMap 

58 

59from .functors import CompositeFunctor, Column 

60 

61log = logging.getLogger(__name__) 

62 

63 

64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None): 

65 """Flattens a dataframe with multilevel column index. 

66 """ 

67 newDf = pd.DataFrame() 

68 # band is the level 0 index 

69 dfBands = df.columns.unique(level=0).values 

70 for band in dfBands: 

71 subdf = df[band] 

72 columnFormat = '{0}{1}' if camelCase else '{0}_{1}' 

73 newColumns = {c: columnFormat.format(band, c) 

74 for c in subdf.columns if c not in noDupCols} 

75 cols = list(newColumns.keys()) 

76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1) 

77 

78 # Band must be present in the input and output or else column is all NaN: 

79 presentBands = dfBands if inputBands is None else list(set(inputBands).intersection(dfBands)) 

80 # Get the unexploded columns from any present band's partition 

81 noDupDf = df[presentBands[0]][noDupCols] 

82 newDf = pd.concat([noDupDf, newDf], axis=1) 

83 return newDf 

84 

85 

86class WriteObjectTableConnections(pipeBase.PipelineTaskConnections, 

87 defaultTemplates={"coaddName": "deep"}, 

88 dimensions=("tract", "patch", "skymap")): 

89 inputCatalogMeas = connectionTypes.Input( 

90 doc="Catalog of source measurements on the deepCoadd.", 

91 dimensions=("tract", "patch", "band", "skymap"), 

92 storageClass="SourceCatalog", 

93 name="{coaddName}Coadd_meas", 

94 multiple=True 

95 ) 

96 inputCatalogForcedSrc = connectionTypes.Input( 

97 doc="Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.", 

98 dimensions=("tract", "patch", "band", "skymap"), 

99 storageClass="SourceCatalog", 

100 name="{coaddName}Coadd_forced_src", 

101 multiple=True 

102 ) 

103 inputCatalogRef = connectionTypes.Input( 

104 doc="Catalog marking the primary detection (which band provides a good shape and position)" 

105 "for each detection in deepCoadd_mergeDet.", 

106 dimensions=("tract", "patch", "skymap"), 

107 storageClass="SourceCatalog", 

108 name="{coaddName}Coadd_ref" 

109 ) 

110 outputCatalog = connectionTypes.Output( 

111 doc="A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, " 

112 "stored as a DataFrame with a multi-level column index per-patch.", 

113 dimensions=("tract", "patch", "skymap"), 

114 storageClass="DataFrame", 

115 name="{coaddName}Coadd_obj" 

116 ) 

117 

118 

119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig, 

120 pipelineConnections=WriteObjectTableConnections): 

121 engine = pexConfig.Field( 

122 dtype=str, 

123 default="pyarrow", 

124 doc="Parquet engine for writing (pyarrow or fastparquet)", 

125 deprecated="This config is no longer used, and will be removed after v26." 

126 ) 

127 coaddName = pexConfig.Field( 

128 dtype=str, 

129 default="deep", 

130 doc="Name of coadd" 

131 ) 

132 

133 

134class WriteObjectTableTask(pipeBase.PipelineTask): 

135 """Write filter-merged source tables as a DataFrame in parquet format. 

136 """ 

137 _DefaultName = "writeObjectTable" 

138 ConfigClass = WriteObjectTableConfig 

139 

140 # Names of table datasets to be merged 

141 inputDatasets = ('forced_src', 'meas', 'ref') 

142 

143 # Tag of output dataset written by `MergeSourcesTask.write` 

144 outputDataset = 'obj' 

145 

146 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

147 inputs = butlerQC.get(inputRefs) 

148 

149 measDict = {ref.dataId['band']: {'meas': cat} for ref, cat in 

150 zip(inputRefs.inputCatalogMeas, inputs['inputCatalogMeas'])} 

151 forcedSourceDict = {ref.dataId['band']: {'forced_src': cat} for ref, cat in 

152 zip(inputRefs.inputCatalogForcedSrc, inputs['inputCatalogForcedSrc'])} 

153 

154 catalogs = {} 

155 for band in measDict.keys(): 

156 catalogs[band] = {'meas': measDict[band]['meas'], 

157 'forced_src': forcedSourceDict[band]['forced_src'], 

158 'ref': inputs['inputCatalogRef']} 

159 dataId = butlerQC.quantum.dataId 

160 df = self.run(catalogs=catalogs, tract=dataId['tract'], patch=dataId['patch']) 

161 outputs = pipeBase.Struct(outputCatalog=df) 

162 butlerQC.put(outputs, outputRefs) 

163 

164 def run(self, catalogs, tract, patch): 

165 """Merge multiple catalogs. 

166 

167 Parameters 

168 ---------- 

169 catalogs : `dict` 

170 Mapping from filter names to dict of catalogs. 

171 tract : int 

172 tractId to use for the tractId column. 

173 patch : str 

174 patchId to use for the patchId column. 

175 

176 Returns 

177 ------- 

178 catalog : `pandas.DataFrame` 

179 Merged dataframe. 

180 """ 

181 dfs = [] 

182 for filt, tableDict in catalogs.items(): 

183 for dataset, table in tableDict.items(): 

184 # Convert afwTable to pandas DataFrame 

185 df = table.asAstropy().to_pandas().set_index('id', drop=True) 

186 

187 # Sort columns by name, to ensure matching schema among patches 

188 df = df.reindex(sorted(df.columns), axis=1) 

189 df = df.assign(tractId=tract, patchId=patch) 

190 

191 # Make columns a 3-level MultiIndex 

192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c) for c in df.columns], 

193 names=('dataset', 'band', 'column')) 

194 dfs.append(df) 

195 

196 # We do this dance and not `pd.concat(dfs)` because the pandas 

197 # concatenation uses infinite memory. 

198 catalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs) 

199 return catalog 

200 

201 

202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, 

203 defaultTemplates={"catalogType": ""}, 

204 dimensions=("instrument", "visit", "detector")): 

205 

206 catalog = connectionTypes.Input( 

207 doc="Input full-depth catalog of sources produced by CalibrateTask", 

208 name="{catalogType}src", 

209 storageClass="SourceCatalog", 

210 dimensions=("instrument", "visit", "detector") 

211 ) 

212 outputCatalog = connectionTypes.Output( 

213 doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is " 

214 "replaced with an index; all other columns are unchanged.", 

215 name="{catalogType}source", 

216 storageClass="DataFrame", 

217 dimensions=("instrument", "visit", "detector") 

218 ) 

219 

220 

221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig, 

222 pipelineConnections=WriteSourceTableConnections): 

223 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

224 

225 

226class WriteSourceTableTask(pipeBase.PipelineTask): 

227 """Write source table to DataFrame Parquet format. 

228 """ 

229 _DefaultName = "writeSourceTable" 

230 ConfigClass = WriteSourceTableConfig 

231 

232 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

233 inputs = butlerQC.get(inputRefs) 

234 inputs['ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id 

235 result = self.run(**inputs) 

236 outputs = pipeBase.Struct(outputCatalog=result.table) 

237 butlerQC.put(outputs, outputRefs) 

238 

239 def run(self, catalog, ccdVisitId=None, **kwargs): 

240 """Convert `src` catalog to DataFrame 

241 

242 Parameters 

243 ---------- 

244 catalog: `afwTable.SourceCatalog` 

245 catalog to be converted 

246 ccdVisitId: `int` 

247 ccdVisitId to be added as a column 

248 **kwargs 

249 Additional keyword arguments are ignored as a convenience for 

250 subclasses that pass the same arguments to several different 

251 methods. 

252 

253 Returns 

254 ------- 

255 result : `~lsst.pipe.base.Struct` 

256 ``table`` 

257 `DataFrame` version of the input catalog 

258 """ 

259 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId) 

260 df = catalog.asAstropy().to_pandas().set_index('id', drop=True) 

261 df['ccdVisitId'] = ccdVisitId 

262 

263 return pipeBase.Struct(table=df) 

264 

265 

266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections, 

267 defaultTemplates={"catalogType": "", 

268 "skyWcsName": "gbdesAstrometricFit", 

269 "photoCalibName": "fgcm"}, 

270 # TODO: remove on DM-39854. 

271 deprecatedTemplates={ 

272 "skyWcsName": "Deprecated; will be removed after v26.", 

273 "photoCalibName": "Deprecated; will be removed after v26." 

274 }, 

275 dimensions=("instrument", "visit", "detector", "skymap")): 

276 skyMap = connectionTypes.Input( 

277 doc="skyMap needed to choose which tract-level calibrations to use when multiple available", 

278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

279 storageClass="SkyMap", 

280 dimensions=("skymap",), 

281 # TODO: remove on DM-39854. 

282 deprecated=( 

283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. " 

284 "Will be removed after v26." 

285 ), 

286 ) 

287 exposure = connectionTypes.Input( 

288 doc="Input exposure to perform photometry on.", 

289 name="calexp", 

290 storageClass="ExposureF", 

291 dimensions=["instrument", "visit", "detector"], 

292 ) 

293 visitSummary = connectionTypes.Input( 

294 doc="Input visit-summary catalog with updated calibration objects.", 

295 name="finalVisitSummary", 

296 storageClass="ExposureCatalog", 

297 dimensions=("instrument", "visit",), 

298 ) 

299 externalSkyWcsTractCatalog = connectionTypes.Input( 

300 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

301 "id for the catalog id, sorted on id for fast lookup."), 

302 name="{skyWcsName}SkyWcsCatalog", 

303 storageClass="ExposureCatalog", 

304 dimensions=["instrument", "visit", "tract"], 

305 multiple=True, 

306 # TODO: remove on DM-39854. 

307 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

308 ) 

309 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

310 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

311 "These catalogs use the detector id for the catalog id, sorted on id for " 

312 "fast lookup."), 

313 name="finalVisitSummary", 

314 storageClass="ExposureCatalog", 

315 dimensions=["instrument", "visit"], 

316 # TODO: remove on DM-39854. 

317 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

318 ) 

319 externalPhotoCalibTractCatalog = connectionTypes.Input( 

320 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

321 "detector id for the catalog id, sorted on id for fast lookup."), 

322 name="{photoCalibName}PhotoCalibCatalog", 

323 storageClass="ExposureCatalog", 

324 dimensions=["instrument", "visit", "tract"], 

325 multiple=True, 

326 # TODO: remove on DM-39854. 

327 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

328 ) 

329 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

330 doc=("Per-visit photometric calibrations computed globally (with no tract " 

331 "information). These catalogs use the detector id for the catalog id, " 

332 "sorted on id for fast lookup."), 

333 name="finalVisitSummary", 

334 storageClass="ExposureCatalog", 

335 dimensions=["instrument", "visit"], 

336 # TODO: remove on DM-39854. 

337 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

338 ) 

339 

340 def __init__(self, *, config=None): 

341 super().__init__(config=config) 

342 # Same connection boilerplate as all other applications of 

343 # Global/Tract calibrations 

344 # TODO: remove all of this on DM-39854. 

345 keepSkyMap = False 

346 if config.doApplyExternalSkyWcs and config.doReevaluateSkyWcs: 

347 if config.useGlobalExternalSkyWcs: 

348 self.inputs.remove("externalSkyWcsTractCatalog") 

349 else: 

350 self.inputs.remove("externalSkyWcsGlobalCatalog") 

351 keepSkyMap = True 

352 else: 

353 self.inputs.remove("externalSkyWcsTractCatalog") 

354 self.inputs.remove("externalSkyWcsGlobalCatalog") 

355 if config.doApplyExternalPhotoCalib and config.doReevaluatePhotoCalib: 

356 if config.useGlobalExternalPhotoCalib: 

357 self.inputs.remove("externalPhotoCalibTractCatalog") 

358 else: 

359 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

360 keepSkyMap = True 

361 else: 

362 self.inputs.remove("externalPhotoCalibTractCatalog") 

363 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

364 if not keepSkyMap: 

365 del self.skyMap 

366 

367 

368class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig, 

369 pipelineConnections=WriteRecalibratedSourceTableConnections): 

370 

371 doReevaluatePhotoCalib = pexConfig.Field( 

372 dtype=bool, 

373 default=True, 

374 doc=("Add or replace local photoCalib columns"), 

375 ) 

376 doReevaluateSkyWcs = pexConfig.Field( 

377 dtype=bool, 

378 default=True, 

379 doc=("Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"), 

380 ) 

381 doApplyExternalPhotoCalib = pexConfig.Field( 

382 dtype=bool, 

383 default=False, 

384 doc=("If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ", 

385 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."), 

386 # TODO: remove on DM-39854. 

387 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.", 

388 ) 

389 doApplyExternalSkyWcs = pexConfig.Field( 

390 dtype=bool, 

391 default=False, 

392 doc=("if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ", 

393 "else use the wcs already attached to the exposure."), 

394 # TODO: remove on DM-39854. 

395 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.", 

396 ) 

397 useGlobalExternalPhotoCalib = pexConfig.Field( 

398 dtype=bool, 

399 default=False, 

400 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

401 "that are not run per-tract. When False, use per-tract photometric " 

402 "calibration files."), 

403 # TODO: remove on DM-39854. 

404 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.", 

405 ) 

406 useGlobalExternalSkyWcs = pexConfig.Field( 

407 dtype=bool, 

408 default=False, 

409 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

410 "that are not run per-tract. When False, use per-tract wcs " 

411 "files."), 

412 # TODO: remove on DM-39854. 

413 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.", 

414 ) 

415 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

416 

417 def validate(self): 

418 super().validate() 

419 if self.doApplyExternalSkyWcs and not self.doReevaluateSkyWcs: 

420 log.warning("doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False" 

421 "External SkyWcs will not be read or evaluated.") 

422 if self.doApplyExternalPhotoCalib and not self.doReevaluatePhotoCalib: 

423 log.warning("doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False." 

424 "External PhotoCalib will not be read or evaluated.") 

425 

426 

427class WriteRecalibratedSourceTableTask(WriteSourceTableTask): 

428 """Write source table to DataFrame Parquet format. 

429 """ 

430 _DefaultName = "writeRecalibratedSourceTable" 

431 ConfigClass = WriteRecalibratedSourceTableConfig 

432 

433 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

434 inputs = butlerQC.get(inputRefs) 

435 

436 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

437 inputs['idGenerator'] = idGenerator 

438 inputs['ccdVisitId'] = idGenerator.catalog_id 

439 

440 if self.config.doReevaluatePhotoCalib or self.config.doReevaluateSkyWcs: 

441 if self.config.doApplyExternalPhotoCalib or self.config.doApplyExternalSkyWcs: 

442 inputs['exposure'] = self.attachCalibs(inputRefs, **inputs) 

443 else: 

444 inputs['exposure'] = self.prepareCalibratedExposure( 

445 exposure=inputs["exposure"], visitSummary=inputs["visitSummary"] 

446 ) 

447 inputs['catalog'] = self.addCalibColumns(**inputs) 

448 

449 result = self.run(**inputs) 

450 outputs = pipeBase.Struct(outputCatalog=result.table) 

451 butlerQC.put(outputs, outputRefs) 

452 

453 # TODO: remove on DM-39854. 

454 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.", 

455 version="v26", category=FutureWarning) 

456 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None, 

457 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

458 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs): 

459 """Apply external calibrations to exposure per configuration 

460 

461 When multiple tract-level calibrations overlap, select the one with the 

462 center closest to detector. 

463 

464 Parameters 

465 ---------- 

466 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`, for dataIds of 

467 tract-level calibs. 

468 skyMap : `~lsst.skymap.BaseSkyMap` 

469 skyMap to lookup tract geometry and WCS. 

470 exposure : `lsst.afw.image.exposure.Exposure` 

471 Input exposure to adjust calibrations. 

472 externalSkyWcsGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

473 Exposure catalog with external skyWcs to be applied per config 

474 externalSkyWcsTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

475 Exposure catalog with external skyWcs to be applied per config 

476 externalPhotoCalibGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

477 Exposure catalog with external photoCalib to be applied per config 

478 externalPhotoCalibTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

479 Exposure catalog with external photoCalib to be applied per config 

480 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

481 Exposure catalog with all calibration objects. WCS and PhotoCalib 

482 are always applied if provided. 

483 **kwargs 

484 Additional keyword arguments are ignored to facilitate passing the 

485 same arguments to several methods. 

486 

487 Returns 

488 ------- 

489 exposure : `lsst.afw.image.exposure.Exposure` 

490 Exposure with adjusted calibrations. 

491 """ 

492 if not self.config.doApplyExternalSkyWcs: 

493 # Do not modify the exposure's SkyWcs 

494 externalSkyWcsCatalog = None 

495 elif self.config.useGlobalExternalSkyWcs: 

496 # Use the global external SkyWcs 

497 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog 

498 self.log.info('Applying global SkyWcs') 

499 else: 

500 # use tract-level external SkyWcs from the closest overlapping tract 

501 inputRef = getattr(inputRefs, 'externalSkyWcsTractCatalog') 

502 tracts = [ref.dataId['tract'] for ref in inputRef] 

503 if len(tracts) == 1: 

504 ind = 0 

505 self.log.info('Applying tract-level SkyWcs from tract %s', tracts[ind]) 

506 else: 

507 if exposure.getWcs() is None: # TODO: could this look-up use the externalPhotoCalib? 

508 raise ValueError("Trying to locate nearest tract, but exposure.wcs is None.") 

509 ind = self.getClosestTract(tracts, skyMap, 

510 exposure.getBBox(), exposure.getWcs()) 

511 self.log.info('Multiple overlapping externalSkyWcsTractCatalogs found (%s). ' 

512 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind]) 

513 

514 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind] 

515 

516 if not self.config.doApplyExternalPhotoCalib: 

517 # Do not modify the exposure's PhotoCalib 

518 externalPhotoCalibCatalog = None 

519 elif self.config.useGlobalExternalPhotoCalib: 

520 # Use the global external PhotoCalib 

521 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog 

522 self.log.info('Applying global PhotoCalib') 

523 else: 

524 # use tract-level external PhotoCalib from the closest overlapping tract 

525 inputRef = getattr(inputRefs, 'externalPhotoCalibTractCatalog') 

526 tracts = [ref.dataId['tract'] for ref in inputRef] 

527 if len(tracts) == 1: 

528 ind = 0 

529 self.log.info('Applying tract-level PhotoCalib from tract %s', tracts[ind]) 

530 else: 

531 ind = self.getClosestTract(tracts, skyMap, 

532 exposure.getBBox(), exposure.getWcs()) 

533 self.log.info('Multiple overlapping externalPhotoCalibTractCatalogs found (%s). ' 

534 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind]) 

535 

536 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind] 

537 

538 return self.prepareCalibratedExposure( 

539 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary 

540 ) 

541 

542 # TODO: remove on DM-39854. 

543 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.", 

544 version="v26", category=FutureWarning) 

545 def getClosestTract(self, tracts, skyMap, bbox, wcs): 

546 """Find the index of the tract closest to detector from list of tractIds 

547 

548 Parameters 

549 ---------- 

550 tracts: `list` [`int`] 

551 Iterable of integer tractIds 

552 skyMap : `~lsst.skymap.BaseSkyMap` 

553 skyMap to lookup tract geometry and wcs 

554 bbox : `~lsst.geom.Box2I` 

555 Detector bbox, center of which will compared to tract centers 

556 wcs : `~lsst.afw.geom.SkyWcs` 

557 Detector Wcs object to map the detector center to SkyCoord 

558 

559 Returns 

560 ------- 

561 index : `int` 

562 """ 

563 if len(tracts) == 1: 

564 return 0 

565 

566 center = wcs.pixelToSky(bbox.getCenter()) 

567 sep = [] 

568 for tractId in tracts: 

569 tract = skyMap[tractId] 

570 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter()) 

571 sep.append(center.separation(tractCenter)) 

572 

573 return np.argmin(sep) 

574 

575 def prepareCalibratedExposure( 

576 self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, visitSummary=None 

577 ): 

578 """Prepare a calibrated exposure and apply external calibrations 

579 if so configured. 

580 

581 Parameters 

582 ---------- 

583 exposure : `lsst.afw.image.exposure.Exposure` 

584 Input exposure to adjust calibrations. 

585 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

586 Exposure catalog with external skyWcs to be applied 

587 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

588 for the catalog id, sorted on id for fast lookup. 

589 Deprecated in favor of ``visitSummary``; will be removed after v26. 

590 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

591 Exposure catalog with external photoCalib to be applied 

592 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

593 id for the catalog id, sorted on id for fast lookup. 

594 Deprecated in favor of ``visitSummary``; will be removed after v26. 

595 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

596 Exposure catalog with all calibration objects. WCS and PhotoCalib 

597 are always applied if ``visitSummary`` is provided and those 

598 components are not `None`. 

599 

600 Returns 

601 ------- 

602 exposure : `lsst.afw.image.exposure.Exposure` 

603 Exposure with adjusted calibrations. 

604 """ 

605 detectorId = exposure.getInfo().getDetector().getId() 

606 

607 if visitSummary is not None: 

608 row = visitSummary.find(detectorId) 

609 if row is None: 

610 raise RuntimeError(f"Visit summary for detector {detectorId} is unexpectedly missing.") 

611 if (photoCalib := row.getPhotoCalib()) is None: 

612 self.log.warning("Detector id %s has None for photoCalib in visit summary; " 

613 "using original photoCalib.", detectorId) 

614 else: 

615 exposure.setPhotoCalib(photoCalib) 

616 if (skyWcs := row.getWcs()) is None: 

617 self.log.warning("Detector id %s has None for skyWcs in visit summary; " 

618 "using original skyWcs.", detectorId) 

619 else: 

620 exposure.setWcs(skyWcs) 

621 

622 if externalPhotoCalibCatalog is not None: 

623 # TODO: remove on DM-39854. 

624 warnings.warn( 

625 "Deprecated in favor of 'visitSummary'; will be removed after v26.", 

626 FutureWarning, 

627 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"), 

628 ) 

629 row = externalPhotoCalibCatalog.find(detectorId) 

630 if row is None: 

631 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; " 

632 "Using original photoCalib.", detectorId) 

633 else: 

634 photoCalib = row.getPhotoCalib() 

635 if photoCalib is None: 

636 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; " 

637 "Using original photoCalib.", detectorId) 

638 else: 

639 exposure.setPhotoCalib(photoCalib) 

640 

641 if externalSkyWcsCatalog is not None: 

642 # TODO: remove on DM-39854. 

643 warnings.warn( 

644 "Deprecated in favor of 'visitSummary'; will be removed after v26.", 

645 FutureWarning, 

646 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"), 

647 ) 

648 row = externalSkyWcsCatalog.find(detectorId) 

649 if row is None: 

650 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; " 

651 "Using original skyWcs.", detectorId) 

652 else: 

653 skyWcs = row.getWcs() 

654 if skyWcs is None: 

655 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; " 

656 "Using original skyWcs.", detectorId) 

657 else: 

658 exposure.setWcs(skyWcs) 

659 

660 return exposure 

661 

662 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs): 

663 """Add replace columns with calibs evaluated at each centroid 

664 

665 Add or replace 'base_LocalWcs' `base_LocalPhotoCalib' columns in a 

666 a source catalog, by rerunning the plugins. 

667 

668 Parameters 

669 ---------- 

670 catalog : `lsst.afw.table.SourceCatalog` 

671 catalog to which calib columns will be added 

672 exposure : `lsst.afw.image.exposure.Exposure` 

673 Exposure with attached PhotoCalibs and SkyWcs attributes to be 

674 reevaluated at local centroids. Pixels are not required. 

675 idGenerator : `lsst.meas.base.IdGenerator` 

676 Object that generates Source IDs and random seeds. 

677 **kwargs 

678 Additional keyword arguments are ignored to facilitate passing the 

679 same arguments to several methods. 

680 

681 Returns 

682 ------- 

683 newCat: `lsst.afw.table.SourceCatalog` 

684 Source Catalog with requested local calib columns 

685 """ 

686 measureConfig = SingleFrameMeasurementTask.ConfigClass() 

687 measureConfig.doReplaceWithNoise = False 

688 

689 # Clear all slots, because we aren't running the relevant plugins. 

690 for slot in measureConfig.slots: 

691 setattr(measureConfig.slots, slot, None) 

692 

693 measureConfig.plugins.names = [] 

694 if self.config.doReevaluateSkyWcs: 

695 measureConfig.plugins.names.add('base_LocalWcs') 

696 self.log.info("Re-evaluating base_LocalWcs plugin") 

697 if self.config.doReevaluatePhotoCalib: 

698 measureConfig.plugins.names.add('base_LocalPhotoCalib') 

699 self.log.info("Re-evaluating base_LocalPhotoCalib plugin") 

700 pluginsNotToCopy = tuple(measureConfig.plugins.names) 

701 

702 # Create a new schema and catalog 

703 # Copy all columns from original except for the ones to reevaluate 

704 aliasMap = catalog.schema.getAliasMap() 

705 mapper = afwTable.SchemaMapper(catalog.schema) 

706 for item in catalog.schema: 

707 if not item.field.getName().startswith(pluginsNotToCopy): 

708 mapper.addMapping(item.key) 

709 

710 schema = mapper.getOutputSchema() 

711 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema) 

712 schema.setAliasMap(aliasMap) 

713 newCat = afwTable.SourceCatalog(schema) 

714 newCat.extend(catalog, mapper=mapper) 

715 

716 # Fluxes in sourceCatalogs are in counts, so there are no fluxes to 

717 # update here. LocalPhotoCalibs are applied during transform tasks. 

718 # Update coord_ra/coord_dec, which are expected to be positions on the 

719 # sky and are used as such in sdm tables without transform 

720 if self.config.doReevaluateSkyWcs and exposure.wcs is not None: 

721 afwTable.updateSourceCoords(exposure.wcs, newCat) 

722 

723 measurement.run(measCat=newCat, exposure=exposure, exposureId=idGenerator.catalog_id) 

724 

725 return newCat 

726 

727 

728class PostprocessAnalysis(object): 

729 """Calculate columns from DataFrames or handles storing DataFrames. 

730 

731 This object manages and organizes an arbitrary set of computations 

732 on a catalog. The catalog is defined by a 

733 `DeferredDatasetHandle` or `InMemoryDatasetHandle` object 

734 (or list thereof), such as a ``deepCoadd_obj`` dataset, and the 

735 computations are defined by a collection of 

736 `~lsst.pipe.tasks.functors.Functor` objects (or, equivalently, a 

737 ``CompositeFunctor``). 

738 

739 After the object is initialized, accessing the ``.df`` attribute (which 

740 holds the `pandas.DataFrame` containing the results of the calculations) 

741 triggers computation of said dataframe. 

742 

743 One of the conveniences of using this object is the ability to define a 

744 desired common filter for all functors. This enables the same functor 

745 collection to be passed to several different `PostprocessAnalysis` objects 

746 without having to change the original functor collection, since the ``filt`` 

747 keyword argument of this object triggers an overwrite of the ``filt`` 

748 property for all functors in the collection. 

749 

750 This object also allows a list of refFlags to be passed, and defines a set 

751 of default refFlags that are always included even if not requested. 

752 

753 If a list of DataFrames or Handles is passed, rather than a single one, 

754 then the calculations will be mapped over all the input catalogs. In 

755 principle, it should be straightforward to parallelize this activity, but 

756 initial tests have failed (see TODO in code comments). 

757 

758 Parameters 

759 ---------- 

760 handles : `~lsst.daf.butler.DeferredDatasetHandle` or 

761 `~lsst.pipe.base.InMemoryDatasetHandle` or 

762 list of these. 

763 Source catalog(s) for computation. 

764 functors : `list`, `dict`, or `~lsst.pipe.tasks.functors.CompositeFunctor` 

765 Computations to do (functors that act on ``handles``). 

766 If a dict, the output 

767 DataFrame will have columns keyed accordingly. 

768 If a list, the column keys will come from the 

769 ``.shortname`` attribute of each functor. 

770 

771 filt : `str`, optional 

772 Filter in which to calculate. If provided, 

773 this will overwrite any existing ``.filt`` attribute 

774 of the provided functors. 

775 

776 flags : `list`, optional 

777 List of flags (per-band) to include in output table. 

778 Taken from the ``meas`` dataset if applied to a multilevel Object Table. 

779 

780 refFlags : `list`, optional 

781 List of refFlags (only reference band) to include in output table. 

782 

783 forcedFlags : `list`, optional 

784 List of flags (per-band) to include in output table. 

785 Taken from the ``forced_src`` dataset if applied to a 

786 multilevel Object Table. Intended for flags from measurement plugins 

787 only run during multi-band forced-photometry. 

788 """ 

789 _defaultRefFlags = [] 

790 _defaultFuncs = () 

791 

792 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None): 

793 self.handles = handles 

794 self.functors = functors 

795 

796 self.filt = filt 

797 self.flags = list(flags) if flags is not None else [] 

798 self.forcedFlags = list(forcedFlags) if forcedFlags is not None else [] 

799 self.refFlags = list(self._defaultRefFlags) 

800 if refFlags is not None: 

801 self.refFlags += list(refFlags) 

802 

803 self._df = None 

804 

805 @property 

806 def defaultFuncs(self): 

807 funcs = dict(self._defaultFuncs) 

808 return funcs 

809 

810 @property 

811 def func(self): 

812 additionalFuncs = self.defaultFuncs 

813 additionalFuncs.update({flag: Column(flag, dataset='forced_src') for flag in self.forcedFlags}) 

814 additionalFuncs.update({flag: Column(flag, dataset='ref') for flag in self.refFlags}) 

815 additionalFuncs.update({flag: Column(flag, dataset='meas') for flag in self.flags}) 

816 

817 if isinstance(self.functors, CompositeFunctor): 

818 func = self.functors 

819 else: 

820 func = CompositeFunctor(self.functors) 

821 

822 func.funcDict.update(additionalFuncs) 

823 func.filt = self.filt 

824 

825 return func 

826 

827 @property 

828 def noDupCols(self): 

829 return [name for name, func in self.func.funcDict.items() if func.noDup or func.dataset == 'ref'] 

830 

831 @property 

832 def df(self): 

833 if self._df is None: 

834 self.compute() 

835 return self._df 

836 

837 def compute(self, dropna=False, pool=None): 

838 # map over multiple handles 

839 if type(self.handles) in (list, tuple): 

840 if pool is None: 

841 dflist = [self.func(handle, dropna=dropna) for handle in self.handles] 

842 else: 

843 # TODO: Figure out why this doesn't work (pyarrow pickling 

844 # issues?) 

845 dflist = pool.map(functools.partial(self.func, dropna=dropna), self.handles) 

846 self._df = pd.concat(dflist) 

847 else: 

848 self._df = self.func(self.handles, dropna=dropna) 

849 

850 return self._df 

851 

852 

853class TransformCatalogBaseConnections(pipeBase.PipelineTaskConnections, 

854 dimensions=()): 

855 """Expected Connections for subclasses of TransformCatalogBaseTask. 

856 

857 Must be subclassed. 

858 """ 

859 inputCatalog = connectionTypes.Input( 

860 name="", 

861 storageClass="DataFrame", 

862 ) 

863 outputCatalog = connectionTypes.Output( 

864 name="", 

865 storageClass="DataFrame", 

866 ) 

867 

868 

869class TransformCatalogBaseConfig(pipeBase.PipelineTaskConfig, 

870 pipelineConnections=TransformCatalogBaseConnections): 

871 functorFile = pexConfig.Field( 

872 dtype=str, 

873 doc="Path to YAML file specifying Science Data Model functors to use " 

874 "when copying columns and computing calibrated values.", 

875 default=None, 

876 optional=True 

877 ) 

878 primaryKey = pexConfig.Field( 

879 dtype=str, 

880 doc="Name of column to be set as the DataFrame index. If None, the index" 

881 "will be named `id`", 

882 default=None, 

883 optional=True 

884 ) 

885 columnsFromDataId = pexConfig.ListField( 

886 dtype=str, 

887 default=None, 

888 optional=True, 

889 doc="Columns to extract from the dataId", 

890 ) 

891 

892 

893class TransformCatalogBaseTask(pipeBase.PipelineTask): 

894 """Base class for transforming/standardizing a catalog by applying functors 

895 that convert units and apply calibrations. 

896 

897 The purpose of this task is to perform a set of computations on an input 

898 ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` that holds a 

899 ``DataFrame`` dataset (such as ``deepCoadd_obj``), and write the results to 

900 a new dataset (which needs to be declared in an ``outputDataset`` 

901 attribute). 

902 

903 The calculations to be performed are defined in a YAML file that specifies 

904 a set of functors to be computed, provided as a ``--functorFile`` config 

905 parameter. An example of such a YAML file is the following: 

906 

907 funcs: 

908 sourceId: 

909 functor: Index 

910 x: 

911 functor: Column 

912 args: slot_Centroid_x 

913 y: 

914 functor: Column 

915 args: slot_Centroid_y 

916 psfFlux: 

917 functor: LocalNanojansky 

918 args: 

919 - slot_PsfFlux_instFlux 

920 - slot_PsfFlux_instFluxErr 

921 - base_LocalPhotoCalib 

922 - base_LocalPhotoCalibErr 

923 psfFluxErr: 

924 functor: LocalNanojanskyErr 

925 args: 

926 - slot_PsfFlux_instFlux 

927 - slot_PsfFlux_instFluxErr 

928 - base_LocalPhotoCalib 

929 - base_LocalPhotoCalibErr 

930 flags: 

931 - detect_isPrimary 

932 

933 The names for each entry under "func" will become the names of columns in 

934 the output dataset. All the functors referenced are defined in 

935 `~lsst.pipe.tasks.functors`. Positional arguments to be passed to each 

936 functor are in the `args` list, and any additional entries for each column 

937 other than "functor" or "args" (e.g., ``'filt'``, ``'dataset'``) are 

938 treated as keyword arguments to be passed to the functor initialization. 

939 

940 The "flags" entry is the default shortcut for `Column` functors. 

941 All columns listed under "flags" will be copied to the output table 

942 untransformed. They can be of any datatype. 

943 In the special case of transforming a multi-level oject table with 

944 band and dataset indices (deepCoadd_obj), these will be taked from the 

945 `meas` dataset and exploded out per band. 

946 

947 There are two special shortcuts that only apply when transforming 

948 multi-level Object (deepCoadd_obj) tables: 

949 - The "refFlags" entry is shortcut for `Column` functor 

950 taken from the `'ref'` dataset if transforming an ObjectTable. 

951 - The "forcedFlags" entry is shortcut for `Column` functors. 

952 taken from the ``forced_src`` dataset if transforming an ObjectTable. 

953 These are expanded out per band. 

954 

955 

956 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object 

957 to organize and excecute the calculations. 

958 """ 

959 @property 

960 def _DefaultName(self): 

961 raise NotImplementedError('Subclass must define "_DefaultName" attribute') 

962 

963 @property 

964 def outputDataset(self): 

965 raise NotImplementedError('Subclass must define "outputDataset" attribute') 

966 

967 @property 

968 def inputDataset(self): 

969 raise NotImplementedError('Subclass must define "inputDataset" attribute') 

970 

971 @property 

972 def ConfigClass(self): 

973 raise NotImplementedError('Subclass must define "ConfigClass" attribute') 

974 

975 def __init__(self, *args, **kwargs): 

976 super().__init__(*args, **kwargs) 

977 if self.config.functorFile: 

978 self.log.info('Loading tranform functor definitions from %s', 

979 self.config.functorFile) 

980 self.funcs = CompositeFunctor.from_file(self.config.functorFile) 

981 self.funcs.update(dict(PostprocessAnalysis._defaultFuncs)) 

982 else: 

983 self.funcs = None 

984 

985 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

986 inputs = butlerQC.get(inputRefs) 

987 if self.funcs is None: 

988 raise ValueError("config.functorFile is None. " 

989 "Must be a valid path to yaml in order to run Task as a PipelineTask.") 

990 result = self.run(handle=inputs['inputCatalog'], funcs=self.funcs, 

991 dataId=outputRefs.outputCatalog.dataId.full) 

992 outputs = pipeBase.Struct(outputCatalog=result) 

993 butlerQC.put(outputs, outputRefs) 

994 

995 def run(self, handle, funcs=None, dataId=None, band=None): 

996 """Do postprocessing calculations 

997 

998 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` or 

999 ``DataFrame`` object and dataId, 

1000 returns a dataframe with results of postprocessing calculations. 

1001 

1002 Parameters 

1003 ---------- 

1004 handles : `~lsst.daf.butler.DeferredDatasetHandle` or 

1005 `~lsst.pipe.base.InMemoryDatasetHandle` or 

1006 `~pandas.DataFrame`, or list of these. 

1007 DataFrames from which calculations are done. 

1008 funcs : `~lsst.pipe.tasks.functors.Functor` 

1009 Functors to apply to the table's columns 

1010 dataId : dict, optional 

1011 Used to add a `patchId` column to the output dataframe. 

1012 band : `str`, optional 

1013 Filter band that is being processed. 

1014 

1015 Returns 

1016 ------- 

1017 df : `pandas.DataFrame` 

1018 """ 

1019 self.log.info("Transforming/standardizing the source table dataId: %s", dataId) 

1020 

1021 df = self.transform(band, handle, funcs, dataId).df 

1022 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df)) 

1023 return df 

1024 

1025 def getFunctors(self): 

1026 return self.funcs 

1027 

1028 def getAnalysis(self, handles, funcs=None, band=None): 

1029 if funcs is None: 

1030 funcs = self.funcs 

1031 analysis = PostprocessAnalysis(handles, funcs, filt=band) 

1032 return analysis 

1033 

1034 def transform(self, band, handles, funcs, dataId): 

1035 analysis = self.getAnalysis(handles, funcs=funcs, band=band) 

1036 df = analysis.df 

1037 if dataId and self.config.columnsFromDataId: 

1038 for key in self.config.columnsFromDataId: 

1039 if key in dataId: 

1040 df[str(key)] = dataId[key] 

1041 else: 

1042 raise ValueError(f"'{key}' in config.columnsFromDataId not found in dataId: {dataId}") 

1043 

1044 if self.config.primaryKey: 

1045 if df.index.name != self.config.primaryKey and self.config.primaryKey in df: 

1046 df.reset_index(inplace=True, drop=True) 

1047 df.set_index(self.config.primaryKey, inplace=True) 

1048 

1049 return pipeBase.Struct( 

1050 df=df, 

1051 analysis=analysis 

1052 ) 

1053 

1054 

1055class TransformObjectCatalogConnections(pipeBase.PipelineTaskConnections, 

1056 defaultTemplates={"coaddName": "deep"}, 

1057 dimensions=("tract", "patch", "skymap")): 

1058 inputCatalog = connectionTypes.Input( 

1059 doc="The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, " 

1060 "stored as a DataFrame with a multi-level column index per-patch.", 

1061 dimensions=("tract", "patch", "skymap"), 

1062 storageClass="DataFrame", 

1063 name="{coaddName}Coadd_obj", 

1064 deferLoad=True, 

1065 ) 

1066 outputCatalog = connectionTypes.Output( 

1067 doc="Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard " 

1068 "data model.", 

1069 dimensions=("tract", "patch", "skymap"), 

1070 storageClass="DataFrame", 

1071 name="objectTable" 

1072 ) 

1073 

1074 

1075class TransformObjectCatalogConfig(TransformCatalogBaseConfig, 

1076 pipelineConnections=TransformObjectCatalogConnections): 

1077 coaddName = pexConfig.Field( 

1078 dtype=str, 

1079 default="deep", 

1080 doc="Name of coadd" 

1081 ) 

1082 # TODO: remove in DM-27177 

1083 filterMap = pexConfig.DictField( 

1084 keytype=str, 

1085 itemtype=str, 

1086 default={}, 

1087 doc=("Dictionary mapping full filter name to short one for column name munging." 

1088 "These filters determine the output columns no matter what filters the " 

1089 "input data actually contain."), 

1090 deprecated=("Coadds are now identified by the band, so this transform is unused." 

1091 "Will be removed after v22.") 

1092 ) 

1093 outputBands = pexConfig.ListField( 

1094 dtype=str, 

1095 default=None, 

1096 optional=True, 

1097 doc=("These bands and only these bands will appear in the output," 

1098 " NaN-filled if the input does not include them." 

1099 " If None, then use all bands found in the input.") 

1100 ) 

1101 camelCase = pexConfig.Field( 

1102 dtype=bool, 

1103 default=False, 

1104 doc=("Write per-band columns names with camelCase, else underscore " 

1105 "For example: gPsFlux instead of g_PsFlux.") 

1106 ) 

1107 multilevelOutput = pexConfig.Field( 

1108 dtype=bool, 

1109 default=False, 

1110 doc=("Whether results dataframe should have a multilevel column index (True) or be flat " 

1111 "and name-munged (False).") 

1112 ) 

1113 goodFlags = pexConfig.ListField( 

1114 dtype=str, 

1115 default=[], 

1116 doc=("List of 'good' flags that should be set False when populating empty tables. " 

1117 "All other flags are considered to be 'bad' flags and will be set to True.") 

1118 ) 

1119 floatFillValue = pexConfig.Field( 

1120 dtype=float, 

1121 default=np.nan, 

1122 doc="Fill value for float fields when populating empty tables." 

1123 ) 

1124 integerFillValue = pexConfig.Field( 

1125 dtype=int, 

1126 default=-1, 

1127 doc="Fill value for integer fields when populating empty tables." 

1128 ) 

1129 

1130 def setDefaults(self): 

1131 super().setDefaults() 

1132 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Object.yaml') 

1133 self.primaryKey = 'objectId' 

1134 self.columnsFromDataId = ['tract', 'patch'] 

1135 self.goodFlags = ['calib_astrometry_used', 

1136 'calib_photometry_reserved', 

1137 'calib_photometry_used', 

1138 'calib_psf_candidate', 

1139 'calib_psf_reserved', 

1140 'calib_psf_used'] 

1141 

1142 

1143class TransformObjectCatalogTask(TransformCatalogBaseTask): 

1144 """Produce a flattened Object Table to match the format specified in 

1145 sdm_schemas. 

1146 

1147 Do the same set of postprocessing calculations on all bands. 

1148 

1149 This is identical to `TransformCatalogBaseTask`, except for that it does 

1150 the specified functor calculations for all filters present in the 

1151 input `deepCoadd_obj` table. Any specific ``"filt"`` keywords specified 

1152 by the YAML file will be superceded. 

1153 """ 

1154 _DefaultName = "transformObjectCatalog" 

1155 ConfigClass = TransformObjectCatalogConfig 

1156 

1157 def run(self, handle, funcs=None, dataId=None, band=None): 

1158 # NOTE: band kwarg is ignored here. 

1159 dfDict = {} 

1160 analysisDict = {} 

1161 templateDf = pd.DataFrame() 

1162 

1163 columns = handle.get(component='columns') 

1164 inputBands = columns.unique(level=1).values 

1165 

1166 outputBands = self.config.outputBands if self.config.outputBands else inputBands 

1167 

1168 # Perform transform for data of filters that exist in the handle dataframe. 

1169 for inputBand in inputBands: 

1170 if inputBand not in outputBands: 

1171 self.log.info("Ignoring %s band data in the input", inputBand) 

1172 continue 

1173 self.log.info("Transforming the catalog of band %s", inputBand) 

1174 result = self.transform(inputBand, handle, funcs, dataId) 

1175 dfDict[inputBand] = result.df 

1176 analysisDict[inputBand] = result.analysis 

1177 if templateDf.empty: 

1178 templateDf = result.df 

1179 

1180 # Put filler values in columns of other wanted bands 

1181 for filt in outputBands: 

1182 if filt not in dfDict: 

1183 self.log.info("Adding empty columns for band %s", filt) 

1184 dfTemp = templateDf.copy() 

1185 for col in dfTemp.columns: 

1186 testValue = dfTemp[col].values[0] 

1187 if isinstance(testValue, (np.bool_, pd.BooleanDtype)): 

1188 # Boolean flag type, check if it is a "good" flag 

1189 if col in self.config.goodFlags: 

1190 fillValue = False 

1191 else: 

1192 fillValue = True 

1193 elif isinstance(testValue, numbers.Integral): 

1194 # Checking numbers.Integral catches all flavors 

1195 # of python, numpy, pandas, etc. integers. 

1196 # We must ensure this is not an unsigned integer. 

1197 if isinstance(testValue, np.unsignedinteger): 

1198 raise ValueError("Parquet tables may not have unsigned integer columns.") 

1199 else: 

1200 fillValue = self.config.integerFillValue 

1201 else: 

1202 fillValue = self.config.floatFillValue 

1203 dfTemp[col].values[:] = fillValue 

1204 dfDict[filt] = dfTemp 

1205 

1206 # This makes a multilevel column index, with band as first level 

1207 df = pd.concat(dfDict, axis=1, names=['band', 'column']) 

1208 

1209 if not self.config.multilevelOutput: 

1210 noDupCols = list(set.union(*[set(v.noDupCols) for v in analysisDict.values()])) 

1211 if self.config.primaryKey in noDupCols: 

1212 noDupCols.remove(self.config.primaryKey) 

1213 if dataId and self.config.columnsFromDataId: 

1214 noDupCols += self.config.columnsFromDataId 

1215 df = flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase, 

1216 inputBands=inputBands) 

1217 

1218 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df)) 

1219 

1220 return df 

1221 

1222 

1223class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections, 

1224 dimensions=("tract", "skymap")): 

1225 inputCatalogs = connectionTypes.Input( 

1226 doc="Per-Patch objectTables conforming to the standard data model.", 

1227 name="objectTable", 

1228 storageClass="DataFrame", 

1229 dimensions=("tract", "patch", "skymap"), 

1230 multiple=True, 

1231 ) 

1232 outputCatalog = connectionTypes.Output( 

1233 doc="Pre-tract horizontal concatenation of the input objectTables", 

1234 name="objectTable_tract", 

1235 storageClass="DataFrame", 

1236 dimensions=("tract", "skymap"), 

1237 ) 

1238 

1239 

1240class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig, 

1241 pipelineConnections=ConsolidateObjectTableConnections): 

1242 coaddName = pexConfig.Field( 

1243 dtype=str, 

1244 default="deep", 

1245 doc="Name of coadd" 

1246 ) 

1247 

1248 

1249class ConsolidateObjectTableTask(pipeBase.PipelineTask): 

1250 """Write patch-merged source tables to a tract-level DataFrame Parquet file. 

1251 

1252 Concatenates `objectTable` list into a per-visit `objectTable_tract`. 

1253 """ 

1254 _DefaultName = "consolidateObjectTable" 

1255 ConfigClass = ConsolidateObjectTableConfig 

1256 

1257 inputDataset = 'objectTable' 

1258 outputDataset = 'objectTable_tract' 

1259 

1260 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1261 inputs = butlerQC.get(inputRefs) 

1262 self.log.info("Concatenating %s per-patch Object Tables", 

1263 len(inputs['inputCatalogs'])) 

1264 df = pd.concat(inputs['inputCatalogs']) 

1265 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs) 

1266 

1267 

1268class TransformSourceTableConnections(pipeBase.PipelineTaskConnections, 

1269 defaultTemplates={"catalogType": ""}, 

1270 dimensions=("instrument", "visit", "detector")): 

1271 

1272 inputCatalog = connectionTypes.Input( 

1273 doc="Wide input catalog of sources produced by WriteSourceTableTask", 

1274 name="{catalogType}source", 

1275 storageClass="DataFrame", 

1276 dimensions=("instrument", "visit", "detector"), 

1277 deferLoad=True 

1278 ) 

1279 outputCatalog = connectionTypes.Output( 

1280 doc="Narrower, per-detector Source Table transformed and converted per a " 

1281 "specified set of functors", 

1282 name="{catalogType}sourceTable", 

1283 storageClass="DataFrame", 

1284 dimensions=("instrument", "visit", "detector") 

1285 ) 

1286 

1287 

1288class TransformSourceTableConfig(TransformCatalogBaseConfig, 

1289 pipelineConnections=TransformSourceTableConnections): 

1290 

1291 def setDefaults(self): 

1292 super().setDefaults() 

1293 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Source.yaml') 

1294 self.primaryKey = 'sourceId' 

1295 self.columnsFromDataId = ['visit', 'detector', 'band', 'physical_filter'] 

1296 

1297 

1298class TransformSourceTableTask(TransformCatalogBaseTask): 

1299 """Transform/standardize a source catalog 

1300 """ 

1301 _DefaultName = "transformSourceTable" 

1302 ConfigClass = TransformSourceTableConfig 

1303 

1304 

1305class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, 

1306 dimensions=("instrument", "visit",), 

1307 defaultTemplates={"calexpType": ""}): 

1308 calexp = connectionTypes.Input( 

1309 doc="Processed exposures used for metadata", 

1310 name="calexp", 

1311 storageClass="ExposureF", 

1312 dimensions=("instrument", "visit", "detector"), 

1313 deferLoad=True, 

1314 multiple=True, 

1315 ) 

1316 visitSummary = connectionTypes.Output( 

1317 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

1318 "detector id for the id and are sorted for fast lookups of a " 

1319 "detector."), 

1320 name="visitSummary", 

1321 storageClass="ExposureCatalog", 

1322 dimensions=("instrument", "visit"), 

1323 ) 

1324 visitSummarySchema = connectionTypes.InitOutput( 

1325 doc="Schema of the visitSummary catalog", 

1326 name="visitSummary_schema", 

1327 storageClass="ExposureCatalog", 

1328 ) 

1329 

1330 

1331class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig, 

1332 pipelineConnections=ConsolidateVisitSummaryConnections): 

1333 """Config for ConsolidateVisitSummaryTask""" 

1334 pass 

1335 

1336 

1337class ConsolidateVisitSummaryTask(pipeBase.PipelineTask): 

1338 """Task to consolidate per-detector visit metadata. 

1339 

1340 This task aggregates the following metadata from all the detectors in a 

1341 single visit into an exposure catalog: 

1342 - The visitInfo. 

1343 - The wcs. 

1344 - The photoCalib. 

1345 - The physical_filter and band (if available). 

1346 - The psf size, shape, and effective area at the center of the detector. 

1347 - The corners of the bounding box in right ascension/declination. 

1348 

1349 Other quantities such as Detector, Psf, ApCorrMap, and TransmissionCurve 

1350 are not persisted here because of storage concerns, and because of their 

1351 limited utility as summary statistics. 

1352 

1353 Tests for this task are performed in ci_hsc_gen3. 

1354 """ 

1355 _DefaultName = "consolidateVisitSummary" 

1356 ConfigClass = ConsolidateVisitSummaryConfig 

1357 

1358 def __init__(self, **kwargs): 

1359 super().__init__(**kwargs) 

1360 self.schema = afwTable.ExposureTable.makeMinimalSchema() 

1361 self.schema.addField('visit', type='L', doc='Visit number') 

1362 self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter') 

1363 self.schema.addField('band', type='String', size=32, doc='Name of band') 

1364 ExposureSummaryStats.update_schema(self.schema) 

1365 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema) 

1366 

1367 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1368 dataRefs = butlerQC.get(inputRefs.calexp) 

1369 visit = dataRefs[0].dataId.byName()['visit'] 

1370 

1371 self.log.debug("Concatenating metadata from %d per-detector calexps (visit %d)", 

1372 len(dataRefs), visit) 

1373 

1374 expCatalog = self._combineExposureMetadata(visit, dataRefs) 

1375 

1376 butlerQC.put(expCatalog, outputRefs.visitSummary) 

1377 

1378 def _combineExposureMetadata(self, visit, dataRefs): 

1379 """Make a combined exposure catalog from a list of dataRefs. 

1380 These dataRefs must point to exposures with wcs, summaryStats, 

1381 and other visit metadata. 

1382 

1383 Parameters 

1384 ---------- 

1385 visit : `int` 

1386 Visit identification number. 

1387 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1388 List of dataRefs in visit. 

1389 

1390 Returns 

1391 ------- 

1392 visitSummary : `lsst.afw.table.ExposureCatalog` 

1393 Exposure catalog with per-detector summary information. 

1394 """ 

1395 cat = afwTable.ExposureCatalog(self.schema) 

1396 cat.resize(len(dataRefs)) 

1397 

1398 cat['visit'] = visit 

1399 

1400 for i, dataRef in enumerate(dataRefs): 

1401 visitInfo = dataRef.get(component='visitInfo') 

1402 filterLabel = dataRef.get(component='filter') 

1403 summaryStats = dataRef.get(component='summaryStats') 

1404 detector = dataRef.get(component='detector') 

1405 wcs = dataRef.get(component='wcs') 

1406 photoCalib = dataRef.get(component='photoCalib') 

1407 detector = dataRef.get(component='detector') 

1408 bbox = dataRef.get(component='bbox') 

1409 validPolygon = dataRef.get(component='validPolygon') 

1410 

1411 rec = cat[i] 

1412 rec.setBBox(bbox) 

1413 rec.setVisitInfo(visitInfo) 

1414 rec.setWcs(wcs) 

1415 rec.setPhotoCalib(photoCalib) 

1416 rec.setValidPolygon(validPolygon) 

1417 

1418 rec['physical_filter'] = filterLabel.physicalLabel if filterLabel.hasPhysicalLabel() else "" 

1419 rec['band'] = filterLabel.bandLabel if filterLabel.hasBandLabel() else "" 

1420 rec.setId(detector.getId()) 

1421 summaryStats.update_record(rec) 

1422 

1423 metadata = dafBase.PropertyList() 

1424 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

1425 # We are looping over existing datarefs, so the following is true 

1426 metadata.add("COMMENT", "Only detectors with data have entries.") 

1427 cat.setMetadata(metadata) 

1428 

1429 cat.sort() 

1430 return cat 

1431 

1432 

1433class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections, 

1434 defaultTemplates={"catalogType": ""}, 

1435 dimensions=("instrument", "visit")): 

1436 inputCatalogs = connectionTypes.Input( 

1437 doc="Input per-detector Source Tables", 

1438 name="{catalogType}sourceTable", 

1439 storageClass="DataFrame", 

1440 dimensions=("instrument", "visit", "detector"), 

1441 multiple=True 

1442 ) 

1443 outputCatalog = connectionTypes.Output( 

1444 doc="Per-visit concatenation of Source Table", 

1445 name="{catalogType}sourceTable_visit", 

1446 storageClass="DataFrame", 

1447 dimensions=("instrument", "visit") 

1448 ) 

1449 

1450 

1451class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig, 

1452 pipelineConnections=ConsolidateSourceTableConnections): 

1453 pass 

1454 

1455 

1456class ConsolidateSourceTableTask(pipeBase.PipelineTask): 

1457 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit` 

1458 """ 

1459 _DefaultName = 'consolidateSourceTable' 

1460 ConfigClass = ConsolidateSourceTableConfig 

1461 

1462 inputDataset = 'sourceTable' 

1463 outputDataset = 'sourceTable_visit' 

1464 

1465 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1466 from .makeWarp import reorderRefs 

1467 

1468 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.inputCatalogs] 

1469 detectorOrder.sort() 

1470 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

1471 inputs = butlerQC.get(inputRefs) 

1472 self.log.info("Concatenating %s per-detector Source Tables", 

1473 len(inputs['inputCatalogs'])) 

1474 df = pd.concat(inputs['inputCatalogs']) 

1475 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs) 

1476 

1477 

1478class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections, 

1479 dimensions=("instrument",), 

1480 defaultTemplates={"calexpType": ""}): 

1481 visitSummaryRefs = connectionTypes.Input( 

1482 doc="Data references for per-visit consolidated exposure metadata", 

1483 name="finalVisitSummary", 

1484 storageClass="ExposureCatalog", 

1485 dimensions=("instrument", "visit"), 

1486 multiple=True, 

1487 deferLoad=True, 

1488 ) 

1489 outputCatalog = connectionTypes.Output( 

1490 doc="CCD and Visit metadata table", 

1491 name="ccdVisitTable", 

1492 storageClass="DataFrame", 

1493 dimensions=("instrument",) 

1494 ) 

1495 

1496 

1497class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig, 

1498 pipelineConnections=MakeCcdVisitTableConnections): 

1499 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

1500 

1501 

1502class MakeCcdVisitTableTask(pipeBase.PipelineTask): 

1503 """Produce a `ccdVisitTable` from the visit summary exposure catalogs. 

1504 """ 

1505 _DefaultName = 'makeCcdVisitTable' 

1506 ConfigClass = MakeCcdVisitTableConfig 

1507 

1508 def run(self, visitSummaryRefs): 

1509 """Make a table of ccd information from the visit summary catalogs. 

1510 

1511 Parameters 

1512 ---------- 

1513 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1514 List of DeferredDatasetHandles pointing to exposure catalogs with 

1515 per-detector summary information. 

1516 

1517 Returns 

1518 ------- 

1519 result : `~lsst.pipe.base.Struct` 

1520 Results struct with attribute: 

1521 

1522 ``outputCatalog`` 

1523 Catalog of ccd and visit information. 

1524 """ 

1525 ccdEntries = [] 

1526 for visitSummaryRef in visitSummaryRefs: 

1527 visitSummary = visitSummaryRef.get() 

1528 visitInfo = visitSummary[0].getVisitInfo() 

1529 

1530 ccdEntry = {} 

1531 summaryTable = visitSummary.asAstropy() 

1532 selectColumns = ['id', 'visit', 'physical_filter', 'band', 'ra', 'dec', 'zenithDistance', 

1533 'zeroPoint', 'psfSigma', 'skyBg', 'skyNoise', 

1534 'astromOffsetMean', 'astromOffsetStd', 'nPsfStar', 

1535 'psfStarDeltaE1Median', 'psfStarDeltaE2Median', 

1536 'psfStarDeltaE1Scatter', 'psfStarDeltaE2Scatter', 

1537 'psfStarDeltaSizeMedian', 'psfStarDeltaSizeScatter', 

1538 'psfStarScaledDeltaSizeScatter', 

1539 'psfTraceRadiusDelta', 'maxDistToNearestPsf'] 

1540 ccdEntry = summaryTable[selectColumns].to_pandas().set_index('id') 

1541 # 'visit' is the human readable visit number. 

1542 # 'visitId' is the key to the visitId table. They are the same. 

1543 # Technically you should join to get the visit from the visit 

1544 # table. 

1545 ccdEntry = ccdEntry.rename(columns={"visit": "visitId"}) 

1546 

1547 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards 

1548 # compatibility. To be removed after September 2023. 

1549 ccdEntry["decl"] = ccdEntry.loc[:, "dec"] 

1550 

1551 ccdEntry['ccdVisitId'] = [ 

1552 self.config.idGenerator.apply( 

1553 visitSummaryRef.dataId, 

1554 detector=detector_id, 

1555 is_exposure=False, 

1556 ).catalog_id # The "catalog ID" here is the ccdVisit ID 

1557 # because it's usually the ID for a whole catalog 

1558 # with a {visit, detector}, and that's the main 

1559 # use case for IdGenerator. This usage for a 

1560 # summary table is rare. 

1561 for detector_id in summaryTable['id'] 

1562 ] 

1563 ccdEntry['detector'] = summaryTable['id'] 

1564 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds() if vR.getWcs() 

1565 else np.nan for vR in visitSummary]) 

1566 ccdEntry["seeing"] = visitSummary['psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds 

1567 

1568 ccdEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees() 

1569 ccdEntry["expMidpt"] = visitInfo.getDate().toPython() 

1570 ccdEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD) 

1571 expTime = visitInfo.getExposureTime() 

1572 ccdEntry['expTime'] = expTime 

1573 ccdEntry["obsStart"] = ccdEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime) 

1574 expTime_days = expTime / (60*60*24) 

1575 ccdEntry["obsStartMJD"] = ccdEntry["expMidptMJD"] - 0.5 * expTime_days 

1576 ccdEntry['darkTime'] = visitInfo.getDarkTime() 

1577 ccdEntry['xSize'] = summaryTable['bbox_max_x'] - summaryTable['bbox_min_x'] 

1578 ccdEntry['ySize'] = summaryTable['bbox_max_y'] - summaryTable['bbox_min_y'] 

1579 ccdEntry['llcra'] = summaryTable['raCorners'][:, 0] 

1580 ccdEntry['llcdec'] = summaryTable['decCorners'][:, 0] 

1581 ccdEntry['ulcra'] = summaryTable['raCorners'][:, 1] 

1582 ccdEntry['ulcdec'] = summaryTable['decCorners'][:, 1] 

1583 ccdEntry['urcra'] = summaryTable['raCorners'][:, 2] 

1584 ccdEntry['urcdec'] = summaryTable['decCorners'][:, 2] 

1585 ccdEntry['lrcra'] = summaryTable['raCorners'][:, 3] 

1586 ccdEntry['lrcdec'] = summaryTable['decCorners'][:, 3] 

1587 # TODO: DM-30618, Add raftName, nExposures, ccdTemp, binX, binY, 

1588 # and flags, and decide if WCS, and llcx, llcy, ulcx, ulcy, etc. 

1589 # values are actually wanted. 

1590 ccdEntries.append(ccdEntry) 

1591 

1592 outputCatalog = pd.concat(ccdEntries) 

1593 outputCatalog.set_index('ccdVisitId', inplace=True, verify_integrity=True) 

1594 return pipeBase.Struct(outputCatalog=outputCatalog) 

1595 

1596 

1597class MakeVisitTableConnections(pipeBase.PipelineTaskConnections, 

1598 dimensions=("instrument",), 

1599 defaultTemplates={"calexpType": ""}): 

1600 visitSummaries = connectionTypes.Input( 

1601 doc="Per-visit consolidated exposure metadata", 

1602 name="finalVisitSummary", 

1603 storageClass="ExposureCatalog", 

1604 dimensions=("instrument", "visit",), 

1605 multiple=True, 

1606 deferLoad=True, 

1607 ) 

1608 outputCatalog = connectionTypes.Output( 

1609 doc="Visit metadata table", 

1610 name="visitTable", 

1611 storageClass="DataFrame", 

1612 dimensions=("instrument",) 

1613 ) 

1614 

1615 

1616class MakeVisitTableConfig(pipeBase.PipelineTaskConfig, 

1617 pipelineConnections=MakeVisitTableConnections): 

1618 pass 

1619 

1620 

1621class MakeVisitTableTask(pipeBase.PipelineTask): 

1622 """Produce a `visitTable` from the visit summary exposure catalogs. 

1623 """ 

1624 _DefaultName = 'makeVisitTable' 

1625 ConfigClass = MakeVisitTableConfig 

1626 

1627 def run(self, visitSummaries): 

1628 """Make a table of visit information from the visit summary catalogs. 

1629 

1630 Parameters 

1631 ---------- 

1632 visitSummaries : `list` of `lsst.afw.table.ExposureCatalog` 

1633 List of exposure catalogs with per-detector summary information. 

1634 Returns 

1635 ------- 

1636 result : `~lsst.pipe.base.Struct` 

1637 Results struct with attribute: 

1638 

1639 ``outputCatalog`` 

1640 Catalog of visit information. 

1641 """ 

1642 visitEntries = [] 

1643 for visitSummary in visitSummaries: 

1644 visitSummary = visitSummary.get() 

1645 visitRow = visitSummary[0] 

1646 visitInfo = visitRow.getVisitInfo() 

1647 

1648 visitEntry = {} 

1649 visitEntry["visitId"] = visitRow['visit'] 

1650 visitEntry["visit"] = visitRow['visit'] 

1651 visitEntry["physical_filter"] = visitRow['physical_filter'] 

1652 visitEntry["band"] = visitRow['band'] 

1653 raDec = visitInfo.getBoresightRaDec() 

1654 visitEntry["ra"] = raDec.getRa().asDegrees() 

1655 visitEntry["dec"] = raDec.getDec().asDegrees() 

1656 

1657 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards 

1658 # compatibility. To be removed after September 2023. 

1659 visitEntry["decl"] = visitEntry["dec"] 

1660 

1661 visitEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees() 

1662 azAlt = visitInfo.getBoresightAzAlt() 

1663 visitEntry["azimuth"] = azAlt.getLongitude().asDegrees() 

1664 visitEntry["altitude"] = azAlt.getLatitude().asDegrees() 

1665 visitEntry["zenithDistance"] = 90 - azAlt.getLatitude().asDegrees() 

1666 visitEntry["airmass"] = visitInfo.getBoresightAirmass() 

1667 expTime = visitInfo.getExposureTime() 

1668 visitEntry["expTime"] = expTime 

1669 visitEntry["expMidpt"] = visitInfo.getDate().toPython() 

1670 visitEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD) 

1671 visitEntry["obsStart"] = visitEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime) 

1672 expTime_days = expTime / (60*60*24) 

1673 visitEntry["obsStartMJD"] = visitEntry["expMidptMJD"] - 0.5 * expTime_days 

1674 visitEntries.append(visitEntry) 

1675 

1676 # TODO: DM-30623, Add programId, exposureType, cameraTemp, 

1677 # mirror1Temp, mirror2Temp, mirror3Temp, domeTemp, externalTemp, 

1678 # dimmSeeing, pwvGPS, pwvMW, flags, nExposures. 

1679 

1680 outputCatalog = pd.DataFrame(data=visitEntries) 

1681 outputCatalog.set_index('visitId', inplace=True, verify_integrity=True) 

1682 return pipeBase.Struct(outputCatalog=outputCatalog) 

1683 

1684 

1685class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections, 

1686 dimensions=("instrument", "visit", "detector", "skymap", "tract")): 

1687 

1688 inputCatalog = connectionTypes.Input( 

1689 doc="Primary per-detector, single-epoch forced-photometry catalog. " 

1690 "By default, it is the output of ForcedPhotCcdTask on calexps", 

1691 name="forced_src", 

1692 storageClass="SourceCatalog", 

1693 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1694 ) 

1695 inputCatalogDiff = connectionTypes.Input( 

1696 doc="Secondary multi-epoch, per-detector, forced photometry catalog. " 

1697 "By default, it is the output of ForcedPhotCcdTask run on image differences.", 

1698 name="forced_diff", 

1699 storageClass="SourceCatalog", 

1700 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1701 ) 

1702 outputCatalog = connectionTypes.Output( 

1703 doc="InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format", 

1704 name="mergedForcedSource", 

1705 storageClass="DataFrame", 

1706 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1707 ) 

1708 

1709 

1710class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig, 

1711 pipelineConnections=WriteForcedSourceTableConnections): 

1712 key = lsst.pex.config.Field( 

1713 doc="Column on which to join the two input tables on and make the primary key of the output", 

1714 dtype=str, 

1715 default="objectId", 

1716 ) 

1717 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

1718 

1719 

1720class WriteForcedSourceTableTask(pipeBase.PipelineTask): 

1721 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format. 

1722 

1723 Because the predecessor ForcedPhotCcdTask operates per-detector, 

1724 per-tract, (i.e., it has tract in its dimensions), detectors 

1725 on the tract boundary may have multiple forced source catalogs. 

1726 

1727 The successor task TransformForcedSourceTable runs per-patch 

1728 and temporally-aggregates overlapping mergedForcedSource catalogs from all 

1729 available multiple epochs. 

1730 """ 

1731 _DefaultName = "writeForcedSourceTable" 

1732 ConfigClass = WriteForcedSourceTableConfig 

1733 

1734 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1735 inputs = butlerQC.get(inputRefs) 

1736 # Add ccdVisitId to allow joining with CcdVisitTable 

1737 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

1738 inputs['ccdVisitId'] = idGenerator.catalog_id 

1739 inputs['band'] = butlerQC.quantum.dataId.full['band'] 

1740 outputs = self.run(**inputs) 

1741 butlerQC.put(outputs, outputRefs) 

1742 

1743 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None): 

1744 dfs = [] 

1745 for table, dataset, in zip((inputCatalog, inputCatalogDiff), ('calexp', 'diff')): 

1746 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=False) 

1747 df = df.reindex(sorted(df.columns), axis=1) 

1748 df['ccdVisitId'] = ccdVisitId if ccdVisitId else pd.NA 

1749 df['band'] = band if band else pd.NA 

1750 df.columns = pd.MultiIndex.from_tuples([(dataset, c) for c in df.columns], 

1751 names=('dataset', 'column')) 

1752 

1753 dfs.append(df) 

1754 

1755 outputCatalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs) 

1756 return pipeBase.Struct(outputCatalog=outputCatalog) 

1757 

1758 

1759class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections, 

1760 dimensions=("instrument", "skymap", "patch", "tract")): 

1761 

1762 inputCatalogs = connectionTypes.Input( 

1763 doc="DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask", 

1764 name="mergedForcedSource", 

1765 storageClass="DataFrame", 

1766 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

1767 multiple=True, 

1768 deferLoad=True 

1769 ) 

1770 referenceCatalog = connectionTypes.Input( 

1771 doc="Reference catalog which was used to seed the forcedPhot. Columns " 

1772 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner " 

1773 "are expected.", 

1774 name="objectTable", 

1775 storageClass="DataFrame", 

1776 dimensions=("tract", "patch", "skymap"), 

1777 deferLoad=True 

1778 ) 

1779 outputCatalog = connectionTypes.Output( 

1780 doc="Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a " 

1781 "specified set of functors", 

1782 name="forcedSourceTable", 

1783 storageClass="DataFrame", 

1784 dimensions=("tract", "patch", "skymap") 

1785 ) 

1786 

1787 

1788class TransformForcedSourceTableConfig(TransformCatalogBaseConfig, 

1789 pipelineConnections=TransformForcedSourceTableConnections): 

1790 referenceColumns = pexConfig.ListField( 

1791 dtype=str, 

1792 default=["detect_isPrimary", "detect_isTractInner", "detect_isPatchInner"], 

1793 optional=True, 

1794 doc="Columns to pull from reference catalog", 

1795 ) 

1796 keyRef = lsst.pex.config.Field( 

1797 doc="Column on which to join the two input tables on and make the primary key of the output", 

1798 dtype=str, 

1799 default="objectId", 

1800 ) 

1801 key = lsst.pex.config.Field( 

1802 doc="Rename the output DataFrame index to this name", 

1803 dtype=str, 

1804 default="forcedSourceId", 

1805 ) 

1806 

1807 def setDefaults(self): 

1808 super().setDefaults() 

1809 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'ForcedSource.yaml') 

1810 self.columnsFromDataId = ['tract', 'patch'] 

1811 

1812 

1813class TransformForcedSourceTableTask(TransformCatalogBaseTask): 

1814 """Transform/standardize a ForcedSource catalog 

1815 

1816 Transforms each wide, per-detector forcedSource DataFrame per the 

1817 specification file (per-camera defaults found in ForcedSource.yaml). 

1818 All epochs that overlap the patch are aggregated into one per-patch 

1819 narrow-DataFrame file. 

1820 

1821 No de-duplication of rows is performed. Duplicate resolutions flags are 

1822 pulled in from the referenceCatalog: `detect_isPrimary`, 

1823 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate 

1824 for analysis or compare duplicates for QA. 

1825 

1826 The resulting table includes multiple bands. Epochs (MJDs) and other useful 

1827 per-visit rows can be retreived by joining with the CcdVisitTable on 

1828 ccdVisitId. 

1829 """ 

1830 _DefaultName = "transformForcedSourceTable" 

1831 ConfigClass = TransformForcedSourceTableConfig 

1832 

1833 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1834 inputs = butlerQC.get(inputRefs) 

1835 if self.funcs is None: 

1836 raise ValueError("config.functorFile is None. " 

1837 "Must be a valid path to yaml in order to run Task as a PipelineTask.") 

1838 outputs = self.run(inputs['inputCatalogs'], inputs['referenceCatalog'], funcs=self.funcs, 

1839 dataId=outputRefs.outputCatalog.dataId.full) 

1840 

1841 butlerQC.put(outputs, outputRefs) 

1842 

1843 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None): 

1844 dfs = [] 

1845 ref = referenceCatalog.get(parameters={"columns": self.config.referenceColumns}) 

1846 self.log.info("Aggregating %s input catalogs" % (len(inputCatalogs))) 

1847 for handle in inputCatalogs: 

1848 result = self.transform(None, handle, funcs, dataId) 

1849 # Filter for only rows that were detected on (overlap) the patch 

1850 dfs.append(result.df.join(ref, how='inner')) 

1851 

1852 outputCatalog = pd.concat(dfs) 

1853 

1854 # Now that we are done joining on config.keyRef 

1855 # Change index to config.key by 

1856 outputCatalog.index.rename(self.config.keyRef, inplace=True) 

1857 # Add config.keyRef to the column list 

1858 outputCatalog.reset_index(inplace=True) 

1859 # Set the forcedSourceId to the index. This is specified in the 

1860 # ForcedSource.yaml 

1861 outputCatalog.set_index("forcedSourceId", inplace=True, verify_integrity=True) 

1862 # Rename it to the config.key 

1863 outputCatalog.index.rename(self.config.key, inplace=True) 

1864 

1865 self.log.info("Made a table of %d columns and %d rows", 

1866 len(outputCatalog.columns), len(outputCatalog)) 

1867 return pipeBase.Struct(outputCatalog=outputCatalog) 

1868 

1869 

1870class ConsolidateTractConnections(pipeBase.PipelineTaskConnections, 

1871 defaultTemplates={"catalogType": ""}, 

1872 dimensions=("instrument", "tract")): 

1873 inputCatalogs = connectionTypes.Input( 

1874 doc="Input per-patch DataFrame Tables to be concatenated", 

1875 name="{catalogType}ForcedSourceTable", 

1876 storageClass="DataFrame", 

1877 dimensions=("tract", "patch", "skymap"), 

1878 multiple=True, 

1879 ) 

1880 

1881 outputCatalog = connectionTypes.Output( 

1882 doc="Output per-tract concatenation of DataFrame Tables", 

1883 name="{catalogType}ForcedSourceTable_tract", 

1884 storageClass="DataFrame", 

1885 dimensions=("tract", "skymap"), 

1886 ) 

1887 

1888 

1889class ConsolidateTractConfig(pipeBase.PipelineTaskConfig, 

1890 pipelineConnections=ConsolidateTractConnections): 

1891 pass 

1892 

1893 

1894class ConsolidateTractTask(pipeBase.PipelineTask): 

1895 """Concatenate any per-patch, dataframe list into a single 

1896 per-tract DataFrame. 

1897 """ 

1898 _DefaultName = 'ConsolidateTract' 

1899 ConfigClass = ConsolidateTractConfig 

1900 

1901 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1902 inputs = butlerQC.get(inputRefs) 

1903 # Not checking at least one inputCatalog exists because that'd be an 

1904 # empty QG. 

1905 self.log.info("Concatenating %s per-patch %s Tables", 

1906 len(inputs['inputCatalogs']), 

1907 inputRefs.inputCatalogs[0].datasetType.name) 

1908 df = pd.concat(inputs['inputCatalogs']) 

1909 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)