Coverage for python/lsst/pipe/tasks/postprocess.py: 27%

692 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-04 13:31 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["WriteObjectTableConfig", "WriteObjectTableTask", 

23 "WriteSourceTableConfig", "WriteSourceTableTask", 

24 "WriteRecalibratedSourceTableConfig", "WriteRecalibratedSourceTableTask", 

25 "PostprocessAnalysis", 

26 "TransformCatalogBaseConfig", "TransformCatalogBaseTask", 

27 "TransformObjectCatalogConfig", "TransformObjectCatalogTask", 

28 "ConsolidateObjectTableConfig", "ConsolidateObjectTableTask", 

29 "TransformSourceTableConfig", "TransformSourceTableTask", 

30 "ConsolidateVisitSummaryConfig", "ConsolidateVisitSummaryTask", 

31 "ConsolidateSourceTableConfig", "ConsolidateSourceTableTask", 

32 "MakeCcdVisitTableConfig", "MakeCcdVisitTableTask", 

33 "MakeVisitTableConfig", "MakeVisitTableTask", 

34 "WriteForcedSourceTableConfig", "WriteForcedSourceTableTask", 

35 "TransformForcedSourceTableConfig", "TransformForcedSourceTableTask", 

36 "ConsolidateTractConfig", "ConsolidateTractTask"] 

37 

38import functools 

39import pandas as pd 

40import logging 

41import numpy as np 

42import numbers 

43import os 

44import warnings 

45 

46from deprecated.sphinx import deprecated 

47 

48import lsst.geom 

49import lsst.pex.config as pexConfig 

50import lsst.pipe.base as pipeBase 

51import lsst.daf.base as dafBase 

52from lsst.utils.introspection import find_outside_stacklevel 

53from lsst.pipe.base import connectionTypes 

54import lsst.afw.table as afwTable 

55from lsst.afw.image import ExposureSummaryStats, ExposureF 

56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig 

57from lsst.skymap import BaseSkyMap 

58 

59from .functors import CompositeFunctor, Column 

60 

61log = logging.getLogger(__name__) 

62 

63 

64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None): 

65 """Flattens a dataframe with multilevel column index. 

66 """ 

67 newDf = pd.DataFrame() 

68 # band is the level 0 index 

69 dfBands = df.columns.unique(level=0).values 

70 for band in dfBands: 

71 subdf = df[band] 

72 columnFormat = '{0}{1}' if camelCase else '{0}_{1}' 

73 newColumns = {c: columnFormat.format(band, c) 

74 for c in subdf.columns if c not in noDupCols} 

75 cols = list(newColumns.keys()) 

76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1) 

77 

78 # Band must be present in the input and output or else column is all NaN: 

79 presentBands = dfBands if inputBands is None else list(set(inputBands).intersection(dfBands)) 

80 # Get the unexploded columns from any present band's partition 

81 noDupDf = df[presentBands[0]][noDupCols] 

82 newDf = pd.concat([noDupDf, newDf], axis=1) 

83 return newDf 

84 

85 

86class WriteObjectTableConnections(pipeBase.PipelineTaskConnections, 

87 defaultTemplates={"coaddName": "deep"}, 

88 dimensions=("tract", "patch", "skymap")): 

89 inputCatalogMeas = connectionTypes.Input( 

90 doc="Catalog of source measurements on the deepCoadd.", 

91 dimensions=("tract", "patch", "band", "skymap"), 

92 storageClass="SourceCatalog", 

93 name="{coaddName}Coadd_meas", 

94 multiple=True 

95 ) 

96 inputCatalogForcedSrc = connectionTypes.Input( 

97 doc="Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.", 

98 dimensions=("tract", "patch", "band", "skymap"), 

99 storageClass="SourceCatalog", 

100 name="{coaddName}Coadd_forced_src", 

101 multiple=True 

102 ) 

103 inputCatalogRef = connectionTypes.Input( 

104 doc="Catalog marking the primary detection (which band provides a good shape and position)" 

105 "for each detection in deepCoadd_mergeDet.", 

106 dimensions=("tract", "patch", "skymap"), 

107 storageClass="SourceCatalog", 

108 name="{coaddName}Coadd_ref" 

109 ) 

110 outputCatalog = connectionTypes.Output( 

111 doc="A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, " 

112 "stored as a DataFrame with a multi-level column index per-patch.", 

113 dimensions=("tract", "patch", "skymap"), 

114 storageClass="DataFrame", 

115 name="{coaddName}Coadd_obj" 

116 ) 

117 

118 

119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig, 

120 pipelineConnections=WriteObjectTableConnections): 

121 engine = pexConfig.Field( 

122 dtype=str, 

123 default="pyarrow", 

124 doc="Parquet engine for writing (pyarrow or fastparquet)", 

125 deprecated="This config is no longer used, and will be removed after v26." 

126 ) 

127 coaddName = pexConfig.Field( 

128 dtype=str, 

129 default="deep", 

130 doc="Name of coadd" 

131 ) 

132 

133 

134class WriteObjectTableTask(pipeBase.PipelineTask): 

135 """Write filter-merged source tables as a DataFrame in parquet format. 

136 """ 

137 _DefaultName = "writeObjectTable" 

138 ConfigClass = WriteObjectTableConfig 

139 

140 # Names of table datasets to be merged 

141 inputDatasets = ('forced_src', 'meas', 'ref') 

142 

143 # Tag of output dataset written by `MergeSourcesTask.write` 

144 outputDataset = 'obj' 

145 

146 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

147 inputs = butlerQC.get(inputRefs) 

148 

149 measDict = {ref.dataId['band']: {'meas': cat} for ref, cat in 

150 zip(inputRefs.inputCatalogMeas, inputs['inputCatalogMeas'])} 

151 forcedSourceDict = {ref.dataId['band']: {'forced_src': cat} for ref, cat in 

152 zip(inputRefs.inputCatalogForcedSrc, inputs['inputCatalogForcedSrc'])} 

153 

154 catalogs = {} 

155 for band in measDict.keys(): 

156 catalogs[band] = {'meas': measDict[band]['meas'], 

157 'forced_src': forcedSourceDict[band]['forced_src'], 

158 'ref': inputs['inputCatalogRef']} 

159 dataId = butlerQC.quantum.dataId 

160 df = self.run(catalogs=catalogs, tract=dataId['tract'], patch=dataId['patch']) 

161 outputs = pipeBase.Struct(outputCatalog=df) 

162 butlerQC.put(outputs, outputRefs) 

163 

164 def run(self, catalogs, tract, patch): 

165 """Merge multiple catalogs. 

166 

167 Parameters 

168 ---------- 

169 catalogs : `dict` 

170 Mapping from filter names to dict of catalogs. 

171 tract : int 

172 tractId to use for the tractId column. 

173 patch : str 

174 patchId to use for the patchId column. 

175 

176 Returns 

177 ------- 

178 catalog : `pandas.DataFrame` 

179 Merged dataframe. 

180 """ 

181 dfs = [] 

182 for filt, tableDict in catalogs.items(): 

183 for dataset, table in tableDict.items(): 

184 # Convert afwTable to pandas DataFrame 

185 df = table.asAstropy().to_pandas().set_index('id', drop=True) 

186 

187 # Sort columns by name, to ensure matching schema among patches 

188 df = df.reindex(sorted(df.columns), axis=1) 

189 df = df.assign(tractId=tract, patchId=patch) 

190 

191 # Make columns a 3-level MultiIndex 

192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c) for c in df.columns], 

193 names=('dataset', 'band', 'column')) 

194 dfs.append(df) 

195 

196 # We do this dance and not `pd.concat(dfs)` because the pandas 

197 # concatenation uses infinite memory. 

198 catalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs) 

199 return catalog 

200 

201 

202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections, 

203 defaultTemplates={"catalogType": ""}, 

204 dimensions=("instrument", "visit", "detector")): 

205 

206 catalog = connectionTypes.Input( 

207 doc="Input full-depth catalog of sources produced by CalibrateTask", 

208 name="{catalogType}src", 

209 storageClass="SourceCatalog", 

210 dimensions=("instrument", "visit", "detector") 

211 ) 

212 outputCatalog = connectionTypes.Output( 

213 doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is " 

214 "replaced with an index; all other columns are unchanged.", 

215 name="{catalogType}source", 

216 storageClass="DataFrame", 

217 dimensions=("instrument", "visit", "detector") 

218 ) 

219 

220 

221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig, 

222 pipelineConnections=WriteSourceTableConnections): 

223 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

224 

225 

226class WriteSourceTableTask(pipeBase.PipelineTask): 

227 """Write source table to DataFrame Parquet format. 

228 """ 

229 _DefaultName = "writeSourceTable" 

230 ConfigClass = WriteSourceTableConfig 

231 

232 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

233 inputs = butlerQC.get(inputRefs) 

234 inputs['ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id 

235 result = self.run(**inputs) 

236 outputs = pipeBase.Struct(outputCatalog=result.table) 

237 butlerQC.put(outputs, outputRefs) 

238 

239 def run(self, catalog, ccdVisitId=None, **kwargs): 

240 """Convert `src` catalog to DataFrame 

241 

242 Parameters 

243 ---------- 

244 catalog: `afwTable.SourceCatalog` 

245 catalog to be converted 

246 ccdVisitId: `int` 

247 ccdVisitId to be added as a column 

248 **kwargs 

249 Additional keyword arguments are ignored as a convenience for 

250 subclasses that pass the same arguments to several different 

251 methods. 

252 

253 Returns 

254 ------- 

255 result : `~lsst.pipe.base.Struct` 

256 ``table`` 

257 `DataFrame` version of the input catalog 

258 """ 

259 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId) 

260 df = catalog.asAstropy().to_pandas().set_index('id', drop=True) 

261 df['ccdVisitId'] = ccdVisitId 

262 

263 return pipeBase.Struct(table=df) 

264 

265 

266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections, 

267 defaultTemplates={"catalogType": "", 

268 "skyWcsName": "gbdesAstrometricFit", 

269 "photoCalibName": "fgcm"}, 

270 # TODO: remove on DM-39854. 

271 deprecatedTemplates={ 

272 "skyWcsName": "Deprecated; will be removed after v26.", 

273 "photoCalibName": "Deprecated; will be removed after v26." 

274 }, 

275 dimensions=("instrument", "visit", "detector", "skymap")): 

276 skyMap = connectionTypes.Input( 

277 doc="skyMap needed to choose which tract-level calibrations to use when multiple available", 

278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

279 storageClass="SkyMap", 

280 dimensions=("skymap",), 

281 # TODO: remove on DM-39854. 

282 deprecated=( 

283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. " 

284 "Will be removed after v26." 

285 ), 

286 ) 

287 exposure = connectionTypes.Input( 

288 doc="Input exposure to perform photometry on.", 

289 name="calexp", 

290 storageClass="ExposureF", 

291 dimensions=["instrument", "visit", "detector"], 

292 # TODO: remove on DM-39584 

293 deprecated=( 

294 "Deprecated, as the `calexp` is not needed and just creates unnecessary i/o. " 

295 "Will be removed after v26." 

296 ), 

297 ) 

298 visitSummary = connectionTypes.Input( 

299 doc="Input visit-summary catalog with updated calibration objects.", 

300 name="finalVisitSummary", 

301 storageClass="ExposureCatalog", 

302 dimensions=("instrument", "visit",), 

303 ) 

304 externalSkyWcsTractCatalog = connectionTypes.Input( 

305 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector " 

306 "id for the catalog id, sorted on id for fast lookup."), 

307 name="{skyWcsName}SkyWcsCatalog", 

308 storageClass="ExposureCatalog", 

309 dimensions=["instrument", "visit", "tract"], 

310 multiple=True, 

311 # TODO: remove on DM-39854. 

312 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

313 ) 

314 externalSkyWcsGlobalCatalog = connectionTypes.Input( 

315 doc=("Per-visit wcs calibrations computed globally (with no tract information). " 

316 "These catalogs use the detector id for the catalog id, sorted on id for " 

317 "fast lookup."), 

318 name="finalVisitSummary", 

319 storageClass="ExposureCatalog", 

320 dimensions=["instrument", "visit"], 

321 # TODO: remove on DM-39854. 

322 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

323 ) 

324 externalPhotoCalibTractCatalog = connectionTypes.Input( 

325 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the " 

326 "detector id for the catalog id, sorted on id for fast lookup."), 

327 name="{photoCalibName}PhotoCalibCatalog", 

328 storageClass="ExposureCatalog", 

329 dimensions=["instrument", "visit", "tract"], 

330 multiple=True, 

331 # TODO: remove on DM-39854. 

332 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

333 ) 

334 externalPhotoCalibGlobalCatalog = connectionTypes.Input( 

335 doc=("Per-visit photometric calibrations computed globally (with no tract " 

336 "information). These catalogs use the detector id for the catalog id, " 

337 "sorted on id for fast lookup."), 

338 name="finalVisitSummary", 

339 storageClass="ExposureCatalog", 

340 dimensions=["instrument", "visit"], 

341 # TODO: remove on DM-39854. 

342 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.", 

343 ) 

344 

345 def __init__(self, *, config=None): 

346 super().__init__(config=config) 

347 # Same connection boilerplate as all other applications of 

348 # Global/Tract calibrations 

349 # TODO: remove all of this on DM-39854. 

350 keepSkyMap = False 

351 keepExposure = False 

352 if config.doApplyExternalSkyWcs and config.doReevaluateSkyWcs: 

353 keepExposure = True 

354 if config.useGlobalExternalSkyWcs: 

355 self.inputs.remove("externalSkyWcsTractCatalog") 

356 else: 

357 self.inputs.remove("externalSkyWcsGlobalCatalog") 

358 keepSkyMap = True 

359 else: 

360 self.inputs.remove("externalSkyWcsTractCatalog") 

361 self.inputs.remove("externalSkyWcsGlobalCatalog") 

362 if config.doApplyExternalPhotoCalib and config.doReevaluatePhotoCalib: 

363 keepExposure = True 

364 if config.useGlobalExternalPhotoCalib: 

365 self.inputs.remove("externalPhotoCalibTractCatalog") 

366 else: 

367 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

368 keepSkyMap = True 

369 else: 

370 self.inputs.remove("externalPhotoCalibTractCatalog") 

371 self.inputs.remove("externalPhotoCalibGlobalCatalog") 

372 if not keepSkyMap: 

373 del self.skyMap 

374 if not keepExposure: 

375 del self.exposure 

376 

377 

378class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig, 

379 pipelineConnections=WriteRecalibratedSourceTableConnections): 

380 

381 doReevaluatePhotoCalib = pexConfig.Field( 

382 dtype=bool, 

383 default=True, 

384 doc=("Add or replace local photoCalib columns"), 

385 ) 

386 doReevaluateSkyWcs = pexConfig.Field( 

387 dtype=bool, 

388 default=True, 

389 doc=("Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"), 

390 ) 

391 doApplyExternalPhotoCalib = pexConfig.Field( 

392 dtype=bool, 

393 default=False, 

394 doc=("If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ", 

395 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."), 

396 # TODO: remove on DM-39854. 

397 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.", 

398 ) 

399 doApplyExternalSkyWcs = pexConfig.Field( 

400 dtype=bool, 

401 default=False, 

402 doc=("if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ", 

403 "else use the wcs already attached to the exposure."), 

404 # TODO: remove on DM-39854. 

405 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.", 

406 ) 

407 useGlobalExternalPhotoCalib = pexConfig.Field( 

408 dtype=bool, 

409 default=False, 

410 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations " 

411 "that are not run per-tract. When False, use per-tract photometric " 

412 "calibration files."), 

413 # TODO: remove on DM-39854. 

414 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.", 

415 ) 

416 useGlobalExternalSkyWcs = pexConfig.Field( 

417 dtype=bool, 

418 default=False, 

419 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations " 

420 "that are not run per-tract. When False, use per-tract wcs " 

421 "files."), 

422 # TODO: remove on DM-39854. 

423 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.", 

424 ) 

425 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

426 

427 def validate(self): 

428 super().validate() 

429 if self.doApplyExternalSkyWcs and not self.doReevaluateSkyWcs: 

430 log.warning("doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False" 

431 "External SkyWcs will not be read or evaluated.") 

432 if self.doApplyExternalPhotoCalib and not self.doReevaluatePhotoCalib: 

433 log.warning("doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False." 

434 "External PhotoCalib will not be read or evaluated.") 

435 

436 

437class WriteRecalibratedSourceTableTask(WriteSourceTableTask): 

438 """Write source table to DataFrame Parquet format. 

439 """ 

440 _DefaultName = "writeRecalibratedSourceTable" 

441 ConfigClass = WriteRecalibratedSourceTableConfig 

442 

443 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

444 inputs = butlerQC.get(inputRefs) 

445 

446 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

447 inputs['idGenerator'] = idGenerator 

448 inputs['ccdVisitId'] = idGenerator.catalog_id 

449 

450 if self.config.doReevaluatePhotoCalib or self.config.doReevaluateSkyWcs: 

451 if self.config.doApplyExternalPhotoCalib or self.config.doApplyExternalSkyWcs: 

452 inputs['exposure'] = self.attachCalibs(inputRefs, **inputs) 

453 else: 

454 # Create an empty exposure that will hold the calibrations. 

455 exposure = ExposureF() 

456 detectorId = butlerQC.quantum.dataId["detector"] 

457 inputs['exposure'] = self.prepareCalibratedExposure( 

458 exposure=exposure, 

459 detectorId=detectorId, 

460 visitSummary=inputs["visitSummary"], 

461 ) 

462 inputs['catalog'] = self.addCalibColumns(**inputs) 

463 

464 result = self.run(**inputs) 

465 outputs = pipeBase.Struct(outputCatalog=result.table) 

466 butlerQC.put(outputs, outputRefs) 

467 

468 # TODO: remove on DM-39854. 

469 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.", 

470 version="v26", category=FutureWarning) 

471 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None, 

472 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None, 

473 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs): 

474 """Apply external calibrations to exposure per configuration 

475 

476 When multiple tract-level calibrations overlap, select the one with the 

477 center closest to detector. 

478 

479 Parameters 

480 ---------- 

481 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`, for dataIds of 

482 tract-level calibs. 

483 skyMap : `~lsst.skymap.BaseSkyMap` 

484 skyMap to lookup tract geometry and WCS. 

485 exposure : `lsst.afw.image.exposure.Exposure` 

486 Input exposure to adjust calibrations. 

487 externalSkyWcsGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

488 Exposure catalog with external skyWcs to be applied per config 

489 externalSkyWcsTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

490 Exposure catalog with external skyWcs to be applied per config 

491 externalPhotoCalibGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

492 Exposure catalog with external photoCalib to be applied per config 

493 externalPhotoCalibTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional 

494 Exposure catalog with external photoCalib to be applied per config 

495 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

496 Exposure catalog with all calibration objects. WCS and PhotoCalib 

497 are always applied if provided. 

498 **kwargs 

499 Additional keyword arguments are ignored to facilitate passing the 

500 same arguments to several methods. 

501 

502 Returns 

503 ------- 

504 exposure : `lsst.afw.image.exposure.Exposure` 

505 Exposure with adjusted calibrations. 

506 """ 

507 if not self.config.doApplyExternalSkyWcs: 

508 # Do not modify the exposure's SkyWcs 

509 externalSkyWcsCatalog = None 

510 elif self.config.useGlobalExternalSkyWcs: 

511 # Use the global external SkyWcs 

512 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog 

513 self.log.info('Applying global SkyWcs') 

514 else: 

515 # use tract-level external SkyWcs from the closest overlapping tract 

516 inputRef = getattr(inputRefs, 'externalSkyWcsTractCatalog') 

517 tracts = [ref.dataId['tract'] for ref in inputRef] 

518 if len(tracts) == 1: 

519 ind = 0 

520 self.log.info('Applying tract-level SkyWcs from tract %s', tracts[ind]) 

521 else: 

522 if exposure.getWcs() is None: # TODO: could this look-up use the externalPhotoCalib? 

523 raise ValueError("Trying to locate nearest tract, but exposure.wcs is None.") 

524 ind = self.getClosestTract(tracts, skyMap, 

525 exposure.getBBox(), exposure.getWcs()) 

526 self.log.info('Multiple overlapping externalSkyWcsTractCatalogs found (%s). ' 

527 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind]) 

528 

529 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind] 

530 

531 if not self.config.doApplyExternalPhotoCalib: 

532 # Do not modify the exposure's PhotoCalib 

533 externalPhotoCalibCatalog = None 

534 elif self.config.useGlobalExternalPhotoCalib: 

535 # Use the global external PhotoCalib 

536 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog 

537 self.log.info('Applying global PhotoCalib') 

538 else: 

539 # use tract-level external PhotoCalib from the closest overlapping tract 

540 inputRef = getattr(inputRefs, 'externalPhotoCalibTractCatalog') 

541 tracts = [ref.dataId['tract'] for ref in inputRef] 

542 if len(tracts) == 1: 

543 ind = 0 

544 self.log.info('Applying tract-level PhotoCalib from tract %s', tracts[ind]) 

545 else: 

546 ind = self.getClosestTract(tracts, skyMap, 

547 exposure.getBBox(), exposure.getWcs()) 

548 self.log.info('Multiple overlapping externalPhotoCalibTractCatalogs found (%s). ' 

549 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind]) 

550 

551 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind] 

552 

553 return self.prepareCalibratedExposure( 

554 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary 

555 ) 

556 

557 # TODO: remove on DM-39854. 

558 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.", 

559 version="v26", category=FutureWarning) 

560 def getClosestTract(self, tracts, skyMap, bbox, wcs): 

561 """Find the index of the tract closest to detector from list of tractIds 

562 

563 Parameters 

564 ---------- 

565 tracts: `list` [`int`] 

566 Iterable of integer tractIds 

567 skyMap : `~lsst.skymap.BaseSkyMap` 

568 skyMap to lookup tract geometry and wcs 

569 bbox : `~lsst.geom.Box2I` 

570 Detector bbox, center of which will compared to tract centers 

571 wcs : `~lsst.afw.geom.SkyWcs` 

572 Detector Wcs object to map the detector center to SkyCoord 

573 

574 Returns 

575 ------- 

576 index : `int` 

577 """ 

578 if len(tracts) == 1: 

579 return 0 

580 

581 center = wcs.pixelToSky(bbox.getCenter()) 

582 sep = [] 

583 for tractId in tracts: 

584 tract = skyMap[tractId] 

585 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter()) 

586 sep.append(center.separation(tractCenter)) 

587 

588 return np.argmin(sep) 

589 

590 def prepareCalibratedExposure( 

591 self, 

592 exposure, 

593 detectorId, 

594 externalSkyWcsCatalog=None, 

595 externalPhotoCalibCatalog=None, 

596 visitSummary=None, 

597 ): 

598 """Prepare a calibrated exposure and apply external calibrations 

599 if so configured. 

600 

601 Parameters 

602 ---------- 

603 exposure : `lsst.afw.image.exposure.Exposure` 

604 Input exposure to adjust calibrations. May be empty. 

605 detectorId : `int` 

606 Detector ID associated with the exposure. 

607 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional 

608 Exposure catalog with external skyWcs to be applied 

609 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id 

610 for the catalog id, sorted on id for fast lookup. 

611 Deprecated in favor of ``visitSummary``; will be removed after v26. 

612 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional 

613 Exposure catalog with external photoCalib to be applied 

614 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector 

615 id for the catalog id, sorted on id for fast lookup. 

616 Deprecated in favor of ``visitSummary``; will be removed after v26. 

617 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

618 Exposure catalog with all calibration objects. WCS and PhotoCalib 

619 are always applied if ``visitSummary`` is provided and those 

620 components are not `None`. 

621 

622 Returns 

623 ------- 

624 exposure : `lsst.afw.image.exposure.Exposure` 

625 Exposure with adjusted calibrations. 

626 """ 

627 if visitSummary is not None: 

628 row = visitSummary.find(detectorId) 

629 if row is None: 

630 raise RuntimeError(f"Visit summary for detector {detectorId} is unexpectedly missing.") 

631 if (photoCalib := row.getPhotoCalib()) is None: 

632 self.log.warning("Detector id %s has None for photoCalib in visit summary; " 

633 "skipping reevaluation of photoCalib.", detectorId) 

634 exposure.setPhotoCalib(None) 

635 else: 

636 exposure.setPhotoCalib(photoCalib) 

637 if (skyWcs := row.getWcs()) is None: 

638 self.log.warning("Detector id %s has None for skyWcs in visit summary; " 

639 "skipping reevaluation of skyWcs.", detectorId) 

640 exposure.setWcs(None) 

641 else: 

642 exposure.setWcs(skyWcs) 

643 

644 if externalPhotoCalibCatalog is not None: 

645 # TODO: remove on DM-39854. 

646 warnings.warn( 

647 "Deprecated in favor of 'visitSummary'; will be removed after v26.", 

648 FutureWarning, 

649 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"), 

650 ) 

651 row = externalPhotoCalibCatalog.find(detectorId) 

652 if row is None: 

653 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; " 

654 "Using original photoCalib.", detectorId) 

655 else: 

656 photoCalib = row.getPhotoCalib() 

657 if photoCalib is None: 

658 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; " 

659 "Using original photoCalib.", detectorId) 

660 else: 

661 exposure.setPhotoCalib(photoCalib) 

662 

663 if externalSkyWcsCatalog is not None: 

664 # TODO: remove on DM-39854. 

665 warnings.warn( 

666 "Deprecated in favor of 'visitSummary'; will be removed after v26.", 

667 FutureWarning, 

668 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"), 

669 ) 

670 row = externalSkyWcsCatalog.find(detectorId) 

671 if row is None: 

672 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; " 

673 "Using original skyWcs.", detectorId) 

674 else: 

675 skyWcs = row.getWcs() 

676 if skyWcs is None: 

677 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; " 

678 "Using original skyWcs.", detectorId) 

679 else: 

680 exposure.setWcs(skyWcs) 

681 

682 return exposure 

683 

684 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs): 

685 """Add replace columns with calibs evaluated at each centroid 

686 

687 Add or replace 'base_LocalWcs' `base_LocalPhotoCalib' columns in a 

688 a source catalog, by rerunning the plugins. 

689 

690 Parameters 

691 ---------- 

692 catalog : `lsst.afw.table.SourceCatalog` 

693 catalog to which calib columns will be added 

694 exposure : `lsst.afw.image.exposure.Exposure` 

695 Exposure with attached PhotoCalibs and SkyWcs attributes to be 

696 reevaluated at local centroids. Pixels are not required. 

697 idGenerator : `lsst.meas.base.IdGenerator` 

698 Object that generates Source IDs and random seeds. 

699 **kwargs 

700 Additional keyword arguments are ignored to facilitate passing the 

701 same arguments to several methods. 

702 

703 Returns 

704 ------- 

705 newCat: `lsst.afw.table.SourceCatalog` 

706 Source Catalog with requested local calib columns 

707 """ 

708 measureConfig = SingleFrameMeasurementTask.ConfigClass() 

709 measureConfig.doReplaceWithNoise = False 

710 

711 # Clear all slots, because we aren't running the relevant plugins. 

712 for slot in measureConfig.slots: 

713 setattr(measureConfig.slots, slot, None) 

714 

715 measureConfig.plugins.names = [] 

716 if self.config.doReevaluateSkyWcs: 

717 measureConfig.plugins.names.add("base_LocalWcs") 

718 self.log.info("Re-evaluating base_LocalWcs plugin") 

719 if self.config.doReevaluatePhotoCalib: 

720 measureConfig.plugins.names.add("base_LocalPhotoCalib") 

721 self.log.info("Re-evaluating base_LocalPhotoCalib plugin") 

722 pluginsNotToCopy = tuple(measureConfig.plugins.names) 

723 

724 # Create a new schema and catalog 

725 # Copy all columns from original except for the ones to reevaluate 

726 aliasMap = catalog.schema.getAliasMap() 

727 mapper = afwTable.SchemaMapper(catalog.schema) 

728 for item in catalog.schema: 

729 if not item.field.getName().startswith(pluginsNotToCopy): 

730 mapper.addMapping(item.key) 

731 

732 schema = mapper.getOutputSchema() 

733 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema) 

734 schema.setAliasMap(aliasMap) 

735 newCat = afwTable.SourceCatalog(schema) 

736 newCat.extend(catalog, mapper=mapper) 

737 

738 # Fluxes in sourceCatalogs are in counts, so there are no fluxes to 

739 # update here. LocalPhotoCalibs are applied during transform tasks. 

740 # Update coord_ra/coord_dec, which are expected to be positions on the 

741 # sky and are used as such in sdm tables without transform 

742 if self.config.doReevaluateSkyWcs and exposure.wcs is not None: 

743 afwTable.updateSourceCoords(exposure.wcs, newCat) 

744 wcsPlugin = measurement.plugins["base_LocalWcs"] 

745 else: 

746 wcsPlugin = None 

747 

748 if self.config.doReevaluatePhotoCalib and exposure.getPhotoCalib() is not None: 

749 pcPlugin = measurement.plugins["base_LocalPhotoCalib"] 

750 else: 

751 pcPlugin = None 

752 

753 for row in newCat: 

754 if wcsPlugin is not None: 

755 wcsPlugin.measure(row, exposure) 

756 if pcPlugin is not None: 

757 pcPlugin.measure(row, exposure) 

758 

759 return newCat 

760 

761 

762class PostprocessAnalysis(object): 

763 """Calculate columns from DataFrames or handles storing DataFrames. 

764 

765 This object manages and organizes an arbitrary set of computations 

766 on a catalog. The catalog is defined by a 

767 `DeferredDatasetHandle` or `InMemoryDatasetHandle` object 

768 (or list thereof), such as a ``deepCoadd_obj`` dataset, and the 

769 computations are defined by a collection of 

770 `~lsst.pipe.tasks.functors.Functor` objects (or, equivalently, a 

771 ``CompositeFunctor``). 

772 

773 After the object is initialized, accessing the ``.df`` attribute (which 

774 holds the `pandas.DataFrame` containing the results of the calculations) 

775 triggers computation of said dataframe. 

776 

777 One of the conveniences of using this object is the ability to define a 

778 desired common filter for all functors. This enables the same functor 

779 collection to be passed to several different `PostprocessAnalysis` objects 

780 without having to change the original functor collection, since the ``filt`` 

781 keyword argument of this object triggers an overwrite of the ``filt`` 

782 property for all functors in the collection. 

783 

784 This object also allows a list of refFlags to be passed, and defines a set 

785 of default refFlags that are always included even if not requested. 

786 

787 If a list of DataFrames or Handles is passed, rather than a single one, 

788 then the calculations will be mapped over all the input catalogs. In 

789 principle, it should be straightforward to parallelize this activity, but 

790 initial tests have failed (see TODO in code comments). 

791 

792 Parameters 

793 ---------- 

794 handles : `~lsst.daf.butler.DeferredDatasetHandle` or 

795 `~lsst.pipe.base.InMemoryDatasetHandle` or 

796 list of these. 

797 Source catalog(s) for computation. 

798 functors : `list`, `dict`, or `~lsst.pipe.tasks.functors.CompositeFunctor` 

799 Computations to do (functors that act on ``handles``). 

800 If a dict, the output 

801 DataFrame will have columns keyed accordingly. 

802 If a list, the column keys will come from the 

803 ``.shortname`` attribute of each functor. 

804 

805 filt : `str`, optional 

806 Filter in which to calculate. If provided, 

807 this will overwrite any existing ``.filt`` attribute 

808 of the provided functors. 

809 

810 flags : `list`, optional 

811 List of flags (per-band) to include in output table. 

812 Taken from the ``meas`` dataset if applied to a multilevel Object Table. 

813 

814 refFlags : `list`, optional 

815 List of refFlags (only reference band) to include in output table. 

816 

817 forcedFlags : `list`, optional 

818 List of flags (per-band) to include in output table. 

819 Taken from the ``forced_src`` dataset if applied to a 

820 multilevel Object Table. Intended for flags from measurement plugins 

821 only run during multi-band forced-photometry. 

822 """ 

823 _defaultRefFlags = [] 

824 _defaultFuncs = () 

825 

826 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None): 

827 self.handles = handles 

828 self.functors = functors 

829 

830 self.filt = filt 

831 self.flags = list(flags) if flags is not None else [] 

832 self.forcedFlags = list(forcedFlags) if forcedFlags is not None else [] 

833 self.refFlags = list(self._defaultRefFlags) 

834 if refFlags is not None: 

835 self.refFlags += list(refFlags) 

836 

837 self._df = None 

838 

839 @property 

840 def defaultFuncs(self): 

841 funcs = dict(self._defaultFuncs) 

842 return funcs 

843 

844 @property 

845 def func(self): 

846 additionalFuncs = self.defaultFuncs 

847 additionalFuncs.update({flag: Column(flag, dataset='forced_src') for flag in self.forcedFlags}) 

848 additionalFuncs.update({flag: Column(flag, dataset='ref') for flag in self.refFlags}) 

849 additionalFuncs.update({flag: Column(flag, dataset='meas') for flag in self.flags}) 

850 

851 if isinstance(self.functors, CompositeFunctor): 

852 func = self.functors 

853 else: 

854 func = CompositeFunctor(self.functors) 

855 

856 func.funcDict.update(additionalFuncs) 

857 func.filt = self.filt 

858 

859 return func 

860 

861 @property 

862 def noDupCols(self): 

863 return [name for name, func in self.func.funcDict.items() if func.noDup or func.dataset == 'ref'] 

864 

865 @property 

866 def df(self): 

867 if self._df is None: 

868 self.compute() 

869 return self._df 

870 

871 def compute(self, dropna=False, pool=None): 

872 # map over multiple handles 

873 if type(self.handles) in (list, tuple): 

874 if pool is None: 

875 dflist = [self.func(handle, dropna=dropna) for handle in self.handles] 

876 else: 

877 # TODO: Figure out why this doesn't work (pyarrow pickling 

878 # issues?) 

879 dflist = pool.map(functools.partial(self.func, dropna=dropna), self.handles) 

880 self._df = pd.concat(dflist) 

881 else: 

882 self._df = self.func(self.handles, dropna=dropna) 

883 

884 return self._df 

885 

886 

887class TransformCatalogBaseConnections(pipeBase.PipelineTaskConnections, 

888 dimensions=()): 

889 """Expected Connections for subclasses of TransformCatalogBaseTask. 

890 

891 Must be subclassed. 

892 """ 

893 inputCatalog = connectionTypes.Input( 

894 name="", 

895 storageClass="DataFrame", 

896 ) 

897 outputCatalog = connectionTypes.Output( 

898 name="", 

899 storageClass="DataFrame", 

900 ) 

901 

902 

903class TransformCatalogBaseConfig(pipeBase.PipelineTaskConfig, 

904 pipelineConnections=TransformCatalogBaseConnections): 

905 functorFile = pexConfig.Field( 

906 dtype=str, 

907 doc="Path to YAML file specifying Science Data Model functors to use " 

908 "when copying columns and computing calibrated values.", 

909 default=None, 

910 optional=True 

911 ) 

912 primaryKey = pexConfig.Field( 

913 dtype=str, 

914 doc="Name of column to be set as the DataFrame index. If None, the index" 

915 "will be named `id`", 

916 default=None, 

917 optional=True 

918 ) 

919 columnsFromDataId = pexConfig.ListField( 

920 dtype=str, 

921 default=None, 

922 optional=True, 

923 doc="Columns to extract from the dataId", 

924 ) 

925 

926 

927class TransformCatalogBaseTask(pipeBase.PipelineTask): 

928 """Base class for transforming/standardizing a catalog by applying functors 

929 that convert units and apply calibrations. 

930 

931 The purpose of this task is to perform a set of computations on an input 

932 ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` that holds a 

933 ``DataFrame`` dataset (such as ``deepCoadd_obj``), and write the results to 

934 a new dataset (which needs to be declared in an ``outputDataset`` 

935 attribute). 

936 

937 The calculations to be performed are defined in a YAML file that specifies 

938 a set of functors to be computed, provided as a ``--functorFile`` config 

939 parameter. An example of such a YAML file is the following: 

940 

941 funcs: 

942 sourceId: 

943 functor: Index 

944 x: 

945 functor: Column 

946 args: slot_Centroid_x 

947 y: 

948 functor: Column 

949 args: slot_Centroid_y 

950 psfFlux: 

951 functor: LocalNanojansky 

952 args: 

953 - slot_PsfFlux_instFlux 

954 - slot_PsfFlux_instFluxErr 

955 - base_LocalPhotoCalib 

956 - base_LocalPhotoCalibErr 

957 psfFluxErr: 

958 functor: LocalNanojanskyErr 

959 args: 

960 - slot_PsfFlux_instFlux 

961 - slot_PsfFlux_instFluxErr 

962 - base_LocalPhotoCalib 

963 - base_LocalPhotoCalibErr 

964 flags: 

965 - detect_isPrimary 

966 

967 The names for each entry under "func" will become the names of columns in 

968 the output dataset. All the functors referenced are defined in 

969 `~lsst.pipe.tasks.functors`. Positional arguments to be passed to each 

970 functor are in the `args` list, and any additional entries for each column 

971 other than "functor" or "args" (e.g., ``'filt'``, ``'dataset'``) are 

972 treated as keyword arguments to be passed to the functor initialization. 

973 

974 The "flags" entry is the default shortcut for `Column` functors. 

975 All columns listed under "flags" will be copied to the output table 

976 untransformed. They can be of any datatype. 

977 In the special case of transforming a multi-level oject table with 

978 band and dataset indices (deepCoadd_obj), these will be taked from the 

979 `meas` dataset and exploded out per band. 

980 

981 There are two special shortcuts that only apply when transforming 

982 multi-level Object (deepCoadd_obj) tables: 

983 - The "refFlags" entry is shortcut for `Column` functor 

984 taken from the `'ref'` dataset if transforming an ObjectTable. 

985 - The "forcedFlags" entry is shortcut for `Column` functors. 

986 taken from the ``forced_src`` dataset if transforming an ObjectTable. 

987 These are expanded out per band. 

988 

989 

990 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object 

991 to organize and excecute the calculations. 

992 """ 

993 @property 

994 def _DefaultName(self): 

995 raise NotImplementedError('Subclass must define "_DefaultName" attribute') 

996 

997 @property 

998 def outputDataset(self): 

999 raise NotImplementedError('Subclass must define "outputDataset" attribute') 

1000 

1001 @property 

1002 def inputDataset(self): 

1003 raise NotImplementedError('Subclass must define "inputDataset" attribute') 

1004 

1005 @property 

1006 def ConfigClass(self): 

1007 raise NotImplementedError('Subclass must define "ConfigClass" attribute') 

1008 

1009 def __init__(self, *args, **kwargs): 

1010 super().__init__(*args, **kwargs) 

1011 if self.config.functorFile: 

1012 self.log.info('Loading tranform functor definitions from %s', 

1013 self.config.functorFile) 

1014 self.funcs = CompositeFunctor.from_file(self.config.functorFile) 

1015 self.funcs.update(dict(PostprocessAnalysis._defaultFuncs)) 

1016 else: 

1017 self.funcs = None 

1018 

1019 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1020 inputs = butlerQC.get(inputRefs) 

1021 if self.funcs is None: 

1022 raise ValueError("config.functorFile is None. " 

1023 "Must be a valid path to yaml in order to run Task as a PipelineTask.") 

1024 result = self.run(handle=inputs['inputCatalog'], funcs=self.funcs, 

1025 dataId=dict(outputRefs.outputCatalog.dataId.mapping)) 

1026 outputs = pipeBase.Struct(outputCatalog=result) 

1027 butlerQC.put(outputs, outputRefs) 

1028 

1029 def run(self, handle, funcs=None, dataId=None, band=None): 

1030 """Do postprocessing calculations 

1031 

1032 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` or 

1033 ``DataFrame`` object and dataId, 

1034 returns a dataframe with results of postprocessing calculations. 

1035 

1036 Parameters 

1037 ---------- 

1038 handles : `~lsst.daf.butler.DeferredDatasetHandle` or 

1039 `~lsst.pipe.base.InMemoryDatasetHandle` or 

1040 `~pandas.DataFrame`, or list of these. 

1041 DataFrames from which calculations are done. 

1042 funcs : `~lsst.pipe.tasks.functors.Functor` 

1043 Functors to apply to the table's columns 

1044 dataId : dict, optional 

1045 Used to add a `patchId` column to the output dataframe. 

1046 band : `str`, optional 

1047 Filter band that is being processed. 

1048 

1049 Returns 

1050 ------- 

1051 df : `pandas.DataFrame` 

1052 """ 

1053 self.log.info("Transforming/standardizing the source table dataId: %s", dataId) 

1054 

1055 df = self.transform(band, handle, funcs, dataId).df 

1056 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df)) 

1057 return df 

1058 

1059 def getFunctors(self): 

1060 return self.funcs 

1061 

1062 def getAnalysis(self, handles, funcs=None, band=None): 

1063 if funcs is None: 

1064 funcs = self.funcs 

1065 analysis = PostprocessAnalysis(handles, funcs, filt=band) 

1066 return analysis 

1067 

1068 def transform(self, band, handles, funcs, dataId): 

1069 analysis = self.getAnalysis(handles, funcs=funcs, band=band) 

1070 df = analysis.df 

1071 if dataId and self.config.columnsFromDataId: 

1072 for key in self.config.columnsFromDataId: 

1073 if key in dataId: 

1074 df[key] = dataId[key] 

1075 else: 

1076 raise ValueError(f"'{key}' in config.columnsFromDataId not found in dataId: {dataId}") 

1077 

1078 if self.config.primaryKey: 

1079 if df.index.name != self.config.primaryKey and self.config.primaryKey in df: 

1080 df.reset_index(inplace=True, drop=True) 

1081 df.set_index(self.config.primaryKey, inplace=True) 

1082 

1083 return pipeBase.Struct( 

1084 df=df, 

1085 analysis=analysis 

1086 ) 

1087 

1088 

1089class TransformObjectCatalogConnections(pipeBase.PipelineTaskConnections, 

1090 defaultTemplates={"coaddName": "deep"}, 

1091 dimensions=("tract", "patch", "skymap")): 

1092 inputCatalog = connectionTypes.Input( 

1093 doc="The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, " 

1094 "stored as a DataFrame with a multi-level column index per-patch.", 

1095 dimensions=("tract", "patch", "skymap"), 

1096 storageClass="DataFrame", 

1097 name="{coaddName}Coadd_obj", 

1098 deferLoad=True, 

1099 ) 

1100 outputCatalog = connectionTypes.Output( 

1101 doc="Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard " 

1102 "data model.", 

1103 dimensions=("tract", "patch", "skymap"), 

1104 storageClass="DataFrame", 

1105 name="objectTable" 

1106 ) 

1107 

1108 

1109class TransformObjectCatalogConfig(TransformCatalogBaseConfig, 

1110 pipelineConnections=TransformObjectCatalogConnections): 

1111 coaddName = pexConfig.Field( 

1112 dtype=str, 

1113 default="deep", 

1114 doc="Name of coadd" 

1115 ) 

1116 # TODO: remove in DM-27177 

1117 filterMap = pexConfig.DictField( 

1118 keytype=str, 

1119 itemtype=str, 

1120 default={}, 

1121 doc=("Dictionary mapping full filter name to short one for column name munging." 

1122 "These filters determine the output columns no matter what filters the " 

1123 "input data actually contain."), 

1124 deprecated=("Coadds are now identified by the band, so this transform is unused." 

1125 "Will be removed after v22.") 

1126 ) 

1127 outputBands = pexConfig.ListField( 

1128 dtype=str, 

1129 default=None, 

1130 optional=True, 

1131 doc=("These bands and only these bands will appear in the output," 

1132 " NaN-filled if the input does not include them." 

1133 " If None, then use all bands found in the input.") 

1134 ) 

1135 camelCase = pexConfig.Field( 

1136 dtype=bool, 

1137 default=False, 

1138 doc=("Write per-band columns names with camelCase, else underscore " 

1139 "For example: gPsFlux instead of g_PsFlux.") 

1140 ) 

1141 multilevelOutput = pexConfig.Field( 

1142 dtype=bool, 

1143 default=False, 

1144 doc=("Whether results dataframe should have a multilevel column index (True) or be flat " 

1145 "and name-munged (False).") 

1146 ) 

1147 goodFlags = pexConfig.ListField( 

1148 dtype=str, 

1149 default=[], 

1150 doc=("List of 'good' flags that should be set False when populating empty tables. " 

1151 "All other flags are considered to be 'bad' flags and will be set to True.") 

1152 ) 

1153 floatFillValue = pexConfig.Field( 

1154 dtype=float, 

1155 default=np.nan, 

1156 doc="Fill value for float fields when populating empty tables." 

1157 ) 

1158 integerFillValue = pexConfig.Field( 

1159 dtype=int, 

1160 default=-1, 

1161 doc="Fill value for integer fields when populating empty tables." 

1162 ) 

1163 

1164 def setDefaults(self): 

1165 super().setDefaults() 

1166 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Object.yaml') 

1167 self.primaryKey = 'objectId' 

1168 self.columnsFromDataId = ['tract', 'patch'] 

1169 self.goodFlags = ['calib_astrometry_used', 

1170 'calib_photometry_reserved', 

1171 'calib_photometry_used', 

1172 'calib_psf_candidate', 

1173 'calib_psf_reserved', 

1174 'calib_psf_used'] 

1175 

1176 

1177class TransformObjectCatalogTask(TransformCatalogBaseTask): 

1178 """Produce a flattened Object Table to match the format specified in 

1179 sdm_schemas. 

1180 

1181 Do the same set of postprocessing calculations on all bands. 

1182 

1183 This is identical to `TransformCatalogBaseTask`, except for that it does 

1184 the specified functor calculations for all filters present in the 

1185 input `deepCoadd_obj` table. Any specific ``"filt"`` keywords specified 

1186 by the YAML file will be superceded. 

1187 """ 

1188 _DefaultName = "transformObjectCatalog" 

1189 ConfigClass = TransformObjectCatalogConfig 

1190 

1191 def run(self, handle, funcs=None, dataId=None, band=None): 

1192 # NOTE: band kwarg is ignored here. 

1193 dfDict = {} 

1194 analysisDict = {} 

1195 templateDf = pd.DataFrame() 

1196 

1197 columns = handle.get(component='columns') 

1198 inputBands = columns.unique(level=1).values 

1199 

1200 outputBands = self.config.outputBands if self.config.outputBands else inputBands 

1201 

1202 # Perform transform for data of filters that exist in the handle dataframe. 

1203 for inputBand in inputBands: 

1204 if inputBand not in outputBands: 

1205 self.log.info("Ignoring %s band data in the input", inputBand) 

1206 continue 

1207 self.log.info("Transforming the catalog of band %s", inputBand) 

1208 result = self.transform(inputBand, handle, funcs, dataId) 

1209 dfDict[inputBand] = result.df 

1210 analysisDict[inputBand] = result.analysis 

1211 if templateDf.empty: 

1212 templateDf = result.df 

1213 

1214 # Put filler values in columns of other wanted bands 

1215 for filt in outputBands: 

1216 if filt not in dfDict: 

1217 self.log.info("Adding empty columns for band %s", filt) 

1218 dfTemp = templateDf.copy() 

1219 for col in dfTemp.columns: 

1220 testValue = dfTemp[col].values[0] 

1221 if isinstance(testValue, (np.bool_, pd.BooleanDtype)): 

1222 # Boolean flag type, check if it is a "good" flag 

1223 if col in self.config.goodFlags: 

1224 fillValue = False 

1225 else: 

1226 fillValue = True 

1227 elif isinstance(testValue, numbers.Integral): 

1228 # Checking numbers.Integral catches all flavors 

1229 # of python, numpy, pandas, etc. integers. 

1230 # We must ensure this is not an unsigned integer. 

1231 if isinstance(testValue, np.unsignedinteger): 

1232 raise ValueError("Parquet tables may not have unsigned integer columns.") 

1233 else: 

1234 fillValue = self.config.integerFillValue 

1235 else: 

1236 fillValue = self.config.floatFillValue 

1237 dfTemp[col].values[:] = fillValue 

1238 dfDict[filt] = dfTemp 

1239 

1240 # This makes a multilevel column index, with band as first level 

1241 df = pd.concat(dfDict, axis=1, names=['band', 'column']) 

1242 

1243 if not self.config.multilevelOutput: 

1244 noDupCols = list(set.union(*[set(v.noDupCols) for v in analysisDict.values()])) 

1245 if self.config.primaryKey in noDupCols: 

1246 noDupCols.remove(self.config.primaryKey) 

1247 if dataId and self.config.columnsFromDataId: 

1248 noDupCols += self.config.columnsFromDataId 

1249 df = flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase, 

1250 inputBands=inputBands) 

1251 

1252 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df)) 

1253 

1254 return df 

1255 

1256 

1257class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections, 

1258 dimensions=("tract", "skymap")): 

1259 inputCatalogs = connectionTypes.Input( 

1260 doc="Per-Patch objectTables conforming to the standard data model.", 

1261 name="objectTable", 

1262 storageClass="DataFrame", 

1263 dimensions=("tract", "patch", "skymap"), 

1264 multiple=True, 

1265 ) 

1266 outputCatalog = connectionTypes.Output( 

1267 doc="Pre-tract horizontal concatenation of the input objectTables", 

1268 name="objectTable_tract", 

1269 storageClass="DataFrame", 

1270 dimensions=("tract", "skymap"), 

1271 ) 

1272 

1273 

1274class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig, 

1275 pipelineConnections=ConsolidateObjectTableConnections): 

1276 coaddName = pexConfig.Field( 

1277 dtype=str, 

1278 default="deep", 

1279 doc="Name of coadd" 

1280 ) 

1281 

1282 

1283class ConsolidateObjectTableTask(pipeBase.PipelineTask): 

1284 """Write patch-merged source tables to a tract-level DataFrame Parquet file. 

1285 

1286 Concatenates `objectTable` list into a per-visit `objectTable_tract`. 

1287 """ 

1288 _DefaultName = "consolidateObjectTable" 

1289 ConfigClass = ConsolidateObjectTableConfig 

1290 

1291 inputDataset = 'objectTable' 

1292 outputDataset = 'objectTable_tract' 

1293 

1294 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1295 inputs = butlerQC.get(inputRefs) 

1296 self.log.info("Concatenating %s per-patch Object Tables", 

1297 len(inputs['inputCatalogs'])) 

1298 df = pd.concat(inputs['inputCatalogs']) 

1299 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs) 

1300 

1301 

1302class TransformSourceTableConnections(pipeBase.PipelineTaskConnections, 

1303 defaultTemplates={"catalogType": ""}, 

1304 dimensions=("instrument", "visit", "detector")): 

1305 

1306 inputCatalog = connectionTypes.Input( 

1307 doc="Wide input catalog of sources produced by WriteSourceTableTask", 

1308 name="{catalogType}source", 

1309 storageClass="DataFrame", 

1310 dimensions=("instrument", "visit", "detector"), 

1311 deferLoad=True 

1312 ) 

1313 outputCatalog = connectionTypes.Output( 

1314 doc="Narrower, per-detector Source Table transformed and converted per a " 

1315 "specified set of functors", 

1316 name="{catalogType}sourceTable", 

1317 storageClass="DataFrame", 

1318 dimensions=("instrument", "visit", "detector") 

1319 ) 

1320 

1321 

1322class TransformSourceTableConfig(TransformCatalogBaseConfig, 

1323 pipelineConnections=TransformSourceTableConnections): 

1324 

1325 def setDefaults(self): 

1326 super().setDefaults() 

1327 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Source.yaml') 

1328 self.primaryKey = 'sourceId' 

1329 self.columnsFromDataId = ['visit', 'detector', 'band', 'physical_filter'] 

1330 

1331 

1332class TransformSourceTableTask(TransformCatalogBaseTask): 

1333 """Transform/standardize a source catalog 

1334 """ 

1335 _DefaultName = "transformSourceTable" 

1336 ConfigClass = TransformSourceTableConfig 

1337 

1338 

1339class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections, 

1340 dimensions=("instrument", "visit",), 

1341 defaultTemplates={"calexpType": ""}): 

1342 calexp = connectionTypes.Input( 

1343 doc="Processed exposures used for metadata", 

1344 name="calexp", 

1345 storageClass="ExposureF", 

1346 dimensions=("instrument", "visit", "detector"), 

1347 deferLoad=True, 

1348 multiple=True, 

1349 ) 

1350 visitSummary = connectionTypes.Output( 

1351 doc=("Per-visit consolidated exposure metadata. These catalogs use " 

1352 "detector id for the id and are sorted for fast lookups of a " 

1353 "detector."), 

1354 name="visitSummary", 

1355 storageClass="ExposureCatalog", 

1356 dimensions=("instrument", "visit"), 

1357 ) 

1358 visitSummarySchema = connectionTypes.InitOutput( 

1359 doc="Schema of the visitSummary catalog", 

1360 name="visitSummary_schema", 

1361 storageClass="ExposureCatalog", 

1362 ) 

1363 

1364 

1365class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig, 

1366 pipelineConnections=ConsolidateVisitSummaryConnections): 

1367 """Config for ConsolidateVisitSummaryTask""" 

1368 pass 

1369 

1370 

1371class ConsolidateVisitSummaryTask(pipeBase.PipelineTask): 

1372 """Task to consolidate per-detector visit metadata. 

1373 

1374 This task aggregates the following metadata from all the detectors in a 

1375 single visit into an exposure catalog: 

1376 - The visitInfo. 

1377 - The wcs. 

1378 - The photoCalib. 

1379 - The physical_filter and band (if available). 

1380 - The psf size, shape, and effective area at the center of the detector. 

1381 - The corners of the bounding box in right ascension/declination. 

1382 

1383 Other quantities such as Detector, Psf, ApCorrMap, and TransmissionCurve 

1384 are not persisted here because of storage concerns, and because of their 

1385 limited utility as summary statistics. 

1386 

1387 Tests for this task are performed in ci_hsc_gen3. 

1388 """ 

1389 _DefaultName = "consolidateVisitSummary" 

1390 ConfigClass = ConsolidateVisitSummaryConfig 

1391 

1392 def __init__(self, **kwargs): 

1393 super().__init__(**kwargs) 

1394 self.schema = afwTable.ExposureTable.makeMinimalSchema() 

1395 self.schema.addField('visit', type='L', doc='Visit number') 

1396 self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter') 

1397 self.schema.addField('band', type='String', size=32, doc='Name of band') 

1398 ExposureSummaryStats.update_schema(self.schema) 

1399 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema) 

1400 

1401 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1402 dataRefs = butlerQC.get(inputRefs.calexp) 

1403 visit = dataRefs[0].dataId['visit'] 

1404 

1405 self.log.debug("Concatenating metadata from %d per-detector calexps (visit %d)", 

1406 len(dataRefs), visit) 

1407 

1408 expCatalog = self._combineExposureMetadata(visit, dataRefs) 

1409 

1410 butlerQC.put(expCatalog, outputRefs.visitSummary) 

1411 

1412 def _combineExposureMetadata(self, visit, dataRefs): 

1413 """Make a combined exposure catalog from a list of dataRefs. 

1414 These dataRefs must point to exposures with wcs, summaryStats, 

1415 and other visit metadata. 

1416 

1417 Parameters 

1418 ---------- 

1419 visit : `int` 

1420 Visit identification number. 

1421 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1422 List of dataRefs in visit. 

1423 

1424 Returns 

1425 ------- 

1426 visitSummary : `lsst.afw.table.ExposureCatalog` 

1427 Exposure catalog with per-detector summary information. 

1428 """ 

1429 cat = afwTable.ExposureCatalog(self.schema) 

1430 cat.resize(len(dataRefs)) 

1431 

1432 cat['visit'] = visit 

1433 

1434 for i, dataRef in enumerate(dataRefs): 

1435 visitInfo = dataRef.get(component='visitInfo') 

1436 filterLabel = dataRef.get(component='filter') 

1437 summaryStats = dataRef.get(component='summaryStats') 

1438 detector = dataRef.get(component='detector') 

1439 wcs = dataRef.get(component='wcs') 

1440 photoCalib = dataRef.get(component='photoCalib') 

1441 detector = dataRef.get(component='detector') 

1442 bbox = dataRef.get(component='bbox') 

1443 validPolygon = dataRef.get(component='validPolygon') 

1444 

1445 rec = cat[i] 

1446 rec.setBBox(bbox) 

1447 rec.setVisitInfo(visitInfo) 

1448 rec.setWcs(wcs) 

1449 rec.setPhotoCalib(photoCalib) 

1450 rec.setValidPolygon(validPolygon) 

1451 

1452 rec['physical_filter'] = filterLabel.physicalLabel if filterLabel.hasPhysicalLabel() else "" 

1453 rec['band'] = filterLabel.bandLabel if filterLabel.hasBandLabel() else "" 

1454 rec.setId(detector.getId()) 

1455 summaryStats.update_record(rec) 

1456 

1457 metadata = dafBase.PropertyList() 

1458 metadata.add("COMMENT", "Catalog id is detector id, sorted.") 

1459 # We are looping over existing datarefs, so the following is true 

1460 metadata.add("COMMENT", "Only detectors with data have entries.") 

1461 cat.setMetadata(metadata) 

1462 

1463 cat.sort() 

1464 return cat 

1465 

1466 

1467class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections, 

1468 defaultTemplates={"catalogType": ""}, 

1469 dimensions=("instrument", "visit")): 

1470 inputCatalogs = connectionTypes.Input( 

1471 doc="Input per-detector Source Tables", 

1472 name="{catalogType}sourceTable", 

1473 storageClass="DataFrame", 

1474 dimensions=("instrument", "visit", "detector"), 

1475 multiple=True 

1476 ) 

1477 outputCatalog = connectionTypes.Output( 

1478 doc="Per-visit concatenation of Source Table", 

1479 name="{catalogType}sourceTable_visit", 

1480 storageClass="DataFrame", 

1481 dimensions=("instrument", "visit") 

1482 ) 

1483 

1484 

1485class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig, 

1486 pipelineConnections=ConsolidateSourceTableConnections): 

1487 pass 

1488 

1489 

1490class ConsolidateSourceTableTask(pipeBase.PipelineTask): 

1491 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit` 

1492 """ 

1493 _DefaultName = 'consolidateSourceTable' 

1494 ConfigClass = ConsolidateSourceTableConfig 

1495 

1496 inputDataset = 'sourceTable' 

1497 outputDataset = 'sourceTable_visit' 

1498 

1499 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1500 from .makeWarp import reorderRefs 

1501 

1502 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.inputCatalogs] 

1503 detectorOrder.sort() 

1504 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector') 

1505 inputs = butlerQC.get(inputRefs) 

1506 self.log.info("Concatenating %s per-detector Source Tables", 

1507 len(inputs['inputCatalogs'])) 

1508 df = pd.concat(inputs['inputCatalogs']) 

1509 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs) 

1510 

1511 

1512class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections, 

1513 dimensions=("instrument",), 

1514 defaultTemplates={"calexpType": ""}): 

1515 visitSummaryRefs = connectionTypes.Input( 

1516 doc="Data references for per-visit consolidated exposure metadata", 

1517 name="finalVisitSummary", 

1518 storageClass="ExposureCatalog", 

1519 dimensions=("instrument", "visit"), 

1520 multiple=True, 

1521 deferLoad=True, 

1522 ) 

1523 outputCatalog = connectionTypes.Output( 

1524 doc="CCD and Visit metadata table", 

1525 name="ccdVisitTable", 

1526 storageClass="DataFrame", 

1527 dimensions=("instrument",) 

1528 ) 

1529 

1530 

1531class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig, 

1532 pipelineConnections=MakeCcdVisitTableConnections): 

1533 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

1534 

1535 

1536class MakeCcdVisitTableTask(pipeBase.PipelineTask): 

1537 """Produce a `ccdVisitTable` from the visit summary exposure catalogs. 

1538 """ 

1539 _DefaultName = 'makeCcdVisitTable' 

1540 ConfigClass = MakeCcdVisitTableConfig 

1541 

1542 def run(self, visitSummaryRefs): 

1543 """Make a table of ccd information from the visit summary catalogs. 

1544 

1545 Parameters 

1546 ---------- 

1547 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle` 

1548 List of DeferredDatasetHandles pointing to exposure catalogs with 

1549 per-detector summary information. 

1550 

1551 Returns 

1552 ------- 

1553 result : `~lsst.pipe.base.Struct` 

1554 Results struct with attribute: 

1555 

1556 ``outputCatalog`` 

1557 Catalog of ccd and visit information. 

1558 """ 

1559 ccdEntries = [] 

1560 for visitSummaryRef in visitSummaryRefs: 

1561 visitSummary = visitSummaryRef.get() 

1562 visitInfo = visitSummary[0].getVisitInfo() 

1563 

1564 ccdEntry = {} 

1565 summaryTable = visitSummary.asAstropy() 

1566 selectColumns = ['id', 'visit', 'physical_filter', 'band', 'ra', 'dec', 'zenithDistance', 

1567 'zeroPoint', 'psfSigma', 'skyBg', 'skyNoise', 

1568 'astromOffsetMean', 'astromOffsetStd', 'nPsfStar', 

1569 'psfStarDeltaE1Median', 'psfStarDeltaE2Median', 

1570 'psfStarDeltaE1Scatter', 'psfStarDeltaE2Scatter', 

1571 'psfStarDeltaSizeMedian', 'psfStarDeltaSizeScatter', 

1572 'psfStarScaledDeltaSizeScatter', 

1573 'psfTraceRadiusDelta', 'maxDistToNearestPsf'] 

1574 ccdEntry = summaryTable[selectColumns].to_pandas().set_index('id') 

1575 # 'visit' is the human readable visit number. 

1576 # 'visitId' is the key to the visitId table. They are the same. 

1577 # Technically you should join to get the visit from the visit 

1578 # table. 

1579 ccdEntry = ccdEntry.rename(columns={"visit": "visitId"}) 

1580 

1581 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards 

1582 # compatibility. To be removed after September 2023. 

1583 ccdEntry["decl"] = ccdEntry.loc[:, "dec"] 

1584 

1585 ccdEntry['ccdVisitId'] = [ 

1586 self.config.idGenerator.apply( 

1587 visitSummaryRef.dataId, 

1588 detector=detector_id, 

1589 is_exposure=False, 

1590 ).catalog_id # The "catalog ID" here is the ccdVisit ID 

1591 # because it's usually the ID for a whole catalog 

1592 # with a {visit, detector}, and that's the main 

1593 # use case for IdGenerator. This usage for a 

1594 # summary table is rare. 

1595 for detector_id in summaryTable['id'] 

1596 ] 

1597 ccdEntry['detector'] = summaryTable['id'] 

1598 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds() if vR.getWcs() 

1599 else np.nan for vR in visitSummary]) 

1600 ccdEntry["seeing"] = visitSummary['psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds 

1601 

1602 ccdEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees() 

1603 ccdEntry["expMidpt"] = visitInfo.getDate().toPython() 

1604 ccdEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD) 

1605 expTime = visitInfo.getExposureTime() 

1606 ccdEntry['expTime'] = expTime 

1607 ccdEntry["obsStart"] = ccdEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime) 

1608 expTime_days = expTime / (60*60*24) 

1609 ccdEntry["obsStartMJD"] = ccdEntry["expMidptMJD"] - 0.5 * expTime_days 

1610 ccdEntry['darkTime'] = visitInfo.getDarkTime() 

1611 ccdEntry['xSize'] = summaryTable['bbox_max_x'] - summaryTable['bbox_min_x'] 

1612 ccdEntry['ySize'] = summaryTable['bbox_max_y'] - summaryTable['bbox_min_y'] 

1613 ccdEntry['llcra'] = summaryTable['raCorners'][:, 0] 

1614 ccdEntry['llcdec'] = summaryTable['decCorners'][:, 0] 

1615 ccdEntry['ulcra'] = summaryTable['raCorners'][:, 1] 

1616 ccdEntry['ulcdec'] = summaryTable['decCorners'][:, 1] 

1617 ccdEntry['urcra'] = summaryTable['raCorners'][:, 2] 

1618 ccdEntry['urcdec'] = summaryTable['decCorners'][:, 2] 

1619 ccdEntry['lrcra'] = summaryTable['raCorners'][:, 3] 

1620 ccdEntry['lrcdec'] = summaryTable['decCorners'][:, 3] 

1621 # TODO: DM-30618, Add raftName, nExposures, ccdTemp, binX, binY, 

1622 # and flags, and decide if WCS, and llcx, llcy, ulcx, ulcy, etc. 

1623 # values are actually wanted. 

1624 ccdEntries.append(ccdEntry) 

1625 

1626 outputCatalog = pd.concat(ccdEntries) 

1627 outputCatalog.set_index('ccdVisitId', inplace=True, verify_integrity=True) 

1628 return pipeBase.Struct(outputCatalog=outputCatalog) 

1629 

1630 

1631class MakeVisitTableConnections(pipeBase.PipelineTaskConnections, 

1632 dimensions=("instrument",), 

1633 defaultTemplates={"calexpType": ""}): 

1634 visitSummaries = connectionTypes.Input( 

1635 doc="Per-visit consolidated exposure metadata", 

1636 name="finalVisitSummary", 

1637 storageClass="ExposureCatalog", 

1638 dimensions=("instrument", "visit",), 

1639 multiple=True, 

1640 deferLoad=True, 

1641 ) 

1642 outputCatalog = connectionTypes.Output( 

1643 doc="Visit metadata table", 

1644 name="visitTable", 

1645 storageClass="DataFrame", 

1646 dimensions=("instrument",) 

1647 ) 

1648 

1649 

1650class MakeVisitTableConfig(pipeBase.PipelineTaskConfig, 

1651 pipelineConnections=MakeVisitTableConnections): 

1652 pass 

1653 

1654 

1655class MakeVisitTableTask(pipeBase.PipelineTask): 

1656 """Produce a `visitTable` from the visit summary exposure catalogs. 

1657 """ 

1658 _DefaultName = 'makeVisitTable' 

1659 ConfigClass = MakeVisitTableConfig 

1660 

1661 def run(self, visitSummaries): 

1662 """Make a table of visit information from the visit summary catalogs. 

1663 

1664 Parameters 

1665 ---------- 

1666 visitSummaries : `list` of `lsst.afw.table.ExposureCatalog` 

1667 List of exposure catalogs with per-detector summary information. 

1668 Returns 

1669 ------- 

1670 result : `~lsst.pipe.base.Struct` 

1671 Results struct with attribute: 

1672 

1673 ``outputCatalog`` 

1674 Catalog of visit information. 

1675 """ 

1676 visitEntries = [] 

1677 for visitSummary in visitSummaries: 

1678 visitSummary = visitSummary.get() 

1679 visitRow = visitSummary[0] 

1680 visitInfo = visitRow.getVisitInfo() 

1681 

1682 visitEntry = {} 

1683 visitEntry["visitId"] = visitRow['visit'] 

1684 visitEntry["visit"] = visitRow['visit'] 

1685 visitEntry["physical_filter"] = visitRow['physical_filter'] 

1686 visitEntry["band"] = visitRow['band'] 

1687 raDec = visitInfo.getBoresightRaDec() 

1688 visitEntry["ra"] = raDec.getRa().asDegrees() 

1689 visitEntry["dec"] = raDec.getDec().asDegrees() 

1690 

1691 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards 

1692 # compatibility. To be removed after September 2023. 

1693 visitEntry["decl"] = visitEntry["dec"] 

1694 

1695 visitEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees() 

1696 azAlt = visitInfo.getBoresightAzAlt() 

1697 visitEntry["azimuth"] = azAlt.getLongitude().asDegrees() 

1698 visitEntry["altitude"] = azAlt.getLatitude().asDegrees() 

1699 visitEntry["zenithDistance"] = 90 - azAlt.getLatitude().asDegrees() 

1700 visitEntry["airmass"] = visitInfo.getBoresightAirmass() 

1701 expTime = visitInfo.getExposureTime() 

1702 visitEntry["expTime"] = expTime 

1703 visitEntry["expMidpt"] = visitInfo.getDate().toPython() 

1704 visitEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD) 

1705 visitEntry["obsStart"] = visitEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime) 

1706 expTime_days = expTime / (60*60*24) 

1707 visitEntry["obsStartMJD"] = visitEntry["expMidptMJD"] - 0.5 * expTime_days 

1708 visitEntries.append(visitEntry) 

1709 

1710 # TODO: DM-30623, Add programId, exposureType, cameraTemp, 

1711 # mirror1Temp, mirror2Temp, mirror3Temp, domeTemp, externalTemp, 

1712 # dimmSeeing, pwvGPS, pwvMW, flags, nExposures. 

1713 

1714 outputCatalog = pd.DataFrame(data=visitEntries) 

1715 outputCatalog.set_index('visitId', inplace=True, verify_integrity=True) 

1716 return pipeBase.Struct(outputCatalog=outputCatalog) 

1717 

1718 

1719class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections, 

1720 dimensions=("instrument", "visit", "detector", "skymap", "tract")): 

1721 

1722 inputCatalog = connectionTypes.Input( 

1723 doc="Primary per-detector, single-epoch forced-photometry catalog. " 

1724 "By default, it is the output of ForcedPhotCcdTask on calexps", 

1725 name="forced_src", 

1726 storageClass="SourceCatalog", 

1727 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1728 ) 

1729 inputCatalogDiff = connectionTypes.Input( 

1730 doc="Secondary multi-epoch, per-detector, forced photometry catalog. " 

1731 "By default, it is the output of ForcedPhotCcdTask run on image differences.", 

1732 name="forced_diff", 

1733 storageClass="SourceCatalog", 

1734 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1735 ) 

1736 outputCatalog = connectionTypes.Output( 

1737 doc="InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format", 

1738 name="mergedForcedSource", 

1739 storageClass="DataFrame", 

1740 dimensions=("instrument", "visit", "detector", "skymap", "tract") 

1741 ) 

1742 

1743 

1744class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig, 

1745 pipelineConnections=WriteForcedSourceTableConnections): 

1746 key = lsst.pex.config.Field( 

1747 doc="Column on which to join the two input tables on and make the primary key of the output", 

1748 dtype=str, 

1749 default="objectId", 

1750 ) 

1751 idGenerator = DetectorVisitIdGeneratorConfig.make_field() 

1752 

1753 

1754class WriteForcedSourceTableTask(pipeBase.PipelineTask): 

1755 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format. 

1756 

1757 Because the predecessor ForcedPhotCcdTask operates per-detector, 

1758 per-tract, (i.e., it has tract in its dimensions), detectors 

1759 on the tract boundary may have multiple forced source catalogs. 

1760 

1761 The successor task TransformForcedSourceTable runs per-patch 

1762 and temporally-aggregates overlapping mergedForcedSource catalogs from all 

1763 available multiple epochs. 

1764 """ 

1765 _DefaultName = "writeForcedSourceTable" 

1766 ConfigClass = WriteForcedSourceTableConfig 

1767 

1768 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1769 inputs = butlerQC.get(inputRefs) 

1770 # Add ccdVisitId to allow joining with CcdVisitTable 

1771 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

1772 inputs['ccdVisitId'] = idGenerator.catalog_id 

1773 inputs['band'] = butlerQC.quantum.dataId['band'] 

1774 outputs = self.run(**inputs) 

1775 butlerQC.put(outputs, outputRefs) 

1776 

1777 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None): 

1778 dfs = [] 

1779 for table, dataset, in zip((inputCatalog, inputCatalogDiff), ('calexp', 'diff')): 

1780 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=False) 

1781 df = df.reindex(sorted(df.columns), axis=1) 

1782 df['ccdVisitId'] = ccdVisitId if ccdVisitId else pd.NA 

1783 df['band'] = band if band else pd.NA 

1784 df.columns = pd.MultiIndex.from_tuples([(dataset, c) for c in df.columns], 

1785 names=('dataset', 'column')) 

1786 

1787 dfs.append(df) 

1788 

1789 outputCatalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs) 

1790 return pipeBase.Struct(outputCatalog=outputCatalog) 

1791 

1792 

1793class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections, 

1794 dimensions=("instrument", "skymap", "patch", "tract")): 

1795 

1796 inputCatalogs = connectionTypes.Input( 

1797 doc="DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask", 

1798 name="mergedForcedSource", 

1799 storageClass="DataFrame", 

1800 dimensions=("instrument", "visit", "detector", "skymap", "tract"), 

1801 multiple=True, 

1802 deferLoad=True 

1803 ) 

1804 referenceCatalog = connectionTypes.Input( 

1805 doc="Reference catalog which was used to seed the forcedPhot. Columns " 

1806 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner " 

1807 "are expected.", 

1808 name="objectTable", 

1809 storageClass="DataFrame", 

1810 dimensions=("tract", "patch", "skymap"), 

1811 deferLoad=True 

1812 ) 

1813 outputCatalog = connectionTypes.Output( 

1814 doc="Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a " 

1815 "specified set of functors", 

1816 name="forcedSourceTable", 

1817 storageClass="DataFrame", 

1818 dimensions=("tract", "patch", "skymap") 

1819 ) 

1820 

1821 

1822class TransformForcedSourceTableConfig(TransformCatalogBaseConfig, 

1823 pipelineConnections=TransformForcedSourceTableConnections): 

1824 referenceColumns = pexConfig.ListField( 

1825 dtype=str, 

1826 default=["detect_isPrimary", "detect_isTractInner", "detect_isPatchInner"], 

1827 optional=True, 

1828 doc="Columns to pull from reference catalog", 

1829 ) 

1830 keyRef = lsst.pex.config.Field( 

1831 doc="Column on which to join the two input tables on and make the primary key of the output", 

1832 dtype=str, 

1833 default="objectId", 

1834 ) 

1835 key = lsst.pex.config.Field( 

1836 doc="Rename the output DataFrame index to this name", 

1837 dtype=str, 

1838 default="forcedSourceId", 

1839 ) 

1840 

1841 def setDefaults(self): 

1842 super().setDefaults() 

1843 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'ForcedSource.yaml') 

1844 self.columnsFromDataId = ['tract', 'patch'] 

1845 

1846 

1847class TransformForcedSourceTableTask(TransformCatalogBaseTask): 

1848 """Transform/standardize a ForcedSource catalog 

1849 

1850 Transforms each wide, per-detector forcedSource DataFrame per the 

1851 specification file (per-camera defaults found in ForcedSource.yaml). 

1852 All epochs that overlap the patch are aggregated into one per-patch 

1853 narrow-DataFrame file. 

1854 

1855 No de-duplication of rows is performed. Duplicate resolutions flags are 

1856 pulled in from the referenceCatalog: `detect_isPrimary`, 

1857 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate 

1858 for analysis or compare duplicates for QA. 

1859 

1860 The resulting table includes multiple bands. Epochs (MJDs) and other useful 

1861 per-visit rows can be retreived by joining with the CcdVisitTable on 

1862 ccdVisitId. 

1863 """ 

1864 _DefaultName = "transformForcedSourceTable" 

1865 ConfigClass = TransformForcedSourceTableConfig 

1866 

1867 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1868 inputs = butlerQC.get(inputRefs) 

1869 if self.funcs is None: 

1870 raise ValueError("config.functorFile is None. " 

1871 "Must be a valid path to yaml in order to run Task as a PipelineTask.") 

1872 outputs = self.run(inputs['inputCatalogs'], inputs['referenceCatalog'], funcs=self.funcs, 

1873 dataId=dict(outputRefs.outputCatalog.dataId.mapping)) 

1874 

1875 butlerQC.put(outputs, outputRefs) 

1876 

1877 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None): 

1878 dfs = [] 

1879 ref = referenceCatalog.get(parameters={"columns": self.config.referenceColumns}) 

1880 self.log.info("Aggregating %s input catalogs" % (len(inputCatalogs))) 

1881 for handle in inputCatalogs: 

1882 result = self.transform(None, handle, funcs, dataId) 

1883 # Filter for only rows that were detected on (overlap) the patch 

1884 dfs.append(result.df.join(ref, how='inner')) 

1885 

1886 outputCatalog = pd.concat(dfs) 

1887 

1888 # Now that we are done joining on config.keyRef 

1889 # Change index to config.key by 

1890 outputCatalog.index.rename(self.config.keyRef, inplace=True) 

1891 # Add config.keyRef to the column list 

1892 outputCatalog.reset_index(inplace=True) 

1893 # Set the forcedSourceId to the index. This is specified in the 

1894 # ForcedSource.yaml 

1895 outputCatalog.set_index("forcedSourceId", inplace=True, verify_integrity=True) 

1896 # Rename it to the config.key 

1897 outputCatalog.index.rename(self.config.key, inplace=True) 

1898 

1899 self.log.info("Made a table of %d columns and %d rows", 

1900 len(outputCatalog.columns), len(outputCatalog)) 

1901 return pipeBase.Struct(outputCatalog=outputCatalog) 

1902 

1903 

1904class ConsolidateTractConnections(pipeBase.PipelineTaskConnections, 

1905 defaultTemplates={"catalogType": ""}, 

1906 dimensions=("instrument", "tract")): 

1907 inputCatalogs = connectionTypes.Input( 

1908 doc="Input per-patch DataFrame Tables to be concatenated", 

1909 name="{catalogType}ForcedSourceTable", 

1910 storageClass="DataFrame", 

1911 dimensions=("tract", "patch", "skymap"), 

1912 multiple=True, 

1913 ) 

1914 

1915 outputCatalog = connectionTypes.Output( 

1916 doc="Output per-tract concatenation of DataFrame Tables", 

1917 name="{catalogType}ForcedSourceTable_tract", 

1918 storageClass="DataFrame", 

1919 dimensions=("tract", "skymap"), 

1920 ) 

1921 

1922 

1923class ConsolidateTractConfig(pipeBase.PipelineTaskConfig, 

1924 pipelineConnections=ConsolidateTractConnections): 

1925 pass 

1926 

1927 

1928class ConsolidateTractTask(pipeBase.PipelineTask): 

1929 """Concatenate any per-patch, dataframe list into a single 

1930 per-tract DataFrame. 

1931 """ 

1932 _DefaultName = 'ConsolidateTract' 

1933 ConfigClass = ConsolidateTractConfig 

1934 

1935 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

1936 inputs = butlerQC.get(inputRefs) 

1937 # Not checking at least one inputCatalog exists because that'd be an 

1938 # empty QG. 

1939 self.log.info("Concatenating %s per-patch %s Tables", 

1940 len(inputs['inputCatalogs']), 

1941 inputRefs.inputCatalogs[0].datasetType.name) 

1942 df = pd.concat(inputs['inputCatalogs']) 

1943 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)