Coverage for python/lsst/pipe/tasks/postprocess.py: 27%
692 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-20 12:29 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-20 12:29 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["WriteObjectTableConfig", "WriteObjectTableTask",
23 "WriteSourceTableConfig", "WriteSourceTableTask",
24 "WriteRecalibratedSourceTableConfig", "WriteRecalibratedSourceTableTask",
25 "PostprocessAnalysis",
26 "TransformCatalogBaseConfig", "TransformCatalogBaseTask",
27 "TransformObjectCatalogConfig", "TransformObjectCatalogTask",
28 "ConsolidateObjectTableConfig", "ConsolidateObjectTableTask",
29 "TransformSourceTableConfig", "TransformSourceTableTask",
30 "ConsolidateVisitSummaryConfig", "ConsolidateVisitSummaryTask",
31 "ConsolidateSourceTableConfig", "ConsolidateSourceTableTask",
32 "MakeCcdVisitTableConfig", "MakeCcdVisitTableTask",
33 "MakeVisitTableConfig", "MakeVisitTableTask",
34 "WriteForcedSourceTableConfig", "WriteForcedSourceTableTask",
35 "TransformForcedSourceTableConfig", "TransformForcedSourceTableTask",
36 "ConsolidateTractConfig", "ConsolidateTractTask"]
38import functools
39import pandas as pd
40import logging
41import numpy as np
42import numbers
43import os
44import warnings
46from deprecated.sphinx import deprecated
48import lsst.geom
49import lsst.pex.config as pexConfig
50import lsst.pipe.base as pipeBase
51import lsst.daf.base as dafBase
52from lsst.utils.introspection import find_outside_stacklevel
53from lsst.pipe.base import connectionTypes
54import lsst.afw.table as afwTable
55from lsst.afw.image import ExposureSummaryStats, ExposureF
56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig
57from lsst.skymap import BaseSkyMap
59from .functors import CompositeFunctor, Column
61log = logging.getLogger(__name__)
64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
65 """Flattens a dataframe with multilevel column index.
66 """
67 newDf = pd.DataFrame()
68 # band is the level 0 index
69 dfBands = df.columns.unique(level=0).values
70 for band in dfBands:
71 subdf = df[band]
72 columnFormat = '{0}{1}' if camelCase else '{0}_{1}'
73 newColumns = {c: columnFormat.format(band, c)
74 for c in subdf.columns if c not in noDupCols}
75 cols = list(newColumns.keys())
76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
78 # Band must be present in the input and output or else column is all NaN:
79 presentBands = dfBands if inputBands is None else list(set(inputBands).intersection(dfBands))
80 # Get the unexploded columns from any present band's partition
81 noDupDf = df[presentBands[0]][noDupCols]
82 newDf = pd.concat([noDupDf, newDf], axis=1)
83 return newDf
86class WriteObjectTableConnections(pipeBase.PipelineTaskConnections,
87 defaultTemplates={"coaddName": "deep"},
88 dimensions=("tract", "patch", "skymap")):
89 inputCatalogMeas = connectionTypes.Input(
90 doc="Catalog of source measurements on the deepCoadd.",
91 dimensions=("tract", "patch", "band", "skymap"),
92 storageClass="SourceCatalog",
93 name="{coaddName}Coadd_meas",
94 multiple=True
95 )
96 inputCatalogForcedSrc = connectionTypes.Input(
97 doc="Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
98 dimensions=("tract", "patch", "band", "skymap"),
99 storageClass="SourceCatalog",
100 name="{coaddName}Coadd_forced_src",
101 multiple=True
102 )
103 inputCatalogRef = connectionTypes.Input(
104 doc="Catalog marking the primary detection (which band provides a good shape and position)"
105 "for each detection in deepCoadd_mergeDet.",
106 dimensions=("tract", "patch", "skymap"),
107 storageClass="SourceCatalog",
108 name="{coaddName}Coadd_ref"
109 )
110 outputCatalog = connectionTypes.Output(
111 doc="A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
112 "stored as a DataFrame with a multi-level column index per-patch.",
113 dimensions=("tract", "patch", "skymap"),
114 storageClass="DataFrame",
115 name="{coaddName}Coadd_obj"
116 )
119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
120 pipelineConnections=WriteObjectTableConnections):
121 engine = pexConfig.Field(
122 dtype=str,
123 default="pyarrow",
124 doc="Parquet engine for writing (pyarrow or fastparquet)",
125 deprecated="This config is no longer used, and will be removed after v26."
126 )
127 coaddName = pexConfig.Field(
128 dtype=str,
129 default="deep",
130 doc="Name of coadd"
131 )
134class WriteObjectTableTask(pipeBase.PipelineTask):
135 """Write filter-merged source tables as a DataFrame in parquet format.
136 """
137 _DefaultName = "writeObjectTable"
138 ConfigClass = WriteObjectTableConfig
140 # Names of table datasets to be merged
141 inputDatasets = ('forced_src', 'meas', 'ref')
143 # Tag of output dataset written by `MergeSourcesTask.write`
144 outputDataset = 'obj'
146 def runQuantum(self, butlerQC, inputRefs, outputRefs):
147 inputs = butlerQC.get(inputRefs)
149 measDict = {ref.dataId['band']: {'meas': cat} for ref, cat in
150 zip(inputRefs.inputCatalogMeas, inputs['inputCatalogMeas'])}
151 forcedSourceDict = {ref.dataId['band']: {'forced_src': cat} for ref, cat in
152 zip(inputRefs.inputCatalogForcedSrc, inputs['inputCatalogForcedSrc'])}
154 catalogs = {}
155 for band in measDict.keys():
156 catalogs[band] = {'meas': measDict[band]['meas'],
157 'forced_src': forcedSourceDict[band]['forced_src'],
158 'ref': inputs['inputCatalogRef']}
159 dataId = butlerQC.quantum.dataId
160 df = self.run(catalogs=catalogs, tract=dataId['tract'], patch=dataId['patch'])
161 outputs = pipeBase.Struct(outputCatalog=df)
162 butlerQC.put(outputs, outputRefs)
164 def run(self, catalogs, tract, patch):
165 """Merge multiple catalogs.
167 Parameters
168 ----------
169 catalogs : `dict`
170 Mapping from filter names to dict of catalogs.
171 tract : int
172 tractId to use for the tractId column.
173 patch : str
174 patchId to use for the patchId column.
176 Returns
177 -------
178 catalog : `pandas.DataFrame`
179 Merged dataframe.
180 """
181 dfs = []
182 for filt, tableDict in catalogs.items():
183 for dataset, table in tableDict.items():
184 # Convert afwTable to pandas DataFrame
185 df = table.asAstropy().to_pandas().set_index('id', drop=True)
187 # Sort columns by name, to ensure matching schema among patches
188 df = df.reindex(sorted(df.columns), axis=1)
189 df = df.assign(tractId=tract, patchId=patch)
191 # Make columns a 3-level MultiIndex
192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c) for c in df.columns],
193 names=('dataset', 'band', 'column'))
194 dfs.append(df)
196 # We do this dance and not `pd.concat(dfs)` because the pandas
197 # concatenation uses infinite memory.
198 catalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs)
199 return catalog
202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
203 defaultTemplates={"catalogType": ""},
204 dimensions=("instrument", "visit", "detector")):
206 catalog = connectionTypes.Input(
207 doc="Input full-depth catalog of sources produced by CalibrateTask",
208 name="{catalogType}src",
209 storageClass="SourceCatalog",
210 dimensions=("instrument", "visit", "detector")
211 )
212 outputCatalog = connectionTypes.Output(
213 doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
214 "replaced with an index; all other columns are unchanged.",
215 name="{catalogType}source",
216 storageClass="DataFrame",
217 dimensions=("instrument", "visit", "detector")
218 )
221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
222 pipelineConnections=WriteSourceTableConnections):
223 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
226class WriteSourceTableTask(pipeBase.PipelineTask):
227 """Write source table to DataFrame Parquet format.
228 """
229 _DefaultName = "writeSourceTable"
230 ConfigClass = WriteSourceTableConfig
232 def runQuantum(self, butlerQC, inputRefs, outputRefs):
233 inputs = butlerQC.get(inputRefs)
234 inputs['ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id
235 result = self.run(**inputs)
236 outputs = pipeBase.Struct(outputCatalog=result.table)
237 butlerQC.put(outputs, outputRefs)
239 def run(self, catalog, ccdVisitId=None, **kwargs):
240 """Convert `src` catalog to DataFrame
242 Parameters
243 ----------
244 catalog: `afwTable.SourceCatalog`
245 catalog to be converted
246 ccdVisitId: `int`
247 ccdVisitId to be added as a column
248 **kwargs
249 Additional keyword arguments are ignored as a convenience for
250 subclasses that pass the same arguments to several different
251 methods.
253 Returns
254 -------
255 result : `~lsst.pipe.base.Struct`
256 ``table``
257 `DataFrame` version of the input catalog
258 """
259 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
260 df = catalog.asAstropy().to_pandas().set_index('id', drop=True)
261 df['ccdVisitId'] = ccdVisitId
263 return pipeBase.Struct(table=df)
266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
267 defaultTemplates={"catalogType": "",
268 "skyWcsName": "gbdesAstrometricFit",
269 "photoCalibName": "fgcm"},
270 # TODO: remove on DM-39854.
271 deprecatedTemplates={
272 "skyWcsName": "Deprecated; will be removed after v26.",
273 "photoCalibName": "Deprecated; will be removed after v26."
274 },
275 dimensions=("instrument", "visit", "detector", "skymap")):
276 skyMap = connectionTypes.Input(
277 doc="skyMap needed to choose which tract-level calibrations to use when multiple available",
278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
279 storageClass="SkyMap",
280 dimensions=("skymap",),
281 # TODO: remove on DM-39854.
282 deprecated=(
283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. "
284 "Will be removed after v26."
285 ),
286 )
287 exposure = connectionTypes.Input(
288 doc="Input exposure to perform photometry on.",
289 name="calexp",
290 storageClass="ExposureF",
291 dimensions=["instrument", "visit", "detector"],
292 # TODO: remove on DM-39584
293 deprecated=(
294 "Deprecated, as the `calexp` is not needed and just creates unnecessary i/o. "
295 "Will be removed after v26."
296 ),
297 )
298 visitSummary = connectionTypes.Input(
299 doc="Input visit-summary catalog with updated calibration objects.",
300 name="finalVisitSummary",
301 storageClass="ExposureCatalog",
302 dimensions=("instrument", "visit",),
303 )
304 externalSkyWcsTractCatalog = connectionTypes.Input(
305 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
306 "id for the catalog id, sorted on id for fast lookup."),
307 name="{skyWcsName}SkyWcsCatalog",
308 storageClass="ExposureCatalog",
309 dimensions=["instrument", "visit", "tract"],
310 multiple=True,
311 # TODO: remove on DM-39854.
312 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
313 )
314 externalSkyWcsGlobalCatalog = connectionTypes.Input(
315 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
316 "These catalogs use the detector id for the catalog id, sorted on id for "
317 "fast lookup."),
318 name="finalVisitSummary",
319 storageClass="ExposureCatalog",
320 dimensions=["instrument", "visit"],
321 # TODO: remove on DM-39854.
322 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
323 )
324 externalPhotoCalibTractCatalog = connectionTypes.Input(
325 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
326 "detector id for the catalog id, sorted on id for fast lookup."),
327 name="{photoCalibName}PhotoCalibCatalog",
328 storageClass="ExposureCatalog",
329 dimensions=["instrument", "visit", "tract"],
330 multiple=True,
331 # TODO: remove on DM-39854.
332 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
333 )
334 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
335 doc=("Per-visit photometric calibrations computed globally (with no tract "
336 "information). These catalogs use the detector id for the catalog id, "
337 "sorted on id for fast lookup."),
338 name="finalVisitSummary",
339 storageClass="ExposureCatalog",
340 dimensions=["instrument", "visit"],
341 # TODO: remove on DM-39854.
342 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
343 )
345 def __init__(self, *, config=None):
346 super().__init__(config=config)
347 # Same connection boilerplate as all other applications of
348 # Global/Tract calibrations
349 # TODO: remove all of this on DM-39854.
350 keepSkyMap = False
351 keepExposure = False
352 if config.doApplyExternalSkyWcs and config.doReevaluateSkyWcs:
353 keepExposure = True
354 if config.useGlobalExternalSkyWcs:
355 self.inputs.remove("externalSkyWcsTractCatalog")
356 else:
357 self.inputs.remove("externalSkyWcsGlobalCatalog")
358 keepSkyMap = True
359 else:
360 self.inputs.remove("externalSkyWcsTractCatalog")
361 self.inputs.remove("externalSkyWcsGlobalCatalog")
362 if config.doApplyExternalPhotoCalib and config.doReevaluatePhotoCalib:
363 keepExposure = True
364 if config.useGlobalExternalPhotoCalib:
365 self.inputs.remove("externalPhotoCalibTractCatalog")
366 else:
367 self.inputs.remove("externalPhotoCalibGlobalCatalog")
368 keepSkyMap = True
369 else:
370 self.inputs.remove("externalPhotoCalibTractCatalog")
371 self.inputs.remove("externalPhotoCalibGlobalCatalog")
372 if not keepSkyMap:
373 del self.skyMap
374 if not keepExposure:
375 del self.exposure
378class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
379 pipelineConnections=WriteRecalibratedSourceTableConnections):
381 doReevaluatePhotoCalib = pexConfig.Field(
382 dtype=bool,
383 default=True,
384 doc=("Add or replace local photoCalib columns"),
385 )
386 doReevaluateSkyWcs = pexConfig.Field(
387 dtype=bool,
388 default=True,
389 doc=("Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"),
390 )
391 doApplyExternalPhotoCalib = pexConfig.Field(
392 dtype=bool,
393 default=False,
394 doc=("If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
395 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
396 # TODO: remove on DM-39854.
397 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
398 )
399 doApplyExternalSkyWcs = pexConfig.Field(
400 dtype=bool,
401 default=False,
402 doc=("if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
403 "else use the wcs already attached to the exposure."),
404 # TODO: remove on DM-39854.
405 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.",
406 )
407 useGlobalExternalPhotoCalib = pexConfig.Field(
408 dtype=bool,
409 default=False,
410 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations "
411 "that are not run per-tract. When False, use per-tract photometric "
412 "calibration files."),
413 # TODO: remove on DM-39854.
414 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
415 )
416 useGlobalExternalSkyWcs = pexConfig.Field(
417 dtype=bool,
418 default=False,
419 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
420 "that are not run per-tract. When False, use per-tract wcs "
421 "files."),
422 # TODO: remove on DM-39854.
423 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.",
424 )
425 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
427 def validate(self):
428 super().validate()
429 if self.doApplyExternalSkyWcs and not self.doReevaluateSkyWcs:
430 log.warning("doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
431 "External SkyWcs will not be read or evaluated.")
432 if self.doApplyExternalPhotoCalib and not self.doReevaluatePhotoCalib:
433 log.warning("doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
434 "External PhotoCalib will not be read or evaluated.")
437class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
438 """Write source table to DataFrame Parquet format.
439 """
440 _DefaultName = "writeRecalibratedSourceTable"
441 ConfigClass = WriteRecalibratedSourceTableConfig
443 def runQuantum(self, butlerQC, inputRefs, outputRefs):
444 inputs = butlerQC.get(inputRefs)
446 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
447 inputs['idGenerator'] = idGenerator
448 inputs['ccdVisitId'] = idGenerator.catalog_id
450 if self.config.doReevaluatePhotoCalib or self.config.doReevaluateSkyWcs:
451 if self.config.doApplyExternalPhotoCalib or self.config.doApplyExternalSkyWcs:
452 inputs['exposure'] = self.attachCalibs(inputRefs, **inputs)
453 else:
454 # Create an empty exposure that will hold the calibrations.
455 exposure = ExposureF()
456 detectorId = butlerQC.quantum.dataId["detector"]
457 inputs['exposure'] = self.prepareCalibratedExposure(
458 exposure=exposure,
459 detectorId=detectorId,
460 visitSummary=inputs["visitSummary"],
461 )
462 inputs['catalog'] = self.addCalibColumns(**inputs)
464 result = self.run(**inputs)
465 outputs = pipeBase.Struct(outputCatalog=result.table)
466 butlerQC.put(outputs, outputRefs)
468 # TODO: remove on DM-39854.
469 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
470 version="v26", category=FutureWarning)
471 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
472 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
473 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs):
474 """Apply external calibrations to exposure per configuration
476 When multiple tract-level calibrations overlap, select the one with the
477 center closest to detector.
479 Parameters
480 ----------
481 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`, for dataIds of
482 tract-level calibs.
483 skyMap : `~lsst.skymap.BaseSkyMap`
484 skyMap to lookup tract geometry and WCS.
485 exposure : `lsst.afw.image.exposure.Exposure`
486 Input exposure to adjust calibrations.
487 externalSkyWcsGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
488 Exposure catalog with external skyWcs to be applied per config
489 externalSkyWcsTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
490 Exposure catalog with external skyWcs to be applied per config
491 externalPhotoCalibGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
492 Exposure catalog with external photoCalib to be applied per config
493 externalPhotoCalibTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
494 Exposure catalog with external photoCalib to be applied per config
495 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
496 Exposure catalog with all calibration objects. WCS and PhotoCalib
497 are always applied if provided.
498 **kwargs
499 Additional keyword arguments are ignored to facilitate passing the
500 same arguments to several methods.
502 Returns
503 -------
504 exposure : `lsst.afw.image.exposure.Exposure`
505 Exposure with adjusted calibrations.
506 """
507 if not self.config.doApplyExternalSkyWcs:
508 # Do not modify the exposure's SkyWcs
509 externalSkyWcsCatalog = None
510 elif self.config.useGlobalExternalSkyWcs:
511 # Use the global external SkyWcs
512 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
513 self.log.info('Applying global SkyWcs')
514 else:
515 # use tract-level external SkyWcs from the closest overlapping tract
516 inputRef = getattr(inputRefs, 'externalSkyWcsTractCatalog')
517 tracts = [ref.dataId['tract'] for ref in inputRef]
518 if len(tracts) == 1:
519 ind = 0
520 self.log.info('Applying tract-level SkyWcs from tract %s', tracts[ind])
521 else:
522 if exposure.getWcs() is None: # TODO: could this look-up use the externalPhotoCalib?
523 raise ValueError("Trying to locate nearest tract, but exposure.wcs is None.")
524 ind = self.getClosestTract(tracts, skyMap,
525 exposure.getBBox(), exposure.getWcs())
526 self.log.info('Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
527 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
529 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
531 if not self.config.doApplyExternalPhotoCalib:
532 # Do not modify the exposure's PhotoCalib
533 externalPhotoCalibCatalog = None
534 elif self.config.useGlobalExternalPhotoCalib:
535 # Use the global external PhotoCalib
536 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
537 self.log.info('Applying global PhotoCalib')
538 else:
539 # use tract-level external PhotoCalib from the closest overlapping tract
540 inputRef = getattr(inputRefs, 'externalPhotoCalibTractCatalog')
541 tracts = [ref.dataId['tract'] for ref in inputRef]
542 if len(tracts) == 1:
543 ind = 0
544 self.log.info('Applying tract-level PhotoCalib from tract %s', tracts[ind])
545 else:
546 ind = self.getClosestTract(tracts, skyMap,
547 exposure.getBBox(), exposure.getWcs())
548 self.log.info('Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
549 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
551 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
553 return self.prepareCalibratedExposure(
554 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary
555 )
557 # TODO: remove on DM-39854.
558 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
559 version="v26", category=FutureWarning)
560 def getClosestTract(self, tracts, skyMap, bbox, wcs):
561 """Find the index of the tract closest to detector from list of tractIds
563 Parameters
564 ----------
565 tracts: `list` [`int`]
566 Iterable of integer tractIds
567 skyMap : `~lsst.skymap.BaseSkyMap`
568 skyMap to lookup tract geometry and wcs
569 bbox : `~lsst.geom.Box2I`
570 Detector bbox, center of which will compared to tract centers
571 wcs : `~lsst.afw.geom.SkyWcs`
572 Detector Wcs object to map the detector center to SkyCoord
574 Returns
575 -------
576 index : `int`
577 """
578 if len(tracts) == 1:
579 return 0
581 center = wcs.pixelToSky(bbox.getCenter())
582 sep = []
583 for tractId in tracts:
584 tract = skyMap[tractId]
585 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
586 sep.append(center.separation(tractCenter))
588 return np.argmin(sep)
590 def prepareCalibratedExposure(
591 self,
592 exposure,
593 detectorId,
594 externalSkyWcsCatalog=None,
595 externalPhotoCalibCatalog=None,
596 visitSummary=None,
597 ):
598 """Prepare a calibrated exposure and apply external calibrations
599 if so configured.
601 Parameters
602 ----------
603 exposure : `lsst.afw.image.exposure.Exposure`
604 Input exposure to adjust calibrations. May be empty.
605 detectorId : `int`
606 Detector ID associated with the exposure.
607 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
608 Exposure catalog with external skyWcs to be applied
609 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
610 for the catalog id, sorted on id for fast lookup.
611 Deprecated in favor of ``visitSummary``; will be removed after v26.
612 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
613 Exposure catalog with external photoCalib to be applied
614 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
615 id for the catalog id, sorted on id for fast lookup.
616 Deprecated in favor of ``visitSummary``; will be removed after v26.
617 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
618 Exposure catalog with all calibration objects. WCS and PhotoCalib
619 are always applied if ``visitSummary`` is provided and those
620 components are not `None`.
622 Returns
623 -------
624 exposure : `lsst.afw.image.exposure.Exposure`
625 Exposure with adjusted calibrations.
626 """
627 if visitSummary is not None:
628 row = visitSummary.find(detectorId)
629 if row is None:
630 raise RuntimeError(f"Visit summary for detector {detectorId} is unexpectedly missing.")
631 if (photoCalib := row.getPhotoCalib()) is None:
632 self.log.warning("Detector id %s has None for photoCalib in visit summary; "
633 "skipping reevaluation of photoCalib.", detectorId)
634 exposure.setPhotoCalib(None)
635 else:
636 exposure.setPhotoCalib(photoCalib)
637 if (skyWcs := row.getWcs()) is None:
638 self.log.warning("Detector id %s has None for skyWcs in visit summary; "
639 "skipping reevaluation of skyWcs.", detectorId)
640 exposure.setWcs(None)
641 else:
642 exposure.setWcs(skyWcs)
644 if externalPhotoCalibCatalog is not None:
645 # TODO: remove on DM-39854.
646 warnings.warn(
647 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
648 FutureWarning,
649 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"),
650 )
651 row = externalPhotoCalibCatalog.find(detectorId)
652 if row is None:
653 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; "
654 "Using original photoCalib.", detectorId)
655 else:
656 photoCalib = row.getPhotoCalib()
657 if photoCalib is None:
658 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
659 "Using original photoCalib.", detectorId)
660 else:
661 exposure.setPhotoCalib(photoCalib)
663 if externalSkyWcsCatalog is not None:
664 # TODO: remove on DM-39854.
665 warnings.warn(
666 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
667 FutureWarning,
668 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"),
669 )
670 row = externalSkyWcsCatalog.find(detectorId)
671 if row is None:
672 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; "
673 "Using original skyWcs.", detectorId)
674 else:
675 skyWcs = row.getWcs()
676 if skyWcs is None:
677 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
678 "Using original skyWcs.", detectorId)
679 else:
680 exposure.setWcs(skyWcs)
682 return exposure
684 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs):
685 """Add replace columns with calibs evaluated at each centroid
687 Add or replace 'base_LocalWcs' `base_LocalPhotoCalib' columns in a
688 a source catalog, by rerunning the plugins.
690 Parameters
691 ----------
692 catalog : `lsst.afw.table.SourceCatalog`
693 catalog to which calib columns will be added
694 exposure : `lsst.afw.image.exposure.Exposure`
695 Exposure with attached PhotoCalibs and SkyWcs attributes to be
696 reevaluated at local centroids. Pixels are not required.
697 idGenerator : `lsst.meas.base.IdGenerator`
698 Object that generates Source IDs and random seeds.
699 **kwargs
700 Additional keyword arguments are ignored to facilitate passing the
701 same arguments to several methods.
703 Returns
704 -------
705 newCat: `lsst.afw.table.SourceCatalog`
706 Source Catalog with requested local calib columns
707 """
708 measureConfig = SingleFrameMeasurementTask.ConfigClass()
709 measureConfig.doReplaceWithNoise = False
711 # Clear all slots, because we aren't running the relevant plugins.
712 for slot in measureConfig.slots:
713 setattr(measureConfig.slots, slot, None)
715 measureConfig.plugins.names = []
716 if self.config.doReevaluateSkyWcs:
717 measureConfig.plugins.names.add("base_LocalWcs")
718 self.log.info("Re-evaluating base_LocalWcs plugin")
719 if self.config.doReevaluatePhotoCalib:
720 measureConfig.plugins.names.add("base_LocalPhotoCalib")
721 self.log.info("Re-evaluating base_LocalPhotoCalib plugin")
722 pluginsNotToCopy = tuple(measureConfig.plugins.names)
724 # Create a new schema and catalog
725 # Copy all columns from original except for the ones to reevaluate
726 aliasMap = catalog.schema.getAliasMap()
727 mapper = afwTable.SchemaMapper(catalog.schema)
728 for item in catalog.schema:
729 if not item.field.getName().startswith(pluginsNotToCopy):
730 mapper.addMapping(item.key)
732 schema = mapper.getOutputSchema()
733 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
734 schema.setAliasMap(aliasMap)
735 newCat = afwTable.SourceCatalog(schema)
736 newCat.extend(catalog, mapper=mapper)
738 # Fluxes in sourceCatalogs are in counts, so there are no fluxes to
739 # update here. LocalPhotoCalibs are applied during transform tasks.
740 # Update coord_ra/coord_dec, which are expected to be positions on the
741 # sky and are used as such in sdm tables without transform
742 if self.config.doReevaluateSkyWcs and exposure.wcs is not None:
743 afwTable.updateSourceCoords(exposure.wcs, newCat)
744 wcsPlugin = measurement.plugins["base_LocalWcs"]
745 else:
746 wcsPlugin = None
748 if self.config.doReevaluatePhotoCalib and exposure.getPhotoCalib() is not None:
749 pcPlugin = measurement.plugins["base_LocalPhotoCalib"]
750 else:
751 pcPlugin = None
753 for row in newCat:
754 if wcsPlugin is not None:
755 wcsPlugin.measure(row, exposure)
756 if pcPlugin is not None:
757 pcPlugin.measure(row, exposure)
759 return newCat
762class PostprocessAnalysis(object):
763 """Calculate columns from DataFrames or handles storing DataFrames.
765 This object manages and organizes an arbitrary set of computations
766 on a catalog. The catalog is defined by a
767 `DeferredDatasetHandle` or `InMemoryDatasetHandle` object
768 (or list thereof), such as a ``deepCoadd_obj`` dataset, and the
769 computations are defined by a collection of
770 `~lsst.pipe.tasks.functors.Functor` objects (or, equivalently, a
771 ``CompositeFunctor``).
773 After the object is initialized, accessing the ``.df`` attribute (which
774 holds the `pandas.DataFrame` containing the results of the calculations)
775 triggers computation of said dataframe.
777 One of the conveniences of using this object is the ability to define a
778 desired common filter for all functors. This enables the same functor
779 collection to be passed to several different `PostprocessAnalysis` objects
780 without having to change the original functor collection, since the ``filt``
781 keyword argument of this object triggers an overwrite of the ``filt``
782 property for all functors in the collection.
784 This object also allows a list of refFlags to be passed, and defines a set
785 of default refFlags that are always included even if not requested.
787 If a list of DataFrames or Handles is passed, rather than a single one,
788 then the calculations will be mapped over all the input catalogs. In
789 principle, it should be straightforward to parallelize this activity, but
790 initial tests have failed (see TODO in code comments).
792 Parameters
793 ----------
794 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
795 `~lsst.pipe.base.InMemoryDatasetHandle` or
796 list of these.
797 Source catalog(s) for computation.
798 functors : `list`, `dict`, or `~lsst.pipe.tasks.functors.CompositeFunctor`
799 Computations to do (functors that act on ``handles``).
800 If a dict, the output
801 DataFrame will have columns keyed accordingly.
802 If a list, the column keys will come from the
803 ``.shortname`` attribute of each functor.
805 filt : `str`, optional
806 Filter in which to calculate. If provided,
807 this will overwrite any existing ``.filt`` attribute
808 of the provided functors.
810 flags : `list`, optional
811 List of flags (per-band) to include in output table.
812 Taken from the ``meas`` dataset if applied to a multilevel Object Table.
814 refFlags : `list`, optional
815 List of refFlags (only reference band) to include in output table.
817 forcedFlags : `list`, optional
818 List of flags (per-band) to include in output table.
819 Taken from the ``forced_src`` dataset if applied to a
820 multilevel Object Table. Intended for flags from measurement plugins
821 only run during multi-band forced-photometry.
822 """
823 _defaultRefFlags = []
824 _defaultFuncs = ()
826 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
827 self.handles = handles
828 self.functors = functors
830 self.filt = filt
831 self.flags = list(flags) if flags is not None else []
832 self.forcedFlags = list(forcedFlags) if forcedFlags is not None else []
833 self.refFlags = list(self._defaultRefFlags)
834 if refFlags is not None:
835 self.refFlags += list(refFlags)
837 self._df = None
839 @property
840 def defaultFuncs(self):
841 funcs = dict(self._defaultFuncs)
842 return funcs
844 @property
845 def func(self):
846 additionalFuncs = self.defaultFuncs
847 additionalFuncs.update({flag: Column(flag, dataset='forced_src') for flag in self.forcedFlags})
848 additionalFuncs.update({flag: Column(flag, dataset='ref') for flag in self.refFlags})
849 additionalFuncs.update({flag: Column(flag, dataset='meas') for flag in self.flags})
851 if isinstance(self.functors, CompositeFunctor):
852 func = self.functors
853 else:
854 func = CompositeFunctor(self.functors)
856 func.funcDict.update(additionalFuncs)
857 func.filt = self.filt
859 return func
861 @property
862 def noDupCols(self):
863 return [name for name, func in self.func.funcDict.items() if func.noDup or func.dataset == 'ref']
865 @property
866 def df(self):
867 if self._df is None:
868 self.compute()
869 return self._df
871 def compute(self, dropna=False, pool=None):
872 # map over multiple handles
873 if type(self.handles) in (list, tuple):
874 if pool is None:
875 dflist = [self.func(handle, dropna=dropna) for handle in self.handles]
876 else:
877 # TODO: Figure out why this doesn't work (pyarrow pickling
878 # issues?)
879 dflist = pool.map(functools.partial(self.func, dropna=dropna), self.handles)
880 self._df = pd.concat(dflist)
881 else:
882 self._df = self.func(self.handles, dropna=dropna)
884 return self._df
887class TransformCatalogBaseConnections(pipeBase.PipelineTaskConnections,
888 dimensions=()):
889 """Expected Connections for subclasses of TransformCatalogBaseTask.
891 Must be subclassed.
892 """
893 inputCatalog = connectionTypes.Input(
894 name="",
895 storageClass="DataFrame",
896 )
897 outputCatalog = connectionTypes.Output(
898 name="",
899 storageClass="DataFrame",
900 )
903class TransformCatalogBaseConfig(pipeBase.PipelineTaskConfig,
904 pipelineConnections=TransformCatalogBaseConnections):
905 functorFile = pexConfig.Field(
906 dtype=str,
907 doc="Path to YAML file specifying Science Data Model functors to use "
908 "when copying columns and computing calibrated values.",
909 default=None,
910 optional=True
911 )
912 primaryKey = pexConfig.Field(
913 dtype=str,
914 doc="Name of column to be set as the DataFrame index. If None, the index"
915 "will be named `id`",
916 default=None,
917 optional=True
918 )
919 columnsFromDataId = pexConfig.ListField(
920 dtype=str,
921 default=None,
922 optional=True,
923 doc="Columns to extract from the dataId",
924 )
927class TransformCatalogBaseTask(pipeBase.PipelineTask):
928 """Base class for transforming/standardizing a catalog by applying functors
929 that convert units and apply calibrations.
931 The purpose of this task is to perform a set of computations on an input
932 ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` that holds a
933 ``DataFrame`` dataset (such as ``deepCoadd_obj``), and write the results to
934 a new dataset (which needs to be declared in an ``outputDataset``
935 attribute).
937 The calculations to be performed are defined in a YAML file that specifies
938 a set of functors to be computed, provided as a ``--functorFile`` config
939 parameter. An example of such a YAML file is the following:
941 funcs:
942 sourceId:
943 functor: Index
944 x:
945 functor: Column
946 args: slot_Centroid_x
947 y:
948 functor: Column
949 args: slot_Centroid_y
950 psfFlux:
951 functor: LocalNanojansky
952 args:
953 - slot_PsfFlux_instFlux
954 - slot_PsfFlux_instFluxErr
955 - base_LocalPhotoCalib
956 - base_LocalPhotoCalibErr
957 psfFluxErr:
958 functor: LocalNanojanskyErr
959 args:
960 - slot_PsfFlux_instFlux
961 - slot_PsfFlux_instFluxErr
962 - base_LocalPhotoCalib
963 - base_LocalPhotoCalibErr
964 flags:
965 - detect_isPrimary
967 The names for each entry under "func" will become the names of columns in
968 the output dataset. All the functors referenced are defined in
969 `~lsst.pipe.tasks.functors`. Positional arguments to be passed to each
970 functor are in the `args` list, and any additional entries for each column
971 other than "functor" or "args" (e.g., ``'filt'``, ``'dataset'``) are
972 treated as keyword arguments to be passed to the functor initialization.
974 The "flags" entry is the default shortcut for `Column` functors.
975 All columns listed under "flags" will be copied to the output table
976 untransformed. They can be of any datatype.
977 In the special case of transforming a multi-level oject table with
978 band and dataset indices (deepCoadd_obj), these will be taked from the
979 `meas` dataset and exploded out per band.
981 There are two special shortcuts that only apply when transforming
982 multi-level Object (deepCoadd_obj) tables:
983 - The "refFlags" entry is shortcut for `Column` functor
984 taken from the `'ref'` dataset if transforming an ObjectTable.
985 - The "forcedFlags" entry is shortcut for `Column` functors.
986 taken from the ``forced_src`` dataset if transforming an ObjectTable.
987 These are expanded out per band.
990 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
991 to organize and excecute the calculations.
992 """
993 @property
994 def _DefaultName(self):
995 raise NotImplementedError('Subclass must define "_DefaultName" attribute')
997 @property
998 def outputDataset(self):
999 raise NotImplementedError('Subclass must define "outputDataset" attribute')
1001 @property
1002 def inputDataset(self):
1003 raise NotImplementedError('Subclass must define "inputDataset" attribute')
1005 @property
1006 def ConfigClass(self):
1007 raise NotImplementedError('Subclass must define "ConfigClass" attribute')
1009 def __init__(self, *args, **kwargs):
1010 super().__init__(*args, **kwargs)
1011 if self.config.functorFile:
1012 self.log.info('Loading tranform functor definitions from %s',
1013 self.config.functorFile)
1014 self.funcs = CompositeFunctor.from_file(self.config.functorFile)
1015 self.funcs.update(dict(PostprocessAnalysis._defaultFuncs))
1016 else:
1017 self.funcs = None
1019 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1020 inputs = butlerQC.get(inputRefs)
1021 if self.funcs is None:
1022 raise ValueError("config.functorFile is None. "
1023 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1024 result = self.run(handle=inputs['inputCatalog'], funcs=self.funcs,
1025 dataId=dict(outputRefs.outputCatalog.dataId.mapping))
1026 outputs = pipeBase.Struct(outputCatalog=result)
1027 butlerQC.put(outputs, outputRefs)
1029 def run(self, handle, funcs=None, dataId=None, band=None):
1030 """Do postprocessing calculations
1032 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` or
1033 ``DataFrame`` object and dataId,
1034 returns a dataframe with results of postprocessing calculations.
1036 Parameters
1037 ----------
1038 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
1039 `~lsst.pipe.base.InMemoryDatasetHandle` or
1040 `~pandas.DataFrame`, or list of these.
1041 DataFrames from which calculations are done.
1042 funcs : `~lsst.pipe.tasks.functors.Functor`
1043 Functors to apply to the table's columns
1044 dataId : dict, optional
1045 Used to add a `patchId` column to the output dataframe.
1046 band : `str`, optional
1047 Filter band that is being processed.
1049 Returns
1050 -------
1051 df : `pandas.DataFrame`
1052 """
1053 self.log.info("Transforming/standardizing the source table dataId: %s", dataId)
1055 df = self.transform(band, handle, funcs, dataId).df
1056 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df))
1057 return df
1059 def getFunctors(self):
1060 return self.funcs
1062 def getAnalysis(self, handles, funcs=None, band=None):
1063 if funcs is None:
1064 funcs = self.funcs
1065 analysis = PostprocessAnalysis(handles, funcs, filt=band)
1066 return analysis
1068 def transform(self, band, handles, funcs, dataId):
1069 analysis = self.getAnalysis(handles, funcs=funcs, band=band)
1070 df = analysis.df
1071 if dataId and self.config.columnsFromDataId:
1072 for key in self.config.columnsFromDataId:
1073 if key in dataId:
1074 df[key] = dataId[key]
1075 else:
1076 raise ValueError(f"'{key}' in config.columnsFromDataId not found in dataId: {dataId}")
1078 if self.config.primaryKey:
1079 if df.index.name != self.config.primaryKey and self.config.primaryKey in df:
1080 df.reset_index(inplace=True, drop=True)
1081 df.set_index(self.config.primaryKey, inplace=True)
1083 return pipeBase.Struct(
1084 df=df,
1085 analysis=analysis
1086 )
1089class TransformObjectCatalogConnections(pipeBase.PipelineTaskConnections,
1090 defaultTemplates={"coaddName": "deep"},
1091 dimensions=("tract", "patch", "skymap")):
1092 inputCatalog = connectionTypes.Input(
1093 doc="The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
1094 "stored as a DataFrame with a multi-level column index per-patch.",
1095 dimensions=("tract", "patch", "skymap"),
1096 storageClass="DataFrame",
1097 name="{coaddName}Coadd_obj",
1098 deferLoad=True,
1099 )
1100 outputCatalog = connectionTypes.Output(
1101 doc="Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
1102 "data model.",
1103 dimensions=("tract", "patch", "skymap"),
1104 storageClass="DataFrame",
1105 name="objectTable"
1106 )
1109class TransformObjectCatalogConfig(TransformCatalogBaseConfig,
1110 pipelineConnections=TransformObjectCatalogConnections):
1111 coaddName = pexConfig.Field(
1112 dtype=str,
1113 default="deep",
1114 doc="Name of coadd"
1115 )
1116 # TODO: remove in DM-27177
1117 filterMap = pexConfig.DictField(
1118 keytype=str,
1119 itemtype=str,
1120 default={},
1121 doc=("Dictionary mapping full filter name to short one for column name munging."
1122 "These filters determine the output columns no matter what filters the "
1123 "input data actually contain."),
1124 deprecated=("Coadds are now identified by the band, so this transform is unused."
1125 "Will be removed after v22.")
1126 )
1127 outputBands = pexConfig.ListField(
1128 dtype=str,
1129 default=None,
1130 optional=True,
1131 doc=("These bands and only these bands will appear in the output,"
1132 " NaN-filled if the input does not include them."
1133 " If None, then use all bands found in the input.")
1134 )
1135 camelCase = pexConfig.Field(
1136 dtype=bool,
1137 default=False,
1138 doc=("Write per-band columns names with camelCase, else underscore "
1139 "For example: gPsFlux instead of g_PsFlux.")
1140 )
1141 multilevelOutput = pexConfig.Field(
1142 dtype=bool,
1143 default=False,
1144 doc=("Whether results dataframe should have a multilevel column index (True) or be flat "
1145 "and name-munged (False).")
1146 )
1147 goodFlags = pexConfig.ListField(
1148 dtype=str,
1149 default=[],
1150 doc=("List of 'good' flags that should be set False when populating empty tables. "
1151 "All other flags are considered to be 'bad' flags and will be set to True.")
1152 )
1153 floatFillValue = pexConfig.Field(
1154 dtype=float,
1155 default=np.nan,
1156 doc="Fill value for float fields when populating empty tables."
1157 )
1158 integerFillValue = pexConfig.Field(
1159 dtype=int,
1160 default=-1,
1161 doc="Fill value for integer fields when populating empty tables."
1162 )
1164 def setDefaults(self):
1165 super().setDefaults()
1166 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Object.yaml')
1167 self.primaryKey = 'objectId'
1168 self.columnsFromDataId = ['tract', 'patch']
1169 self.goodFlags = ['calib_astrometry_used',
1170 'calib_photometry_reserved',
1171 'calib_photometry_used',
1172 'calib_psf_candidate',
1173 'calib_psf_reserved',
1174 'calib_psf_used']
1177class TransformObjectCatalogTask(TransformCatalogBaseTask):
1178 """Produce a flattened Object Table to match the format specified in
1179 sdm_schemas.
1181 Do the same set of postprocessing calculations on all bands.
1183 This is identical to `TransformCatalogBaseTask`, except for that it does
1184 the specified functor calculations for all filters present in the
1185 input `deepCoadd_obj` table. Any specific ``"filt"`` keywords specified
1186 by the YAML file will be superceded.
1187 """
1188 _DefaultName = "transformObjectCatalog"
1189 ConfigClass = TransformObjectCatalogConfig
1191 def run(self, handle, funcs=None, dataId=None, band=None):
1192 # NOTE: band kwarg is ignored here.
1193 dfDict = {}
1194 analysisDict = {}
1195 templateDf = pd.DataFrame()
1197 columns = handle.get(component='columns')
1198 inputBands = columns.unique(level=1).values
1200 outputBands = self.config.outputBands if self.config.outputBands else inputBands
1202 # Perform transform for data of filters that exist in the handle dataframe.
1203 for inputBand in inputBands:
1204 if inputBand not in outputBands:
1205 self.log.info("Ignoring %s band data in the input", inputBand)
1206 continue
1207 self.log.info("Transforming the catalog of band %s", inputBand)
1208 result = self.transform(inputBand, handle, funcs, dataId)
1209 dfDict[inputBand] = result.df
1210 analysisDict[inputBand] = result.analysis
1211 if templateDf.empty:
1212 templateDf = result.df
1214 # Put filler values in columns of other wanted bands
1215 for filt in outputBands:
1216 if filt not in dfDict:
1217 self.log.info("Adding empty columns for band %s", filt)
1218 dfTemp = templateDf.copy()
1219 for col in dfTemp.columns:
1220 testValue = dfTemp[col].values[0]
1221 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1222 # Boolean flag type, check if it is a "good" flag
1223 if col in self.config.goodFlags:
1224 fillValue = False
1225 else:
1226 fillValue = True
1227 elif isinstance(testValue, numbers.Integral):
1228 # Checking numbers.Integral catches all flavors
1229 # of python, numpy, pandas, etc. integers.
1230 # We must ensure this is not an unsigned integer.
1231 if isinstance(testValue, np.unsignedinteger):
1232 raise ValueError("Parquet tables may not have unsigned integer columns.")
1233 else:
1234 fillValue = self.config.integerFillValue
1235 else:
1236 fillValue = self.config.floatFillValue
1237 dfTemp[col].values[:] = fillValue
1238 dfDict[filt] = dfTemp
1240 # This makes a multilevel column index, with band as first level
1241 df = pd.concat(dfDict, axis=1, names=['band', 'column'])
1243 if not self.config.multilevelOutput:
1244 noDupCols = list(set.union(*[set(v.noDupCols) for v in analysisDict.values()]))
1245 if self.config.primaryKey in noDupCols:
1246 noDupCols.remove(self.config.primaryKey)
1247 if dataId and self.config.columnsFromDataId:
1248 noDupCols += self.config.columnsFromDataId
1249 df = flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1250 inputBands=inputBands)
1252 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df))
1254 return df
1257class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1258 dimensions=("tract", "skymap")):
1259 inputCatalogs = connectionTypes.Input(
1260 doc="Per-Patch objectTables conforming to the standard data model.",
1261 name="objectTable",
1262 storageClass="DataFrame",
1263 dimensions=("tract", "patch", "skymap"),
1264 multiple=True,
1265 )
1266 outputCatalog = connectionTypes.Output(
1267 doc="Pre-tract horizontal concatenation of the input objectTables",
1268 name="objectTable_tract",
1269 storageClass="DataFrame",
1270 dimensions=("tract", "skymap"),
1271 )
1274class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1275 pipelineConnections=ConsolidateObjectTableConnections):
1276 coaddName = pexConfig.Field(
1277 dtype=str,
1278 default="deep",
1279 doc="Name of coadd"
1280 )
1283class ConsolidateObjectTableTask(pipeBase.PipelineTask):
1284 """Write patch-merged source tables to a tract-level DataFrame Parquet file.
1286 Concatenates `objectTable` list into a per-visit `objectTable_tract`.
1287 """
1288 _DefaultName = "consolidateObjectTable"
1289 ConfigClass = ConsolidateObjectTableConfig
1291 inputDataset = 'objectTable'
1292 outputDataset = 'objectTable_tract'
1294 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1295 inputs = butlerQC.get(inputRefs)
1296 self.log.info("Concatenating %s per-patch Object Tables",
1297 len(inputs['inputCatalogs']))
1298 df = pd.concat(inputs['inputCatalogs'])
1299 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1302class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1303 defaultTemplates={"catalogType": ""},
1304 dimensions=("instrument", "visit", "detector")):
1306 inputCatalog = connectionTypes.Input(
1307 doc="Wide input catalog of sources produced by WriteSourceTableTask",
1308 name="{catalogType}source",
1309 storageClass="DataFrame",
1310 dimensions=("instrument", "visit", "detector"),
1311 deferLoad=True
1312 )
1313 outputCatalog = connectionTypes.Output(
1314 doc="Narrower, per-detector Source Table transformed and converted per a "
1315 "specified set of functors",
1316 name="{catalogType}sourceTable",
1317 storageClass="DataFrame",
1318 dimensions=("instrument", "visit", "detector")
1319 )
1322class TransformSourceTableConfig(TransformCatalogBaseConfig,
1323 pipelineConnections=TransformSourceTableConnections):
1325 def setDefaults(self):
1326 super().setDefaults()
1327 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Source.yaml')
1328 self.primaryKey = 'sourceId'
1329 self.columnsFromDataId = ['visit', 'detector', 'band', 'physical_filter']
1332class TransformSourceTableTask(TransformCatalogBaseTask):
1333 """Transform/standardize a source catalog
1334 """
1335 _DefaultName = "transformSourceTable"
1336 ConfigClass = TransformSourceTableConfig
1339class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1340 dimensions=("instrument", "visit",),
1341 defaultTemplates={"calexpType": ""}):
1342 calexp = connectionTypes.Input(
1343 doc="Processed exposures used for metadata",
1344 name="calexp",
1345 storageClass="ExposureF",
1346 dimensions=("instrument", "visit", "detector"),
1347 deferLoad=True,
1348 multiple=True,
1349 )
1350 visitSummary = connectionTypes.Output(
1351 doc=("Per-visit consolidated exposure metadata. These catalogs use "
1352 "detector id for the id and are sorted for fast lookups of a "
1353 "detector."),
1354 name="visitSummary",
1355 storageClass="ExposureCatalog",
1356 dimensions=("instrument", "visit"),
1357 )
1358 visitSummarySchema = connectionTypes.InitOutput(
1359 doc="Schema of the visitSummary catalog",
1360 name="visitSummary_schema",
1361 storageClass="ExposureCatalog",
1362 )
1365class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1366 pipelineConnections=ConsolidateVisitSummaryConnections):
1367 """Config for ConsolidateVisitSummaryTask"""
1368 pass
1371class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
1372 """Task to consolidate per-detector visit metadata.
1374 This task aggregates the following metadata from all the detectors in a
1375 single visit into an exposure catalog:
1376 - The visitInfo.
1377 - The wcs.
1378 - The photoCalib.
1379 - The physical_filter and band (if available).
1380 - The psf size, shape, and effective area at the center of the detector.
1381 - The corners of the bounding box in right ascension/declination.
1383 Other quantities such as Detector, Psf, ApCorrMap, and TransmissionCurve
1384 are not persisted here because of storage concerns, and because of their
1385 limited utility as summary statistics.
1387 Tests for this task are performed in ci_hsc_gen3.
1388 """
1389 _DefaultName = "consolidateVisitSummary"
1390 ConfigClass = ConsolidateVisitSummaryConfig
1392 def __init__(self, **kwargs):
1393 super().__init__(**kwargs)
1394 self.schema = afwTable.ExposureTable.makeMinimalSchema()
1395 self.schema.addField('visit', type='L', doc='Visit number')
1396 self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter')
1397 self.schema.addField('band', type='String', size=32, doc='Name of band')
1398 ExposureSummaryStats.update_schema(self.schema)
1399 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
1401 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1402 dataRefs = butlerQC.get(inputRefs.calexp)
1403 visit = dataRefs[0].dataId['visit']
1405 self.log.debug("Concatenating metadata from %d per-detector calexps (visit %d)",
1406 len(dataRefs), visit)
1408 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1410 butlerQC.put(expCatalog, outputRefs.visitSummary)
1412 def _combineExposureMetadata(self, visit, dataRefs):
1413 """Make a combined exposure catalog from a list of dataRefs.
1414 These dataRefs must point to exposures with wcs, summaryStats,
1415 and other visit metadata.
1417 Parameters
1418 ----------
1419 visit : `int`
1420 Visit identification number.
1421 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1422 List of dataRefs in visit.
1424 Returns
1425 -------
1426 visitSummary : `lsst.afw.table.ExposureCatalog`
1427 Exposure catalog with per-detector summary information.
1428 """
1429 cat = afwTable.ExposureCatalog(self.schema)
1430 cat.resize(len(dataRefs))
1432 cat['visit'] = visit
1434 for i, dataRef in enumerate(dataRefs):
1435 visitInfo = dataRef.get(component='visitInfo')
1436 filterLabel = dataRef.get(component='filter')
1437 summaryStats = dataRef.get(component='summaryStats')
1438 detector = dataRef.get(component='detector')
1439 wcs = dataRef.get(component='wcs')
1440 photoCalib = dataRef.get(component='photoCalib')
1441 detector = dataRef.get(component='detector')
1442 bbox = dataRef.get(component='bbox')
1443 validPolygon = dataRef.get(component='validPolygon')
1445 rec = cat[i]
1446 rec.setBBox(bbox)
1447 rec.setVisitInfo(visitInfo)
1448 rec.setWcs(wcs)
1449 rec.setPhotoCalib(photoCalib)
1450 rec.setValidPolygon(validPolygon)
1452 rec['physical_filter'] = filterLabel.physicalLabel if filterLabel.hasPhysicalLabel() else ""
1453 rec['band'] = filterLabel.bandLabel if filterLabel.hasBandLabel() else ""
1454 rec.setId(detector.getId())
1455 summaryStats.update_record(rec)
1457 metadata = dafBase.PropertyList()
1458 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1459 # We are looping over existing datarefs, so the following is true
1460 metadata.add("COMMENT", "Only detectors with data have entries.")
1461 cat.setMetadata(metadata)
1463 cat.sort()
1464 return cat
1467class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1468 defaultTemplates={"catalogType": ""},
1469 dimensions=("instrument", "visit")):
1470 inputCatalogs = connectionTypes.Input(
1471 doc="Input per-detector Source Tables",
1472 name="{catalogType}sourceTable",
1473 storageClass="DataFrame",
1474 dimensions=("instrument", "visit", "detector"),
1475 multiple=True
1476 )
1477 outputCatalog = connectionTypes.Output(
1478 doc="Per-visit concatenation of Source Table",
1479 name="{catalogType}sourceTable_visit",
1480 storageClass="DataFrame",
1481 dimensions=("instrument", "visit")
1482 )
1485class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1486 pipelineConnections=ConsolidateSourceTableConnections):
1487 pass
1490class ConsolidateSourceTableTask(pipeBase.PipelineTask):
1491 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1492 """
1493 _DefaultName = 'consolidateSourceTable'
1494 ConfigClass = ConsolidateSourceTableConfig
1496 inputDataset = 'sourceTable'
1497 outputDataset = 'sourceTable_visit'
1499 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1500 from .makeWarp import reorderRefs
1502 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.inputCatalogs]
1503 detectorOrder.sort()
1504 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
1505 inputs = butlerQC.get(inputRefs)
1506 self.log.info("Concatenating %s per-detector Source Tables",
1507 len(inputs['inputCatalogs']))
1508 df = pd.concat(inputs['inputCatalogs'])
1509 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1512class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1513 dimensions=("instrument",),
1514 defaultTemplates={"calexpType": ""}):
1515 visitSummaryRefs = connectionTypes.Input(
1516 doc="Data references for per-visit consolidated exposure metadata",
1517 name="finalVisitSummary",
1518 storageClass="ExposureCatalog",
1519 dimensions=("instrument", "visit"),
1520 multiple=True,
1521 deferLoad=True,
1522 )
1523 outputCatalog = connectionTypes.Output(
1524 doc="CCD and Visit metadata table",
1525 name="ccdVisitTable",
1526 storageClass="DataFrame",
1527 dimensions=("instrument",)
1528 )
1531class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1532 pipelineConnections=MakeCcdVisitTableConnections):
1533 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1536class MakeCcdVisitTableTask(pipeBase.PipelineTask):
1537 """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
1538 """
1539 _DefaultName = 'makeCcdVisitTable'
1540 ConfigClass = MakeCcdVisitTableConfig
1542 def run(self, visitSummaryRefs):
1543 """Make a table of ccd information from the visit summary catalogs.
1545 Parameters
1546 ----------
1547 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1548 List of DeferredDatasetHandles pointing to exposure catalogs with
1549 per-detector summary information.
1551 Returns
1552 -------
1553 result : `~lsst.pipe.base.Struct`
1554 Results struct with attribute:
1556 ``outputCatalog``
1557 Catalog of ccd and visit information.
1558 """
1559 ccdEntries = []
1560 for visitSummaryRef in visitSummaryRefs:
1561 visitSummary = visitSummaryRef.get()
1562 visitInfo = visitSummary[0].getVisitInfo()
1564 ccdEntry = {}
1565 summaryTable = visitSummary.asAstropy()
1566 selectColumns = ['id', 'visit', 'physical_filter', 'band', 'ra', 'dec', 'zenithDistance',
1567 'zeroPoint', 'psfSigma', 'skyBg', 'skyNoise',
1568 'astromOffsetMean', 'astromOffsetStd', 'nPsfStar',
1569 'psfStarDeltaE1Median', 'psfStarDeltaE2Median',
1570 'psfStarDeltaE1Scatter', 'psfStarDeltaE2Scatter',
1571 'psfStarDeltaSizeMedian', 'psfStarDeltaSizeScatter',
1572 'psfStarScaledDeltaSizeScatter',
1573 'psfTraceRadiusDelta', 'maxDistToNearestPsf',
1574 'effTime', 'effTimePsfSigmaScale',
1575 'effTimeSkyBgScale', 'effTimeZeroPointScale']
1576 ccdEntry = summaryTable[selectColumns].to_pandas().set_index('id')
1577 # 'visit' is the human readable visit number.
1578 # 'visitId' is the key to the visitId table. They are the same.
1579 # Technically you should join to get the visit from the visit
1580 # table.
1581 ccdEntry = ccdEntry.rename(columns={"visit": "visitId"})
1583 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards
1584 # compatibility. To be removed after September 2023.
1585 ccdEntry["decl"] = ccdEntry.loc[:, "dec"]
1587 ccdEntry['ccdVisitId'] = [
1588 self.config.idGenerator.apply(
1589 visitSummaryRef.dataId,
1590 detector=detector_id,
1591 is_exposure=False,
1592 ).catalog_id # The "catalog ID" here is the ccdVisit ID
1593 # because it's usually the ID for a whole catalog
1594 # with a {visit, detector}, and that's the main
1595 # use case for IdGenerator. This usage for a
1596 # summary table is rare.
1597 for detector_id in summaryTable['id']
1598 ]
1599 ccdEntry['detector'] = summaryTable['id']
1600 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds() if vR.getWcs()
1601 else np.nan for vR in visitSummary])
1602 ccdEntry["seeing"] = visitSummary['psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1604 ccdEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1605 ccdEntry["expMidpt"] = visitInfo.getDate().toPython()
1606 ccdEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1607 expTime = visitInfo.getExposureTime()
1608 ccdEntry['expTime'] = expTime
1609 ccdEntry["obsStart"] = ccdEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1610 expTime_days = expTime / (60*60*24)
1611 ccdEntry["obsStartMJD"] = ccdEntry["expMidptMJD"] - 0.5 * expTime_days
1612 ccdEntry['darkTime'] = visitInfo.getDarkTime()
1613 ccdEntry['xSize'] = summaryTable['bbox_max_x'] - summaryTable['bbox_min_x']
1614 ccdEntry['ySize'] = summaryTable['bbox_max_y'] - summaryTable['bbox_min_y']
1615 ccdEntry['llcra'] = summaryTable['raCorners'][:, 0]
1616 ccdEntry['llcdec'] = summaryTable['decCorners'][:, 0]
1617 ccdEntry['ulcra'] = summaryTable['raCorners'][:, 1]
1618 ccdEntry['ulcdec'] = summaryTable['decCorners'][:, 1]
1619 ccdEntry['urcra'] = summaryTable['raCorners'][:, 2]
1620 ccdEntry['urcdec'] = summaryTable['decCorners'][:, 2]
1621 ccdEntry['lrcra'] = summaryTable['raCorners'][:, 3]
1622 ccdEntry['lrcdec'] = summaryTable['decCorners'][:, 3]
1623 # TODO: DM-30618, Add raftName, nExposures, ccdTemp, binX, binY,
1624 # and flags, and decide if WCS, and llcx, llcy, ulcx, ulcy, etc.
1625 # values are actually wanted.
1626 ccdEntries.append(ccdEntry)
1628 outputCatalog = pd.concat(ccdEntries)
1629 outputCatalog.set_index('ccdVisitId', inplace=True, verify_integrity=True)
1630 return pipeBase.Struct(outputCatalog=outputCatalog)
1633class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1634 dimensions=("instrument",),
1635 defaultTemplates={"calexpType": ""}):
1636 visitSummaries = connectionTypes.Input(
1637 doc="Per-visit consolidated exposure metadata",
1638 name="finalVisitSummary",
1639 storageClass="ExposureCatalog",
1640 dimensions=("instrument", "visit",),
1641 multiple=True,
1642 deferLoad=True,
1643 )
1644 outputCatalog = connectionTypes.Output(
1645 doc="Visit metadata table",
1646 name="visitTable",
1647 storageClass="DataFrame",
1648 dimensions=("instrument",)
1649 )
1652class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1653 pipelineConnections=MakeVisitTableConnections):
1654 pass
1657class MakeVisitTableTask(pipeBase.PipelineTask):
1658 """Produce a `visitTable` from the visit summary exposure catalogs.
1659 """
1660 _DefaultName = 'makeVisitTable'
1661 ConfigClass = MakeVisitTableConfig
1663 def run(self, visitSummaries):
1664 """Make a table of visit information from the visit summary catalogs.
1666 Parameters
1667 ----------
1668 visitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
1669 List of exposure catalogs with per-detector summary information.
1670 Returns
1671 -------
1672 result : `~lsst.pipe.base.Struct`
1673 Results struct with attribute:
1675 ``outputCatalog``
1676 Catalog of visit information.
1677 """
1678 visitEntries = []
1679 for visitSummary in visitSummaries:
1680 visitSummary = visitSummary.get()
1681 visitRow = visitSummary[0]
1682 visitInfo = visitRow.getVisitInfo()
1684 visitEntry = {}
1685 visitEntry["visitId"] = visitRow['visit']
1686 visitEntry["visit"] = visitRow['visit']
1687 visitEntry["physical_filter"] = visitRow['physical_filter']
1688 visitEntry["band"] = visitRow['band']
1689 raDec = visitInfo.getBoresightRaDec()
1690 visitEntry["ra"] = raDec.getRa().asDegrees()
1691 visitEntry["dec"] = raDec.getDec().asDegrees()
1693 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards
1694 # compatibility. To be removed after September 2023.
1695 visitEntry["decl"] = visitEntry["dec"]
1697 visitEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1698 azAlt = visitInfo.getBoresightAzAlt()
1699 visitEntry["azimuth"] = azAlt.getLongitude().asDegrees()
1700 visitEntry["altitude"] = azAlt.getLatitude().asDegrees()
1701 visitEntry["zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1702 visitEntry["airmass"] = visitInfo.getBoresightAirmass()
1703 expTime = visitInfo.getExposureTime()
1704 visitEntry["expTime"] = expTime
1705 visitEntry["expMidpt"] = visitInfo.getDate().toPython()
1706 visitEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1707 visitEntry["obsStart"] = visitEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1708 expTime_days = expTime / (60*60*24)
1709 visitEntry["obsStartMJD"] = visitEntry["expMidptMJD"] - 0.5 * expTime_days
1710 visitEntries.append(visitEntry)
1712 # TODO: DM-30623, Add programId, exposureType, cameraTemp,
1713 # mirror1Temp, mirror2Temp, mirror3Temp, domeTemp, externalTemp,
1714 # dimmSeeing, pwvGPS, pwvMW, flags, nExposures.
1716 outputCatalog = pd.DataFrame(data=visitEntries)
1717 outputCatalog.set_index('visitId', inplace=True, verify_integrity=True)
1718 return pipeBase.Struct(outputCatalog=outputCatalog)
1721class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1722 dimensions=("instrument", "visit", "detector", "skymap", "tract")):
1724 inputCatalog = connectionTypes.Input(
1725 doc="Primary per-detector, single-epoch forced-photometry catalog. "
1726 "By default, it is the output of ForcedPhotCcdTask on calexps",
1727 name="forced_src",
1728 storageClass="SourceCatalog",
1729 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1730 )
1731 inputCatalogDiff = connectionTypes.Input(
1732 doc="Secondary multi-epoch, per-detector, forced photometry catalog. "
1733 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1734 name="forced_diff",
1735 storageClass="SourceCatalog",
1736 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1737 )
1738 outputCatalog = connectionTypes.Output(
1739 doc="InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format",
1740 name="mergedForcedSource",
1741 storageClass="DataFrame",
1742 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1743 )
1746class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1747 pipelineConnections=WriteForcedSourceTableConnections):
1748 key = lsst.pex.config.Field(
1749 doc="Column on which to join the two input tables on and make the primary key of the output",
1750 dtype=str,
1751 default="objectId",
1752 )
1753 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1756class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1757 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format.
1759 Because the predecessor ForcedPhotCcdTask operates per-detector,
1760 per-tract, (i.e., it has tract in its dimensions), detectors
1761 on the tract boundary may have multiple forced source catalogs.
1763 The successor task TransformForcedSourceTable runs per-patch
1764 and temporally-aggregates overlapping mergedForcedSource catalogs from all
1765 available multiple epochs.
1766 """
1767 _DefaultName = "writeForcedSourceTable"
1768 ConfigClass = WriteForcedSourceTableConfig
1770 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1771 inputs = butlerQC.get(inputRefs)
1772 # Add ccdVisitId to allow joining with CcdVisitTable
1773 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
1774 inputs['ccdVisitId'] = idGenerator.catalog_id
1775 inputs['band'] = butlerQC.quantum.dataId['band']
1776 outputs = self.run(**inputs)
1777 butlerQC.put(outputs, outputRefs)
1779 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1780 dfs = []
1781 for table, dataset, in zip((inputCatalog, inputCatalogDiff), ('calexp', 'diff')):
1782 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=False)
1783 df = df.reindex(sorted(df.columns), axis=1)
1784 df['ccdVisitId'] = ccdVisitId if ccdVisitId else pd.NA
1785 df['band'] = band if band else pd.NA
1786 df.columns = pd.MultiIndex.from_tuples([(dataset, c) for c in df.columns],
1787 names=('dataset', 'column'))
1789 dfs.append(df)
1791 outputCatalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs)
1792 return pipeBase.Struct(outputCatalog=outputCatalog)
1795class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1796 dimensions=("instrument", "skymap", "patch", "tract")):
1798 inputCatalogs = connectionTypes.Input(
1799 doc="DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask",
1800 name="mergedForcedSource",
1801 storageClass="DataFrame",
1802 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
1803 multiple=True,
1804 deferLoad=True
1805 )
1806 referenceCatalog = connectionTypes.Input(
1807 doc="Reference catalog which was used to seed the forcedPhot. Columns "
1808 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1809 "are expected.",
1810 name="objectTable",
1811 storageClass="DataFrame",
1812 dimensions=("tract", "patch", "skymap"),
1813 deferLoad=True
1814 )
1815 outputCatalog = connectionTypes.Output(
1816 doc="Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1817 "specified set of functors",
1818 name="forcedSourceTable",
1819 storageClass="DataFrame",
1820 dimensions=("tract", "patch", "skymap")
1821 )
1824class TransformForcedSourceTableConfig(TransformCatalogBaseConfig,
1825 pipelineConnections=TransformForcedSourceTableConnections):
1826 referenceColumns = pexConfig.ListField(
1827 dtype=str,
1828 default=["detect_isPrimary", "detect_isTractInner", "detect_isPatchInner"],
1829 optional=True,
1830 doc="Columns to pull from reference catalog",
1831 )
1832 keyRef = lsst.pex.config.Field(
1833 doc="Column on which to join the two input tables on and make the primary key of the output",
1834 dtype=str,
1835 default="objectId",
1836 )
1837 key = lsst.pex.config.Field(
1838 doc="Rename the output DataFrame index to this name",
1839 dtype=str,
1840 default="forcedSourceId",
1841 )
1843 def setDefaults(self):
1844 super().setDefaults()
1845 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'ForcedSource.yaml')
1846 self.columnsFromDataId = ['tract', 'patch']
1849class TransformForcedSourceTableTask(TransformCatalogBaseTask):
1850 """Transform/standardize a ForcedSource catalog
1852 Transforms each wide, per-detector forcedSource DataFrame per the
1853 specification file (per-camera defaults found in ForcedSource.yaml).
1854 All epochs that overlap the patch are aggregated into one per-patch
1855 narrow-DataFrame file.
1857 No de-duplication of rows is performed. Duplicate resolutions flags are
1858 pulled in from the referenceCatalog: `detect_isPrimary`,
1859 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
1860 for analysis or compare duplicates for QA.
1862 The resulting table includes multiple bands. Epochs (MJDs) and other useful
1863 per-visit rows can be retreived by joining with the CcdVisitTable on
1864 ccdVisitId.
1865 """
1866 _DefaultName = "transformForcedSourceTable"
1867 ConfigClass = TransformForcedSourceTableConfig
1869 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1870 inputs = butlerQC.get(inputRefs)
1871 if self.funcs is None:
1872 raise ValueError("config.functorFile is None. "
1873 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1874 outputs = self.run(inputs['inputCatalogs'], inputs['referenceCatalog'], funcs=self.funcs,
1875 dataId=dict(outputRefs.outputCatalog.dataId.mapping))
1877 butlerQC.put(outputs, outputRefs)
1879 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
1880 dfs = []
1881 ref = referenceCatalog.get(parameters={"columns": self.config.referenceColumns})
1882 self.log.info("Aggregating %s input catalogs" % (len(inputCatalogs)))
1883 for handle in inputCatalogs:
1884 result = self.transform(None, handle, funcs, dataId)
1885 # Filter for only rows that were detected on (overlap) the patch
1886 dfs.append(result.df.join(ref, how='inner'))
1888 outputCatalog = pd.concat(dfs)
1890 # Now that we are done joining on config.keyRef
1891 # Change index to config.key by
1892 outputCatalog.index.rename(self.config.keyRef, inplace=True)
1893 # Add config.keyRef to the column list
1894 outputCatalog.reset_index(inplace=True)
1895 # Set the forcedSourceId to the index. This is specified in the
1896 # ForcedSource.yaml
1897 outputCatalog.set_index("forcedSourceId", inplace=True, verify_integrity=True)
1898 # Rename it to the config.key
1899 outputCatalog.index.rename(self.config.key, inplace=True)
1901 self.log.info("Made a table of %d columns and %d rows",
1902 len(outputCatalog.columns), len(outputCatalog))
1903 return pipeBase.Struct(outputCatalog=outputCatalog)
1906class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
1907 defaultTemplates={"catalogType": ""},
1908 dimensions=("instrument", "tract")):
1909 inputCatalogs = connectionTypes.Input(
1910 doc="Input per-patch DataFrame Tables to be concatenated",
1911 name="{catalogType}ForcedSourceTable",
1912 storageClass="DataFrame",
1913 dimensions=("tract", "patch", "skymap"),
1914 multiple=True,
1915 )
1917 outputCatalog = connectionTypes.Output(
1918 doc="Output per-tract concatenation of DataFrame Tables",
1919 name="{catalogType}ForcedSourceTable_tract",
1920 storageClass="DataFrame",
1921 dimensions=("tract", "skymap"),
1922 )
1925class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
1926 pipelineConnections=ConsolidateTractConnections):
1927 pass
1930class ConsolidateTractTask(pipeBase.PipelineTask):
1931 """Concatenate any per-patch, dataframe list into a single
1932 per-tract DataFrame.
1933 """
1934 _DefaultName = 'ConsolidateTract'
1935 ConfigClass = ConsolidateTractConfig
1937 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1938 inputs = butlerQC.get(inputRefs)
1939 # Not checking at least one inputCatalog exists because that'd be an
1940 # empty QG.
1941 self.log.info("Concatenating %s per-patch %s Tables",
1942 len(inputs['inputCatalogs']),
1943 inputRefs.inputCatalogs[0].datasetType.name)
1944 df = pd.concat(inputs['inputCatalogs'])
1945 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)