Coverage for python/lsst/pipe/tasks/postprocess.py: 28%
675 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-06 03:31 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-06 03:31 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["WriteObjectTableConfig", "WriteObjectTableTask",
23 "WriteSourceTableConfig", "WriteSourceTableTask",
24 "WriteRecalibratedSourceTableConfig", "WriteRecalibratedSourceTableTask",
25 "PostprocessAnalysis",
26 "TransformCatalogBaseConfig", "TransformCatalogBaseTask",
27 "TransformObjectCatalogConfig", "TransformObjectCatalogTask",
28 "ConsolidateObjectTableConfig", "ConsolidateObjectTableTask",
29 "TransformSourceTableConfig", "TransformSourceTableTask",
30 "ConsolidateVisitSummaryConfig", "ConsolidateVisitSummaryTask",
31 "ConsolidateSourceTableConfig", "ConsolidateSourceTableTask",
32 "MakeCcdVisitTableConfig", "MakeCcdVisitTableTask",
33 "MakeVisitTableConfig", "MakeVisitTableTask",
34 "WriteForcedSourceTableConfig", "WriteForcedSourceTableTask",
35 "TransformForcedSourceTableConfig", "TransformForcedSourceTableTask",
36 "ConsolidateTractConfig", "ConsolidateTractTask"]
38import functools
39import pandas as pd
40import logging
41import numpy as np
42import numbers
43import os
44import warnings
46from deprecated.sphinx import deprecated
48import lsst.geom
49import lsst.pex.config as pexConfig
50import lsst.pipe.base as pipeBase
51import lsst.daf.base as dafBase
52from lsst.utils.introspection import find_outside_stacklevel
53from lsst.pipe.base import connectionTypes
54import lsst.afw.table as afwTable
55from lsst.afw.image import ExposureSummaryStats
56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig
57from lsst.skymap import BaseSkyMap
59from .functors import CompositeFunctor, Column
61log = logging.getLogger(__name__)
64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
65 """Flattens a dataframe with multilevel column index.
66 """
67 newDf = pd.DataFrame()
68 # band is the level 0 index
69 dfBands = df.columns.unique(level=0).values
70 for band in dfBands:
71 subdf = df[band]
72 columnFormat = '{0}{1}' if camelCase else '{0}_{1}'
73 newColumns = {c: columnFormat.format(band, c)
74 for c in subdf.columns if c not in noDupCols}
75 cols = list(newColumns.keys())
76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
78 # Band must be present in the input and output or else column is all NaN:
79 presentBands = dfBands if inputBands is None else list(set(inputBands).intersection(dfBands))
80 # Get the unexploded columns from any present band's partition
81 noDupDf = df[presentBands[0]][noDupCols]
82 newDf = pd.concat([noDupDf, newDf], axis=1)
83 return newDf
86class WriteObjectTableConnections(pipeBase.PipelineTaskConnections,
87 defaultTemplates={"coaddName": "deep"},
88 dimensions=("tract", "patch", "skymap")):
89 inputCatalogMeas = connectionTypes.Input(
90 doc="Catalog of source measurements on the deepCoadd.",
91 dimensions=("tract", "patch", "band", "skymap"),
92 storageClass="SourceCatalog",
93 name="{coaddName}Coadd_meas",
94 multiple=True
95 )
96 inputCatalogForcedSrc = connectionTypes.Input(
97 doc="Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
98 dimensions=("tract", "patch", "band", "skymap"),
99 storageClass="SourceCatalog",
100 name="{coaddName}Coadd_forced_src",
101 multiple=True
102 )
103 inputCatalogRef = connectionTypes.Input(
104 doc="Catalog marking the primary detection (which band provides a good shape and position)"
105 "for each detection in deepCoadd_mergeDet.",
106 dimensions=("tract", "patch", "skymap"),
107 storageClass="SourceCatalog",
108 name="{coaddName}Coadd_ref"
109 )
110 outputCatalog = connectionTypes.Output(
111 doc="A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
112 "stored as a DataFrame with a multi-level column index per-patch.",
113 dimensions=("tract", "patch", "skymap"),
114 storageClass="DataFrame",
115 name="{coaddName}Coadd_obj"
116 )
119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
120 pipelineConnections=WriteObjectTableConnections):
121 engine = pexConfig.Field(
122 dtype=str,
123 default="pyarrow",
124 doc="Parquet engine for writing (pyarrow or fastparquet)",
125 deprecated="This config is no longer used, and will be removed after v26."
126 )
127 coaddName = pexConfig.Field(
128 dtype=str,
129 default="deep",
130 doc="Name of coadd"
131 )
134class WriteObjectTableTask(pipeBase.PipelineTask):
135 """Write filter-merged source tables as a DataFrame in parquet format.
136 """
137 _DefaultName = "writeObjectTable"
138 ConfigClass = WriteObjectTableConfig
140 # Names of table datasets to be merged
141 inputDatasets = ('forced_src', 'meas', 'ref')
143 # Tag of output dataset written by `MergeSourcesTask.write`
144 outputDataset = 'obj'
146 def runQuantum(self, butlerQC, inputRefs, outputRefs):
147 inputs = butlerQC.get(inputRefs)
149 measDict = {ref.dataId['band']: {'meas': cat} for ref, cat in
150 zip(inputRefs.inputCatalogMeas, inputs['inputCatalogMeas'])}
151 forcedSourceDict = {ref.dataId['band']: {'forced_src': cat} for ref, cat in
152 zip(inputRefs.inputCatalogForcedSrc, inputs['inputCatalogForcedSrc'])}
154 catalogs = {}
155 for band in measDict.keys():
156 catalogs[band] = {'meas': measDict[band]['meas'],
157 'forced_src': forcedSourceDict[band]['forced_src'],
158 'ref': inputs['inputCatalogRef']}
159 dataId = butlerQC.quantum.dataId
160 df = self.run(catalogs=catalogs, tract=dataId['tract'], patch=dataId['patch'])
161 outputs = pipeBase.Struct(outputCatalog=df)
162 butlerQC.put(outputs, outputRefs)
164 def run(self, catalogs, tract, patch):
165 """Merge multiple catalogs.
167 Parameters
168 ----------
169 catalogs : `dict`
170 Mapping from filter names to dict of catalogs.
171 tract : int
172 tractId to use for the tractId column.
173 patch : str
174 patchId to use for the patchId column.
176 Returns
177 -------
178 catalog : `pandas.DataFrame`
179 Merged dataframe.
180 """
181 dfs = []
182 for filt, tableDict in catalogs.items():
183 for dataset, table in tableDict.items():
184 # Convert afwTable to pandas DataFrame
185 df = table.asAstropy().to_pandas().set_index('id', drop=True)
187 # Sort columns by name, to ensure matching schema among patches
188 df = df.reindex(sorted(df.columns), axis=1)
189 df = df.assign(tractId=tract, patchId=patch)
191 # Make columns a 3-level MultiIndex
192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c) for c in df.columns],
193 names=('dataset', 'band', 'column'))
194 dfs.append(df)
196 # We do this dance and not `pd.concat(dfs)` because the pandas
197 # concatenation uses infinite memory.
198 catalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs)
199 return catalog
202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
203 defaultTemplates={"catalogType": ""},
204 dimensions=("instrument", "visit", "detector")):
206 catalog = connectionTypes.Input(
207 doc="Input full-depth catalog of sources produced by CalibrateTask",
208 name="{catalogType}src",
209 storageClass="SourceCatalog",
210 dimensions=("instrument", "visit", "detector")
211 )
212 outputCatalog = connectionTypes.Output(
213 doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
214 "replaced with an index; all other columns are unchanged.",
215 name="{catalogType}source",
216 storageClass="DataFrame",
217 dimensions=("instrument", "visit", "detector")
218 )
221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
222 pipelineConnections=WriteSourceTableConnections):
223 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
226class WriteSourceTableTask(pipeBase.PipelineTask):
227 """Write source table to DataFrame Parquet format.
228 """
229 _DefaultName = "writeSourceTable"
230 ConfigClass = WriteSourceTableConfig
232 def runQuantum(self, butlerQC, inputRefs, outputRefs):
233 inputs = butlerQC.get(inputRefs)
234 inputs['ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id
235 result = self.run(**inputs)
236 outputs = pipeBase.Struct(outputCatalog=result.table)
237 butlerQC.put(outputs, outputRefs)
239 def run(self, catalog, ccdVisitId=None, **kwargs):
240 """Convert `src` catalog to DataFrame
242 Parameters
243 ----------
244 catalog: `afwTable.SourceCatalog`
245 catalog to be converted
246 ccdVisitId: `int`
247 ccdVisitId to be added as a column
248 **kwargs
249 Additional keyword arguments are ignored as a convenience for
250 subclasses that pass the same arguments to several different
251 methods.
253 Returns
254 -------
255 result : `~lsst.pipe.base.Struct`
256 ``table``
257 `DataFrame` version of the input catalog
258 """
259 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
260 df = catalog.asAstropy().to_pandas().set_index('id', drop=True)
261 df['ccdVisitId'] = ccdVisitId
263 return pipeBase.Struct(table=df)
266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
267 defaultTemplates={"catalogType": "",
268 "skyWcsName": "gbdesAstrometricFit",
269 "photoCalibName": "fgcm"},
270 # TODO: remove on DM-39854.
271 deprecatedTemplates={
272 "skyWcsName": "Deprecated; will be removed after v26.",
273 "photoCalibName": "Deprecated; will be removed after v26."
274 },
275 dimensions=("instrument", "visit", "detector", "skymap")):
276 skyMap = connectionTypes.Input(
277 doc="skyMap needed to choose which tract-level calibrations to use when multiple available",
278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
279 storageClass="SkyMap",
280 dimensions=("skymap",),
281 # TODO: remove on DM-39854.
282 deprecated=(
283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. "
284 "Will be removed after v26."
285 ),
286 )
287 exposure = connectionTypes.Input(
288 doc="Input exposure to perform photometry on.",
289 name="calexp",
290 storageClass="ExposureF",
291 dimensions=["instrument", "visit", "detector"],
292 )
293 visitSummary = connectionTypes.Input(
294 doc="Input visit-summary catalog with updated calibration objects.",
295 name="finalVisitSummary",
296 storageClass="ExposureCatalog",
297 dimensions=("instrument", "visit",),
298 )
299 externalSkyWcsTractCatalog = connectionTypes.Input(
300 doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
301 "id for the catalog id, sorted on id for fast lookup."),
302 name="{skyWcsName}SkyWcsCatalog",
303 storageClass="ExposureCatalog",
304 dimensions=["instrument", "visit", "tract"],
305 multiple=True,
306 # TODO: remove on DM-39854.
307 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
308 )
309 externalSkyWcsGlobalCatalog = connectionTypes.Input(
310 doc=("Per-visit wcs calibrations computed globally (with no tract information). "
311 "These catalogs use the detector id for the catalog id, sorted on id for "
312 "fast lookup."),
313 name="finalVisitSummary",
314 storageClass="ExposureCatalog",
315 dimensions=["instrument", "visit"],
316 # TODO: remove on DM-39854.
317 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
318 )
319 externalPhotoCalibTractCatalog = connectionTypes.Input(
320 doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
321 "detector id for the catalog id, sorted on id for fast lookup."),
322 name="{photoCalibName}PhotoCalibCatalog",
323 storageClass="ExposureCatalog",
324 dimensions=["instrument", "visit", "tract"],
325 multiple=True,
326 # TODO: remove on DM-39854.
327 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
328 )
329 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
330 doc=("Per-visit photometric calibrations computed globally (with no tract "
331 "information). These catalogs use the detector id for the catalog id, "
332 "sorted on id for fast lookup."),
333 name="finalVisitSummary",
334 storageClass="ExposureCatalog",
335 dimensions=["instrument", "visit"],
336 # TODO: remove on DM-39854.
337 deprecated="Deprecated in favor of 'visitSummary'. Will be removed after v26.",
338 )
340 def __init__(self, *, config=None):
341 super().__init__(config=config)
342 # Same connection boilerplate as all other applications of
343 # Global/Tract calibrations
344 # TODO: remove all of this on DM-39854.
345 keepSkyMap = False
346 if config.doApplyExternalSkyWcs and config.doReevaluateSkyWcs:
347 if config.useGlobalExternalSkyWcs:
348 self.inputs.remove("externalSkyWcsTractCatalog")
349 else:
350 self.inputs.remove("externalSkyWcsGlobalCatalog")
351 keepSkyMap = True
352 else:
353 self.inputs.remove("externalSkyWcsTractCatalog")
354 self.inputs.remove("externalSkyWcsGlobalCatalog")
355 if config.doApplyExternalPhotoCalib and config.doReevaluatePhotoCalib:
356 if config.useGlobalExternalPhotoCalib:
357 self.inputs.remove("externalPhotoCalibTractCatalog")
358 else:
359 self.inputs.remove("externalPhotoCalibGlobalCatalog")
360 keepSkyMap = True
361 else:
362 self.inputs.remove("externalPhotoCalibTractCatalog")
363 self.inputs.remove("externalPhotoCalibGlobalCatalog")
364 if not keepSkyMap:
365 del self.skyMap
368class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
369 pipelineConnections=WriteRecalibratedSourceTableConnections):
371 doReevaluatePhotoCalib = pexConfig.Field(
372 dtype=bool,
373 default=True,
374 doc=("Add or replace local photoCalib columns"),
375 )
376 doReevaluateSkyWcs = pexConfig.Field(
377 dtype=bool,
378 default=True,
379 doc=("Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"),
380 )
381 doApplyExternalPhotoCalib = pexConfig.Field(
382 dtype=bool,
383 default=False,
384 doc=("If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
385 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
386 # TODO: remove on DM-39854.
387 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
388 )
389 doApplyExternalSkyWcs = pexConfig.Field(
390 dtype=bool,
391 default=False,
392 doc=("if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
393 "else use the wcs already attached to the exposure."),
394 # TODO: remove on DM-39854.
395 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.",
396 )
397 useGlobalExternalPhotoCalib = pexConfig.Field(
398 dtype=bool,
399 default=False,
400 doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations "
401 "that are not run per-tract. When False, use per-tract photometric "
402 "calibration files."),
403 # TODO: remove on DM-39854.
404 deprecated="Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
405 )
406 useGlobalExternalSkyWcs = pexConfig.Field(
407 dtype=bool,
408 default=False,
409 doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
410 "that are not run per-tract. When False, use per-tract wcs "
411 "files."),
412 # TODO: remove on DM-39854.
413 deprecated="Deprecated along with the external WCS connections. Will be removed after v26.",
414 )
415 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
417 def validate(self):
418 super().validate()
419 if self.doApplyExternalSkyWcs and not self.doReevaluateSkyWcs:
420 log.warning("doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
421 "External SkyWcs will not be read or evaluated.")
422 if self.doApplyExternalPhotoCalib and not self.doReevaluatePhotoCalib:
423 log.warning("doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
424 "External PhotoCalib will not be read or evaluated.")
427class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
428 """Write source table to DataFrame Parquet format.
429 """
430 _DefaultName = "writeRecalibratedSourceTable"
431 ConfigClass = WriteRecalibratedSourceTableConfig
433 def runQuantum(self, butlerQC, inputRefs, outputRefs):
434 inputs = butlerQC.get(inputRefs)
436 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
437 inputs['idGenerator'] = idGenerator
438 inputs['ccdVisitId'] = idGenerator.catalog_id
440 if self.config.doReevaluatePhotoCalib or self.config.doReevaluateSkyWcs:
441 if self.config.doApplyExternalPhotoCalib or self.config.doApplyExternalSkyWcs:
442 inputs['exposure'] = self.attachCalibs(inputRefs, **inputs)
443 else:
444 inputs['exposure'] = self.prepareCalibratedExposure(
445 exposure=inputs["exposure"], visitSummary=inputs["visitSummary"]
446 )
447 inputs['catalog'] = self.addCalibColumns(**inputs)
449 result = self.run(**inputs)
450 outputs = pipeBase.Struct(outputCatalog=result.table)
451 butlerQC.put(outputs, outputRefs)
453 # TODO: remove on DM-39854.
454 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
455 version="v26", category=FutureWarning)
456 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
457 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
458 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs):
459 """Apply external calibrations to exposure per configuration
461 When multiple tract-level calibrations overlap, select the one with the
462 center closest to detector.
464 Parameters
465 ----------
466 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`, for dataIds of
467 tract-level calibs.
468 skyMap : `~lsst.skymap.BaseSkyMap`
469 skyMap to lookup tract geometry and WCS.
470 exposure : `lsst.afw.image.exposure.Exposure`
471 Input exposure to adjust calibrations.
472 externalSkyWcsGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
473 Exposure catalog with external skyWcs to be applied per config
474 externalSkyWcsTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
475 Exposure catalog with external skyWcs to be applied per config
476 externalPhotoCalibGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
477 Exposure catalog with external photoCalib to be applied per config
478 externalPhotoCalibTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
479 Exposure catalog with external photoCalib to be applied per config
480 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
481 Exposure catalog with all calibration objects. WCS and PhotoCalib
482 are always applied if provided.
483 **kwargs
484 Additional keyword arguments are ignored to facilitate passing the
485 same arguments to several methods.
487 Returns
488 -------
489 exposure : `lsst.afw.image.exposure.Exposure`
490 Exposure with adjusted calibrations.
491 """
492 if not self.config.doApplyExternalSkyWcs:
493 # Do not modify the exposure's SkyWcs
494 externalSkyWcsCatalog = None
495 elif self.config.useGlobalExternalSkyWcs:
496 # Use the global external SkyWcs
497 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
498 self.log.info('Applying global SkyWcs')
499 else:
500 # use tract-level external SkyWcs from the closest overlapping tract
501 inputRef = getattr(inputRefs, 'externalSkyWcsTractCatalog')
502 tracts = [ref.dataId['tract'] for ref in inputRef]
503 if len(tracts) == 1:
504 ind = 0
505 self.log.info('Applying tract-level SkyWcs from tract %s', tracts[ind])
506 else:
507 if exposure.getWcs() is None: # TODO: could this look-up use the externalPhotoCalib?
508 raise ValueError("Trying to locate nearest tract, but exposure.wcs is None.")
509 ind = self.getClosestTract(tracts, skyMap,
510 exposure.getBBox(), exposure.getWcs())
511 self.log.info('Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
512 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
514 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
516 if not self.config.doApplyExternalPhotoCalib:
517 # Do not modify the exposure's PhotoCalib
518 externalPhotoCalibCatalog = None
519 elif self.config.useGlobalExternalPhotoCalib:
520 # Use the global external PhotoCalib
521 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
522 self.log.info('Applying global PhotoCalib')
523 else:
524 # use tract-level external PhotoCalib from the closest overlapping tract
525 inputRef = getattr(inputRefs, 'externalPhotoCalibTractCatalog')
526 tracts = [ref.dataId['tract'] for ref in inputRef]
527 if len(tracts) == 1:
528 ind = 0
529 self.log.info('Applying tract-level PhotoCalib from tract %s', tracts[ind])
530 else:
531 ind = self.getClosestTract(tracts, skyMap,
532 exposure.getBBox(), exposure.getWcs())
533 self.log.info('Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
534 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
536 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
538 return self.prepareCalibratedExposure(
539 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary
540 )
542 # TODO: remove on DM-39854.
543 @deprecated("Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
544 version="v26", category=FutureWarning)
545 def getClosestTract(self, tracts, skyMap, bbox, wcs):
546 """Find the index of the tract closest to detector from list of tractIds
548 Parameters
549 ----------
550 tracts: `list` [`int`]
551 Iterable of integer tractIds
552 skyMap : `~lsst.skymap.BaseSkyMap`
553 skyMap to lookup tract geometry and wcs
554 bbox : `~lsst.geom.Box2I`
555 Detector bbox, center of which will compared to tract centers
556 wcs : `~lsst.afw.geom.SkyWcs`
557 Detector Wcs object to map the detector center to SkyCoord
559 Returns
560 -------
561 index : `int`
562 """
563 if len(tracts) == 1:
564 return 0
566 center = wcs.pixelToSky(bbox.getCenter())
567 sep = []
568 for tractId in tracts:
569 tract = skyMap[tractId]
570 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
571 sep.append(center.separation(tractCenter))
573 return np.argmin(sep)
575 def prepareCalibratedExposure(
576 self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, visitSummary=None
577 ):
578 """Prepare a calibrated exposure and apply external calibrations
579 if so configured.
581 Parameters
582 ----------
583 exposure : `lsst.afw.image.exposure.Exposure`
584 Input exposure to adjust calibrations.
585 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
586 Exposure catalog with external skyWcs to be applied
587 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
588 for the catalog id, sorted on id for fast lookup.
589 Deprecated in favor of ``visitSummary``; will be removed after v26.
590 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
591 Exposure catalog with external photoCalib to be applied
592 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
593 id for the catalog id, sorted on id for fast lookup.
594 Deprecated in favor of ``visitSummary``; will be removed after v26.
595 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
596 Exposure catalog with all calibration objects. WCS and PhotoCalib
597 are always applied if ``visitSummary`` is provided and those
598 components are not `None`.
600 Returns
601 -------
602 exposure : `lsst.afw.image.exposure.Exposure`
603 Exposure with adjusted calibrations.
604 """
605 detectorId = exposure.getInfo().getDetector().getId()
607 if visitSummary is not None:
608 row = visitSummary.find(detectorId)
609 if row is None:
610 raise RuntimeError(f"Visit summary for detector {detectorId} is unexpectedly missing.")
611 if (photoCalib := row.getPhotoCalib()) is None:
612 self.log.warning("Detector id %s has None for photoCalib in visit summary; "
613 "using original photoCalib.", detectorId)
614 else:
615 exposure.setPhotoCalib(photoCalib)
616 if (skyWcs := row.getWcs()) is None:
617 self.log.warning("Detector id %s has None for skyWcs in visit summary; "
618 "using original skyWcs.", detectorId)
619 else:
620 exposure.setWcs(skyWcs)
622 if externalPhotoCalibCatalog is not None:
623 # TODO: remove on DM-39854.
624 warnings.warn(
625 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
626 FutureWarning,
627 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"),
628 )
629 row = externalPhotoCalibCatalog.find(detectorId)
630 if row is None:
631 self.log.warning("Detector id %s not found in externalPhotoCalibCatalog; "
632 "Using original photoCalib.", detectorId)
633 else:
634 photoCalib = row.getPhotoCalib()
635 if photoCalib is None:
636 self.log.warning("Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
637 "Using original photoCalib.", detectorId)
638 else:
639 exposure.setPhotoCalib(photoCalib)
641 if externalSkyWcsCatalog is not None:
642 # TODO: remove on DM-39854.
643 warnings.warn(
644 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
645 FutureWarning,
646 stacklevel=find_outside_stacklevel("lsst.pipe.tasks.postprocessing"),
647 )
648 row = externalSkyWcsCatalog.find(detectorId)
649 if row is None:
650 self.log.warning("Detector id %s not found in externalSkyWcsCatalog; "
651 "Using original skyWcs.", detectorId)
652 else:
653 skyWcs = row.getWcs()
654 if skyWcs is None:
655 self.log.warning("Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
656 "Using original skyWcs.", detectorId)
657 else:
658 exposure.setWcs(skyWcs)
660 return exposure
662 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs):
663 """Add replace columns with calibs evaluated at each centroid
665 Add or replace 'base_LocalWcs' `base_LocalPhotoCalib' columns in a
666 a source catalog, by rerunning the plugins.
668 Parameters
669 ----------
670 catalog : `lsst.afw.table.SourceCatalog`
671 catalog to which calib columns will be added
672 exposure : `lsst.afw.image.exposure.Exposure`
673 Exposure with attached PhotoCalibs and SkyWcs attributes to be
674 reevaluated at local centroids. Pixels are not required.
675 idGenerator : `lsst.meas.base.IdGenerator`
676 Object that generates Source IDs and random seeds.
677 **kwargs
678 Additional keyword arguments are ignored to facilitate passing the
679 same arguments to several methods.
681 Returns
682 -------
683 newCat: `lsst.afw.table.SourceCatalog`
684 Source Catalog with requested local calib columns
685 """
686 measureConfig = SingleFrameMeasurementTask.ConfigClass()
687 measureConfig.doReplaceWithNoise = False
689 # Clear all slots, because we aren't running the relevant plugins.
690 for slot in measureConfig.slots:
691 setattr(measureConfig.slots, slot, None)
693 measureConfig.plugins.names = []
694 if self.config.doReevaluateSkyWcs:
695 measureConfig.plugins.names.add('base_LocalWcs')
696 self.log.info("Re-evaluating base_LocalWcs plugin")
697 if self.config.doReevaluatePhotoCalib:
698 measureConfig.plugins.names.add('base_LocalPhotoCalib')
699 self.log.info("Re-evaluating base_LocalPhotoCalib plugin")
700 pluginsNotToCopy = tuple(measureConfig.plugins.names)
702 # Create a new schema and catalog
703 # Copy all columns from original except for the ones to reevaluate
704 aliasMap = catalog.schema.getAliasMap()
705 mapper = afwTable.SchemaMapper(catalog.schema)
706 for item in catalog.schema:
707 if not item.field.getName().startswith(pluginsNotToCopy):
708 mapper.addMapping(item.key)
710 schema = mapper.getOutputSchema()
711 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
712 schema.setAliasMap(aliasMap)
713 newCat = afwTable.SourceCatalog(schema)
714 newCat.extend(catalog, mapper=mapper)
716 # Fluxes in sourceCatalogs are in counts, so there are no fluxes to
717 # update here. LocalPhotoCalibs are applied during transform tasks.
718 # Update coord_ra/coord_dec, which are expected to be positions on the
719 # sky and are used as such in sdm tables without transform
720 if self.config.doReevaluateSkyWcs and exposure.wcs is not None:
721 afwTable.updateSourceCoords(exposure.wcs, newCat)
723 measurement.run(measCat=newCat, exposure=exposure, exposureId=idGenerator.catalog_id)
725 return newCat
728class PostprocessAnalysis(object):
729 """Calculate columns from DataFrames or handles storing DataFrames.
731 This object manages and organizes an arbitrary set of computations
732 on a catalog. The catalog is defined by a
733 `DeferredDatasetHandle` or `InMemoryDatasetHandle` object
734 (or list thereof), such as a ``deepCoadd_obj`` dataset, and the
735 computations are defined by a collection of
736 `~lsst.pipe.tasks.functors.Functor` objects (or, equivalently, a
737 ``CompositeFunctor``).
739 After the object is initialized, accessing the ``.df`` attribute (which
740 holds the `pandas.DataFrame` containing the results of the calculations)
741 triggers computation of said dataframe.
743 One of the conveniences of using this object is the ability to define a
744 desired common filter for all functors. This enables the same functor
745 collection to be passed to several different `PostprocessAnalysis` objects
746 without having to change the original functor collection, since the ``filt``
747 keyword argument of this object triggers an overwrite of the ``filt``
748 property for all functors in the collection.
750 This object also allows a list of refFlags to be passed, and defines a set
751 of default refFlags that are always included even if not requested.
753 If a list of DataFrames or Handles is passed, rather than a single one,
754 then the calculations will be mapped over all the input catalogs. In
755 principle, it should be straightforward to parallelize this activity, but
756 initial tests have failed (see TODO in code comments).
758 Parameters
759 ----------
760 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
761 `~lsst.pipe.base.InMemoryDatasetHandle` or
762 list of these.
763 Source catalog(s) for computation.
764 functors : `list`, `dict`, or `~lsst.pipe.tasks.functors.CompositeFunctor`
765 Computations to do (functors that act on ``handles``).
766 If a dict, the output
767 DataFrame will have columns keyed accordingly.
768 If a list, the column keys will come from the
769 ``.shortname`` attribute of each functor.
771 filt : `str`, optional
772 Filter in which to calculate. If provided,
773 this will overwrite any existing ``.filt`` attribute
774 of the provided functors.
776 flags : `list`, optional
777 List of flags (per-band) to include in output table.
778 Taken from the ``meas`` dataset if applied to a multilevel Object Table.
780 refFlags : `list`, optional
781 List of refFlags (only reference band) to include in output table.
783 forcedFlags : `list`, optional
784 List of flags (per-band) to include in output table.
785 Taken from the ``forced_src`` dataset if applied to a
786 multilevel Object Table. Intended for flags from measurement plugins
787 only run during multi-band forced-photometry.
788 """
789 _defaultRefFlags = []
790 _defaultFuncs = ()
792 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
793 self.handles = handles
794 self.functors = functors
796 self.filt = filt
797 self.flags = list(flags) if flags is not None else []
798 self.forcedFlags = list(forcedFlags) if forcedFlags is not None else []
799 self.refFlags = list(self._defaultRefFlags)
800 if refFlags is not None:
801 self.refFlags += list(refFlags)
803 self._df = None
805 @property
806 def defaultFuncs(self):
807 funcs = dict(self._defaultFuncs)
808 return funcs
810 @property
811 def func(self):
812 additionalFuncs = self.defaultFuncs
813 additionalFuncs.update({flag: Column(flag, dataset='forced_src') for flag in self.forcedFlags})
814 additionalFuncs.update({flag: Column(flag, dataset='ref') for flag in self.refFlags})
815 additionalFuncs.update({flag: Column(flag, dataset='meas') for flag in self.flags})
817 if isinstance(self.functors, CompositeFunctor):
818 func = self.functors
819 else:
820 func = CompositeFunctor(self.functors)
822 func.funcDict.update(additionalFuncs)
823 func.filt = self.filt
825 return func
827 @property
828 def noDupCols(self):
829 return [name for name, func in self.func.funcDict.items() if func.noDup or func.dataset == 'ref']
831 @property
832 def df(self):
833 if self._df is None:
834 self.compute()
835 return self._df
837 def compute(self, dropna=False, pool=None):
838 # map over multiple handles
839 if type(self.handles) in (list, tuple):
840 if pool is None:
841 dflist = [self.func(handle, dropna=dropna) for handle in self.handles]
842 else:
843 # TODO: Figure out why this doesn't work (pyarrow pickling
844 # issues?)
845 dflist = pool.map(functools.partial(self.func, dropna=dropna), self.handles)
846 self._df = pd.concat(dflist)
847 else:
848 self._df = self.func(self.handles, dropna=dropna)
850 return self._df
853class TransformCatalogBaseConnections(pipeBase.PipelineTaskConnections,
854 dimensions=()):
855 """Expected Connections for subclasses of TransformCatalogBaseTask.
857 Must be subclassed.
858 """
859 inputCatalog = connectionTypes.Input(
860 name="",
861 storageClass="DataFrame",
862 )
863 outputCatalog = connectionTypes.Output(
864 name="",
865 storageClass="DataFrame",
866 )
869class TransformCatalogBaseConfig(pipeBase.PipelineTaskConfig,
870 pipelineConnections=TransformCatalogBaseConnections):
871 functorFile = pexConfig.Field(
872 dtype=str,
873 doc="Path to YAML file specifying Science Data Model functors to use "
874 "when copying columns and computing calibrated values.",
875 default=None,
876 optional=True
877 )
878 primaryKey = pexConfig.Field(
879 dtype=str,
880 doc="Name of column to be set as the DataFrame index. If None, the index"
881 "will be named `id`",
882 default=None,
883 optional=True
884 )
885 columnsFromDataId = pexConfig.ListField(
886 dtype=str,
887 default=None,
888 optional=True,
889 doc="Columns to extract from the dataId",
890 )
893class TransformCatalogBaseTask(pipeBase.PipelineTask):
894 """Base class for transforming/standardizing a catalog by applying functors
895 that convert units and apply calibrations.
897 The purpose of this task is to perform a set of computations on an input
898 ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` that holds a
899 ``DataFrame`` dataset (such as ``deepCoadd_obj``), and write the results to
900 a new dataset (which needs to be declared in an ``outputDataset``
901 attribute).
903 The calculations to be performed are defined in a YAML file that specifies
904 a set of functors to be computed, provided as a ``--functorFile`` config
905 parameter. An example of such a YAML file is the following:
907 funcs:
908 sourceId:
909 functor: Index
910 x:
911 functor: Column
912 args: slot_Centroid_x
913 y:
914 functor: Column
915 args: slot_Centroid_y
916 psfFlux:
917 functor: LocalNanojansky
918 args:
919 - slot_PsfFlux_instFlux
920 - slot_PsfFlux_instFluxErr
921 - base_LocalPhotoCalib
922 - base_LocalPhotoCalibErr
923 psfFluxErr:
924 functor: LocalNanojanskyErr
925 args:
926 - slot_PsfFlux_instFlux
927 - slot_PsfFlux_instFluxErr
928 - base_LocalPhotoCalib
929 - base_LocalPhotoCalibErr
930 flags:
931 - detect_isPrimary
933 The names for each entry under "func" will become the names of columns in
934 the output dataset. All the functors referenced are defined in
935 `~lsst.pipe.tasks.functors`. Positional arguments to be passed to each
936 functor are in the `args` list, and any additional entries for each column
937 other than "functor" or "args" (e.g., ``'filt'``, ``'dataset'``) are
938 treated as keyword arguments to be passed to the functor initialization.
940 The "flags" entry is the default shortcut for `Column` functors.
941 All columns listed under "flags" will be copied to the output table
942 untransformed. They can be of any datatype.
943 In the special case of transforming a multi-level oject table with
944 band and dataset indices (deepCoadd_obj), these will be taked from the
945 `meas` dataset and exploded out per band.
947 There are two special shortcuts that only apply when transforming
948 multi-level Object (deepCoadd_obj) tables:
949 - The "refFlags" entry is shortcut for `Column` functor
950 taken from the `'ref'` dataset if transforming an ObjectTable.
951 - The "forcedFlags" entry is shortcut for `Column` functors.
952 taken from the ``forced_src`` dataset if transforming an ObjectTable.
953 These are expanded out per band.
956 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
957 to organize and excecute the calculations.
958 """
959 @property
960 def _DefaultName(self):
961 raise NotImplementedError('Subclass must define "_DefaultName" attribute')
963 @property
964 def outputDataset(self):
965 raise NotImplementedError('Subclass must define "outputDataset" attribute')
967 @property
968 def inputDataset(self):
969 raise NotImplementedError('Subclass must define "inputDataset" attribute')
971 @property
972 def ConfigClass(self):
973 raise NotImplementedError('Subclass must define "ConfigClass" attribute')
975 def __init__(self, *args, **kwargs):
976 super().__init__(*args, **kwargs)
977 if self.config.functorFile:
978 self.log.info('Loading tranform functor definitions from %s',
979 self.config.functorFile)
980 self.funcs = CompositeFunctor.from_file(self.config.functorFile)
981 self.funcs.update(dict(PostprocessAnalysis._defaultFuncs))
982 else:
983 self.funcs = None
985 def runQuantum(self, butlerQC, inputRefs, outputRefs):
986 inputs = butlerQC.get(inputRefs)
987 if self.funcs is None:
988 raise ValueError("config.functorFile is None. "
989 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
990 result = self.run(handle=inputs['inputCatalog'], funcs=self.funcs,
991 dataId=outputRefs.outputCatalog.dataId.full)
992 outputs = pipeBase.Struct(outputCatalog=result)
993 butlerQC.put(outputs, outputRefs)
995 def run(self, handle, funcs=None, dataId=None, band=None):
996 """Do postprocessing calculations
998 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` or
999 ``DataFrame`` object and dataId,
1000 returns a dataframe with results of postprocessing calculations.
1002 Parameters
1003 ----------
1004 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
1005 `~lsst.pipe.base.InMemoryDatasetHandle` or
1006 `~pandas.DataFrame`, or list of these.
1007 DataFrames from which calculations are done.
1008 funcs : `~lsst.pipe.tasks.functors.Functor`
1009 Functors to apply to the table's columns
1010 dataId : dict, optional
1011 Used to add a `patchId` column to the output dataframe.
1012 band : `str`, optional
1013 Filter band that is being processed.
1015 Returns
1016 -------
1017 df : `pandas.DataFrame`
1018 """
1019 self.log.info("Transforming/standardizing the source table dataId: %s", dataId)
1021 df = self.transform(band, handle, funcs, dataId).df
1022 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df))
1023 return df
1025 def getFunctors(self):
1026 return self.funcs
1028 def getAnalysis(self, handles, funcs=None, band=None):
1029 if funcs is None:
1030 funcs = self.funcs
1031 analysis = PostprocessAnalysis(handles, funcs, filt=band)
1032 return analysis
1034 def transform(self, band, handles, funcs, dataId):
1035 analysis = self.getAnalysis(handles, funcs=funcs, band=band)
1036 df = analysis.df
1037 if dataId and self.config.columnsFromDataId:
1038 for key in self.config.columnsFromDataId:
1039 if key in dataId:
1040 df[str(key)] = dataId[key]
1041 else:
1042 raise ValueError(f"'{key}' in config.columnsFromDataId not found in dataId: {dataId}")
1044 if self.config.primaryKey:
1045 if df.index.name != self.config.primaryKey and self.config.primaryKey in df:
1046 df.reset_index(inplace=True, drop=True)
1047 df.set_index(self.config.primaryKey, inplace=True)
1049 return pipeBase.Struct(
1050 df=df,
1051 analysis=analysis
1052 )
1055class TransformObjectCatalogConnections(pipeBase.PipelineTaskConnections,
1056 defaultTemplates={"coaddName": "deep"},
1057 dimensions=("tract", "patch", "skymap")):
1058 inputCatalog = connectionTypes.Input(
1059 doc="The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
1060 "stored as a DataFrame with a multi-level column index per-patch.",
1061 dimensions=("tract", "patch", "skymap"),
1062 storageClass="DataFrame",
1063 name="{coaddName}Coadd_obj",
1064 deferLoad=True,
1065 )
1066 outputCatalog = connectionTypes.Output(
1067 doc="Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
1068 "data model.",
1069 dimensions=("tract", "patch", "skymap"),
1070 storageClass="DataFrame",
1071 name="objectTable"
1072 )
1075class TransformObjectCatalogConfig(TransformCatalogBaseConfig,
1076 pipelineConnections=TransformObjectCatalogConnections):
1077 coaddName = pexConfig.Field(
1078 dtype=str,
1079 default="deep",
1080 doc="Name of coadd"
1081 )
1082 # TODO: remove in DM-27177
1083 filterMap = pexConfig.DictField(
1084 keytype=str,
1085 itemtype=str,
1086 default={},
1087 doc=("Dictionary mapping full filter name to short one for column name munging."
1088 "These filters determine the output columns no matter what filters the "
1089 "input data actually contain."),
1090 deprecated=("Coadds are now identified by the band, so this transform is unused."
1091 "Will be removed after v22.")
1092 )
1093 outputBands = pexConfig.ListField(
1094 dtype=str,
1095 default=None,
1096 optional=True,
1097 doc=("These bands and only these bands will appear in the output,"
1098 " NaN-filled if the input does not include them."
1099 " If None, then use all bands found in the input.")
1100 )
1101 camelCase = pexConfig.Field(
1102 dtype=bool,
1103 default=False,
1104 doc=("Write per-band columns names with camelCase, else underscore "
1105 "For example: gPsFlux instead of g_PsFlux.")
1106 )
1107 multilevelOutput = pexConfig.Field(
1108 dtype=bool,
1109 default=False,
1110 doc=("Whether results dataframe should have a multilevel column index (True) or be flat "
1111 "and name-munged (False).")
1112 )
1113 goodFlags = pexConfig.ListField(
1114 dtype=str,
1115 default=[],
1116 doc=("List of 'good' flags that should be set False when populating empty tables. "
1117 "All other flags are considered to be 'bad' flags and will be set to True.")
1118 )
1119 floatFillValue = pexConfig.Field(
1120 dtype=float,
1121 default=np.nan,
1122 doc="Fill value for float fields when populating empty tables."
1123 )
1124 integerFillValue = pexConfig.Field(
1125 dtype=int,
1126 default=-1,
1127 doc="Fill value for integer fields when populating empty tables."
1128 )
1130 def setDefaults(self):
1131 super().setDefaults()
1132 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Object.yaml')
1133 self.primaryKey = 'objectId'
1134 self.columnsFromDataId = ['tract', 'patch']
1135 self.goodFlags = ['calib_astrometry_used',
1136 'calib_photometry_reserved',
1137 'calib_photometry_used',
1138 'calib_psf_candidate',
1139 'calib_psf_reserved',
1140 'calib_psf_used']
1143class TransformObjectCatalogTask(TransformCatalogBaseTask):
1144 """Produce a flattened Object Table to match the format specified in
1145 sdm_schemas.
1147 Do the same set of postprocessing calculations on all bands.
1149 This is identical to `TransformCatalogBaseTask`, except for that it does
1150 the specified functor calculations for all filters present in the
1151 input `deepCoadd_obj` table. Any specific ``"filt"`` keywords specified
1152 by the YAML file will be superceded.
1153 """
1154 _DefaultName = "transformObjectCatalog"
1155 ConfigClass = TransformObjectCatalogConfig
1157 def run(self, handle, funcs=None, dataId=None, band=None):
1158 # NOTE: band kwarg is ignored here.
1159 dfDict = {}
1160 analysisDict = {}
1161 templateDf = pd.DataFrame()
1163 columns = handle.get(component='columns')
1164 inputBands = columns.unique(level=1).values
1166 outputBands = self.config.outputBands if self.config.outputBands else inputBands
1168 # Perform transform for data of filters that exist in the handle dataframe.
1169 for inputBand in inputBands:
1170 if inputBand not in outputBands:
1171 self.log.info("Ignoring %s band data in the input", inputBand)
1172 continue
1173 self.log.info("Transforming the catalog of band %s", inputBand)
1174 result = self.transform(inputBand, handle, funcs, dataId)
1175 dfDict[inputBand] = result.df
1176 analysisDict[inputBand] = result.analysis
1177 if templateDf.empty:
1178 templateDf = result.df
1180 # Put filler values in columns of other wanted bands
1181 for filt in outputBands:
1182 if filt not in dfDict:
1183 self.log.info("Adding empty columns for band %s", filt)
1184 dfTemp = templateDf.copy()
1185 for col in dfTemp.columns:
1186 testValue = dfTemp[col].values[0]
1187 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1188 # Boolean flag type, check if it is a "good" flag
1189 if col in self.config.goodFlags:
1190 fillValue = False
1191 else:
1192 fillValue = True
1193 elif isinstance(testValue, numbers.Integral):
1194 # Checking numbers.Integral catches all flavors
1195 # of python, numpy, pandas, etc. integers.
1196 # We must ensure this is not an unsigned integer.
1197 if isinstance(testValue, np.unsignedinteger):
1198 raise ValueError("Parquet tables may not have unsigned integer columns.")
1199 else:
1200 fillValue = self.config.integerFillValue
1201 else:
1202 fillValue = self.config.floatFillValue
1203 dfTemp[col].values[:] = fillValue
1204 dfDict[filt] = dfTemp
1206 # This makes a multilevel column index, with band as first level
1207 df = pd.concat(dfDict, axis=1, names=['band', 'column'])
1209 if not self.config.multilevelOutput:
1210 noDupCols = list(set.union(*[set(v.noDupCols) for v in analysisDict.values()]))
1211 if self.config.primaryKey in noDupCols:
1212 noDupCols.remove(self.config.primaryKey)
1213 if dataId and self.config.columnsFromDataId:
1214 noDupCols += self.config.columnsFromDataId
1215 df = flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1216 inputBands=inputBands)
1218 self.log.info("Made a table of %d columns and %d rows", len(df.columns), len(df))
1220 return df
1223class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1224 dimensions=("tract", "skymap")):
1225 inputCatalogs = connectionTypes.Input(
1226 doc="Per-Patch objectTables conforming to the standard data model.",
1227 name="objectTable",
1228 storageClass="DataFrame",
1229 dimensions=("tract", "patch", "skymap"),
1230 multiple=True,
1231 )
1232 outputCatalog = connectionTypes.Output(
1233 doc="Pre-tract horizontal concatenation of the input objectTables",
1234 name="objectTable_tract",
1235 storageClass="DataFrame",
1236 dimensions=("tract", "skymap"),
1237 )
1240class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1241 pipelineConnections=ConsolidateObjectTableConnections):
1242 coaddName = pexConfig.Field(
1243 dtype=str,
1244 default="deep",
1245 doc="Name of coadd"
1246 )
1249class ConsolidateObjectTableTask(pipeBase.PipelineTask):
1250 """Write patch-merged source tables to a tract-level DataFrame Parquet file.
1252 Concatenates `objectTable` list into a per-visit `objectTable_tract`.
1253 """
1254 _DefaultName = "consolidateObjectTable"
1255 ConfigClass = ConsolidateObjectTableConfig
1257 inputDataset = 'objectTable'
1258 outputDataset = 'objectTable_tract'
1260 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1261 inputs = butlerQC.get(inputRefs)
1262 self.log.info("Concatenating %s per-patch Object Tables",
1263 len(inputs['inputCatalogs']))
1264 df = pd.concat(inputs['inputCatalogs'])
1265 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1268class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1269 defaultTemplates={"catalogType": ""},
1270 dimensions=("instrument", "visit", "detector")):
1272 inputCatalog = connectionTypes.Input(
1273 doc="Wide input catalog of sources produced by WriteSourceTableTask",
1274 name="{catalogType}source",
1275 storageClass="DataFrame",
1276 dimensions=("instrument", "visit", "detector"),
1277 deferLoad=True
1278 )
1279 outputCatalog = connectionTypes.Output(
1280 doc="Narrower, per-detector Source Table transformed and converted per a "
1281 "specified set of functors",
1282 name="{catalogType}sourceTable",
1283 storageClass="DataFrame",
1284 dimensions=("instrument", "visit", "detector")
1285 )
1288class TransformSourceTableConfig(TransformCatalogBaseConfig,
1289 pipelineConnections=TransformSourceTableConnections):
1291 def setDefaults(self):
1292 super().setDefaults()
1293 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'Source.yaml')
1294 self.primaryKey = 'sourceId'
1295 self.columnsFromDataId = ['visit', 'detector', 'band', 'physical_filter']
1298class TransformSourceTableTask(TransformCatalogBaseTask):
1299 """Transform/standardize a source catalog
1300 """
1301 _DefaultName = "transformSourceTable"
1302 ConfigClass = TransformSourceTableConfig
1305class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1306 dimensions=("instrument", "visit",),
1307 defaultTemplates={"calexpType": ""}):
1308 calexp = connectionTypes.Input(
1309 doc="Processed exposures used for metadata",
1310 name="calexp",
1311 storageClass="ExposureF",
1312 dimensions=("instrument", "visit", "detector"),
1313 deferLoad=True,
1314 multiple=True,
1315 )
1316 visitSummary = connectionTypes.Output(
1317 doc=("Per-visit consolidated exposure metadata. These catalogs use "
1318 "detector id for the id and are sorted for fast lookups of a "
1319 "detector."),
1320 name="visitSummary",
1321 storageClass="ExposureCatalog",
1322 dimensions=("instrument", "visit"),
1323 )
1324 visitSummarySchema = connectionTypes.InitOutput(
1325 doc="Schema of the visitSummary catalog",
1326 name="visitSummary_schema",
1327 storageClass="ExposureCatalog",
1328 )
1331class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1332 pipelineConnections=ConsolidateVisitSummaryConnections):
1333 """Config for ConsolidateVisitSummaryTask"""
1334 pass
1337class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
1338 """Task to consolidate per-detector visit metadata.
1340 This task aggregates the following metadata from all the detectors in a
1341 single visit into an exposure catalog:
1342 - The visitInfo.
1343 - The wcs.
1344 - The photoCalib.
1345 - The physical_filter and band (if available).
1346 - The psf size, shape, and effective area at the center of the detector.
1347 - The corners of the bounding box in right ascension/declination.
1349 Other quantities such as Detector, Psf, ApCorrMap, and TransmissionCurve
1350 are not persisted here because of storage concerns, and because of their
1351 limited utility as summary statistics.
1353 Tests for this task are performed in ci_hsc_gen3.
1354 """
1355 _DefaultName = "consolidateVisitSummary"
1356 ConfigClass = ConsolidateVisitSummaryConfig
1358 def __init__(self, **kwargs):
1359 super().__init__(**kwargs)
1360 self.schema = afwTable.ExposureTable.makeMinimalSchema()
1361 self.schema.addField('visit', type='L', doc='Visit number')
1362 self.schema.addField('physical_filter', type='String', size=32, doc='Physical filter')
1363 self.schema.addField('band', type='String', size=32, doc='Name of band')
1364 ExposureSummaryStats.update_schema(self.schema)
1365 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
1367 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1368 dataRefs = butlerQC.get(inputRefs.calexp)
1369 visit = dataRefs[0].dataId.byName()['visit']
1371 self.log.debug("Concatenating metadata from %d per-detector calexps (visit %d)",
1372 len(dataRefs), visit)
1374 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1376 butlerQC.put(expCatalog, outputRefs.visitSummary)
1378 def _combineExposureMetadata(self, visit, dataRefs):
1379 """Make a combined exposure catalog from a list of dataRefs.
1380 These dataRefs must point to exposures with wcs, summaryStats,
1381 and other visit metadata.
1383 Parameters
1384 ----------
1385 visit : `int`
1386 Visit identification number.
1387 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1388 List of dataRefs in visit.
1390 Returns
1391 -------
1392 visitSummary : `lsst.afw.table.ExposureCatalog`
1393 Exposure catalog with per-detector summary information.
1394 """
1395 cat = afwTable.ExposureCatalog(self.schema)
1396 cat.resize(len(dataRefs))
1398 cat['visit'] = visit
1400 for i, dataRef in enumerate(dataRefs):
1401 visitInfo = dataRef.get(component='visitInfo')
1402 filterLabel = dataRef.get(component='filter')
1403 summaryStats = dataRef.get(component='summaryStats')
1404 detector = dataRef.get(component='detector')
1405 wcs = dataRef.get(component='wcs')
1406 photoCalib = dataRef.get(component='photoCalib')
1407 detector = dataRef.get(component='detector')
1408 bbox = dataRef.get(component='bbox')
1409 validPolygon = dataRef.get(component='validPolygon')
1411 rec = cat[i]
1412 rec.setBBox(bbox)
1413 rec.setVisitInfo(visitInfo)
1414 rec.setWcs(wcs)
1415 rec.setPhotoCalib(photoCalib)
1416 rec.setValidPolygon(validPolygon)
1418 rec['physical_filter'] = filterLabel.physicalLabel if filterLabel.hasPhysicalLabel() else ""
1419 rec['band'] = filterLabel.bandLabel if filterLabel.hasBandLabel() else ""
1420 rec.setId(detector.getId())
1421 summaryStats.update_record(rec)
1423 metadata = dafBase.PropertyList()
1424 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
1425 # We are looping over existing datarefs, so the following is true
1426 metadata.add("COMMENT", "Only detectors with data have entries.")
1427 cat.setMetadata(metadata)
1429 cat.sort()
1430 return cat
1433class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1434 defaultTemplates={"catalogType": ""},
1435 dimensions=("instrument", "visit")):
1436 inputCatalogs = connectionTypes.Input(
1437 doc="Input per-detector Source Tables",
1438 name="{catalogType}sourceTable",
1439 storageClass="DataFrame",
1440 dimensions=("instrument", "visit", "detector"),
1441 multiple=True
1442 )
1443 outputCatalog = connectionTypes.Output(
1444 doc="Per-visit concatenation of Source Table",
1445 name="{catalogType}sourceTable_visit",
1446 storageClass="DataFrame",
1447 dimensions=("instrument", "visit")
1448 )
1451class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1452 pipelineConnections=ConsolidateSourceTableConnections):
1453 pass
1456class ConsolidateSourceTableTask(pipeBase.PipelineTask):
1457 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1458 """
1459 _DefaultName = 'consolidateSourceTable'
1460 ConfigClass = ConsolidateSourceTableConfig
1462 inputDataset = 'sourceTable'
1463 outputDataset = 'sourceTable_visit'
1465 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1466 from .makeWarp import reorderRefs
1468 detectorOrder = [ref.dataId['detector'] for ref in inputRefs.inputCatalogs]
1469 detectorOrder.sort()
1470 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey='detector')
1471 inputs = butlerQC.get(inputRefs)
1472 self.log.info("Concatenating %s per-detector Source Tables",
1473 len(inputs['inputCatalogs']))
1474 df = pd.concat(inputs['inputCatalogs'])
1475 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1478class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1479 dimensions=("instrument",),
1480 defaultTemplates={"calexpType": ""}):
1481 visitSummaryRefs = connectionTypes.Input(
1482 doc="Data references for per-visit consolidated exposure metadata",
1483 name="finalVisitSummary",
1484 storageClass="ExposureCatalog",
1485 dimensions=("instrument", "visit"),
1486 multiple=True,
1487 deferLoad=True,
1488 )
1489 outputCatalog = connectionTypes.Output(
1490 doc="CCD and Visit metadata table",
1491 name="ccdVisitTable",
1492 storageClass="DataFrame",
1493 dimensions=("instrument",)
1494 )
1497class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1498 pipelineConnections=MakeCcdVisitTableConnections):
1499 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1502class MakeCcdVisitTableTask(pipeBase.PipelineTask):
1503 """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
1504 """
1505 _DefaultName = 'makeCcdVisitTable'
1506 ConfigClass = MakeCcdVisitTableConfig
1508 def run(self, visitSummaryRefs):
1509 """Make a table of ccd information from the visit summary catalogs.
1511 Parameters
1512 ----------
1513 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1514 List of DeferredDatasetHandles pointing to exposure catalogs with
1515 per-detector summary information.
1517 Returns
1518 -------
1519 result : `~lsst.pipe.base.Struct`
1520 Results struct with attribute:
1522 ``outputCatalog``
1523 Catalog of ccd and visit information.
1524 """
1525 ccdEntries = []
1526 for visitSummaryRef in visitSummaryRefs:
1527 visitSummary = visitSummaryRef.get()
1528 visitInfo = visitSummary[0].getVisitInfo()
1530 ccdEntry = {}
1531 summaryTable = visitSummary.asAstropy()
1532 selectColumns = ['id', 'visit', 'physical_filter', 'band', 'ra', 'dec', 'zenithDistance',
1533 'zeroPoint', 'psfSigma', 'skyBg', 'skyNoise',
1534 'astromOffsetMean', 'astromOffsetStd', 'nPsfStar',
1535 'psfStarDeltaE1Median', 'psfStarDeltaE2Median',
1536 'psfStarDeltaE1Scatter', 'psfStarDeltaE2Scatter',
1537 'psfStarDeltaSizeMedian', 'psfStarDeltaSizeScatter',
1538 'psfStarScaledDeltaSizeScatter',
1539 'psfTraceRadiusDelta', 'maxDistToNearestPsf']
1540 ccdEntry = summaryTable[selectColumns].to_pandas().set_index('id')
1541 # 'visit' is the human readable visit number.
1542 # 'visitId' is the key to the visitId table. They are the same.
1543 # Technically you should join to get the visit from the visit
1544 # table.
1545 ccdEntry = ccdEntry.rename(columns={"visit": "visitId"})
1547 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards
1548 # compatibility. To be removed after September 2023.
1549 ccdEntry["decl"] = ccdEntry.loc[:, "dec"]
1551 ccdEntry['ccdVisitId'] = [
1552 self.config.idGenerator.apply(
1553 visitSummaryRef.dataId,
1554 detector=detector_id,
1555 is_exposure=False,
1556 ).catalog_id # The "catalog ID" here is the ccdVisit ID
1557 # because it's usually the ID for a whole catalog
1558 # with a {visit, detector}, and that's the main
1559 # use case for IdGenerator. This usage for a
1560 # summary table is rare.
1561 for detector_id in summaryTable['id']
1562 ]
1563 ccdEntry['detector'] = summaryTable['id']
1564 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds() if vR.getWcs()
1565 else np.nan for vR in visitSummary])
1566 ccdEntry["seeing"] = visitSummary['psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1568 ccdEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1569 ccdEntry["expMidpt"] = visitInfo.getDate().toPython()
1570 ccdEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1571 expTime = visitInfo.getExposureTime()
1572 ccdEntry['expTime'] = expTime
1573 ccdEntry["obsStart"] = ccdEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1574 expTime_days = expTime / (60*60*24)
1575 ccdEntry["obsStartMJD"] = ccdEntry["expMidptMJD"] - 0.5 * expTime_days
1576 ccdEntry['darkTime'] = visitInfo.getDarkTime()
1577 ccdEntry['xSize'] = summaryTable['bbox_max_x'] - summaryTable['bbox_min_x']
1578 ccdEntry['ySize'] = summaryTable['bbox_max_y'] - summaryTable['bbox_min_y']
1579 ccdEntry['llcra'] = summaryTable['raCorners'][:, 0]
1580 ccdEntry['llcdec'] = summaryTable['decCorners'][:, 0]
1581 ccdEntry['ulcra'] = summaryTable['raCorners'][:, 1]
1582 ccdEntry['ulcdec'] = summaryTable['decCorners'][:, 1]
1583 ccdEntry['urcra'] = summaryTable['raCorners'][:, 2]
1584 ccdEntry['urcdec'] = summaryTable['decCorners'][:, 2]
1585 ccdEntry['lrcra'] = summaryTable['raCorners'][:, 3]
1586 ccdEntry['lrcdec'] = summaryTable['decCorners'][:, 3]
1587 # TODO: DM-30618, Add raftName, nExposures, ccdTemp, binX, binY,
1588 # and flags, and decide if WCS, and llcx, llcy, ulcx, ulcy, etc.
1589 # values are actually wanted.
1590 ccdEntries.append(ccdEntry)
1592 outputCatalog = pd.concat(ccdEntries)
1593 outputCatalog.set_index('ccdVisitId', inplace=True, verify_integrity=True)
1594 return pipeBase.Struct(outputCatalog=outputCatalog)
1597class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1598 dimensions=("instrument",),
1599 defaultTemplates={"calexpType": ""}):
1600 visitSummaries = connectionTypes.Input(
1601 doc="Per-visit consolidated exposure metadata",
1602 name="finalVisitSummary",
1603 storageClass="ExposureCatalog",
1604 dimensions=("instrument", "visit",),
1605 multiple=True,
1606 deferLoad=True,
1607 )
1608 outputCatalog = connectionTypes.Output(
1609 doc="Visit metadata table",
1610 name="visitTable",
1611 storageClass="DataFrame",
1612 dimensions=("instrument",)
1613 )
1616class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1617 pipelineConnections=MakeVisitTableConnections):
1618 pass
1621class MakeVisitTableTask(pipeBase.PipelineTask):
1622 """Produce a `visitTable` from the visit summary exposure catalogs.
1623 """
1624 _DefaultName = 'makeVisitTable'
1625 ConfigClass = MakeVisitTableConfig
1627 def run(self, visitSummaries):
1628 """Make a table of visit information from the visit summary catalogs.
1630 Parameters
1631 ----------
1632 visitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
1633 List of exposure catalogs with per-detector summary information.
1634 Returns
1635 -------
1636 result : `~lsst.pipe.base.Struct`
1637 Results struct with attribute:
1639 ``outputCatalog``
1640 Catalog of visit information.
1641 """
1642 visitEntries = []
1643 for visitSummary in visitSummaries:
1644 visitSummary = visitSummary.get()
1645 visitRow = visitSummary[0]
1646 visitInfo = visitRow.getVisitInfo()
1648 visitEntry = {}
1649 visitEntry["visitId"] = visitRow['visit']
1650 visitEntry["visit"] = visitRow['visit']
1651 visitEntry["physical_filter"] = visitRow['physical_filter']
1652 visitEntry["band"] = visitRow['band']
1653 raDec = visitInfo.getBoresightRaDec()
1654 visitEntry["ra"] = raDec.getRa().asDegrees()
1655 visitEntry["dec"] = raDec.getDec().asDegrees()
1657 # RFC-924: Temporarily keep a duplicate "decl" entry for backwards
1658 # compatibility. To be removed after September 2023.
1659 visitEntry["decl"] = visitEntry["dec"]
1661 visitEntry["skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1662 azAlt = visitInfo.getBoresightAzAlt()
1663 visitEntry["azimuth"] = azAlt.getLongitude().asDegrees()
1664 visitEntry["altitude"] = azAlt.getLatitude().asDegrees()
1665 visitEntry["zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1666 visitEntry["airmass"] = visitInfo.getBoresightAirmass()
1667 expTime = visitInfo.getExposureTime()
1668 visitEntry["expTime"] = expTime
1669 visitEntry["expMidpt"] = visitInfo.getDate().toPython()
1670 visitEntry["expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1671 visitEntry["obsStart"] = visitEntry["expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1672 expTime_days = expTime / (60*60*24)
1673 visitEntry["obsStartMJD"] = visitEntry["expMidptMJD"] - 0.5 * expTime_days
1674 visitEntries.append(visitEntry)
1676 # TODO: DM-30623, Add programId, exposureType, cameraTemp,
1677 # mirror1Temp, mirror2Temp, mirror3Temp, domeTemp, externalTemp,
1678 # dimmSeeing, pwvGPS, pwvMW, flags, nExposures.
1680 outputCatalog = pd.DataFrame(data=visitEntries)
1681 outputCatalog.set_index('visitId', inplace=True, verify_integrity=True)
1682 return pipeBase.Struct(outputCatalog=outputCatalog)
1685class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1686 dimensions=("instrument", "visit", "detector", "skymap", "tract")):
1688 inputCatalog = connectionTypes.Input(
1689 doc="Primary per-detector, single-epoch forced-photometry catalog. "
1690 "By default, it is the output of ForcedPhotCcdTask on calexps",
1691 name="forced_src",
1692 storageClass="SourceCatalog",
1693 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1694 )
1695 inputCatalogDiff = connectionTypes.Input(
1696 doc="Secondary multi-epoch, per-detector, forced photometry catalog. "
1697 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1698 name="forced_diff",
1699 storageClass="SourceCatalog",
1700 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1701 )
1702 outputCatalog = connectionTypes.Output(
1703 doc="InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format",
1704 name="mergedForcedSource",
1705 storageClass="DataFrame",
1706 dimensions=("instrument", "visit", "detector", "skymap", "tract")
1707 )
1710class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1711 pipelineConnections=WriteForcedSourceTableConnections):
1712 key = lsst.pex.config.Field(
1713 doc="Column on which to join the two input tables on and make the primary key of the output",
1714 dtype=str,
1715 default="objectId",
1716 )
1717 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1720class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1721 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format.
1723 Because the predecessor ForcedPhotCcdTask operates per-detector,
1724 per-tract, (i.e., it has tract in its dimensions), detectors
1725 on the tract boundary may have multiple forced source catalogs.
1727 The successor task TransformForcedSourceTable runs per-patch
1728 and temporally-aggregates overlapping mergedForcedSource catalogs from all
1729 available multiple epochs.
1730 """
1731 _DefaultName = "writeForcedSourceTable"
1732 ConfigClass = WriteForcedSourceTableConfig
1734 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1735 inputs = butlerQC.get(inputRefs)
1736 # Add ccdVisitId to allow joining with CcdVisitTable
1737 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
1738 inputs['ccdVisitId'] = idGenerator.catalog_id
1739 inputs['band'] = butlerQC.quantum.dataId.full['band']
1740 outputs = self.run(**inputs)
1741 butlerQC.put(outputs, outputRefs)
1743 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1744 dfs = []
1745 for table, dataset, in zip((inputCatalog, inputCatalogDiff), ('calexp', 'diff')):
1746 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=False)
1747 df = df.reindex(sorted(df.columns), axis=1)
1748 df['ccdVisitId'] = ccdVisitId if ccdVisitId else pd.NA
1749 df['band'] = band if band else pd.NA
1750 df.columns = pd.MultiIndex.from_tuples([(dataset, c) for c in df.columns],
1751 names=('dataset', 'column'))
1753 dfs.append(df)
1755 outputCatalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs)
1756 return pipeBase.Struct(outputCatalog=outputCatalog)
1759class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1760 dimensions=("instrument", "skymap", "patch", "tract")):
1762 inputCatalogs = connectionTypes.Input(
1763 doc="DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask",
1764 name="mergedForcedSource",
1765 storageClass="DataFrame",
1766 dimensions=("instrument", "visit", "detector", "skymap", "tract"),
1767 multiple=True,
1768 deferLoad=True
1769 )
1770 referenceCatalog = connectionTypes.Input(
1771 doc="Reference catalog which was used to seed the forcedPhot. Columns "
1772 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1773 "are expected.",
1774 name="objectTable",
1775 storageClass="DataFrame",
1776 dimensions=("tract", "patch", "skymap"),
1777 deferLoad=True
1778 )
1779 outputCatalog = connectionTypes.Output(
1780 doc="Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1781 "specified set of functors",
1782 name="forcedSourceTable",
1783 storageClass="DataFrame",
1784 dimensions=("tract", "patch", "skymap")
1785 )
1788class TransformForcedSourceTableConfig(TransformCatalogBaseConfig,
1789 pipelineConnections=TransformForcedSourceTableConnections):
1790 referenceColumns = pexConfig.ListField(
1791 dtype=str,
1792 default=["detect_isPrimary", "detect_isTractInner", "detect_isPatchInner"],
1793 optional=True,
1794 doc="Columns to pull from reference catalog",
1795 )
1796 keyRef = lsst.pex.config.Field(
1797 doc="Column on which to join the two input tables on and make the primary key of the output",
1798 dtype=str,
1799 default="objectId",
1800 )
1801 key = lsst.pex.config.Field(
1802 doc="Rename the output DataFrame index to this name",
1803 dtype=str,
1804 default="forcedSourceId",
1805 )
1807 def setDefaults(self):
1808 super().setDefaults()
1809 self.functorFile = os.path.join('$PIPE_TASKS_DIR', 'schemas', 'ForcedSource.yaml')
1810 self.columnsFromDataId = ['tract', 'patch']
1813class TransformForcedSourceTableTask(TransformCatalogBaseTask):
1814 """Transform/standardize a ForcedSource catalog
1816 Transforms each wide, per-detector forcedSource DataFrame per the
1817 specification file (per-camera defaults found in ForcedSource.yaml).
1818 All epochs that overlap the patch are aggregated into one per-patch
1819 narrow-DataFrame file.
1821 No de-duplication of rows is performed. Duplicate resolutions flags are
1822 pulled in from the referenceCatalog: `detect_isPrimary`,
1823 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
1824 for analysis or compare duplicates for QA.
1826 The resulting table includes multiple bands. Epochs (MJDs) and other useful
1827 per-visit rows can be retreived by joining with the CcdVisitTable on
1828 ccdVisitId.
1829 """
1830 _DefaultName = "transformForcedSourceTable"
1831 ConfigClass = TransformForcedSourceTableConfig
1833 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1834 inputs = butlerQC.get(inputRefs)
1835 if self.funcs is None:
1836 raise ValueError("config.functorFile is None. "
1837 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1838 outputs = self.run(inputs['inputCatalogs'], inputs['referenceCatalog'], funcs=self.funcs,
1839 dataId=outputRefs.outputCatalog.dataId.full)
1841 butlerQC.put(outputs, outputRefs)
1843 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
1844 dfs = []
1845 ref = referenceCatalog.get(parameters={"columns": self.config.referenceColumns})
1846 self.log.info("Aggregating %s input catalogs" % (len(inputCatalogs)))
1847 for handle in inputCatalogs:
1848 result = self.transform(None, handle, funcs, dataId)
1849 # Filter for only rows that were detected on (overlap) the patch
1850 dfs.append(result.df.join(ref, how='inner'))
1852 outputCatalog = pd.concat(dfs)
1854 # Now that we are done joining on config.keyRef
1855 # Change index to config.key by
1856 outputCatalog.index.rename(self.config.keyRef, inplace=True)
1857 # Add config.keyRef to the column list
1858 outputCatalog.reset_index(inplace=True)
1859 # Set the forcedSourceId to the index. This is specified in the
1860 # ForcedSource.yaml
1861 outputCatalog.set_index("forcedSourceId", inplace=True, verify_integrity=True)
1862 # Rename it to the config.key
1863 outputCatalog.index.rename(self.config.key, inplace=True)
1865 self.log.info("Made a table of %d columns and %d rows",
1866 len(outputCatalog.columns), len(outputCatalog))
1867 return pipeBase.Struct(outputCatalog=outputCatalog)
1870class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
1871 defaultTemplates={"catalogType": ""},
1872 dimensions=("instrument", "tract")):
1873 inputCatalogs = connectionTypes.Input(
1874 doc="Input per-patch DataFrame Tables to be concatenated",
1875 name="{catalogType}ForcedSourceTable",
1876 storageClass="DataFrame",
1877 dimensions=("tract", "patch", "skymap"),
1878 multiple=True,
1879 )
1881 outputCatalog = connectionTypes.Output(
1882 doc="Output per-tract concatenation of DataFrame Tables",
1883 name="{catalogType}ForcedSourceTable_tract",
1884 storageClass="DataFrame",
1885 dimensions=("tract", "skymap"),
1886 )
1889class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
1890 pipelineConnections=ConsolidateTractConnections):
1891 pass
1894class ConsolidateTractTask(pipeBase.PipelineTask):
1895 """Concatenate any per-patch, dataframe list into a single
1896 per-tract DataFrame.
1897 """
1898 _DefaultName = 'ConsolidateTract'
1899 ConfigClass = ConsolidateTractConfig
1901 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1902 inputs = butlerQC.get(inputRefs)
1903 # Not checking at least one inputCatalog exists because that'd be an
1904 # empty QG.
1905 self.log.info("Concatenating %s per-patch %s Tables",
1906 len(inputs['inputCatalogs']),
1907 inputRefs.inputCatalogs[0].datasetType.name)
1908 df = pd.concat(inputs['inputCatalogs'])
1909 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)