22__all__ = [
"WriteObjectTableConfig",
"WriteObjectTableTask",
23 "WriteSourceTableConfig",
"WriteSourceTableTask",
24 "WriteRecalibratedSourceTableConfig",
"WriteRecalibratedSourceTableTask",
25 "PostprocessAnalysis",
26 "TransformCatalogBaseConfig",
"TransformCatalogBaseTask",
27 "TransformObjectCatalogConfig",
"TransformObjectCatalogTask",
28 "ConsolidateObjectTableConfig",
"ConsolidateObjectTableTask",
29 "TransformSourceTableConfig",
"TransformSourceTableTask",
30 "ConsolidateVisitSummaryConfig",
"ConsolidateVisitSummaryTask",
31 "ConsolidateSourceTableConfig",
"ConsolidateSourceTableTask",
32 "MakeCcdVisitTableConfig",
"MakeCcdVisitTableTask",
33 "MakeVisitTableConfig",
"MakeVisitTableTask",
34 "WriteForcedSourceTableConfig",
"WriteForcedSourceTableTask",
35 "TransformForcedSourceTableConfig",
"TransformForcedSourceTableTask",
36 "ConsolidateTractConfig",
"ConsolidateTractTask"]
46from deprecated.sphinx
import deprecated
50import lsst.pipe.base
as pipeBase
52from lsst.utils.introspection
import find_outside_stacklevel
53from lsst.pipe.base
import connectionTypes
56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig
59from .functors
import CompositeFunctor, Column
61log = logging.getLogger(__name__)
64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
65 """Flattens a dataframe with multilevel column index.
67 newDf = pd.DataFrame()
69 dfBands = df.columns.unique(level=0).values
72 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
73 newColumns = {c: columnFormat.format(band, c)
74 for c
in subdf.columns
if c
not in noDupCols}
75 cols = list(newColumns.keys())
76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
79 presentBands = dfBands
if inputBands
is None else list(set(inputBands).intersection(dfBands))
81 noDupDf = df[presentBands[0]][noDupCols]
82 newDf = pd.concat([noDupDf, newDf], axis=1)
87 defaultTemplates={
"coaddName":
"deep"},
88 dimensions=(
"tract",
"patch",
"skymap")):
89 inputCatalogMeas = connectionTypes.Input(
90 doc=
"Catalog of source measurements on the deepCoadd.",
91 dimensions=(
"tract",
"patch",
"band",
"skymap"),
92 storageClass=
"SourceCatalog",
93 name=
"{coaddName}Coadd_meas",
96 inputCatalogForcedSrc = connectionTypes.Input(
97 doc=
"Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
98 dimensions=(
"tract",
"patch",
"band",
"skymap"),
99 storageClass=
"SourceCatalog",
100 name=
"{coaddName}Coadd_forced_src",
103 inputCatalogRef = connectionTypes.Input(
104 doc=
"Catalog marking the primary detection (which band provides a good shape and position)"
105 "for each detection in deepCoadd_mergeDet.",
106 dimensions=(
"tract",
"patch",
"skymap"),
107 storageClass=
"SourceCatalog",
108 name=
"{coaddName}Coadd_ref"
110 outputCatalog = connectionTypes.Output(
111 doc=
"A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
112 "stored as a DataFrame with a multi-level column index per-patch.",
113 dimensions=(
"tract",
"patch",
"skymap"),
114 storageClass=
"DataFrame",
115 name=
"{coaddName}Coadd_obj"
119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
120 pipelineConnections=WriteObjectTableConnections):
121 engine = pexConfig.Field(
124 doc=
"Parquet engine for writing (pyarrow or fastparquet)",
125 deprecated=
"This config is no longer used, and will be removed after v26."
127 coaddName = pexConfig.Field(
134class WriteObjectTableTask(pipeBase.PipelineTask):
135 """Write filter-merged source tables as a DataFrame in parquet format.
137 _DefaultName =
"writeObjectTable"
138 ConfigClass = WriteObjectTableConfig
141 inputDatasets = (
'forced_src',
'meas',
'ref')
144 outputDataset =
'obj'
146 def runQuantum(self, butlerQC, inputRefs, outputRefs):
147 inputs = butlerQC.get(inputRefs)
149 measDict = {ref.dataId[
'band']: {
'meas': cat}
for ref, cat
in
150 zip(inputRefs.inputCatalogMeas, inputs[
'inputCatalogMeas'])}
151 forcedSourceDict = {ref.dataId[
'band']: {
'forced_src': cat}
for ref, cat
in
152 zip(inputRefs.inputCatalogForcedSrc, inputs[
'inputCatalogForcedSrc'])}
155 for band
in measDict.keys():
156 catalogs[band] = {
'meas': measDict[band][
'meas'],
157 'forced_src': forcedSourceDict[band][
'forced_src'],
158 'ref': inputs[
'inputCatalogRef']}
159 dataId = butlerQC.quantum.dataId
160 df = self.run(catalogs=catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
161 outputs = pipeBase.Struct(outputCatalog=df)
162 butlerQC.put(outputs, outputRefs)
164 def run(self, catalogs, tract, patch):
165 """Merge multiple catalogs.
170 Mapping from filter names to dict of catalogs.
172 tractId to use for the tractId column.
174 patchId to use for the patchId column.
178 catalog : `pandas.DataFrame`
182 for filt, tableDict
in catalogs.items():
183 for dataset, table
in tableDict.items():
185 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
188 df = df.reindex(sorted(df.columns), axis=1)
189 df = df.assign(tractId=tract, patchId=patch)
192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
193 names=(
'dataset',
'band',
'column'))
198 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
203 defaultTemplates={
"catalogType":
""},
204 dimensions=(
"instrument",
"visit",
"detector")):
206 catalog = connectionTypes.Input(
207 doc=
"Input full-depth catalog of sources produced by CalibrateTask",
208 name=
"{catalogType}src",
209 storageClass=
"SourceCatalog",
210 dimensions=(
"instrument",
"visit",
"detector")
212 outputCatalog = connectionTypes.Output(
213 doc=
"Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
214 "replaced with an index; all other columns are unchanged.",
215 name=
"{catalogType}source",
216 storageClass=
"DataFrame",
217 dimensions=(
"instrument",
"visit",
"detector")
221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
222 pipelineConnections=WriteSourceTableConnections):
223 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
226class WriteSourceTableTask(pipeBase.PipelineTask):
227 """Write source table to DataFrame Parquet format.
229 _DefaultName =
"writeSourceTable"
230 ConfigClass = WriteSourceTableConfig
232 def runQuantum(self, butlerQC, inputRefs, outputRefs):
233 inputs = butlerQC.get(inputRefs)
234 inputs[
'ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id
235 result = self.run(**inputs)
236 outputs = pipeBase.Struct(outputCatalog=result.table)
237 butlerQC.put(outputs, outputRefs)
239 def run(self, catalog, ccdVisitId=None, **kwargs):
240 """Convert `src` catalog to DataFrame
244 catalog: `afwTable.SourceCatalog`
245 catalog to be converted
247 ccdVisitId to be added as a column
249 Additional keyword arguments are ignored as a convenience for
250 subclasses that pass the same arguments to several different
255 result : `~lsst.pipe.base.Struct`
257 `DataFrame` version of the input catalog
259 self.log.info(
"Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
260 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
261 df[
'ccdVisitId'] = ccdVisitId
263 return pipeBase.Struct(table=df)
266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
267 defaultTemplates={
"catalogType":
"",
268 "skyWcsName":
"gbdesAstrometricFit",
269 "photoCalibName":
"fgcm"},
271 deprecatedTemplates={
272 "skyWcsName":
"Deprecated; will be removed after v26.",
273 "photoCalibName":
"Deprecated; will be removed after v26."
275 dimensions=(
"instrument",
"visit",
"detector",
"skymap")):
276 skyMap = connectionTypes.Input(
277 doc=
"skyMap needed to choose which tract-level calibrations to use when multiple available",
278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
279 storageClass=
"SkyMap",
280 dimensions=(
"skymap",),
283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. "
284 "Will be removed after v26."
287 exposure = connectionTypes.Input(
288 doc=
"Input exposure to perform photometry on.",
290 storageClass=
"ExposureF",
291 dimensions=[
"instrument",
"visit",
"detector"],
294 "Deprecated, as the `calexp` is not needed and just creates unnecessary i/o. "
295 "Will be removed after v26."
298 visitSummary = connectionTypes.Input(
299 doc=
"Input visit-summary catalog with updated calibration objects.",
300 name=
"finalVisitSummary",
301 storageClass=
"ExposureCatalog",
302 dimensions=(
"instrument",
"visit",),
304 externalSkyWcsTractCatalog = connectionTypes.Input(
305 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
306 "id for the catalog id, sorted on id for fast lookup."),
307 name=
"{skyWcsName}SkyWcsCatalog",
308 storageClass=
"ExposureCatalog",
309 dimensions=[
"instrument",
"visit",
"tract"],
312 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
314 externalSkyWcsGlobalCatalog = connectionTypes.Input(
315 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
316 "These catalogs use the detector id for the catalog id, sorted on id for "
318 name=
"finalVisitSummary",
319 storageClass=
"ExposureCatalog",
320 dimensions=[
"instrument",
"visit"],
322 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
324 externalPhotoCalibTractCatalog = connectionTypes.Input(
325 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
326 "detector id for the catalog id, sorted on id for fast lookup."),
327 name=
"{photoCalibName}PhotoCalibCatalog",
328 storageClass=
"ExposureCatalog",
329 dimensions=[
"instrument",
"visit",
"tract"],
332 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
334 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
335 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
336 "information). These catalogs use the detector id for the catalog id, "
337 "sorted on id for fast lookup."),
338 name=
"finalVisitSummary",
339 storageClass=
"ExposureCatalog",
340 dimensions=[
"instrument",
"visit"],
342 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
345 def __init__(self, *, config=None):
346 super().__init__(config=config)
352 if config.doApplyExternalSkyWcs
and config.doReevaluateSkyWcs:
354 if config.useGlobalExternalSkyWcs:
355 self.inputs.remove(
"externalSkyWcsTractCatalog")
357 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
360 self.inputs.remove(
"externalSkyWcsTractCatalog")
361 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
362 if config.doApplyExternalPhotoCalib
and config.doReevaluatePhotoCalib:
364 if config.useGlobalExternalPhotoCalib:
365 self.inputs.remove(
"externalPhotoCalibTractCatalog")
367 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
370 self.inputs.remove(
"externalPhotoCalibTractCatalog")
371 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
378class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
379 pipelineConnections=WriteRecalibratedSourceTableConnections):
381 doReevaluatePhotoCalib = pexConfig.Field(
384 doc=(
"Add or replace local photoCalib columns"),
386 doReevaluateSkyWcs = pexConfig.Field(
389 doc=(
"Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"),
391 doApplyExternalPhotoCalib = pexConfig.Field(
394 doc=(
"If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
395 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
397 deprecated=
"Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
399 doApplyExternalSkyWcs = pexConfig.Field(
402 doc=(
"if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
403 "else use the wcs already attached to the exposure."),
405 deprecated=
"Deprecated along with the external WCS connections. Will be removed after v26.",
407 useGlobalExternalPhotoCalib = pexConfig.Field(
410 doc=(
"When using doApplyExternalPhotoCalib, use 'global' calibrations "
411 "that are not run per-tract. When False, use per-tract photometric "
412 "calibration files."),
414 deprecated=
"Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
416 useGlobalExternalSkyWcs = pexConfig.Field(
419 doc=(
"When using doApplyExternalSkyWcs, use 'global' calibrations "
420 "that are not run per-tract. When False, use per-tract wcs "
423 deprecated=
"Deprecated along with the external WCS connections. Will be removed after v26.",
425 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
429 if self.doApplyExternalSkyWcs
and not self.doReevaluateSkyWcs:
430 log.warning(
"doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
431 "External SkyWcs will not be read or evaluated.")
432 if self.doApplyExternalPhotoCalib
and not self.doReevaluatePhotoCalib:
433 log.warning(
"doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
434 "External PhotoCalib will not be read or evaluated.")
437class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
438 """Write source table to DataFrame Parquet format.
440 _DefaultName =
"writeRecalibratedSourceTable"
441 ConfigClass = WriteRecalibratedSourceTableConfig
443 def runQuantum(self, butlerQC, inputRefs, outputRefs):
444 inputs = butlerQC.get(inputRefs)
446 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
447 inputs[
'idGenerator'] = idGenerator
448 inputs[
'ccdVisitId'] = idGenerator.catalog_id
450 if self.config.doReevaluatePhotoCalib
or self.config.doReevaluateSkyWcs:
451 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
452 inputs[
'exposure'] = self.attachCalibs(inputRefs, **inputs)
455 exposure = ExposureF()
456 detectorId = butlerQC.quantum.dataId[
"detector"]
457 inputs[
'exposure'] = self.prepareCalibratedExposure(
459 detectorId=detectorId,
460 visitSummary=inputs[
"visitSummary"],
462 inputs[
'catalog'] = self.addCalibColumns(**inputs)
464 result = self.run(**inputs)
465 outputs = pipeBase.Struct(outputCatalog=result.table)
466 butlerQC.put(outputs, outputRefs)
469 @deprecated(
"Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
470 version=
"v26", category=FutureWarning)
471 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
472 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
473 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs):
474 """Apply external calibrations to exposure per configuration
476 When multiple tract-level calibrations overlap, select the one with the
477 center closest to detector.
481 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`, for dataIds of
483 skyMap : `~lsst.skymap.BaseSkyMap`
484 skyMap to lookup tract geometry and WCS.
485 exposure : `lsst.afw.image.exposure.Exposure`
486 Input exposure to adjust calibrations.
487 externalSkyWcsGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
488 Exposure catalog with external skyWcs to be applied per config
489 externalSkyWcsTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
490 Exposure catalog with external skyWcs to be applied per config
491 externalPhotoCalibGlobalCatalog : `~lsst.afw.table.ExposureCatalog`, optional
492 Exposure catalog with external photoCalib to be applied per config
493 externalPhotoCalibTractCatalog : `~lsst.afw.table.ExposureCatalog`, optional
494 Exposure catalog with external photoCalib to be applied per config
495 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
496 Exposure catalog with all calibration objects. WCS and PhotoCalib
497 are always applied if provided.
499 Additional keyword arguments are ignored to facilitate passing the
500 same arguments to several methods.
504 exposure : `lsst.afw.image.exposure.Exposure`
505 Exposure with adjusted calibrations.
507 if not self.config.doApplyExternalSkyWcs:
509 externalSkyWcsCatalog =
None
510 elif self.config.useGlobalExternalSkyWcs:
512 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
513 self.log.info(
'Applying global SkyWcs')
516 inputRef = getattr(inputRefs,
'externalSkyWcsTractCatalog')
517 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
520 self.log.info(
'Applying tract-level SkyWcs from tract %s', tracts[ind])
522 if exposure.getWcs()
is None:
523 raise ValueError(
"Trying to locate nearest tract, but exposure.wcs is None.")
524 ind = self.getClosestTract(tracts, skyMap,
525 exposure.getBBox(), exposure.getWcs())
526 self.log.info(
'Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
527 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
529 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
531 if not self.config.doApplyExternalPhotoCalib:
533 externalPhotoCalibCatalog =
None
534 elif self.config.useGlobalExternalPhotoCalib:
536 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
537 self.log.info(
'Applying global PhotoCalib')
540 inputRef = getattr(inputRefs,
'externalPhotoCalibTractCatalog')
541 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
544 self.log.info(
'Applying tract-level PhotoCalib from tract %s', tracts[ind])
546 ind = self.getClosestTract(tracts, skyMap,
547 exposure.getBBox(), exposure.getWcs())
548 self.log.info(
'Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
549 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
551 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
553 return self.prepareCalibratedExposure(
554 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary
558 @deprecated(
"Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
559 version=
"v26", category=FutureWarning)
560 def getClosestTract(self, tracts, skyMap, bbox, wcs):
561 """Find the index of the tract closest to detector from list of tractIds
565 tracts: `list` [`int`]
566 Iterable of integer tractIds
567 skyMap : `~lsst.skymap.BaseSkyMap`
568 skyMap to lookup tract geometry and wcs
569 bbox : `~lsst.geom.Box2I`
570 Detector bbox, center of which will compared to tract centers
571 wcs : `~lsst.afw.geom.SkyWcs`
572 Detector Wcs object to map the detector center to SkyCoord
581 center = wcs.pixelToSky(bbox.getCenter())
583 for tractId
in tracts:
584 tract = skyMap[tractId]
585 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
586 sep.append(center.separation(tractCenter))
588 return np.argmin(sep)
590 def prepareCalibratedExposure(
594 externalSkyWcsCatalog=None,
595 externalPhotoCalibCatalog=None,
598 """Prepare a calibrated exposure and apply external calibrations
603 exposure : `lsst.afw.image.exposure.Exposure`
604 Input exposure to adjust calibrations. May be empty.
606 Detector ID associated with the exposure.
607 externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`, optional
608 Exposure catalog with external skyWcs to be applied
609 if config.doApplyExternalSkyWcs=True. Catalog uses the detector id
610 for the catalog id, sorted on id for fast lookup.
611 Deprecated in favor of ``visitSummary``; will be removed after v26.
612 externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`, optional
613 Exposure catalog with external photoCalib to be applied
614 if config.doApplyExternalPhotoCalib=True. Catalog uses the detector
615 id for the catalog id, sorted on id for fast lookup.
616 Deprecated in favor of ``visitSummary``; will be removed after v26.
617 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
618 Exposure catalog with all calibration objects. WCS and PhotoCalib
619 are always applied if ``visitSummary`` is provided and those
620 components are not `None`.
624 exposure : `lsst.afw.image.exposure.Exposure`
625 Exposure with adjusted calibrations.
627 if visitSummary
is not None:
628 row = visitSummary.find(detectorId)
630 raise RuntimeError(f
"Visit summary for detector {detectorId} is unexpectedly missing.")
631 if (photoCalib := row.getPhotoCalib())
is None:
632 self.log.warning(
"Detector id %s has None for photoCalib in visit summary; "
633 "skipping reevaluation of photoCalib.", detectorId)
634 exposure.setPhotoCalib(
None)
636 exposure.setPhotoCalib(photoCalib)
637 if (skyWcs := row.getWcs())
is None:
638 self.log.warning(
"Detector id %s has None for skyWcs in visit summary; "
639 "skipping reevaluation of skyWcs.", detectorId)
640 exposure.setWcs(
None)
642 exposure.setWcs(skyWcs)
644 if externalPhotoCalibCatalog
is not None:
647 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
649 stacklevel=find_outside_stacklevel(
"lsst.pipe.tasks.postprocessing"),
651 row = externalPhotoCalibCatalog.find(detectorId)
653 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog; "
654 "Using original photoCalib.", detectorId)
656 photoCalib = row.getPhotoCalib()
657 if photoCalib
is None:
658 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
659 "Using original photoCalib.", detectorId)
661 exposure.setPhotoCalib(photoCalib)
663 if externalSkyWcsCatalog
is not None:
666 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
668 stacklevel=find_outside_stacklevel(
"lsst.pipe.tasks.postprocessing"),
670 row = externalSkyWcsCatalog.find(detectorId)
672 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog; "
673 "Using original skyWcs.", detectorId)
675 skyWcs = row.getWcs()
677 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
678 "Using original skyWcs.", detectorId)
680 exposure.setWcs(skyWcs)
684 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs):
685 """Add replace columns with calibs evaluated at each centroid
687 Add or replace 'base_LocalWcs' `base_LocalPhotoCalib' columns in a
688 a source catalog, by rerunning the plugins.
692 catalog : `lsst.afw.table.SourceCatalog`
693 catalog to which calib columns will be added
694 exposure : `lsst.afw.image.exposure.Exposure`
695 Exposure with attached PhotoCalibs and SkyWcs attributes to be
696 reevaluated at local centroids. Pixels are not required.
697 idGenerator : `lsst.meas.base.IdGenerator`
698 Object that generates Source IDs and random seeds.
700 Additional keyword arguments are ignored to facilitate passing the
701 same arguments to several methods.
705 newCat: `lsst.afw.table.SourceCatalog`
706 Source Catalog with requested local calib columns
708 measureConfig = SingleFrameMeasurementTask.ConfigClass()
709 measureConfig.doReplaceWithNoise =
False
712 for slot
in measureConfig.slots:
713 setattr(measureConfig.slots, slot,
None)
715 measureConfig.plugins.names = []
716 if self.config.doReevaluateSkyWcs:
717 measureConfig.plugins.names.add(
"base_LocalWcs")
718 self.log.info(
"Re-evaluating base_LocalWcs plugin")
719 if self.config.doReevaluatePhotoCalib:
720 measureConfig.plugins.names.add(
"base_LocalPhotoCalib")
721 self.log.info(
"Re-evaluating base_LocalPhotoCalib plugin")
722 pluginsNotToCopy = tuple(measureConfig.plugins.names)
726 aliasMap = catalog.schema.getAliasMap()
727 mapper = afwTable.SchemaMapper(catalog.schema)
728 for item
in catalog.schema:
729 if not item.field.getName().startswith(pluginsNotToCopy):
730 mapper.addMapping(item.key)
732 schema = mapper.getOutputSchema()
733 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
734 schema.setAliasMap(aliasMap)
735 newCat = afwTable.SourceCatalog(schema)
736 newCat.extend(catalog, mapper=mapper)
742 if self.config.doReevaluateSkyWcs
and exposure.wcs
is not None:
743 afwTable.updateSourceCoords(exposure.wcs, newCat)
744 wcsPlugin = measurement.plugins[
"base_LocalWcs"]
748 if self.config.doReevaluatePhotoCalib
and exposure.getPhotoCalib()
is not None:
749 pcPlugin = measurement.plugins[
"base_LocalPhotoCalib"]
754 if wcsPlugin
is not None:
755 wcsPlugin.measure(row, exposure)
756 if pcPlugin
is not None:
757 pcPlugin.measure(row, exposure)
763 """Calculate columns from DataFrames or handles storing DataFrames.
765 This object manages and organizes an arbitrary set of computations
766 on a catalog. The catalog is defined by a
767 `DeferredDatasetHandle` or `InMemoryDatasetHandle` object
768 (or list thereof), such as a ``deepCoadd_obj`` dataset, and the
769 computations are defined by a collection of
770 `~lsst.pipe.tasks.functors.Functor` objects (or, equivalently, a
771 ``CompositeFunctor``).
773 After the object is initialized, accessing the ``.df`` attribute (which
774 holds the `pandas.DataFrame` containing the results of the calculations)
775 triggers computation of said dataframe.
777 One of the conveniences of using this object is the ability to define a
778 desired common filter for all functors. This enables the same functor
779 collection to be passed to several different `PostprocessAnalysis` objects
780 without having to change the original functor collection, since the ``filt``
781 keyword argument of this object triggers an overwrite of the ``filt``
782 property for all functors in the collection.
784 This object also allows a list of refFlags to be passed, and defines a set
785 of default refFlags that are always included even if not requested.
787 If a list of DataFrames or Handles is passed, rather than a single one,
788 then the calculations will be mapped over all the input catalogs. In
789 principle, it should be straightforward to parallelize this activity, but
790 initial tests have failed (see TODO in code comments).
794 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
795 `~lsst.pipe.base.InMemoryDatasetHandle` or
797 Source catalog(s) for computation.
798 functors : `list`, `dict`, or `~lsst.pipe.tasks.functors.CompositeFunctor`
799 Computations to do (functors that act on ``handles``).
800 If a dict, the output
801 DataFrame will have columns keyed accordingly.
802 If a list, the column keys will come from the
803 ``.shortname`` attribute of each functor.
805 filt : `str`, optional
806 Filter in which to calculate. If provided,
807 this will overwrite any existing ``.filt`` attribute
808 of the provided functors.
810 flags : `list`, optional
811 List of flags (per-band) to include in output table.
812 Taken from the ``meas`` dataset if applied to a multilevel Object Table.
814 refFlags : `list`, optional
815 List of refFlags (only reference band) to include in output table.
817 forcedFlags : `list`, optional
818 List of flags (per-band) to include in output table.
819 Taken from the ``forced_src`` dataset if applied to a
820 multilevel Object Table. Intended for flags from measurement plugins
821 only run during multi-band forced-photometry.
823 _defaultRefFlags = []
826 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
831 self.
flags = list(flags)
if flags
is not None else []
832 self.
forcedFlags = list(forcedFlags)
if forcedFlags
is not None else []
834 if refFlags
is not None:
847 additionalFuncs.update({flag:
Column(flag, dataset=
'forced_src')
for flag
in self.
forcedFlags})
848 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.
refFlags})
849 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.
flags})
851 if isinstance(self.
functors, CompositeFunctor):
856 func.funcDict.update(additionalFuncs)
857 func.filt = self.
filt
863 return [name
for name, func
in self.
func.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
873 if type(self.
handles)
in (list, tuple):
875 dflist = [self.
func(handle, dropna=dropna)
for handle
in self.
handles]
879 dflist = pool.map(functools.partial(self.
func, dropna=dropna), self.
handles)
880 self.
_df = pd.concat(dflist)
889 """Expected Connections for subclasses of TransformCatalogBaseTask.
893 inputCatalog = connectionTypes.Input(
895 storageClass=
"DataFrame",
897 outputCatalog = connectionTypes.Output(
899 storageClass=
"DataFrame",
904 pipelineConnections=TransformCatalogBaseConnections):
905 functorFile = pexConfig.Field(
907 doc=
"Path to YAML file specifying Science Data Model functors to use "
908 "when copying columns and computing calibrated values.",
912 primaryKey = pexConfig.Field(
914 doc=
"Name of column to be set as the DataFrame index. If None, the index"
915 "will be named `id`",
919 columnsFromDataId = pexConfig.ListField(
923 doc=
"Columns to extract from the dataId",
928 """Base class for transforming/standardizing a catalog by applying functors
929 that convert units and apply calibrations.
931 The purpose of this task is to perform a set of computations on an input
932 ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` that holds a
933 ``DataFrame`` dataset (such as ``deepCoadd_obj``), and write the results to
934 a new dataset (which needs to be declared in an ``outputDataset``
937 The calculations to be performed are defined in a YAML file that specifies
938 a set of functors to be computed, provided as a ``--functorFile`` config
939 parameter. An example of such a YAML file is the following:
946 args: slot_Centroid_x
949 args: slot_Centroid_y
951 functor: LocalNanojansky
953 - slot_PsfFlux_instFlux
954 - slot_PsfFlux_instFluxErr
955 - base_LocalPhotoCalib
956 - base_LocalPhotoCalibErr
958 functor: LocalNanojanskyErr
960 - slot_PsfFlux_instFlux
961 - slot_PsfFlux_instFluxErr
962 - base_LocalPhotoCalib
963 - base_LocalPhotoCalibErr
967 The names for each entry under "func" will become the names of columns in
968 the output dataset. All the functors referenced are defined in
969 `~lsst.pipe.tasks.functors`. Positional arguments to be passed to each
970 functor are in the `args` list, and any additional entries for each column
971 other than "functor" or "args" (e.g., ``'filt'``, ``'dataset'``) are
972 treated as keyword arguments to be passed to the functor initialization.
974 The "flags" entry is the default shortcut for `Column` functors.
975 All columns listed under "flags" will be copied to the output table
976 untransformed. They can be of any datatype.
977 In the special case of transforming a multi-level oject table with
978 band and dataset indices (deepCoadd_obj), these will be taked from the
979 `meas` dataset and exploded out per band.
981 There are two special shortcuts that only apply when transforming
982 multi-level Object (deepCoadd_obj) tables:
983 - The "refFlags" entry is shortcut for `Column` functor
984 taken from the `'ref'` dataset if transforming an ObjectTable.
985 - The "forcedFlags" entry is shortcut for `Column` functors.
986 taken from the ``forced_src`` dataset if transforming an ObjectTable.
987 These are expanded out per band.
990 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
991 to organize and excecute the calculations.
995 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
999 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
1003 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
1007 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
1011 if self.config.functorFile:
1012 self.log.info(
'Loading tranform functor definitions from %s',
1013 self.config.functorFile)
1014 self.
funcs = CompositeFunctor.from_file(self.config.functorFile)
1015 self.
funcs.update(dict(PostprocessAnalysis._defaultFuncs))
1020 inputs = butlerQC.get(inputRefs)
1021 if self.
funcs is None:
1022 raise ValueError(
"config.functorFile is None. "
1023 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1024 result = self.
run(handle=inputs[
'inputCatalog'], funcs=self.
funcs,
1025 dataId=dict(outputRefs.outputCatalog.dataId.mapping))
1026 outputs = pipeBase.Struct(outputCatalog=result)
1027 butlerQC.put(outputs, outputRefs)
1029 def run(self, handle, funcs=None, dataId=None, band=None):
1030 """Do postprocessing calculations
1032 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle`` or
1033 ``DataFrame`` object and dataId,
1034 returns a dataframe with results of postprocessing calculations.
1038 handles : `~lsst.daf.butler.DeferredDatasetHandle` or
1039 `~lsst.pipe.base.InMemoryDatasetHandle` or
1040 `~pandas.DataFrame`, or list of these.
1041 DataFrames from which calculations are done.
1042 funcs : `~lsst.pipe.tasks.functors.Functor`
1043 Functors to apply to the table's columns
1044 dataId : dict, optional
1045 Used to add a `patchId` column to the output dataframe.
1046 band : `str`, optional
1047 Filter band that is being processed.
1051 df : `pandas.DataFrame`
1053 self.log.info(
"Transforming/standardizing the source table dataId: %s", dataId)
1055 df = self.
transform(band, handle, funcs, dataId).df
1056 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1068 def transform(self, band, handles, funcs, dataId):
1069 analysis = self.
getAnalysis(handles, funcs=funcs, band=band)
1071 if dataId
and self.config.columnsFromDataId:
1072 for key
in self.config.columnsFromDataId:
1074 df[key] = dataId[key]
1076 raise ValueError(f
"'{key}' in config.columnsFromDataId not found in dataId: {dataId}")
1078 if self.config.primaryKey:
1079 if df.index.name != self.config.primaryKey
and self.config.primaryKey
in df:
1080 df.reset_index(inplace=
True, drop=
True)
1081 df.set_index(self.config.primaryKey, inplace=
True)
1083 return pipeBase.Struct(
1090 defaultTemplates={
"coaddName":
"deep"},
1091 dimensions=(
"tract",
"patch",
"skymap")):
1092 inputCatalog = connectionTypes.Input(
1093 doc=
"The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
1094 "stored as a DataFrame with a multi-level column index per-patch.",
1095 dimensions=(
"tract",
"patch",
"skymap"),
1096 storageClass=
"DataFrame",
1097 name=
"{coaddName}Coadd_obj",
1100 outputCatalog = connectionTypes.Output(
1101 doc=
"Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
1103 dimensions=(
"tract",
"patch",
"skymap"),
1104 storageClass=
"DataFrame",
1110 pipelineConnections=TransformObjectCatalogConnections):
1111 coaddName = pexConfig.Field(
1117 filterMap = pexConfig.DictField(
1121 doc=(
"Dictionary mapping full filter name to short one for column name munging."
1122 "These filters determine the output columns no matter what filters the "
1123 "input data actually contain."),
1124 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
1125 "Will be removed after v22.")
1127 outputBands = pexConfig.ListField(
1131 doc=(
"These bands and only these bands will appear in the output,"
1132 " NaN-filled if the input does not include them."
1133 " If None, then use all bands found in the input.")
1135 camelCase = pexConfig.Field(
1138 doc=(
"Write per-band columns names with camelCase, else underscore "
1139 "For example: gPsFlux instead of g_PsFlux.")
1141 multilevelOutput = pexConfig.Field(
1144 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
1145 "and name-munged (False).")
1147 goodFlags = pexConfig.ListField(
1150 doc=(
"List of 'good' flags that should be set False when populating empty tables. "
1151 "All other flags are considered to be 'bad' flags and will be set to True.")
1153 floatFillValue = pexConfig.Field(
1156 doc=
"Fill value for float fields when populating empty tables."
1158 integerFillValue = pexConfig.Field(
1161 doc=
"Fill value for integer fields when populating empty tables."
1164 def setDefaults(self):
1165 super().setDefaults()
1166 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Object.yaml')
1167 self.primaryKey =
'objectId'
1168 self.columnsFromDataId = [
'tract',
'patch']
1169 self.goodFlags = [
'calib_astrometry_used',
1170 'calib_photometry_reserved',
1171 'calib_photometry_used',
1172 'calib_psf_candidate',
1173 'calib_psf_reserved',
1178 """Produce a flattened Object Table to match the format specified in
1181 Do the same set of postprocessing calculations on all bands.
1183 This is identical to `TransformCatalogBaseTask`, except for that it does
1184 the specified functor calculations for all filters present in the
1185 input `deepCoadd_obj` table. Any specific ``"filt"`` keywords specified
1186 by the YAML file will be superceded.
1188 _DefaultName =
"transformObjectCatalog"
1189 ConfigClass = TransformObjectCatalogConfig
1191 def run(self, handle, funcs=None, dataId=None, band=None):
1195 templateDf = pd.DataFrame()
1197 columns = handle.get(component=
'columns')
1198 inputBands = columns.unique(level=1).values
1200 outputBands = self.config.outputBands
if self.config.outputBands
else inputBands
1203 for inputBand
in inputBands:
1204 if inputBand
not in outputBands:
1205 self.log.info(
"Ignoring %s band data in the input", inputBand)
1207 self.log.info(
"Transforming the catalog of band %s", inputBand)
1208 result = self.transform(inputBand, handle, funcs, dataId)
1209 dfDict[inputBand] = result.df
1210 analysisDict[inputBand] = result.analysis
1211 if templateDf.empty:
1212 templateDf = result.df
1215 for filt
in outputBands:
1216 if filt
not in dfDict:
1217 self.log.info(
"Adding empty columns for band %s", filt)
1218 dfTemp = templateDf.copy()
1219 for col
in dfTemp.columns:
1220 testValue = dfTemp[col].values[0]
1221 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1223 if col
in self.config.goodFlags:
1227 elif isinstance(testValue, numbers.Integral):
1231 if isinstance(testValue, np.unsignedinteger):
1232 raise ValueError(
"Parquet tables may not have unsigned integer columns.")
1234 fillValue = self.config.integerFillValue
1236 fillValue = self.config.floatFillValue
1237 dfTemp[col].values[:] = fillValue
1238 dfDict[filt] = dfTemp
1241 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
1243 if not self.config.multilevelOutput:
1244 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
1245 if self.config.primaryKey
in noDupCols:
1246 noDupCols.remove(self.config.primaryKey)
1247 if dataId
and self.config.columnsFromDataId:
1248 noDupCols += self.config.columnsFromDataId
1249 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1250 inputBands=inputBands)
1252 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1257class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1258 dimensions=(
"tract",
"skymap")):
1259 inputCatalogs = connectionTypes.Input(
1260 doc=
"Per-Patch objectTables conforming to the standard data model.",
1262 storageClass=
"DataFrame",
1263 dimensions=(
"tract",
"patch",
"skymap"),
1266 outputCatalog = connectionTypes.Output(
1267 doc=
"Pre-tract horizontal concatenation of the input objectTables",
1268 name=
"objectTable_tract",
1269 storageClass=
"DataFrame",
1270 dimensions=(
"tract",
"skymap"),
1274class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1275 pipelineConnections=ConsolidateObjectTableConnections):
1276 coaddName = pexConfig.Field(
1283class ConsolidateObjectTableTask(pipeBase.PipelineTask):
1284 """Write patch-merged source tables to a tract-level DataFrame Parquet file.
1286 Concatenates `objectTable` list into a per-visit `objectTable_tract`.
1288 _DefaultName =
"consolidateObjectTable"
1289 ConfigClass = ConsolidateObjectTableConfig
1291 inputDataset =
'objectTable'
1292 outputDataset =
'objectTable_tract'
1294 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1295 inputs = butlerQC.get(inputRefs)
1296 self.log.info(
"Concatenating %s per-patch Object Tables",
1297 len(inputs[
'inputCatalogs']))
1298 df = pd.concat(inputs[
'inputCatalogs'])
1299 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1302class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1303 defaultTemplates={
"catalogType":
""},
1304 dimensions=(
"instrument",
"visit",
"detector")):
1306 inputCatalog = connectionTypes.Input(
1307 doc=
"Wide input catalog of sources produced by WriteSourceTableTask",
1308 name=
"{catalogType}source",
1309 storageClass=
"DataFrame",
1310 dimensions=(
"instrument",
"visit",
"detector"),
1313 outputCatalog = connectionTypes.Output(
1314 doc=
"Narrower, per-detector Source Table transformed and converted per a "
1315 "specified set of functors",
1316 name=
"{catalogType}sourceTable",
1317 storageClass=
"DataFrame",
1318 dimensions=(
"instrument",
"visit",
"detector")
1323 pipelineConnections=TransformSourceTableConnections):
1325 def setDefaults(self):
1326 super().setDefaults()
1327 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Source.yaml')
1328 self.primaryKey =
'sourceId'
1329 self.columnsFromDataId = [
'visit',
'detector',
'band',
'physical_filter']
1333 """Transform/standardize a source catalog
1335 _DefaultName =
"transformSourceTable"
1336 ConfigClass = TransformSourceTableConfig
1339class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1340 dimensions=(
"instrument",
"visit",),
1341 defaultTemplates={
"calexpType":
""}):
1342 calexp = connectionTypes.Input(
1343 doc=
"Processed exposures used for metadata",
1345 storageClass=
"ExposureF",
1346 dimensions=(
"instrument",
"visit",
"detector"),
1350 visitSummary = connectionTypes.Output(
1351 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
1352 "detector id for the id and are sorted for fast lookups of a "
1354 name=
"visitSummary",
1355 storageClass=
"ExposureCatalog",
1356 dimensions=(
"instrument",
"visit"),
1358 visitSummarySchema = connectionTypes.InitOutput(
1359 doc=
"Schema of the visitSummary catalog",
1360 name=
"visitSummary_schema",
1361 storageClass=
"ExposureCatalog",
1365class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1366 pipelineConnections=ConsolidateVisitSummaryConnections):
1367 """Config for ConsolidateVisitSummaryTask"""
1371class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
1372 """Task to consolidate per-detector visit metadata.
1374 This task aggregates the following metadata from all the detectors in a
1375 single visit into an exposure catalog:
1379 - The physical_filter and band (if available).
1380 - The psf size, shape, and effective area at the center of the detector.
1381 - The corners of the bounding box in right ascension/declination.
1383 Other quantities such as Detector, Psf, ApCorrMap, and TransmissionCurve
1384 are not persisted here because of storage concerns, and because of their
1385 limited utility as summary statistics.
1387 Tests for this task are performed in ci_hsc_gen3.
1389 _DefaultName =
"consolidateVisitSummary"
1390 ConfigClass = ConsolidateVisitSummaryConfig
1392 def __init__(self, **kwargs):
1393 super().__init__(**kwargs)
1394 self.schema = afwTable.ExposureTable.makeMinimalSchema()
1395 self.schema.addField(
'visit', type=
'L', doc=
'Visit number')
1396 self.schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
1397 self.schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
1398 ExposureSummaryStats.update_schema(self.schema)
1399 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
1401 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1402 dataRefs = butlerQC.get(inputRefs.calexp)
1403 visit = dataRefs[0].dataId[
'visit']
1405 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)",
1406 len(dataRefs), visit)
1408 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1410 butlerQC.put(expCatalog, outputRefs.visitSummary)
1412 def _combineExposureMetadata(self, visit, dataRefs):
1413 """Make a combined exposure catalog from a list of dataRefs.
1414 These dataRefs must point to exposures with wcs, summaryStats,
1415 and other visit metadata.
1420 Visit identification number.
1421 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1422 List of dataRefs in visit.
1426 visitSummary : `lsst.afw.table.ExposureCatalog`
1427 Exposure catalog with per-detector summary information.
1429 cat = afwTable.ExposureCatalog(self.schema)
1430 cat.resize(len(dataRefs))
1432 cat[
'visit'] = visit
1434 for i, dataRef
in enumerate(dataRefs):
1435 visitInfo = dataRef.get(component=
'visitInfo')
1436 filterLabel = dataRef.get(component=
'filter')
1437 summaryStats = dataRef.get(component=
'summaryStats')
1438 detector = dataRef.get(component=
'detector')
1439 wcs = dataRef.get(component=
'wcs')
1440 photoCalib = dataRef.get(component=
'photoCalib')
1441 detector = dataRef.get(component=
'detector')
1442 bbox = dataRef.get(component=
'bbox')
1443 validPolygon = dataRef.get(component=
'validPolygon')
1447 rec.setVisitInfo(visitInfo)
1449 rec.setPhotoCalib(photoCalib)
1450 rec.setValidPolygon(validPolygon)
1452 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
1453 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
1454 rec.setId(detector.getId())
1455 summaryStats.update_record(rec)
1457 metadata = dafBase.PropertyList()
1458 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1460 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1461 cat.setMetadata(metadata)
1467class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1468 defaultTemplates={
"catalogType":
""},
1469 dimensions=(
"instrument",
"visit")):
1470 inputCatalogs = connectionTypes.Input(
1471 doc=
"Input per-detector Source Tables",
1472 name=
"{catalogType}sourceTable",
1473 storageClass=
"DataFrame",
1474 dimensions=(
"instrument",
"visit",
"detector"),
1477 outputCatalog = connectionTypes.Output(
1478 doc=
"Per-visit concatenation of Source Table",
1479 name=
"{catalogType}sourceTable_visit",
1480 storageClass=
"DataFrame",
1481 dimensions=(
"instrument",
"visit")
1485class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1486 pipelineConnections=ConsolidateSourceTableConnections):
1490class ConsolidateSourceTableTask(pipeBase.PipelineTask):
1491 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1493 _DefaultName =
'consolidateSourceTable'
1494 ConfigClass = ConsolidateSourceTableConfig
1496 inputDataset =
'sourceTable'
1497 outputDataset =
'sourceTable_visit'
1499 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1500 from .makeWarp
import reorderRefs
1502 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.inputCatalogs]
1503 detectorOrder.sort()
1504 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
1505 inputs = butlerQC.get(inputRefs)
1506 self.log.info(
"Concatenating %s per-detector Source Tables",
1507 len(inputs[
'inputCatalogs']))
1508 df = pd.concat(inputs[
'inputCatalogs'])
1509 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1512class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1513 dimensions=(
"instrument",),
1514 defaultTemplates={
"calexpType":
""}):
1515 visitSummaryRefs = connectionTypes.Input(
1516 doc=
"Data references for per-visit consolidated exposure metadata",
1517 name=
"finalVisitSummary",
1518 storageClass=
"ExposureCatalog",
1519 dimensions=(
"instrument",
"visit"),
1523 outputCatalog = connectionTypes.Output(
1524 doc=
"CCD and Visit metadata table",
1525 name=
"ccdVisitTable",
1526 storageClass=
"DataFrame",
1527 dimensions=(
"instrument",)
1531class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1532 pipelineConnections=MakeCcdVisitTableConnections):
1533 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1536class MakeCcdVisitTableTask(pipeBase.PipelineTask):
1537 """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
1539 _DefaultName =
'makeCcdVisitTable'
1540 ConfigClass = MakeCcdVisitTableConfig
1542 def run(self, visitSummaryRefs):
1543 """Make a table of ccd information from the visit summary catalogs.
1547 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1548 List of DeferredDatasetHandles pointing to exposure catalogs with
1549 per-detector summary information.
1553 result : `~lsst.pipe.base.Struct`
1554 Results struct with attribute:
1557 Catalog of ccd and visit information.
1560 for visitSummaryRef
in visitSummaryRefs:
1561 visitSummary = visitSummaryRef.get()
1562 visitInfo = visitSummary[0].getVisitInfo()
1565 summaryTable = visitSummary.asAstropy()
1566 selectColumns = [
'id',
'visit',
'physical_filter',
'band',
'ra',
'dec',
'zenithDistance',
1567 'zeroPoint',
'psfSigma',
'skyBg',
'skyNoise',
1568 'astromOffsetMean',
'astromOffsetStd',
'nPsfStar',
1569 'psfStarDeltaE1Median',
'psfStarDeltaE2Median',
1570 'psfStarDeltaE1Scatter',
'psfStarDeltaE2Scatter',
1571 'psfStarDeltaSizeMedian',
'psfStarDeltaSizeScatter',
1572 'psfStarScaledDeltaSizeScatter',
1573 'psfTraceRadiusDelta',
'maxDistToNearestPsf',
1574 'effTime',
'effTimePsfSigmaScale',
1575 'effTimeSkyBgScale',
'effTimeZeroPointScale']
1576 ccdEntry = summaryTable[selectColumns].to_pandas().set_index(
'id')
1581 ccdEntry = ccdEntry.rename(columns={
"visit":
"visitId"})
1585 ccdEntry[
"decl"] = ccdEntry.loc[:,
"dec"]
1587 ccdEntry[
'ccdVisitId'] = [
1588 self.config.idGenerator.apply(
1589 visitSummaryRef.dataId,
1590 detector=detector_id,
1597 for detector_id
in summaryTable[
'id']
1599 ccdEntry[
'detector'] = summaryTable[
'id']
1600 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds()
if vR.getWcs()
1601 else np.nan
for vR
in visitSummary])
1602 ccdEntry[
"seeing"] = visitSummary[
'psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1604 ccdEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1605 ccdEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1606 ccdEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1607 expTime = visitInfo.getExposureTime()
1608 ccdEntry[
'expTime'] = expTime
1609 ccdEntry[
"obsStart"] = ccdEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1610 expTime_days = expTime / (60*60*24)
1611 ccdEntry[
"obsStartMJD"] = ccdEntry[
"expMidptMJD"] - 0.5 * expTime_days
1612 ccdEntry[
'darkTime'] = visitInfo.getDarkTime()
1613 ccdEntry[
'xSize'] = summaryTable[
'bbox_max_x'] - summaryTable[
'bbox_min_x']
1614 ccdEntry[
'ySize'] = summaryTable[
'bbox_max_y'] - summaryTable[
'bbox_min_y']
1615 ccdEntry[
'llcra'] = summaryTable[
'raCorners'][:, 0]
1616 ccdEntry[
'llcdec'] = summaryTable[
'decCorners'][:, 0]
1617 ccdEntry[
'ulcra'] = summaryTable[
'raCorners'][:, 1]
1618 ccdEntry[
'ulcdec'] = summaryTable[
'decCorners'][:, 1]
1619 ccdEntry[
'urcra'] = summaryTable[
'raCorners'][:, 2]
1620 ccdEntry[
'urcdec'] = summaryTable[
'decCorners'][:, 2]
1621 ccdEntry[
'lrcra'] = summaryTable[
'raCorners'][:, 3]
1622 ccdEntry[
'lrcdec'] = summaryTable[
'decCorners'][:, 3]
1626 ccdEntries.append(ccdEntry)
1628 outputCatalog = pd.concat(ccdEntries)
1629 outputCatalog.set_index(
'ccdVisitId', inplace=
True, verify_integrity=
True)
1630 return pipeBase.Struct(outputCatalog=outputCatalog)
1633class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1634 dimensions=(
"instrument",),
1635 defaultTemplates={
"calexpType":
""}):
1636 visitSummaries = connectionTypes.Input(
1637 doc=
"Per-visit consolidated exposure metadata",
1638 name=
"finalVisitSummary",
1639 storageClass=
"ExposureCatalog",
1640 dimensions=(
"instrument",
"visit",),
1644 outputCatalog = connectionTypes.Output(
1645 doc=
"Visit metadata table",
1647 storageClass=
"DataFrame",
1648 dimensions=(
"instrument",)
1652class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1653 pipelineConnections=MakeVisitTableConnections):
1657class MakeVisitTableTask(pipeBase.PipelineTask):
1658 """Produce a `visitTable` from the visit summary exposure catalogs.
1660 _DefaultName =
'makeVisitTable'
1661 ConfigClass = MakeVisitTableConfig
1663 def run(self, visitSummaries):
1664 """Make a table of visit information from the visit summary catalogs.
1668 visitSummaries : `list` of `lsst.afw.table.ExposureCatalog`
1669 List of exposure catalogs with per-detector summary information.
1672 result : `~lsst.pipe.base.Struct`
1673 Results struct with attribute:
1676 Catalog of visit information.
1679 for visitSummary
in visitSummaries:
1680 visitSummary = visitSummary.get()
1681 visitRow = visitSummary[0]
1682 visitInfo = visitRow.getVisitInfo()
1685 visitEntry[
"visitId"] = visitRow[
'visit']
1686 visitEntry[
"visit"] = visitRow[
'visit']
1687 visitEntry[
"physical_filter"] = visitRow[
'physical_filter']
1688 visitEntry[
"band"] = visitRow[
'band']
1689 raDec = visitInfo.getBoresightRaDec()
1690 visitEntry[
"ra"] = raDec.getRa().asDegrees()
1691 visitEntry[
"dec"] = raDec.getDec().asDegrees()
1695 visitEntry[
"decl"] = visitEntry[
"dec"]
1697 visitEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1698 azAlt = visitInfo.getBoresightAzAlt()
1699 visitEntry[
"azimuth"] = azAlt.getLongitude().asDegrees()
1700 visitEntry[
"altitude"] = azAlt.getLatitude().asDegrees()
1701 visitEntry[
"zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1702 visitEntry[
"airmass"] = visitInfo.getBoresightAirmass()
1703 expTime = visitInfo.getExposureTime()
1704 visitEntry[
"expTime"] = expTime
1705 visitEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1706 visitEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1707 visitEntry[
"obsStart"] = visitEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1708 expTime_days = expTime / (60*60*24)
1709 visitEntry[
"obsStartMJD"] = visitEntry[
"expMidptMJD"] - 0.5 * expTime_days
1710 visitEntries.append(visitEntry)
1716 outputCatalog = pd.DataFrame(data=visitEntries)
1717 outputCatalog.set_index(
'visitId', inplace=
True, verify_integrity=
True)
1718 return pipeBase.Struct(outputCatalog=outputCatalog)
1721class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1722 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")):
1724 inputCatalog = connectionTypes.Input(
1725 doc=
"Primary per-detector, single-epoch forced-photometry catalog. "
1726 "By default, it is the output of ForcedPhotCcdTask on calexps",
1728 storageClass=
"SourceCatalog",
1729 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1731 inputCatalogDiff = connectionTypes.Input(
1732 doc=
"Secondary multi-epoch, per-detector, forced photometry catalog. "
1733 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1735 storageClass=
"SourceCatalog",
1736 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1738 outputCatalog = connectionTypes.Output(
1739 doc=
"InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format",
1740 name=
"mergedForcedSource",
1741 storageClass=
"DataFrame",
1742 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1746class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1747 pipelineConnections=WriteForcedSourceTableConnections):
1748 key = lsst.pex.config.Field(
1749 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1753 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1756class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1757 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format.
1759 Because the predecessor ForcedPhotCcdTask operates per-detector,
1760 per-tract, (i.e., it has tract in its dimensions), detectors
1761 on the tract boundary may have multiple forced source catalogs.
1763 The successor task TransformForcedSourceTable runs per-patch
1764 and temporally-aggregates overlapping mergedForcedSource catalogs from all
1765 available multiple epochs.
1767 _DefaultName =
"writeForcedSourceTable"
1768 ConfigClass = WriteForcedSourceTableConfig
1770 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1771 inputs = butlerQC.get(inputRefs)
1773 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
1774 inputs[
'ccdVisitId'] = idGenerator.catalog_id
1775 inputs[
'band'] = butlerQC.quantum.dataId[
'band']
1776 outputs = self.run(**inputs)
1777 butlerQC.put(outputs, outputRefs)
1779 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1781 for table, dataset,
in zip((inputCatalog, inputCatalogDiff), (
'calexp',
'diff')):
1782 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=
False)
1783 df = df.reindex(sorted(df.columns), axis=1)
1784 df[
'ccdVisitId'] = ccdVisitId
if ccdVisitId
else pd.NA
1785 df[
'band'] = band
if band
else pd.NA
1786 df.columns = pd.MultiIndex.from_tuples([(dataset, c)
for c
in df.columns],
1787 names=(
'dataset',
'column'))
1791 outputCatalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
1792 return pipeBase.Struct(outputCatalog=outputCatalog)
1795class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1796 dimensions=(
"instrument",
"skymap",
"patch",
"tract")):
1798 inputCatalogs = connectionTypes.Input(
1799 doc=
"DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask",
1800 name=
"mergedForcedSource",
1801 storageClass=
"DataFrame",
1802 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
1806 referenceCatalog = connectionTypes.Input(
1807 doc=
"Reference catalog which was used to seed the forcedPhot. Columns "
1808 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1811 storageClass=
"DataFrame",
1812 dimensions=(
"tract",
"patch",
"skymap"),
1815 outputCatalog = connectionTypes.Output(
1816 doc=
"Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1817 "specified set of functors",
1818 name=
"forcedSourceTable",
1819 storageClass=
"DataFrame",
1820 dimensions=(
"tract",
"patch",
"skymap")
1825 pipelineConnections=TransformForcedSourceTableConnections):
1826 referenceColumns = pexConfig.ListField(
1828 default=[
"detect_isPrimary",
"detect_isTractInner",
"detect_isPatchInner"],
1830 doc=
"Columns to pull from reference catalog",
1832 keyRef = lsst.pex.config.Field(
1833 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1837 key = lsst.pex.config.Field(
1838 doc=
"Rename the output DataFrame index to this name",
1840 default=
"forcedSourceId",
1843 def setDefaults(self):
1844 super().setDefaults()
1845 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'ForcedSource.yaml')
1846 self.columnsFromDataId = [
'tract',
'patch']
1850 """Transform/standardize a ForcedSource catalog
1852 Transforms each wide, per-detector forcedSource DataFrame per the
1853 specification file (per-camera defaults found in ForcedSource.yaml).
1854 All epochs that overlap the patch are aggregated into one per-patch
1855 narrow-DataFrame file.
1857 No de-duplication of rows is performed. Duplicate resolutions flags are
1858 pulled in from the referenceCatalog: `detect_isPrimary`,
1859 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
1860 for analysis or compare duplicates for QA.
1862 The resulting table includes multiple bands. Epochs (MJDs) and other useful
1863 per-visit rows can be retreived by joining with the CcdVisitTable on
1866 _DefaultName =
"transformForcedSourceTable"
1867 ConfigClass = TransformForcedSourceTableConfig
1869 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1870 inputs = butlerQC.get(inputRefs)
1871 if self.funcs
is None:
1872 raise ValueError(
"config.functorFile is None. "
1873 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1874 outputs = self.run(inputs[
'inputCatalogs'], inputs[
'referenceCatalog'], funcs=self.funcs,
1875 dataId=dict(outputRefs.outputCatalog.dataId.mapping))
1877 butlerQC.put(outputs, outputRefs)
1879 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
1881 ref = referenceCatalog.get(parameters={
"columns": self.config.referenceColumns})
1882 self.log.info(
"Aggregating %s input catalogs" % (len(inputCatalogs)))
1883 for handle
in inputCatalogs:
1884 result = self.transform(
None, handle, funcs, dataId)
1886 dfs.append(result.df.join(ref, how=
'inner'))
1888 outputCatalog = pd.concat(dfs)
1892 outputCatalog.index.rename(self.config.keyRef, inplace=
True)
1894 outputCatalog.reset_index(inplace=
True)
1897 outputCatalog.set_index(
"forcedSourceId", inplace=
True, verify_integrity=
True)
1899 outputCatalog.index.rename(self.config.key, inplace=
True)
1901 self.log.info(
"Made a table of %d columns and %d rows",
1902 len(outputCatalog.columns), len(outputCatalog))
1903 return pipeBase.Struct(outputCatalog=outputCatalog)
1906class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
1907 defaultTemplates={
"catalogType":
""},
1908 dimensions=(
"instrument",
"tract")):
1909 inputCatalogs = connectionTypes.Input(
1910 doc=
"Input per-patch DataFrame Tables to be concatenated",
1911 name=
"{catalogType}ForcedSourceTable",
1912 storageClass=
"DataFrame",
1913 dimensions=(
"tract",
"patch",
"skymap"),
1917 outputCatalog = connectionTypes.Output(
1918 doc=
"Output per-tract concatenation of DataFrame Tables",
1919 name=
"{catalogType}ForcedSourceTable_tract",
1920 storageClass=
"DataFrame",
1921 dimensions=(
"tract",
"skymap"),
1925class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
1926 pipelineConnections=ConsolidateTractConnections):
1930class ConsolidateTractTask(pipeBase.PipelineTask):
1931 """Concatenate any per-patch, dataframe list into a single
1932 per-tract DataFrame.
1934 _DefaultName =
'ConsolidateTract'
1935 ConfigClass = ConsolidateTractConfig
1937 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1938 inputs = butlerQC.get(inputRefs)
1941 self.log.info(
"Concatenating %s per-patch %s Tables",
1942 len(inputs[
'inputCatalogs']),
1943 inputRefs.inputCatalogs[0].datasetType.name)
1944 df = pd.concat(inputs[
'inputCatalogs'])
1945 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
compute(self, dropna=False, pool=None)
__init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None)
run(self, handle, funcs=None, dataId=None, band=None)
runQuantum(self, butlerQC, inputRefs, outputRefs)
transform(self, band, handles, funcs, dataId)
getAnalysis(self, handles, funcs=None, band=None)
__init__(self, *args, **kwargs)
flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None)