22__all__ = [
"WriteObjectTableConfig",
"WriteObjectTableTask",
23 "WriteSourceTableConfig",
"WriteSourceTableTask",
24 "WriteRecalibratedSourceTableConfig",
"WriteRecalibratedSourceTableTask",
25 "PostprocessAnalysis",
26 "TransformCatalogBaseConfig",
"TransformCatalogBaseTask",
27 "TransformObjectCatalogConfig",
"TransformObjectCatalogTask",
28 "ConsolidateObjectTableConfig",
"ConsolidateObjectTableTask",
29 "TransformSourceTableConfig",
"TransformSourceTableTask",
30 "ConsolidateVisitSummaryConfig",
"ConsolidateVisitSummaryTask",
31 "ConsolidateSourceTableConfig",
"ConsolidateSourceTableTask",
32 "MakeCcdVisitTableConfig",
"MakeCcdVisitTableTask",
33 "MakeVisitTableConfig",
"MakeVisitTableTask",
34 "WriteForcedSourceTableConfig",
"WriteForcedSourceTableTask",
35 "TransformForcedSourceTableConfig",
"TransformForcedSourceTableTask",
36 "ConsolidateTractConfig",
"ConsolidateTractTask"]
46from deprecated.sphinx
import deprecated
50import lsst.pipe.base
as pipeBase
52from lsst.utils.introspection
import find_outside_stacklevel
53from lsst.pipe.base
import connectionTypes
56from lsst.meas.base import SingleFrameMeasurementTask, DetectorVisitIdGeneratorConfig
59from .functors
import CompositeFunctor, Column
61log = logging.getLogger(__name__)
64def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
65 """Flattens a dataframe with multilevel column index.
67 newDf = pd.DataFrame()
69 dfBands = df.columns.unique(level=0).values
72 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
73 newColumns = {c: columnFormat.format(band, c)
74 for c
in subdf.columns
if c
not in noDupCols}
75 cols = list(newColumns.keys())
76 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
79 presentBands = dfBands
if inputBands
is None else list(set(inputBands).intersection(dfBands))
81 noDupDf = df[presentBands[0]][noDupCols]
82 newDf = pd.concat([noDupDf, newDf], axis=1)
87 defaultTemplates={
"coaddName":
"deep"},
88 dimensions=(
"tract",
"patch",
"skymap")):
89 inputCatalogMeas = connectionTypes.Input(
90 doc=
"Catalog of source measurements on the deepCoadd.",
91 dimensions=(
"tract",
"patch",
"band",
"skymap"),
92 storageClass=
"SourceCatalog",
93 name=
"{coaddName}Coadd_meas",
96 inputCatalogForcedSrc = connectionTypes.Input(
97 doc=
"Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
98 dimensions=(
"tract",
"patch",
"band",
"skymap"),
99 storageClass=
"SourceCatalog",
100 name=
"{coaddName}Coadd_forced_src",
103 inputCatalogRef = connectionTypes.Input(
104 doc=
"Catalog marking the primary detection (which band provides a good shape and position)"
105 "for each detection in deepCoadd_mergeDet.",
106 dimensions=(
"tract",
"patch",
"skymap"),
107 storageClass=
"SourceCatalog",
108 name=
"{coaddName}Coadd_ref"
110 outputCatalog = connectionTypes.Output(
111 doc=
"A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
112 "stored as a DataFrame with a multi-level column index per-patch.",
113 dimensions=(
"tract",
"patch",
"skymap"),
114 storageClass=
"DataFrame",
115 name=
"{coaddName}Coadd_obj"
119class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
120 pipelineConnections=WriteObjectTableConnections):
121 engine = pexConfig.Field(
124 doc=
"Parquet engine for writing (pyarrow or fastparquet)",
125 deprecated=
"This config is no longer used, and will be removed after v26."
127 coaddName = pexConfig.Field(
134class WriteObjectTableTask(pipeBase.PipelineTask):
135 """Write filter-merged source tables as a DataFrame in parquet format.
137 _DefaultName = "writeObjectTable"
138 ConfigClass = WriteObjectTableConfig
141 inputDatasets = (
'forced_src',
'meas',
'ref')
144 outputDataset =
'obj'
146 def runQuantum(self, butlerQC, inputRefs, outputRefs):
147 inputs = butlerQC.get(inputRefs)
149 measDict = {ref.dataId[
'band']: {
'meas': cat}
for ref, cat
in
150 zip(inputRefs.inputCatalogMeas, inputs[
'inputCatalogMeas'])}
151 forcedSourceDict = {ref.dataId[
'band']: {
'forced_src': cat}
for ref, cat
in
152 zip(inputRefs.inputCatalogForcedSrc, inputs[
'inputCatalogForcedSrc'])}
155 for band
in measDict.keys():
156 catalogs[band] = {
'meas': measDict[band][
'meas'],
157 'forced_src': forcedSourceDict[band][
'forced_src'],
158 'ref': inputs[
'inputCatalogRef']}
159 dataId = butlerQC.quantum.dataId
160 df = self.run(catalogs=catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
161 outputs = pipeBase.Struct(outputCatalog=df)
162 butlerQC.put(outputs, outputRefs)
164 def run(self, catalogs, tract, patch):
165 """Merge multiple catalogs.
170 Mapping from filter names to dict of catalogs.
172 tractId to use
for the tractId column.
174 patchId to use
for the patchId column.
178 catalog : `pandas.DataFrame`
182 for filt, tableDict
in catalogs.items():
183 for dataset, table
in tableDict.items():
185 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
188 df = df.reindex(sorted(df.columns), axis=1)
189 df = df.assign(tractId=tract, patchId=patch)
192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
193 names=(
'dataset',
'band',
'column'))
198 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
202class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
203 defaultTemplates={
"catalogType":
""},
204 dimensions=(
"instrument",
"visit",
"detector")):
206 catalog = connectionTypes.Input(
207 doc=
"Input full-depth catalog of sources produced by CalibrateTask",
208 name=
"{catalogType}src",
209 storageClass=
"SourceCatalog",
210 dimensions=(
"instrument",
"visit",
"detector")
212 outputCatalog = connectionTypes.Output(
213 doc=
"Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
214 "replaced with an index; all other columns are unchanged.",
215 name=
"{catalogType}source",
216 storageClass=
"DataFrame",
217 dimensions=(
"instrument",
"visit",
"detector")
221class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
222 pipelineConnections=WriteSourceTableConnections):
223 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
226class WriteSourceTableTask(pipeBase.PipelineTask):
227 """Write source table to DataFrame Parquet format.
229 _DefaultName = "writeSourceTable"
230 ConfigClass = WriteSourceTableConfig
232 def runQuantum(self, butlerQC, inputRefs, outputRefs):
233 inputs = butlerQC.get(inputRefs)
234 inputs[
'ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id
235 result = self.run(**inputs)
236 outputs = pipeBase.Struct(outputCatalog=result.table)
237 butlerQC.put(outputs, outputRefs)
239 def run(self, catalog, ccdVisitId=None, **kwargs):
240 """Convert `src` catalog to DataFrame
244 catalog: `afwTable.SourceCatalog`
245 catalog to be converted
247 ccdVisitId to be added as a column
249 Additional keyword arguments are ignored
as a convenience
for
250 subclasses that
pass the same arguments to several different
255 result : `~lsst.pipe.base.Struct`
257 `DataFrame` version of the input catalog
259 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
260 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
261 df[
'ccdVisitId'] = ccdVisitId
263 return pipeBase.Struct(table=df)
266class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
267 defaultTemplates={
"catalogType":
"",
268 "skyWcsName":
"gbdesAstrometricFit",
269 "photoCalibName":
"fgcm"},
271 deprecatedTemplates={
272 "skyWcsName":
"Deprecated; will be removed after v26.",
273 "photoCalibName":
"Deprecated; will be removed after v26."
275 dimensions=(
"instrument",
"visit",
"detector",
"skymap")):
276 skyMap = connectionTypes.Input(
277 doc=
"skyMap needed to choose which tract-level calibrations to use when multiple available",
278 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
279 storageClass=
"SkyMap",
280 dimensions=(
"skymap",),
283 "Deprecated, since 'visitSummary' already resolves calibrations across tracts. "
284 "Will be removed after v26."
287 exposure = connectionTypes.Input(
288 doc=
"Input exposure to perform photometry on.",
290 storageClass=
"ExposureF",
291 dimensions=[
"instrument",
"visit",
"detector"],
293 visitSummary = connectionTypes.Input(
294 doc=
"Input visit-summary catalog with updated calibration objects.",
295 name=
"finalVisitSummary",
296 storageClass=
"ExposureCatalog",
297 dimensions=(
"instrument",
"visit",),
299 externalSkyWcsTractCatalog = connectionTypes.Input(
300 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
301 "id for the catalog id, sorted on id for fast lookup."),
302 name=
"{skyWcsName}SkyWcsCatalog",
303 storageClass=
"ExposureCatalog",
304 dimensions=[
"instrument",
"visit",
"tract"],
307 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
309 externalSkyWcsGlobalCatalog = connectionTypes.Input(
310 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
311 "These catalogs use the detector id for the catalog id, sorted on id for "
313 name=
"finalVisitSummary",
314 storageClass=
"ExposureCatalog",
315 dimensions=[
"instrument",
"visit"],
317 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
319 externalPhotoCalibTractCatalog = connectionTypes.Input(
320 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
321 "detector id for the catalog id, sorted on id for fast lookup."),
322 name=
"{photoCalibName}PhotoCalibCatalog",
323 storageClass=
"ExposureCatalog",
324 dimensions=[
"instrument",
"visit",
"tract"],
327 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
329 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
330 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
331 "information). These catalogs use the detector id for the catalog id, "
332 "sorted on id for fast lookup."),
333 name=
"finalVisitSummary",
334 storageClass=
"ExposureCatalog",
335 dimensions=[
"instrument",
"visit"],
337 deprecated=
"Deprecated in favor of 'visitSummary'. Will be removed after v26.",
340 def __init__(self, *, config=None):
341 super().__init__(config=config)
346 if config.doApplyExternalSkyWcs
and config.doReevaluateSkyWcs:
347 if config.useGlobalExternalSkyWcs:
348 self.inputs.remove(
"externalSkyWcsTractCatalog")
350 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
353 self.inputs.remove(
"externalSkyWcsTractCatalog")
354 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
355 if config.doApplyExternalPhotoCalib
and config.doReevaluatePhotoCalib:
356 if config.useGlobalExternalPhotoCalib:
357 self.inputs.remove(
"externalPhotoCalibTractCatalog")
359 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
362 self.inputs.remove(
"externalPhotoCalibTractCatalog")
363 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
368class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
369 pipelineConnections=WriteRecalibratedSourceTableConnections):
371 doReevaluatePhotoCalib = pexConfig.Field(
374 doc=(
"Add or replace local photoCalib columns"),
376 doReevaluateSkyWcs = pexConfig.Field(
379 doc=(
"Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec"),
381 doApplyExternalPhotoCalib = pexConfig.Field(
384 doc=(
"If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
385 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
387 deprecated=
"Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
389 doApplyExternalSkyWcs = pexConfig.Field(
392 doc=(
"if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
393 "else use the wcs already attached to the exposure."),
395 deprecated=
"Deprecated along with the external WCS connections. Will be removed after v26.",
397 useGlobalExternalPhotoCalib = pexConfig.Field(
400 doc=(
"When using doApplyExternalPhotoCalib, use 'global' calibrations "
401 "that are not run per-tract. When False, use per-tract photometric "
402 "calibration files."),
404 deprecated=
"Deprecated along with the external PhotoCalib connections. Will be removed after v26.",
406 useGlobalExternalSkyWcs = pexConfig.Field(
409 doc=(
"When using doApplyExternalSkyWcs, use 'global' calibrations "
410 "that are not run per-tract. When False, use per-tract wcs "
413 deprecated=
"Deprecated along with the external WCS connections. Will be removed after v26.",
415 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
419 if self.doApplyExternalSkyWcs
and not self.doReevaluateSkyWcs:
420 log.warning(
"doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
421 "External SkyWcs will not be read or evaluated.")
422 if self.doApplyExternalPhotoCalib
and not self.doReevaluatePhotoCalib:
423 log.warning(
"doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
424 "External PhotoCalib will not be read or evaluated.")
427class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
428 """Write source table to DataFrame Parquet format.
430 _DefaultName = "writeRecalibratedSourceTable"
431 ConfigClass = WriteRecalibratedSourceTableConfig
433 def runQuantum(self, butlerQC, inputRefs, outputRefs):
434 inputs = butlerQC.get(inputRefs)
436 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
437 inputs[
'idGenerator'] = idGenerator
438 inputs[
'ccdVisitId'] = idGenerator.catalog_id
440 if self.config.doReevaluatePhotoCalib
or self.config.doReevaluateSkyWcs:
441 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
442 inputs[
'exposure'] = self.attachCalibs(inputRefs, **inputs)
444 inputs[
'exposure'] = self.prepareCalibratedExposure(
445 exposure=inputs[
"exposure"], visitSummary=inputs[
"visitSummary"]
447 inputs[
'catalog'] = self.addCalibColumns(**inputs)
449 result = self.run(**inputs)
450 outputs = pipeBase.Struct(outputCatalog=result.table)
451 butlerQC.put(outputs, outputRefs)
454 @deprecated(
"Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
455 version=
"v26", category=FutureWarning)
456 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
457 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
458 externalPhotoCalibTractCatalog=None, visitSummary=None, **kwargs):
459 """Apply external calibrations to exposure per configuration
461 When multiple tract-level calibrations overlap, select the one with the
462 center closest to detector.
466 inputRefs : `~lsst.pipe.base.InputQuantizedConnection`,
for dataIds of
468 skyMap : `~lsst.skymap.BaseSkyMap`
469 skyMap to lookup tract geometry
and WCS.
470 exposure : `lsst.afw.image.exposure.Exposure`
471 Input exposure to adjust calibrations.
473 Exposure catalog
with external skyWcs to be applied per config
475 Exposure catalog
with external skyWcs to be applied per config
477 Exposure catalog
with external photoCalib to be applied per config
479 Exposure catalog
with external photoCalib to be applied per config
481 Exposure catalog
with all calibration objects. WCS
and PhotoCalib
482 are always applied
if provided.
484 Additional keyword arguments are ignored to facilitate passing the
485 same arguments to several methods.
489 exposure : `lsst.afw.image.exposure.Exposure`
490 Exposure
with adjusted calibrations.
492 if not self.config.doApplyExternalSkyWcs:
494 externalSkyWcsCatalog =
None
495 elif self.config.useGlobalExternalSkyWcs:
497 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
498 self.log.info(
'Applying global SkyWcs')
501 inputRef = getattr(inputRefs,
'externalSkyWcsTractCatalog')
502 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
505 self.log.info(
'Applying tract-level SkyWcs from tract %s', tracts[ind])
507 if exposure.getWcs()
is None:
508 raise ValueError(
"Trying to locate nearest tract, but exposure.wcs is None.")
509 ind = self.getClosestTract(tracts, skyMap,
510 exposure.getBBox(), exposure.getWcs())
511 self.log.info(
'Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
512 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
514 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
516 if not self.config.doApplyExternalPhotoCalib:
518 externalPhotoCalibCatalog =
None
519 elif self.config.useGlobalExternalPhotoCalib:
521 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
522 self.log.info(
'Applying global PhotoCalib')
525 inputRef = getattr(inputRefs,
'externalPhotoCalibTractCatalog')
526 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
529 self.log.info(
'Applying tract-level PhotoCalib from tract %s', tracts[ind])
531 ind = self.getClosestTract(tracts, skyMap,
532 exposure.getBBox(), exposure.getWcs())
533 self.log.info(
'Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
534 'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
536 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
538 return self.prepareCalibratedExposure(
539 exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog, visitSummary
543 @deprecated(
"Deprecated in favor of exclusively using visit summaries; will be removed after v26.",
544 version=
"v26", category=FutureWarning)
545 def getClosestTract(self, tracts, skyMap, bbox, wcs):
546 """Find the index of the tract closest to detector from list of tractIds
550 tracts: `list` [`int`]
551 Iterable of integer tractIds
552 skyMap : `~lsst.skymap.BaseSkyMap`
553 skyMap to lookup tract geometry and wcs
555 Detector bbox, center of which will compared to tract centers
557 Detector Wcs object to map the detector center to SkyCoord
566 center = wcs.pixelToSky(bbox.getCenter())
568 for tractId
in tracts:
569 tract = skyMap[tractId]
570 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
571 sep.append(center.separation(tractCenter))
573 return np.argmin(sep)
575 def prepareCalibratedExposure(
576 self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None, visitSummary=None
578 """Prepare a calibrated exposure and apply external calibrations
583 exposure : `lsst.afw.image.exposure.Exposure`
584 Input exposure to adjust calibrations.
586 Exposure catalog
with external skyWcs to be applied
587 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
588 for the catalog id, sorted on id
for fast lookup.
589 Deprecated
in favor of ``visitSummary``; will be removed after v26.
591 Exposure catalog
with external photoCalib to be applied
592 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
593 id
for the catalog id, sorted on id
for fast lookup.
594 Deprecated
in favor of ``visitSummary``; will be removed after v26.
596 Exposure catalog
with all calibration objects. WCS
and PhotoCalib
597 are always applied
if ``visitSummary``
is provided
and those
598 components are
not `
None`.
602 exposure : `lsst.afw.image.exposure.Exposure`
603 Exposure
with adjusted calibrations.
605 detectorId = exposure.getInfo().getDetector().getId()
607 if visitSummary
is not None:
608 row = visitSummary.find(detectorId)
610 raise RuntimeError(f
"Visit summary for detector {detectorId} is unexpectedly missing.")
611 if (photoCalib := row.getPhotoCalib())
is None:
612 self.log.warning(
"Detector id %s has None for photoCalib in visit summary; "
613 "using original photoCalib.", detectorId)
615 exposure.setPhotoCalib(photoCalib)
616 if (skyWcs := row.getWcs())
is None:
617 self.log.warning(
"Detector id %s has None for skyWcs in visit summary; "
618 "using original skyWcs.", detectorId)
620 exposure.setWcs(skyWcs)
622 if externalPhotoCalibCatalog
is not None:
625 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
627 stacklevel=find_outside_stacklevel(
"lsst.pipe.tasks.postprocessing"),
629 row = externalPhotoCalibCatalog.find(detectorId)
631 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog; "
632 "Using original photoCalib.", detectorId)
634 photoCalib = row.getPhotoCalib()
635 if photoCalib
is None:
636 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
637 "Using original photoCalib.", detectorId)
639 exposure.setPhotoCalib(photoCalib)
641 if externalSkyWcsCatalog
is not None:
644 "Deprecated in favor of 'visitSummary'; will be removed after v26.",
646 stacklevel=find_outside_stacklevel(
"lsst.pipe.tasks.postprocessing"),
648 row = externalSkyWcsCatalog.find(detectorId)
650 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog; "
651 "Using original skyWcs.", detectorId)
653 skyWcs = row.getWcs()
655 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
656 "Using original skyWcs.", detectorId)
658 exposure.setWcs(skyWcs)
662 def addCalibColumns(self, catalog, exposure, idGenerator, **kwargs):
663 """Add replace columns with calibs evaluated at each centroid
665 Add or replace
'base_LocalWcs' `base_LocalPhotoCalib
' columns in a
666 a source catalog, by rerunning the plugins.
671 catalog to which calib columns will be added
672 exposure : `lsst.afw.image.exposure.Exposure`
673 Exposure with attached PhotoCalibs
and SkyWcs attributes to be
674 reevaluated at local centroids. Pixels are
not required.
675 idGenerator : `lsst.meas.base.IdGenerator`
676 Object that generates Source IDs
and random seeds.
678 Additional keyword arguments are ignored to facilitate passing the
679 same arguments to several methods.
684 Source Catalog
with requested local calib columns
686 measureConfig = SingleFrameMeasurementTask.ConfigClass()
687 measureConfig.doReplaceWithNoise = False
690 for slot
in measureConfig.slots:
691 setattr(measureConfig.slots, slot,
None)
693 measureConfig.plugins.names = []
694 if self.config.doReevaluateSkyWcs:
695 measureConfig.plugins.names.add(
'base_LocalWcs')
696 self.log.info(
"Re-evaluating base_LocalWcs plugin")
697 if self.config.doReevaluatePhotoCalib:
698 measureConfig.plugins.names.add(
'base_LocalPhotoCalib')
699 self.log.info(
"Re-evaluating base_LocalPhotoCalib plugin")
700 pluginsNotToCopy = tuple(measureConfig.plugins.names)
704 aliasMap = catalog.schema.getAliasMap()
705 mapper = afwTable.SchemaMapper(catalog.schema)
706 for item
in catalog.schema:
707 if not item.field.getName().startswith(pluginsNotToCopy):
708 mapper.addMapping(item.key)
710 schema = mapper.getOutputSchema()
711 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
712 schema.setAliasMap(aliasMap)
713 newCat = afwTable.SourceCatalog(schema)
714 newCat.extend(catalog, mapper=mapper)
720 if self.config.doReevaluateSkyWcs
and exposure.wcs
is not None:
721 afwTable.updateSourceCoords(exposure.wcs, newCat)
723 measurement.run(measCat=newCat, exposure=exposure, exposureId=idGenerator.catalog_id)
729 """Calculate columns from DataFrames or handles storing DataFrames.
731 This object manages and organizes an arbitrary set of computations
732 on a catalog. The catalog
is defined by a
733 `DeferredDatasetHandle`
or `InMemoryDatasetHandle` object
734 (
or list thereof), such
as a ``deepCoadd_obj`` dataset,
and the
735 computations are defined by a collection of
737 ``CompositeFunctor``).
739 After the object
is initialized, accessing the ``.df`` attribute (which
740 holds the `pandas.DataFrame` containing the results of the calculations)
741 triggers computation of said dataframe.
743 One of the conveniences of using this object
is the ability to define a
744 desired common filter
for all functors. This enables the same functor
745 collection to be passed to several different `PostprocessAnalysis` objects
746 without having to change the original functor collection, since the ``filt``
747 keyword argument of this object triggers an overwrite of the ``filt``
748 property
for all functors
in the collection.
750 This object also allows a list of refFlags to be passed,
and defines a set
751 of default refFlags that are always included even
if not requested.
753 If a list of DataFrames
or Handles
is passed, rather than a single one,
754 then the calculations will be mapped over all the input catalogs. In
755 principle, it should be straightforward to parallelize this activity, but
756 initial tests have failed (see TODO
in code comments).
760 handles : `~lsst.daf.butler.DeferredDatasetHandle`
or
761 `~lsst.pipe.base.InMemoryDatasetHandle`
or
763 Source
catalog(s)
for computation.
765 Computations to do (functors that act on ``handles``).
766 If a dict, the output
767 DataFrame will have columns keyed accordingly.
768 If a list, the column keys will come
from the
769 ``.shortname`` attribute of each functor.
771 filt : `str`, optional
772 Filter
in which to calculate. If provided,
773 this will overwrite any existing ``.filt`` attribute
774 of the provided functors.
776 flags : `list`, optional
777 List of flags (per-band) to include
in output table.
778 Taken
from the ``meas`` dataset
if applied to a multilevel Object Table.
780 refFlags : `list`, optional
781 List of refFlags (only reference band) to include
in output table.
783 forcedFlags : `list`, optional
784 List of flags (per-band) to include
in output table.
785 Taken
from the ``forced_src`` dataset
if applied to a
786 multilevel Object Table. Intended
for flags
from measurement plugins
787 only run during multi-band forced-photometry.
789 _defaultRefFlags = []
792 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
797 self.
flags = list(flags)
if flags
is not None else []
798 self.
forcedFlags = list(forcedFlags)
if forcedFlags
is not None else []
800 if refFlags
is not None:
813 additionalFuncs.update({flag:
Column(flag, dataset=
'forced_src')
for flag
in self.
forcedFlags})
814 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.
refFlags})
815 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.
flags})
817 if isinstance(self.
functors, CompositeFunctor):
822 func.funcDict.update(additionalFuncs)
823 func.filt = self.
filt
829 return [name
for name, func
in self.
func.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
839 if type(self.
handles)
in (list, tuple):
841 dflist = [self.
func(handle, dropna=dropna)
for handle
in self.
handles]
845 dflist = pool.map(functools.partial(self.
func, dropna=dropna), self.
handles)
846 self.
_df = pd.concat(dflist)
855 """Expected Connections for subclasses of TransformCatalogBaseTask.
859 inputCatalog = connectionTypes.Input(
861 storageClass=
"DataFrame",
863 outputCatalog = connectionTypes.Output(
865 storageClass=
"DataFrame",
870 pipelineConnections=TransformCatalogBaseConnections):
871 functorFile = pexConfig.Field(
873 doc=
"Path to YAML file specifying Science Data Model functors to use "
874 "when copying columns and computing calibrated values.",
878 primaryKey = pexConfig.Field(
880 doc=
"Name of column to be set as the DataFrame index. If None, the index"
881 "will be named `id`",
885 columnsFromDataId = pexConfig.ListField(
889 doc=
"Columns to extract from the dataId",
894 """Base class for transforming/standardizing a catalog by applying functors
895 that convert units and apply calibrations.
897 The purpose of this task
is to perform a set of computations on an input
898 ``DeferredDatasetHandle``
or ``InMemoryDatasetHandle`` that holds a
899 ``DataFrame`` dataset (such
as ``deepCoadd_obj``),
and write the results to
900 a new dataset (which needs to be declared
in an ``outputDataset``
903 The calculations to be performed are defined
in a YAML file that specifies
904 a set of functors to be computed, provided
as a ``--functorFile`` config
905 parameter. An example of such a YAML file
is the following:
912 args: slot_Centroid_x
915 args: slot_Centroid_y
917 functor: LocalNanojansky
919 - slot_PsfFlux_instFlux
920 - slot_PsfFlux_instFluxErr
921 - base_LocalPhotoCalib
922 - base_LocalPhotoCalibErr
924 functor: LocalNanojanskyErr
926 - slot_PsfFlux_instFlux
927 - slot_PsfFlux_instFluxErr
928 - base_LocalPhotoCalib
929 - base_LocalPhotoCalibErr
933 The names
for each entry under
"func" will become the names of columns
in
934 the output dataset. All the functors referenced are defined
in
936 functor are
in the `args` list,
and any additional entries
for each column
937 other than
"functor" or "args" (e.g., ``
'filt'``, ``
'dataset'``) are
938 treated
as keyword arguments to be passed to the functor initialization.
940 The
"flags" entry
is the default shortcut
for `Column` functors.
941 All columns listed under
"flags" will be copied to the output table
942 untransformed. They can be of any datatype.
943 In the special case of transforming a multi-level oject table
with
944 band
and dataset indices (deepCoadd_obj), these will be taked
from the
945 `meas` dataset
and exploded out per band.
947 There are two special shortcuts that only apply when transforming
948 multi-level Object (deepCoadd_obj) tables:
949 - The
"refFlags" entry
is shortcut
for `Column` functor
950 taken
from the `
'ref'` dataset
if transforming an ObjectTable.
951 - The
"forcedFlags" entry
is shortcut
for `Column` functors.
952 taken
from the ``forced_src`` dataset
if transforming an ObjectTable.
953 These are expanded out per band.
957 to organize
and excecute the calculations.
961 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
965 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
969 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
972 def ConfigClass(self):
973 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
977 if self.config.functorFile:
978 self.log.info(
'Loading tranform functor definitions from %s',
979 self.config.functorFile)
980 self.
funcs = CompositeFunctor.from_file(self.config.functorFile)
981 self.
funcs.update(dict(PostprocessAnalysis._defaultFuncs))
986 inputs = butlerQC.get(inputRefs)
987 if self.
funcs is None:
988 raise ValueError(
"config.functorFile is None. "
989 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
990 result = self.
run(handle=inputs[
'inputCatalog'], funcs=self.
funcs,
991 dataId=outputRefs.outputCatalog.dataId.full)
992 outputs = pipeBase.Struct(outputCatalog=result)
993 butlerQC.put(outputs, outputRefs)
995 def run(self, handle, funcs=None, dataId=None, band=None):
996 """Do postprocessing calculations
998 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle``
or
999 ``DataFrame`` object
and dataId,
1000 returns a dataframe
with results of postprocessing calculations.
1004 handles : `~lsst.daf.butler.DeferredDatasetHandle`
or
1005 `~lsst.pipe.base.InMemoryDatasetHandle`
or
1006 `~pandas.DataFrame`,
or list of these.
1007 DataFrames
from which calculations are done.
1009 Functors to apply to the table
's columns
1010 dataId : dict, optional
1011 Used to add a `patchId` column to the output dataframe.
1012 band : `str`, optional
1013 Filter band that is being processed.
1017 df : `pandas.DataFrame`
1019 self.log.info("Transforming/standardizing the source table dataId: %s", dataId)
1021 df = self.
transform(band, handle, funcs, dataId).df
1022 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1034 def transform(self, band, handles, funcs, dataId):
1035 analysis = self.
getAnalysis(handles, funcs=funcs, band=band)
1037 if dataId
and self.config.columnsFromDataId:
1038 for key
in self.config.columnsFromDataId:
1040 df[str(key)] = dataId[key]
1042 raise ValueError(f
"'{key}' in config.columnsFromDataId not found in dataId: {dataId}")
1044 if self.config.primaryKey:
1045 if df.index.name != self.config.primaryKey
and self.config.primaryKey
in df:
1046 df.reset_index(inplace=
True, drop=
True)
1047 df.set_index(self.config.primaryKey, inplace=
True)
1049 return pipeBase.Struct(
1056 defaultTemplates={
"coaddName":
"deep"},
1057 dimensions=(
"tract",
"patch",
"skymap")):
1058 inputCatalog = connectionTypes.Input(
1059 doc=
"The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
1060 "stored as a DataFrame with a multi-level column index per-patch.",
1061 dimensions=(
"tract",
"patch",
"skymap"),
1062 storageClass=
"DataFrame",
1063 name=
"{coaddName}Coadd_obj",
1066 outputCatalog = connectionTypes.Output(
1067 doc=
"Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
1069 dimensions=(
"tract",
"patch",
"skymap"),
1070 storageClass=
"DataFrame",
1076 pipelineConnections=TransformObjectCatalogConnections):
1077 coaddName = pexConfig.Field(
1083 filterMap = pexConfig.DictField(
1087 doc=(
"Dictionary mapping full filter name to short one for column name munging."
1088 "These filters determine the output columns no matter what filters the "
1089 "input data actually contain."),
1090 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
1091 "Will be removed after v22.")
1093 outputBands = pexConfig.ListField(
1097 doc=(
"These bands and only these bands will appear in the output,"
1098 " NaN-filled if the input does not include them."
1099 " If None, then use all bands found in the input.")
1101 camelCase = pexConfig.Field(
1104 doc=(
"Write per-band columns names with camelCase, else underscore "
1105 "For example: gPsFlux instead of g_PsFlux.")
1107 multilevelOutput = pexConfig.Field(
1110 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
1111 "and name-munged (False).")
1113 goodFlags = pexConfig.ListField(
1116 doc=(
"List of 'good' flags that should be set False when populating empty tables. "
1117 "All other flags are considered to be 'bad' flags and will be set to True.")
1119 floatFillValue = pexConfig.Field(
1122 doc=
"Fill value for float fields when populating empty tables."
1124 integerFillValue = pexConfig.Field(
1127 doc=
"Fill value for integer fields when populating empty tables."
1130 def setDefaults(self):
1131 super().setDefaults()
1132 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Object.yaml')
1133 self.primaryKey =
'objectId'
1134 self.columnsFromDataId = [
'tract',
'patch']
1135 self.goodFlags = [
'calib_astrometry_used',
1136 'calib_photometry_reserved',
1137 'calib_photometry_used',
1138 'calib_psf_candidate',
1139 'calib_psf_reserved',
1144 """Produce a flattened Object Table to match the format specified in
1147 Do the same set of postprocessing calculations on all bands.
1149 This is identical to `TransformCatalogBaseTask`,
except for that it does
1150 the specified functor calculations
for all filters present
in the
1151 input `deepCoadd_obj` table. Any specific ``
"filt"`` keywords specified
1152 by the YAML file will be superceded.
1154 _DefaultName = "transformObjectCatalog"
1155 ConfigClass = TransformObjectCatalogConfig
1157 def run(self, handle, funcs=None, dataId=None, band=None):
1161 templateDf = pd.DataFrame()
1163 columns = handle.get(component=
'columns')
1164 inputBands = columns.unique(level=1).values
1166 outputBands = self.config.outputBands
if self.config.outputBands
else inputBands
1169 for inputBand
in inputBands:
1170 if inputBand
not in outputBands:
1171 self.log.info(
"Ignoring %s band data in the input", inputBand)
1173 self.log.info(
"Transforming the catalog of band %s", inputBand)
1174 result = self.transform(inputBand, handle, funcs, dataId)
1175 dfDict[inputBand] = result.df
1176 analysisDict[inputBand] = result.analysis
1177 if templateDf.empty:
1178 templateDf = result.df
1181 for filt
in outputBands:
1182 if filt
not in dfDict:
1183 self.log.info(
"Adding empty columns for band %s", filt)
1184 dfTemp = templateDf.copy()
1185 for col
in dfTemp.columns:
1186 testValue = dfTemp[col].values[0]
1187 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1189 if col
in self.config.goodFlags:
1193 elif isinstance(testValue, numbers.Integral):
1197 if isinstance(testValue, np.unsignedinteger):
1198 raise ValueError(
"Parquet tables may not have unsigned integer columns.")
1200 fillValue = self.config.integerFillValue
1202 fillValue = self.config.floatFillValue
1203 dfTemp[col].values[:] = fillValue
1204 dfDict[filt] = dfTemp
1207 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
1209 if not self.config.multilevelOutput:
1210 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
1211 if self.config.primaryKey
in noDupCols:
1212 noDupCols.remove(self.config.primaryKey)
1213 if dataId
and self.config.columnsFromDataId:
1214 noDupCols += self.config.columnsFromDataId
1215 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1216 inputBands=inputBands)
1218 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1223class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1224 dimensions=(
"tract",
"skymap")):
1225 inputCatalogs = connectionTypes.Input(
1226 doc=
"Per-Patch objectTables conforming to the standard data model.",
1228 storageClass=
"DataFrame",
1229 dimensions=(
"tract",
"patch",
"skymap"),
1232 outputCatalog = connectionTypes.Output(
1233 doc=
"Pre-tract horizontal concatenation of the input objectTables",
1234 name=
"objectTable_tract",
1235 storageClass=
"DataFrame",
1236 dimensions=(
"tract",
"skymap"),
1240class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1241 pipelineConnections=ConsolidateObjectTableConnections):
1242 coaddName = pexConfig.Field(
1249class ConsolidateObjectTableTask(pipeBase.PipelineTask):
1250 """Write patch-merged source tables to a tract-level DataFrame Parquet file.
1252 Concatenates `objectTable` list into a per-visit `objectTable_tract`.
1254 _DefaultName = "consolidateObjectTable"
1255 ConfigClass = ConsolidateObjectTableConfig
1257 inputDataset =
'objectTable'
1258 outputDataset =
'objectTable_tract'
1260 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1261 inputs = butlerQC.get(inputRefs)
1262 self.log.info(
"Concatenating %s per-patch Object Tables",
1263 len(inputs[
'inputCatalogs']))
1264 df = pd.concat(inputs[
'inputCatalogs'])
1265 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1268class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1269 defaultTemplates={
"catalogType":
""},
1270 dimensions=(
"instrument",
"visit",
"detector")):
1272 inputCatalog = connectionTypes.Input(
1273 doc=
"Wide input catalog of sources produced by WriteSourceTableTask",
1274 name=
"{catalogType}source",
1275 storageClass=
"DataFrame",
1276 dimensions=(
"instrument",
"visit",
"detector"),
1279 outputCatalog = connectionTypes.Output(
1280 doc=
"Narrower, per-detector Source Table transformed and converted per a "
1281 "specified set of functors",
1282 name=
"{catalogType}sourceTable",
1283 storageClass=
"DataFrame",
1284 dimensions=(
"instrument",
"visit",
"detector")
1289 pipelineConnections=TransformSourceTableConnections):
1291 def setDefaults(self):
1292 super().setDefaults()
1293 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Source.yaml')
1294 self.primaryKey =
'sourceId'
1295 self.columnsFromDataId = [
'visit',
'detector',
'band',
'physical_filter']
1299 """Transform/standardize a source catalog
1301 _DefaultName = "transformSourceTable"
1302 ConfigClass = TransformSourceTableConfig
1305class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1306 dimensions=(
"instrument",
"visit",),
1307 defaultTemplates={
"calexpType":
""}):
1308 calexp = connectionTypes.Input(
1309 doc=
"Processed exposures used for metadata",
1311 storageClass=
"ExposureF",
1312 dimensions=(
"instrument",
"visit",
"detector"),
1316 visitSummary = connectionTypes.Output(
1317 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
1318 "detector id for the id and are sorted for fast lookups of a "
1320 name=
"visitSummary",
1321 storageClass=
"ExposureCatalog",
1322 dimensions=(
"instrument",
"visit"),
1324 visitSummarySchema = connectionTypes.InitOutput(
1325 doc=
"Schema of the visitSummary catalog",
1326 name=
"visitSummary_schema",
1327 storageClass=
"ExposureCatalog",
1331class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1332 pipelineConnections=ConsolidateVisitSummaryConnections):
1333 """Config for ConsolidateVisitSummaryTask"""
1337class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
1338 """Task to consolidate per-detector visit metadata.
1340 This task aggregates the following metadata from all the detectors
in a
1341 single visit into an exposure catalog:
1345 - The physical_filter
and band (
if available).
1346 - The psf size, shape,
and effective area at the center of the detector.
1347 - The corners of the bounding box
in right ascension/declination.
1349 Other quantities such
as Detector, Psf, ApCorrMap,
and TransmissionCurve
1350 are
not persisted here because of storage concerns,
and because of their
1351 limited utility
as summary statistics.
1353 Tests
for this task are performed
in ci_hsc_gen3.
1355 _DefaultName = "consolidateVisitSummary"
1356 ConfigClass = ConsolidateVisitSummaryConfig
1358 def __init__(self, **kwargs):
1359 super().__init__(**kwargs)
1360 self.schema = afwTable.ExposureTable.makeMinimalSchema()
1361 self.schema.addField(
'visit', type=
'L', doc=
'Visit number')
1362 self.schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
1363 self.schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
1364 ExposureSummaryStats.update_schema(self.schema)
1365 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
1367 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1368 dataRefs = butlerQC.get(inputRefs.calexp)
1369 visit = dataRefs[0].dataId.byName()[
'visit']
1371 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)",
1372 len(dataRefs), visit)
1374 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1376 butlerQC.put(expCatalog, outputRefs.visitSummary)
1378 def _combineExposureMetadata(self, visit, dataRefs):
1379 """Make a combined exposure catalog from a list of dataRefs.
1380 These dataRefs must point to exposures with wcs, summaryStats,
1381 and other visit metadata.
1386 Visit identification number.
1387 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1388 List of dataRefs
in visit.
1393 Exposure catalog
with per-detector summary information.
1395 cat = afwTable.ExposureCatalog(self.schema)
1396 cat.resize(len(dataRefs))
1398 cat['visit'] = visit
1400 for i, dataRef
in enumerate(dataRefs):
1401 visitInfo = dataRef.get(component=
'visitInfo')
1402 filterLabel = dataRef.get(component=
'filter')
1403 summaryStats = dataRef.get(component=
'summaryStats')
1404 detector = dataRef.get(component=
'detector')
1405 wcs = dataRef.get(component=
'wcs')
1406 photoCalib = dataRef.get(component=
'photoCalib')
1407 detector = dataRef.get(component=
'detector')
1408 bbox = dataRef.get(component=
'bbox')
1409 validPolygon = dataRef.get(component=
'validPolygon')
1413 rec.setVisitInfo(visitInfo)
1415 rec.setPhotoCalib(photoCalib)
1416 rec.setValidPolygon(validPolygon)
1418 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
1419 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
1420 rec.setId(detector.getId())
1421 summaryStats.update_record(rec)
1423 metadata = dafBase.PropertyList()
1424 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1426 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1427 cat.setMetadata(metadata)
1433class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1434 defaultTemplates={
"catalogType":
""},
1435 dimensions=(
"instrument",
"visit")):
1436 inputCatalogs = connectionTypes.Input(
1437 doc=
"Input per-detector Source Tables",
1438 name=
"{catalogType}sourceTable",
1439 storageClass=
"DataFrame",
1440 dimensions=(
"instrument",
"visit",
"detector"),
1443 outputCatalog = connectionTypes.Output(
1444 doc=
"Per-visit concatenation of Source Table",
1445 name=
"{catalogType}sourceTable_visit",
1446 storageClass=
"DataFrame",
1447 dimensions=(
"instrument",
"visit")
1451class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1452 pipelineConnections=ConsolidateSourceTableConnections):
1456class ConsolidateSourceTableTask(pipeBase.PipelineTask):
1457 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1459 _DefaultName = 'consolidateSourceTable'
1460 ConfigClass = ConsolidateSourceTableConfig
1462 inputDataset =
'sourceTable'
1463 outputDataset =
'sourceTable_visit'
1465 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1466 from .makeWarp
import reorderRefs
1468 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.inputCatalogs]
1469 detectorOrder.sort()
1470 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
1471 inputs = butlerQC.get(inputRefs)
1472 self.log.info(
"Concatenating %s per-detector Source Tables",
1473 len(inputs[
'inputCatalogs']))
1474 df = pd.concat(inputs[
'inputCatalogs'])
1475 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1478class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1479 dimensions=(
"instrument",),
1480 defaultTemplates={
"calexpType":
""}):
1481 visitSummaryRefs = connectionTypes.Input(
1482 doc=
"Data references for per-visit consolidated exposure metadata",
1483 name=
"finalVisitSummary",
1484 storageClass=
"ExposureCatalog",
1485 dimensions=(
"instrument",
"visit"),
1489 outputCatalog = connectionTypes.Output(
1490 doc=
"CCD and Visit metadata table",
1491 name=
"ccdVisitTable",
1492 storageClass=
"DataFrame",
1493 dimensions=(
"instrument",)
1497class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1498 pipelineConnections=MakeCcdVisitTableConnections):
1499 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1502class MakeCcdVisitTableTask(pipeBase.PipelineTask):
1503 """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
1505 _DefaultName = 'makeCcdVisitTable'
1506 ConfigClass = MakeCcdVisitTableConfig
1508 def run(self, visitSummaryRefs):
1509 """Make a table of ccd information from the visit summary catalogs.
1513 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1514 List of DeferredDatasetHandles pointing to exposure catalogs with
1515 per-detector summary information.
1519 result : `~lsst.pipe.base.Struct`
1520 Results struct
with attribute:
1523 Catalog of ccd
and visit information.
1526 for visitSummaryRef
in visitSummaryRefs:
1527 visitSummary = visitSummaryRef.get()
1528 visitInfo = visitSummary[0].getVisitInfo()
1531 summaryTable = visitSummary.asAstropy()
1532 selectColumns = [
'id',
'visit',
'physical_filter',
'band',
'ra',
'dec',
'zenithDistance',
1533 'zeroPoint',
'psfSigma',
'skyBg',
'skyNoise',
1534 'astromOffsetMean',
'astromOffsetStd',
'nPsfStar',
1535 'psfStarDeltaE1Median',
'psfStarDeltaE2Median',
1536 'psfStarDeltaE1Scatter',
'psfStarDeltaE2Scatter',
1537 'psfStarDeltaSizeMedian',
'psfStarDeltaSizeScatter',
1538 'psfStarScaledDeltaSizeScatter',
1539 'psfTraceRadiusDelta',
'maxDistToNearestPsf']
1540 ccdEntry = summaryTable[selectColumns].to_pandas().set_index(
'id')
1545 ccdEntry = ccdEntry.rename(columns={
"visit":
"visitId"})
1549 ccdEntry[
"decl"] = ccdEntry.loc[:,
"dec"]
1551 ccdEntry[
'ccdVisitId'] = [
1552 self.config.idGenerator.apply(
1553 visitSummaryRef.dataId,
1554 detector=detector_id,
1561 for detector_id
in summaryTable[
'id']
1563 ccdEntry[
'detector'] = summaryTable[
'id']
1564 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds()
if vR.getWcs()
1565 else np.nan
for vR
in visitSummary])
1566 ccdEntry[
"seeing"] = visitSummary[
'psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1568 ccdEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1569 ccdEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1570 ccdEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1571 expTime = visitInfo.getExposureTime()
1572 ccdEntry[
'expTime'] = expTime
1573 ccdEntry[
"obsStart"] = ccdEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1574 expTime_days = expTime / (60*60*24)
1575 ccdEntry[
"obsStartMJD"] = ccdEntry[
"expMidptMJD"] - 0.5 * expTime_days
1576 ccdEntry[
'darkTime'] = visitInfo.getDarkTime()
1577 ccdEntry[
'xSize'] = summaryTable[
'bbox_max_x'] - summaryTable[
'bbox_min_x']
1578 ccdEntry[
'ySize'] = summaryTable[
'bbox_max_y'] - summaryTable[
'bbox_min_y']
1579 ccdEntry[
'llcra'] = summaryTable[
'raCorners'][:, 0]
1580 ccdEntry[
'llcdec'] = summaryTable[
'decCorners'][:, 0]
1581 ccdEntry[
'ulcra'] = summaryTable[
'raCorners'][:, 1]
1582 ccdEntry[
'ulcdec'] = summaryTable[
'decCorners'][:, 1]
1583 ccdEntry[
'urcra'] = summaryTable[
'raCorners'][:, 2]
1584 ccdEntry[
'urcdec'] = summaryTable[
'decCorners'][:, 2]
1585 ccdEntry[
'lrcra'] = summaryTable[
'raCorners'][:, 3]
1586 ccdEntry[
'lrcdec'] = summaryTable[
'decCorners'][:, 3]
1590 ccdEntries.append(ccdEntry)
1592 outputCatalog = pd.concat(ccdEntries)
1593 outputCatalog.set_index(
'ccdVisitId', inplace=
True, verify_integrity=
True)
1594 return pipeBase.Struct(outputCatalog=outputCatalog)
1597class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1598 dimensions=(
"instrument",),
1599 defaultTemplates={
"calexpType":
""}):
1600 visitSummaries = connectionTypes.Input(
1601 doc=
"Per-visit consolidated exposure metadata",
1602 name=
"finalVisitSummary",
1603 storageClass=
"ExposureCatalog",
1604 dimensions=(
"instrument",
"visit",),
1608 outputCatalog = connectionTypes.Output(
1609 doc=
"Visit metadata table",
1611 storageClass=
"DataFrame",
1612 dimensions=(
"instrument",)
1616class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1617 pipelineConnections=MakeVisitTableConnections):
1621class MakeVisitTableTask(pipeBase.PipelineTask):
1622 """Produce a `visitTable` from the visit summary exposure catalogs.
1624 _DefaultName = 'makeVisitTable'
1625 ConfigClass = MakeVisitTableConfig
1627 def run(self, visitSummaries):
1628 """Make a table of visit information from the visit summary catalogs.
1633 List of exposure catalogs with per-detector summary information.
1636 result : `~lsst.pipe.base.Struct`
1637 Results struct
with attribute:
1640 Catalog of visit information.
1643 for visitSummary
in visitSummaries:
1644 visitSummary = visitSummary.get()
1645 visitRow = visitSummary[0]
1646 visitInfo = visitRow.getVisitInfo()
1649 visitEntry[
"visitId"] = visitRow[
'visit']
1650 visitEntry[
"visit"] = visitRow[
'visit']
1651 visitEntry[
"physical_filter"] = visitRow[
'physical_filter']
1652 visitEntry[
"band"] = visitRow[
'band']
1653 raDec = visitInfo.getBoresightRaDec()
1654 visitEntry[
"ra"] = raDec.getRa().asDegrees()
1655 visitEntry[
"dec"] = raDec.getDec().asDegrees()
1659 visitEntry[
"decl"] = visitEntry[
"dec"]
1661 visitEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1662 azAlt = visitInfo.getBoresightAzAlt()
1663 visitEntry[
"azimuth"] = azAlt.getLongitude().asDegrees()
1664 visitEntry[
"altitude"] = azAlt.getLatitude().asDegrees()
1665 visitEntry[
"zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1666 visitEntry[
"airmass"] = visitInfo.getBoresightAirmass()
1667 expTime = visitInfo.getExposureTime()
1668 visitEntry[
"expTime"] = expTime
1669 visitEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1670 visitEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1671 visitEntry[
"obsStart"] = visitEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1672 expTime_days = expTime / (60*60*24)
1673 visitEntry[
"obsStartMJD"] = visitEntry[
"expMidptMJD"] - 0.5 * expTime_days
1674 visitEntries.append(visitEntry)
1680 outputCatalog = pd.DataFrame(data=visitEntries)
1681 outputCatalog.set_index(
'visitId', inplace=
True, verify_integrity=
True)
1682 return pipeBase.Struct(outputCatalog=outputCatalog)
1685class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1686 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")):
1688 inputCatalog = connectionTypes.Input(
1689 doc=
"Primary per-detector, single-epoch forced-photometry catalog. "
1690 "By default, it is the output of ForcedPhotCcdTask on calexps",
1692 storageClass=
"SourceCatalog",
1693 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1695 inputCatalogDiff = connectionTypes.Input(
1696 doc=
"Secondary multi-epoch, per-detector, forced photometry catalog. "
1697 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1699 storageClass=
"SourceCatalog",
1700 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1702 outputCatalog = connectionTypes.Output(
1703 doc=
"InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format",
1704 name=
"mergedForcedSource",
1705 storageClass=
"DataFrame",
1706 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1710class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1711 pipelineConnections=WriteForcedSourceTableConnections):
1712 key = lsst.pex.config.Field(
1713 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1717 idGenerator = DetectorVisitIdGeneratorConfig.make_field()
1720class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1721 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format.
1723 Because the predecessor ForcedPhotCcdTask operates per-detector,
1724 per-tract, (i.e., it has tract in its dimensions), detectors
1725 on the tract boundary may have multiple forced source catalogs.
1727 The successor task TransformForcedSourceTable runs per-patch
1728 and temporally-aggregates overlapping mergedForcedSource catalogs
from all
1729 available multiple epochs.
1731 _DefaultName = "writeForcedSourceTable"
1732 ConfigClass = WriteForcedSourceTableConfig
1734 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1735 inputs = butlerQC.get(inputRefs)
1737 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
1738 inputs[
'ccdVisitId'] = idGenerator.catalog_id
1739 inputs[
'band'] = butlerQC.quantum.dataId.full[
'band']
1740 outputs = self.run(**inputs)
1741 butlerQC.put(outputs, outputRefs)
1743 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1745 for table, dataset,
in zip((inputCatalog, inputCatalogDiff), (
'calexp',
'diff')):
1746 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=
False)
1747 df = df.reindex(sorted(df.columns), axis=1)
1748 df[
'ccdVisitId'] = ccdVisitId
if ccdVisitId
else pd.NA
1749 df[
'band'] = band
if band
else pd.NA
1750 df.columns = pd.MultiIndex.from_tuples([(dataset, c)
for c
in df.columns],
1751 names=(
'dataset',
'column'))
1755 outputCatalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
1756 return pipeBase.Struct(outputCatalog=outputCatalog)
1759class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1760 dimensions=(
"instrument",
"skymap",
"patch",
"tract")):
1762 inputCatalogs = connectionTypes.Input(
1763 doc=
"DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask",
1764 name=
"mergedForcedSource",
1765 storageClass=
"DataFrame",
1766 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
1770 referenceCatalog = connectionTypes.Input(
1771 doc=
"Reference catalog which was used to seed the forcedPhot. Columns "
1772 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1775 storageClass=
"DataFrame",
1776 dimensions=(
"tract",
"patch",
"skymap"),
1779 outputCatalog = connectionTypes.Output(
1780 doc=
"Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1781 "specified set of functors",
1782 name=
"forcedSourceTable",
1783 storageClass=
"DataFrame",
1784 dimensions=(
"tract",
"patch",
"skymap")
1789 pipelineConnections=TransformForcedSourceTableConnections):
1790 referenceColumns = pexConfig.ListField(
1792 default=[
"detect_isPrimary",
"detect_isTractInner",
"detect_isPatchInner"],
1794 doc=
"Columns to pull from reference catalog",
1796 keyRef = lsst.pex.config.Field(
1797 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1801 key = lsst.pex.config.Field(
1802 doc=
"Rename the output DataFrame index to this name",
1804 default=
"forcedSourceId",
1807 def setDefaults(self):
1808 super().setDefaults()
1809 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'ForcedSource.yaml')
1810 self.columnsFromDataId = [
'tract',
'patch']
1814 """Transform/standardize a ForcedSource catalog
1816 Transforms each wide, per-detector forcedSource DataFrame per the
1817 specification file (per-camera defaults found in ForcedSource.yaml).
1818 All epochs that overlap the patch are aggregated into one per-patch
1819 narrow-DataFrame file.
1821 No de-duplication of rows
is performed. Duplicate resolutions flags are
1822 pulled
in from the referenceCatalog: `detect_isPrimary`,
1823 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
1824 for analysis
or compare duplicates
for QA.
1826 The resulting table includes multiple bands. Epochs (MJDs)
and other useful
1827 per-visit rows can be retreived by joining
with the CcdVisitTable on
1830 _DefaultName = "transformForcedSourceTable"
1831 ConfigClass = TransformForcedSourceTableConfig
1833 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1834 inputs = butlerQC.get(inputRefs)
1835 if self.funcs
is None:
1836 raise ValueError(
"config.functorFile is None. "
1837 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1838 outputs = self.run(inputs[
'inputCatalogs'], inputs[
'referenceCatalog'], funcs=self.funcs,
1839 dataId=outputRefs.outputCatalog.dataId.full)
1841 butlerQC.put(outputs, outputRefs)
1843 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
1845 ref = referenceCatalog.get(parameters={
"columns": self.config.referenceColumns})
1846 self.log.info(
"Aggregating %s input catalogs" % (len(inputCatalogs)))
1847 for handle
in inputCatalogs:
1848 result = self.transform(
None, handle, funcs, dataId)
1850 dfs.append(result.df.join(ref, how=
'inner'))
1852 outputCatalog = pd.concat(dfs)
1856 outputCatalog.index.rename(self.config.keyRef, inplace=
True)
1858 outputCatalog.reset_index(inplace=
True)
1861 outputCatalog.set_index(
"forcedSourceId", inplace=
True, verify_integrity=
True)
1863 outputCatalog.index.rename(self.config.key, inplace=
True)
1865 self.log.info(
"Made a table of %d columns and %d rows",
1866 len(outputCatalog.columns), len(outputCatalog))
1867 return pipeBase.Struct(outputCatalog=outputCatalog)
1870class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
1871 defaultTemplates={
"catalogType":
""},
1872 dimensions=(
"instrument",
"tract")):
1873 inputCatalogs = connectionTypes.Input(
1874 doc=
"Input per-patch DataFrame Tables to be concatenated",
1875 name=
"{catalogType}ForcedSourceTable",
1876 storageClass=
"DataFrame",
1877 dimensions=(
"tract",
"patch",
"skymap"),
1881 outputCatalog = connectionTypes.Output(
1882 doc=
"Output per-tract concatenation of DataFrame Tables",
1883 name=
"{catalogType}ForcedSourceTable_tract",
1884 storageClass=
"DataFrame",
1885 dimensions=(
"tract",
"skymap"),
1889class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
1890 pipelineConnections=ConsolidateTractConnections):
1894class ConsolidateTractTask(pipeBase.PipelineTask):
1895 """Concatenate any per-patch, dataframe list into a single
1896 per-tract DataFrame.
1898 _DefaultName = 'ConsolidateTract'
1899 ConfigClass = ConsolidateTractConfig
1901 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1902 inputs = butlerQC.get(inputRefs)
1905 self.log.info(
"Concatenating %s per-patch %s Tables",
1906 len(inputs[
'inputCatalogs']),
1907 inputRefs.inputCatalogs[0].datasetType.name)
1908 df = pd.concat(inputs[
'inputCatalogs'])
1909 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
compute(self, dropna=False, pool=None)
__init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None)
run(self, handle, funcs=None, dataId=None, band=None)
runQuantum(self, butlerQC, inputRefs, outputRefs)
transform(self, band, handles, funcs, dataId)
getAnalysis(self, handles, funcs=None, band=None)
__init__(self, *args, **kwargs)
flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None)