22__all__ = [
"WriteObjectTableConfig",
"WriteObjectTableTask",
23 "WriteSourceTableConfig",
"WriteSourceTableTask",
24 "WriteRecalibratedSourceTableConfig",
"WriteRecalibratedSourceTableTask",
25 "PostprocessAnalysis",
26 "TransformCatalogBaseConfig",
"TransformCatalogBaseTask",
27 "TransformObjectCatalogConfig",
"TransformObjectCatalogTask",
28 "ConsolidateObjectTableConfig",
"ConsolidateObjectTableTask",
29 "TransformSourceTableConfig",
"TransformSourceTableTask",
30 "ConsolidateVisitSummaryConfig",
"ConsolidateVisitSummaryTask",
31 "ConsolidateSourceTableConfig",
"ConsolidateSourceTableTask",
32 "MakeCcdVisitTableConfig",
"MakeCcdVisitTableTask",
33 "MakeVisitTableConfig",
"MakeVisitTableTask",
34 "WriteForcedSourceTableConfig",
"WriteForcedSourceTableTask",
35 "TransformForcedSourceTableConfig",
"TransformForcedSourceTableTask",
36 "ConsolidateTractConfig",
"ConsolidateTractTask"]
49from lsst.obs.base
import ExposureIdInfo
54from lsst.daf.butler
import DataCoordinate
57from .functors
import CompositeFunctor, Column
59log = logging.getLogger(__name__)
62def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
63 """Flattens a dataframe with multilevel column index.
65 newDf = pd.DataFrame()
67 dfBands = df.columns.unique(level=0).values
70 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
71 newColumns = {c: columnFormat.format(band, c)
72 for c
in subdf.columns
if c
not in noDupCols}
73 cols = list(newColumns.keys())
74 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
77 presentBands = dfBands
if inputBands
is None else list(set(inputBands).intersection(dfBands))
79 noDupDf = df[presentBands[0]][noDupCols]
80 newDf = pd.concat([noDupDf, newDf], axis=1)
85 defaultTemplates={
"coaddName":
"deep"},
86 dimensions=(
"tract",
"patch",
"skymap")):
87 inputCatalogMeas = connectionTypes.Input(
88 doc=
"Catalog of source measurements on the deepCoadd.",
89 dimensions=(
"tract",
"patch",
"band",
"skymap"),
90 storageClass=
"SourceCatalog",
91 name=
"{coaddName}Coadd_meas",
94 inputCatalogForcedSrc = connectionTypes.Input(
95 doc=
"Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
96 dimensions=(
"tract",
"patch",
"band",
"skymap"),
97 storageClass=
"SourceCatalog",
98 name=
"{coaddName}Coadd_forced_src",
101 inputCatalogRef = connectionTypes.Input(
102 doc=
"Catalog marking the primary detection (which band provides a good shape and position)"
103 "for each detection in deepCoadd_mergeDet.",
104 dimensions=(
"tract",
"patch",
"skymap"),
105 storageClass=
"SourceCatalog",
106 name=
"{coaddName}Coadd_ref"
108 outputCatalog = connectionTypes.Output(
109 doc=
"A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
110 "stored as a DataFrame with a multi-level column index per-patch.",
111 dimensions=(
"tract",
"patch",
"skymap"),
112 storageClass=
"DataFrame",
113 name=
"{coaddName}Coadd_obj"
117class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
118 pipelineConnections=WriteObjectTableConnections):
119 engine = pexConfig.Field(
122 doc=
"Parquet engine for writing (pyarrow or fastparquet)",
123 deprecated=
"This config is no longer used, and will be removed after v26."
125 coaddName = pexConfig.Field(
132class WriteObjectTableTask(pipeBase.PipelineTask):
133 """Write filter-merged source tables as a DataFrame in parquet format.
135 _DefaultName = "writeObjectTable"
136 ConfigClass = WriteObjectTableConfig
139 inputDatasets = (
'forced_src',
'meas',
'ref')
142 outputDataset =
'obj'
144 def runQuantum(self, butlerQC, inputRefs, outputRefs):
145 inputs = butlerQC.get(inputRefs)
147 measDict = {ref.dataId[
'band']: {
'meas': cat}
for ref, cat
in
148 zip(inputRefs.inputCatalogMeas, inputs[
'inputCatalogMeas'])}
149 forcedSourceDict = {ref.dataId[
'band']: {
'forced_src': cat}
for ref, cat
in
150 zip(inputRefs.inputCatalogForcedSrc, inputs[
'inputCatalogForcedSrc'])}
153 for band
in measDict.keys():
154 catalogs[band] = {
'meas': measDict[band][
'meas'],
155 'forced_src': forcedSourceDict[band][
'forced_src'],
156 'ref': inputs[
'inputCatalogRef']}
157 dataId = butlerQC.quantum.dataId
158 df = self.run(catalogs=catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
159 outputs = pipeBase.Struct(outputCatalog=df)
160 butlerQC.put(outputs, outputRefs)
162 def run(self, catalogs, tract, patch):
163 """Merge multiple catalogs.
168 Mapping from filter names to dict of catalogs.
170 tractId to use
for the tractId column.
172 patchId to use
for the patchId column.
176 catalog : `pandas.DataFrame`
181 for filt, tableDict
in catalogs.items():
182 for dataset, table
in tableDict.items():
184 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
187 df = df.reindex(sorted(df.columns), axis=1)
188 df[
'tractId'] = tract
189 df[
'patchId'] = patch
192 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
193 names=(
'dataset',
'band',
'column'))
196 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
200class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
201 defaultTemplates={
"catalogType":
""},
202 dimensions=(
"instrument",
"visit",
"detector")):
204 catalog = connectionTypes.Input(
205 doc=
"Input full-depth catalog of sources produced by CalibrateTask",
206 name=
"{catalogType}src",
207 storageClass=
"SourceCatalog",
208 dimensions=(
"instrument",
"visit",
"detector")
210 outputCatalog = connectionTypes.Output(
211 doc=
"Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
212 "replaced with an index; all other columns are unchanged.",
213 name=
"{catalogType}source",
214 storageClass=
"DataFrame",
215 dimensions=(
"instrument",
"visit",
"detector")
219class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
220 pipelineConnections=WriteSourceTableConnections):
224class WriteSourceTableTask(pipeBase.PipelineTask):
225 """Write source table to DataFrame Parquet format.
227 _DefaultName = "writeSourceTable"
228 ConfigClass = WriteSourceTableConfig
230 def runQuantum(self, butlerQC, inputRefs, outputRefs):
231 inputs = butlerQC.get(inputRefs)
232 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
233 result = self.run(**inputs)
234 outputs = pipeBase.Struct(outputCatalog=result.table)
235 butlerQC.put(outputs, outputRefs)
237 def run(self, catalog, ccdVisitId=None, **kwargs):
238 """Convert `src` catalog to DataFrame
242 catalog: `afwTable.SourceCatalog`
243 catalog to be converted
245 ccdVisitId to be added as a column
249 result : `lsst.pipe.base.Struct`
251 `DataFrame` version of the input catalog
253 self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
254 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
255 df[
'ccdVisitId'] = ccdVisitId
256 return pipeBase.Struct(table=df)
259class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
260 defaultTemplates={
"catalogType":
"",
261 "skyWcsName":
"gbdesAstrometricFit",
262 "photoCalibName":
"fgcm"},
263 dimensions=(
"instrument",
"visit",
"detector",
"skymap")):
264 skyMap = connectionTypes.Input(
265 doc=
"skyMap needed to choose which tract-level calibrations to use when multiple available",
266 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
267 storageClass=
"SkyMap",
268 dimensions=(
"skymap",),
270 exposure = connectionTypes.Input(
271 doc=
"Input exposure to perform photometry on.",
273 storageClass=
"ExposureF",
274 dimensions=[
"instrument",
"visit",
"detector"],
276 externalSkyWcsTractCatalog = connectionTypes.Input(
277 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
278 "id for the catalog id, sorted on id for fast lookup."),
279 name=
"{skyWcsName}SkyWcsCatalog",
280 storageClass=
"ExposureCatalog",
281 dimensions=[
"instrument",
"visit",
"tract"],
284 externalSkyWcsGlobalCatalog = connectionTypes.Input(
285 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
286 "These catalogs use the detector id for the catalog id, sorted on id for "
288 name=
"finalVisitSummary",
289 storageClass=
"ExposureCatalog",
290 dimensions=[
"instrument",
"visit"],
292 externalPhotoCalibTractCatalog = connectionTypes.Input(
293 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
294 "detector id for the catalog id, sorted on id for fast lookup."),
295 name=
"{photoCalibName}PhotoCalibCatalog",
296 storageClass=
"ExposureCatalog",
297 dimensions=[
"instrument",
"visit",
"tract"],
300 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
301 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
302 "information). These catalogs use the detector id for the catalog id, "
303 "sorted on id for fast lookup."),
304 name=
"finalVisitSummary",
305 storageClass=
"ExposureCatalog",
306 dimensions=[
"instrument",
"visit"],
309 def __init__(self, *, config=None):
310 super().__init__(config=config)
313 if config.doApplyExternalSkyWcs
and config.doReevaluateSkyWcs:
314 if config.useGlobalExternalSkyWcs:
315 self.inputs.remove(
"externalSkyWcsTractCatalog")
317 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
319 self.inputs.remove(
"externalSkyWcsTractCatalog")
320 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
321 if config.doApplyExternalPhotoCalib
and config.doReevaluatePhotoCalib:
322 if config.useGlobalExternalPhotoCalib:
323 self.inputs.remove(
"externalPhotoCalibTractCatalog")
325 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
327 self.inputs.remove(
"externalPhotoCalibTractCatalog")
328 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
331class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
332 pipelineConnections=WriteRecalibratedSourceTableConnections):
334 doReevaluatePhotoCalib = pexConfig.Field(
337 doc=(
"Add or replace local photoCalib columns")
339 doReevaluateSkyWcs = pexConfig.Field(
342 doc=(
"Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec")
344 doApplyExternalPhotoCalib = pexConfig.Field(
347 doc=(
"If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
348 "algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
350 doApplyExternalSkyWcs = pexConfig.Field(
353 doc=(
"if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
354 "else use the wcs already attached to the exposure."),
356 useGlobalExternalPhotoCalib = pexConfig.Field(
359 doc=(
"When using doApplyExternalPhotoCalib, use 'global' calibrations "
360 "that are not run per-tract. When False, use per-tract photometric "
361 "calibration files.")
363 useGlobalExternalSkyWcs = pexConfig.Field(
366 doc=(
"When using doApplyExternalSkyWcs, use 'global' calibrations "
367 "that are not run per-tract. When False, use per-tract wcs "
373 if self.doApplyExternalSkyWcs
and not self.doReevaluateSkyWcs:
374 log.warning(
"doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
375 "External SkyWcs will not be read or evaluated.")
376 if self.doApplyExternalPhotoCalib
and not self.doReevaluatePhotoCalib:
377 log.warning(
"doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
378 "External PhotoCalib will not be read or evaluated.")
381class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
382 """Write source table to DataFrame Parquet format.
384 _DefaultName = "writeRecalibratedSourceTable"
385 ConfigClass = WriteRecalibratedSourceTableConfig
387 def runQuantum(self, butlerQC, inputRefs, outputRefs):
388 inputs = butlerQC.get(inputRefs)
389 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
390 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
392 if self.config.doReevaluatePhotoCalib
or self.config.doReevaluateSkyWcs:
393 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
394 inputs[
'exposure'] = self.attachCalibs(inputRefs, **inputs)
396 inputs[
'catalog'] = self.addCalibColumns(**inputs)
398 result = self.run(**inputs)
399 outputs = pipeBase.Struct(outputCatalog=result.table)
400 butlerQC.put(outputs, outputRefs)
402 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
403 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
404 externalPhotoCalibTractCatalog=None, **kwargs):
405 """Apply external calibrations to exposure per configuration
407 When multiple tract-level calibrations overlap, select the one with the
408 center closest to detector.
412 inputRefs : `lsst.pipe.base.InputQuantizedConnection`,
for dataIds of
414 skyMap : `lsst.skymap.SkyMap`
415 exposure : `lsst.afw.image.exposure.Exposure`
416 Input exposure to adjust calibrations.
418 Exposure catalog
with external skyWcs to be applied per config
420 Exposure catalog
with external skyWcs to be applied per config
422 Exposure catalog
with external photoCalib to be applied per config
428 exposure : `lsst.afw.image.exposure.Exposure`
429 Exposure
with adjusted calibrations.
431 if not self.config.doApplyExternalSkyWcs:
433 externalSkyWcsCatalog =
None
434 elif self.config.useGlobalExternalSkyWcs:
436 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
437 self.log.info(
'Applying global SkyWcs')
440 inputRef = getattr(inputRefs,
'externalSkyWcsTractCatalog')
441 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
444 self.log.info(
'Applying tract-level SkyWcs from tract %s', tracts[ind])
446 if exposure.getWcs()
is None:
447 raise ValueError(
"Trying to locate nearest tract, but exposure.wcs is None.")
448 ind = self.getClosestTract(tracts, skyMap,
449 exposure.getBBox(), exposure.getWcs())
450 self.log.info(
'Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
451 'Applying closest to detector center: tract=%s',
str(tracts), tracts[ind])
453 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
455 if not self.config.doApplyExternalPhotoCalib:
457 externalPhotoCalibCatalog =
None
458 elif self.config.useGlobalExternalPhotoCalib:
460 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
461 self.log.info(
'Applying global PhotoCalib')
464 inputRef = getattr(inputRefs,
'externalPhotoCalibTractCatalog')
465 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
468 self.log.info(
'Applying tract-level PhotoCalib from tract %s', tracts[ind])
470 ind = self.getClosestTract(tracts, skyMap,
471 exposure.getBBox(), exposure.getWcs())
472 self.log.info(
'Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
473 'Applying closest to detector center: tract=%s',
str(tracts), tracts[ind])
475 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
477 return self.prepareCalibratedExposure(exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog)
479 def getClosestTract(self, tracts, skyMap, bbox, wcs):
480 """Find the index of the tract closest to detector from list of tractIds
484 tracts: `list` [`int`]
485 Iterable of integer tractIds
486 skyMap : `lsst.skymap.SkyMap`
487 skyMap to lookup tract geometry and wcs
489 Detector bbox, center of which will compared to tract centers
491 Detector Wcs object to map the detector center to SkyCoord
500 center = wcs.pixelToSky(bbox.getCenter())
502 for tractId
in tracts:
503 tract = skyMap[tractId]
504 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
505 sep.append(center.separation(tractCenter))
507 return np.argmin(sep)
509 def prepareCalibratedExposure(self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None):
510 """Prepare a calibrated exposure and apply external calibrations
515 exposure : `lsst.afw.image.exposure.Exposure`
516 Input exposure to adjust calibrations.
518 Exposure catalog
with external skyWcs to be applied
519 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
520 for the catalog id, sorted on id
for fast lookup.
522 Exposure catalog
with external photoCalib to be applied
523 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
524 id
for the catalog id, sorted on id
for fast lookup.
528 exposure : `lsst.afw.image.exposure.Exposure`
529 Exposure
with adjusted calibrations.
531 detectorId = exposure.getInfo().getDetector().getId()
533 if externalPhotoCalibCatalog
is not None:
534 row = externalPhotoCalibCatalog.find(detectorId)
536 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog; "
537 "Using original photoCalib.", detectorId)
539 photoCalib = row.getPhotoCalib()
540 if photoCalib
is None:
541 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
542 "Using original photoCalib.", detectorId)
544 exposure.setPhotoCalib(photoCalib)
546 if externalSkyWcsCatalog
is not None:
547 row = externalSkyWcsCatalog.find(detectorId)
549 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog; "
550 "Using original skyWcs.", detectorId)
552 skyWcs = row.getWcs()
554 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
555 "Using original skyWcs.", detectorId)
557 exposure.setWcs(skyWcs)
561 def addCalibColumns(self, catalog, exposure, exposureIdInfo, **kwargs):
562 """Add replace columns with calibs evaluated at each centroid
564 Add or replace
'base_LocalWcs' `base_LocalPhotoCalib
' columns in a
565 a source catalog, by rerunning the plugins.
570 catalog to which calib columns will be added
571 exposure : `lsst.afw.image.exposure.Exposure`
572 Exposure with attached PhotoCalibs
and SkyWcs attributes to be
573 reevaluated at local centroids. Pixels are
not required.
574 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
579 Source Catalog
with requested local calib columns
581 measureConfig = SingleFrameMeasurementTask.ConfigClass()
582 measureConfig.doReplaceWithNoise = False
585 for slot
in measureConfig.slots:
586 setattr(measureConfig.slots, slot,
None)
588 measureConfig.plugins.names = []
589 if self.config.doReevaluateSkyWcs:
590 measureConfig.plugins.names.add(
'base_LocalWcs')
591 self.log.info(
"Re-evaluating base_LocalWcs plugin")
592 if self.config.doReevaluatePhotoCalib:
593 measureConfig.plugins.names.add(
'base_LocalPhotoCalib')
594 self.log.info(
"Re-evaluating base_LocalPhotoCalib plugin")
595 pluginsNotToCopy = tuple(measureConfig.plugins.names)
599 aliasMap = catalog.schema.getAliasMap()
600 mapper = afwTable.SchemaMapper(catalog.schema)
601 for item
in catalog.schema:
602 if not item.field.getName().startswith(pluginsNotToCopy):
603 mapper.addMapping(item.key)
605 schema = mapper.getOutputSchema()
606 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
607 schema.setAliasMap(aliasMap)
608 newCat = afwTable.SourceCatalog(schema)
609 newCat.extend(catalog, mapper=mapper)
615 if self.config.doReevaluateSkyWcs
and exposure.wcs
is not None:
616 afwTable.updateSourceCoords(exposure.wcs, newCat)
618 measurement.run(measCat=newCat, exposure=exposure, exposureId=exposureIdInfo.expId)
624 """Calculate columns from DataFrames or handles storing DataFrames.
626 This object manages and organizes an arbitrary set of computations
627 on a catalog. The catalog
is defined by a
628 `DeferredDatasetHandle`
or `InMemoryDatasetHandle` object
629 (
or list thereof), such
as a ``deepCoadd_obj`` dataset,
and the
630 computations are defined by a collection of `lsst.pipe.tasks.functor.Functor`
631 objects (
or, equivalently, a ``CompositeFunctor``).
633 After the object
is initialized, accessing the ``.df`` attribute (which
634 holds the `pandas.DataFrame` containing the results of the calculations)
635 triggers computation of said dataframe.
637 One of the conveniences of using this object
is the ability to define a
638 desired common filter
for all functors. This enables the same functor
639 collection to be passed to several different `PostprocessAnalysis` objects
640 without having to change the original functor collection, since the ``filt``
641 keyword argument of this object triggers an overwrite of the ``filt``
642 property
for all functors
in the collection.
644 This object also allows a list of refFlags to be passed,
and defines a set
645 of default refFlags that are always included even
if not requested.
647 If a list of DataFrames
or Handles
is passed, rather than a single one,
648 then the calculations will be mapped over all the input catalogs. In
649 principle, it should be straightforward to parallelize this activity, but
650 initial tests have failed (see TODO
in code comments).
654 handles : `lsst.daf.butler.DeferredDatasetHandle`
or
655 `lsst.pipe.base.InMemoryDatasetHandle`
or
657 Source
catalog(s)
for computation.
659 Computations to do (functors that act on ``handles``).
660 If a dict, the output
661 DataFrame will have columns keyed accordingly.
662 If a list, the column keys will come
from the
663 ``.shortname`` attribute of each functor.
665 filt : `str`, optional
666 Filter
in which to calculate. If provided,
667 this will overwrite any existing ``.filt`` attribute
668 of the provided functors.
670 flags : `list`, optional
671 List of flags (per-band) to include
in output table.
672 Taken
from the ``meas`` dataset
if applied to a multilevel Object Table.
674 refFlags : `list`, optional
675 List of refFlags (only reference band) to include
in output table.
677 forcedFlags : `list`, optional
678 List of flags (per-band) to include
in output table.
679 Taken
from the ``forced_src`` dataset
if applied to a
680 multilevel Object Table. Intended
for flags
from measurement plugins
681 only run during multi-band forced-photometry.
683 _defaultRefFlags = []
686 def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
691 self.
flags = list(flags)
if flags
is not None else []
692 self.
forcedFlags = list(forcedFlags)
if forcedFlags
is not None else []
694 if refFlags
is not None:
707 additionalFuncs.update({flag:
Column(flag, dataset=
'forced_src')
for flag
in self.
forcedFlags})
708 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.
refFlags})
709 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.
flags})
711 if isinstance(self.
functors, CompositeFunctor):
716 func.funcDict.update(additionalFuncs)
717 func.filt = self.
filt
723 return [name
for name, func
in self.
func.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
733 if type(self.
handles)
in (list, tuple):
735 dflist = [self.
func(handle, dropna=dropna)
for handle
in self.
handles]
739 dflist = pool.map(functools.partial(self.
func, dropna=dropna), self.
handles)
740 self.
_df = pd.concat(dflist)
749 """Expected Connections for subclasses of TransformCatalogBaseTask.
753 inputCatalog = connectionTypes.Input(
755 storageClass=
"DataFrame",
757 outputCatalog = connectionTypes.Output(
759 storageClass=
"DataFrame",
764 pipelineConnections=TransformCatalogBaseConnections):
765 functorFile = pexConfig.Field(
767 doc=
"Path to YAML file specifying Science Data Model functors to use "
768 "when copying columns and computing calibrated values.",
772 primaryKey = pexConfig.Field(
774 doc=
"Name of column to be set as the DataFrame index. If None, the index"
775 "will be named `id`",
779 columnsFromDataId = pexConfig.ListField(
783 doc=
"Columns to extract from the dataId",
788 """Base class for transforming/standardizing a catalog
790 by applying functors that convert units and apply calibrations.
791 The purpose of this task
is to perform a set of computations on
792 an input ``DeferredDatasetHandle``
or ``InMemoryDatasetHandle`` that holds
793 a ``DataFrame`` dataset (such
as ``deepCoadd_obj``),
and write the
794 results to a new dataset (which needs to be declared
in an ``outputDataset``
797 The calculations to be performed are defined
in a YAML file that specifies
798 a set of functors to be computed, provided
as
799 a ``--functorFile`` config parameter. An example of such a YAML file
824 - base_InputCount_value
827 functor: DeconvolvedMoments
832 - merge_measurement_i
833 - merge_measurement_r
834 - merge_measurement_z
835 - merge_measurement_y
836 - merge_measurement_g
837 - base_PixelFlags_flag_inexact_psfCenter
840 The names
for each entry under
"func" will become the names of columns
in
841 the output dataset. All the functors referenced are defined
in
843 functor are
in the `args` list,
and any additional entries
for each column
844 other than
"functor" or "args" (e.g., ``
'filt'``, ``
'dataset'``) are treated
as
845 keyword arguments to be passed to the functor initialization.
847 The
"flags" entry
is the default shortcut
for `Column` functors.
848 All columns listed under
"flags" will be copied to the output table
849 untransformed. They can be of any datatype.
850 In the special case of transforming a multi-level oject table
with
851 band
and dataset indices (deepCoadd_obj), these will be taked
from the
852 `meas` dataset
and exploded out per band.
854 There are two special shortcuts that only apply when transforming
855 multi-level Object (deepCoadd_obj) tables:
856 - The
"refFlags" entry
is shortcut
for `Column` functor
857 taken
from the `
'ref'` dataset
if transforming an ObjectTable.
858 - The
"forcedFlags" entry
is shortcut
for `Column` functors.
859 taken
from the ``forced_src`` dataset
if transforming an ObjectTable.
860 These are expanded out per band.
864 to organize
and excecute the calculations.
867 def _DefaultName(self):
868 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
872 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
876 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
879 def ConfigClass(self):
880 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
884 if self.config.functorFile:
885 self.log.info(
'Loading tranform functor definitions from %s',
886 self.config.functorFile)
887 self.
funcs = CompositeFunctor.from_file(self.config.functorFile)
888 self.
funcs.update(dict(PostprocessAnalysis._defaultFuncs))
893 inputs = butlerQC.get(inputRefs)
894 if self.
funcs is None:
895 raise ValueError(
"config.functorFile is None. "
896 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
897 result = self.
run(handle=inputs[
'inputCatalog'], funcs=self.
funcs,
898 dataId=outputRefs.outputCatalog.dataId.full)
899 outputs = pipeBase.Struct(outputCatalog=result)
900 butlerQC.put(outputs, outputRefs)
902 def run(self, handle, funcs=None, dataId=None, band=None):
903 """Do postprocessing calculations
905 Takes a ``DeferredDatasetHandle`` or ``InMemoryDatasetHandle``
or
906 ``DataFrame`` object
and dataId,
907 returns a dataframe
with results of postprocessing calculations.
911 handles : `lsst.daf.butler.DeferredDatasetHandle`
or
912 `lsst.pipe.base.InMemoryDatasetHandle`
or
913 `pandas.DataFrame`,
or list of these.
914 DataFrames
from which calculations are done.
915 funcs : `lsst.pipe.tasks.functors.Functors`
916 Functors to apply to the table
's columns
917 dataId : dict, optional
918 Used to add a `patchId` column to the output dataframe.
919 band : `str`, optional
920 Filter band that is being processed.
924 df : `pandas.DataFrame`
926 self.log.info("Transforming/standardizing the source table dataId: %s", dataId)
928 df = self.
transform(band, handle, funcs, dataId).df
929 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
941 def transform(self, band, handles, funcs, dataId):
942 analysis = self.
getAnalysis(handles, funcs=funcs, band=band)
944 if dataId
and self.config.columnsFromDataId:
945 for key
in self.config.columnsFromDataId:
947 df[
str(key)] = dataId[key]
949 raise ValueError(f
"'{key}' in config.columnsFromDataId not found in dataId: {dataId}")
951 if self.config.primaryKey:
952 if df.index.name != self.config.primaryKey
and self.config.primaryKey
in df:
953 df.reset_index(inplace=
True, drop=
True)
954 df.set_index(self.config.primaryKey, inplace=
True)
956 return pipeBase.Struct(
963 defaultTemplates={
"coaddName":
"deep"},
964 dimensions=(
"tract",
"patch",
"skymap")):
965 inputCatalog = connectionTypes.Input(
966 doc=
"The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
967 "stored as a DataFrame with a multi-level column index per-patch.",
968 dimensions=(
"tract",
"patch",
"skymap"),
969 storageClass=
"DataFrame",
970 name=
"{coaddName}Coadd_obj",
973 outputCatalog = connectionTypes.Output(
974 doc=
"Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
976 dimensions=(
"tract",
"patch",
"skymap"),
977 storageClass=
"DataFrame",
983 pipelineConnections=TransformObjectCatalogConnections):
984 coaddName = pexConfig.Field(
990 filterMap = pexConfig.DictField(
994 doc=(
"Dictionary mapping full filter name to short one for column name munging."
995 "These filters determine the output columns no matter what filters the "
996 "input data actually contain."),
997 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
998 "Will be removed after v22.")
1000 outputBands = pexConfig.ListField(
1004 doc=(
"These bands and only these bands will appear in the output,"
1005 " NaN-filled if the input does not include them."
1006 " If None, then use all bands found in the input.")
1008 camelCase = pexConfig.Field(
1011 doc=(
"Write per-band columns names with camelCase, else underscore "
1012 "For example: gPsFlux instead of g_PsFlux.")
1014 multilevelOutput = pexConfig.Field(
1017 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
1018 "and name-munged (False).")
1020 goodFlags = pexConfig.ListField(
1023 doc=(
"List of 'good' flags that should be set False when populating empty tables. "
1024 "All other flags are considered to be 'bad' flags and will be set to True.")
1026 floatFillValue = pexConfig.Field(
1029 doc=
"Fill value for float fields when populating empty tables."
1031 integerFillValue = pexConfig.Field(
1034 doc=
"Fill value for integer fields when populating empty tables."
1037 def setDefaults(self):
1038 super().setDefaults()
1039 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Object.yaml')
1040 self.primaryKey =
'objectId'
1041 self.columnsFromDataId = [
'tract',
'patch']
1042 self.goodFlags = [
'calib_astrometry_used',
1043 'calib_photometry_reserved',
1044 'calib_photometry_used',
1045 'calib_psf_candidate',
1046 'calib_psf_reserved',
1051 """Produce a flattened Object Table to match the format specified in
1054 Do the same set of postprocessing calculations on all bands.
1056 This is identical to `TransformCatalogBaseTask`,
except for that it does
1057 the specified functor calculations
for all filters present
in the
1058 input `deepCoadd_obj` table. Any specific ``
"filt"`` keywords specified
1059 by the YAML file will be superceded.
1061 _DefaultName = "transformObjectCatalog"
1062 ConfigClass = TransformObjectCatalogConfig
1064 def run(self, handle, funcs=None, dataId=None, band=None):
1068 templateDf = pd.DataFrame()
1070 columns = handle.get(component=
'columns')
1071 inputBands = columns.unique(level=1).values
1073 outputBands = self.config.outputBands
if self.config.outputBands
else inputBands
1076 for inputBand
in inputBands:
1077 if inputBand
not in outputBands:
1078 self.log.info(
"Ignoring %s band data in the input", inputBand)
1080 self.log.info(
"Transforming the catalog of band %s", inputBand)
1081 result = self.transform(inputBand, handle, funcs, dataId)
1082 dfDict[inputBand] = result.df
1083 analysisDict[inputBand] = result.analysis
1084 if templateDf.empty:
1085 templateDf = result.df
1088 for filt
in outputBands:
1089 if filt
not in dfDict:
1090 self.log.info(
"Adding empty columns for band %s", filt)
1091 dfTemp = templateDf.copy()
1092 for col
in dfTemp.columns:
1093 testValue = dfTemp[col].values[0]
1094 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1096 if col
in self.config.goodFlags:
1100 elif isinstance(testValue, numbers.Integral):
1104 if isinstance(testValue, np.unsignedinteger):
1105 raise ValueError(
"Parquet tables may not have unsigned integer columns.")
1107 fillValue = self.config.integerFillValue
1109 fillValue = self.config.floatFillValue
1110 dfTemp[col].values[:] = fillValue
1111 dfDict[filt] = dfTemp
1114 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
1116 if not self.config.multilevelOutput:
1117 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
1118 if self.config.primaryKey
in noDupCols:
1119 noDupCols.remove(self.config.primaryKey)
1120 if dataId
and self.config.columnsFromDataId:
1121 noDupCols += self.config.columnsFromDataId
1122 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1123 inputBands=inputBands)
1125 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1130class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1131 dimensions=(
"tract",
"skymap")):
1132 inputCatalogs = connectionTypes.Input(
1133 doc=
"Per-Patch objectTables conforming to the standard data model.",
1135 storageClass=
"DataFrame",
1136 dimensions=(
"tract",
"patch",
"skymap"),
1139 outputCatalog = connectionTypes.Output(
1140 doc=
"Pre-tract horizontal concatenation of the input objectTables",
1141 name=
"objectTable_tract",
1142 storageClass=
"DataFrame",
1143 dimensions=(
"tract",
"skymap"),
1147class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1148 pipelineConnections=ConsolidateObjectTableConnections):
1149 coaddName = pexConfig.Field(
1156class ConsolidateObjectTableTask(pipeBase.PipelineTask):
1157 """Write patch-merged source tables to a tract-level DataFrame Parquet file.
1159 Concatenates `objectTable` list into a per-visit `objectTable_tract`.
1161 _DefaultName = "consolidateObjectTable"
1162 ConfigClass = ConsolidateObjectTableConfig
1164 inputDataset =
'objectTable'
1165 outputDataset =
'objectTable_tract'
1167 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1168 inputs = butlerQC.get(inputRefs)
1169 self.log.info(
"Concatenating %s per-patch Object Tables",
1170 len(inputs[
'inputCatalogs']))
1171 df = pd.concat(inputs[
'inputCatalogs'])
1172 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1175class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1176 defaultTemplates={
"catalogType":
""},
1177 dimensions=(
"instrument",
"visit",
"detector")):
1179 inputCatalog = connectionTypes.Input(
1180 doc=
"Wide input catalog of sources produced by WriteSourceTableTask",
1181 name=
"{catalogType}source",
1182 storageClass=
"DataFrame",
1183 dimensions=(
"instrument",
"visit",
"detector"),
1186 outputCatalog = connectionTypes.Output(
1187 doc=
"Narrower, per-detector Source Table transformed and converted per a "
1188 "specified set of functors",
1189 name=
"{catalogType}sourceTable",
1190 storageClass=
"DataFrame",
1191 dimensions=(
"instrument",
"visit",
"detector")
1196 pipelineConnections=TransformSourceTableConnections):
1198 def setDefaults(self):
1199 super().setDefaults()
1200 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Source.yaml')
1201 self.primaryKey =
'sourceId'
1202 self.columnsFromDataId = [
'visit',
'detector',
'band',
'physical_filter']
1206 """Transform/standardize a source catalog
1208 _DefaultName = "transformSourceTable"
1209 ConfigClass = TransformSourceTableConfig
1212class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1213 dimensions=(
"instrument",
"visit",),
1214 defaultTemplates={
"calexpType":
""}):
1215 calexp = connectionTypes.Input(
1216 doc=
"Processed exposures used for metadata",
1218 storageClass=
"ExposureF",
1219 dimensions=(
"instrument",
"visit",
"detector"),
1223 visitSummary = connectionTypes.Output(
1224 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
1225 "detector id for the id and are sorted for fast lookups of a "
1227 name=
"visitSummary",
1228 storageClass=
"ExposureCatalog",
1229 dimensions=(
"instrument",
"visit"),
1231 visitSummarySchema = connectionTypes.InitOutput(
1232 doc=
"Schema of the visitSummary catalog",
1233 name=
"visitSummary_schema",
1234 storageClass=
"ExposureCatalog",
1238class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1239 pipelineConnections=ConsolidateVisitSummaryConnections):
1240 """Config for ConsolidateVisitSummaryTask"""
1244class ConsolidateVisitSummaryTask(pipeBase.PipelineTask):
1245 """Task to consolidate per-detector visit metadata.
1247 This task aggregates the following metadata from all the detectors
in a
1248 single visit into an exposure catalog:
1252 - The physical_filter
and band (
if available).
1253 - The psf size, shape,
and effective area at the center of the detector.
1254 - The corners of the bounding box
in right ascension/declination.
1256 Other quantities such
as Detector, Psf, ApCorrMap,
and TransmissionCurve
1257 are
not persisted here because of storage concerns,
and because of their
1258 limited utility
as summary statistics.
1260 Tests
for this task are performed
in ci_hsc_gen3.
1262 _DefaultName = "consolidateVisitSummary"
1263 ConfigClass = ConsolidateVisitSummaryConfig
1265 def __init__(self, **kwargs):
1266 super().__init__(**kwargs)
1267 self.schema = afwTable.ExposureTable.makeMinimalSchema()
1268 self.schema.addField(
'visit', type=
'L', doc=
'Visit number')
1269 self.schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
1270 self.schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
1271 ExposureSummaryStats.update_schema(self.schema)
1272 self.visitSummarySchema = afwTable.ExposureCatalog(self.schema)
1274 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1275 dataRefs = butlerQC.get(inputRefs.calexp)
1276 visit = dataRefs[0].dataId.byName()[
'visit']
1278 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)",
1279 len(dataRefs), visit)
1281 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1283 butlerQC.put(expCatalog, outputRefs.visitSummary)
1285 def _combineExposureMetadata(self, visit, dataRefs):
1286 """Make a combined exposure catalog from a list of dataRefs.
1287 These dataRefs must point to exposures with wcs, summaryStats,
1288 and other visit metadata.
1293 Visit identification number.
1294 dataRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1295 List of dataRefs
in visit.
1300 Exposure catalog
with per-detector summary information.
1302 cat = afwTable.ExposureCatalog(self.schema)
1303 cat.resize(len(dataRefs))
1305 cat['visit'] = visit
1307 for i, dataRef
in enumerate(dataRefs):
1308 visitInfo = dataRef.get(component=
'visitInfo')
1309 filterLabel = dataRef.get(component=
'filter')
1310 summaryStats = dataRef.get(component=
'summaryStats')
1311 detector = dataRef.get(component=
'detector')
1312 wcs = dataRef.get(component=
'wcs')
1313 photoCalib = dataRef.get(component=
'photoCalib')
1314 detector = dataRef.get(component=
'detector')
1315 bbox = dataRef.get(component=
'bbox')
1316 validPolygon = dataRef.get(component=
'validPolygon')
1320 rec.setVisitInfo(visitInfo)
1322 rec.setPhotoCalib(photoCalib)
1323 rec.setValidPolygon(validPolygon)
1325 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
1326 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
1327 rec.setId(detector.getId())
1328 summaryStats.update_record(rec)
1330 metadata = dafBase.PropertyList()
1331 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1333 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1334 cat.setMetadata(metadata)
1340class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1341 defaultTemplates={
"catalogType":
""},
1342 dimensions=(
"instrument",
"visit")):
1343 inputCatalogs = connectionTypes.Input(
1344 doc=
"Input per-detector Source Tables",
1345 name=
"{catalogType}sourceTable",
1346 storageClass=
"DataFrame",
1347 dimensions=(
"instrument",
"visit",
"detector"),
1350 outputCatalog = connectionTypes.Output(
1351 doc=
"Per-visit concatenation of Source Table",
1352 name=
"{catalogType}sourceTable_visit",
1353 storageClass=
"DataFrame",
1354 dimensions=(
"instrument",
"visit")
1358class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1359 pipelineConnections=ConsolidateSourceTableConnections):
1363class ConsolidateSourceTableTask(pipeBase.PipelineTask):
1364 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1366 _DefaultName = 'consolidateSourceTable'
1367 ConfigClass = ConsolidateSourceTableConfig
1369 inputDataset =
'sourceTable'
1370 outputDataset =
'sourceTable_visit'
1372 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1373 from .makeWarp
import reorderRefs
1375 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.inputCatalogs]
1376 detectorOrder.sort()
1377 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
1378 inputs = butlerQC.get(inputRefs)
1379 self.log.info(
"Concatenating %s per-detector Source Tables",
1380 len(inputs[
'inputCatalogs']))
1381 df = pd.concat(inputs[
'inputCatalogs'])
1382 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1385class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1386 dimensions=(
"instrument",),
1387 defaultTemplates={
"calexpType":
""}):
1388 visitSummaryRefs = connectionTypes.Input(
1389 doc=
"Data references for per-visit consolidated exposure metadata",
1390 name=
"finalVisitSummary",
1391 storageClass=
"ExposureCatalog",
1392 dimensions=(
"instrument",
"visit"),
1396 outputCatalog = connectionTypes.Output(
1397 doc=
"CCD and Visit metadata table",
1398 name=
"ccdVisitTable",
1399 storageClass=
"DataFrame",
1400 dimensions=(
"instrument",)
1404class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1405 pipelineConnections=MakeCcdVisitTableConnections):
1409class MakeCcdVisitTableTask(pipeBase.PipelineTask):
1410 """Produce a `ccdVisitTable` from the visit summary exposure catalogs.
1412 _DefaultName = 'makeCcdVisitTable'
1413 ConfigClass = MakeCcdVisitTableConfig
1415 def run(self, visitSummaryRefs):
1416 """Make a table of ccd information from the visit summary catalogs.
1420 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1421 List of DeferredDatasetHandles pointing to exposure catalogs with
1422 per-detector summary information.
1426 result : `lsst.pipe.Base.Struct`
1427 Results struct
with attribute:
1430 Catalog of ccd
and visit information.
1433 for visitSummaryRef
in visitSummaryRefs:
1434 visitSummary = visitSummaryRef.get()
1435 visitInfo = visitSummary[0].getVisitInfo()
1438 summaryTable = visitSummary.asAstropy()
1439 selectColumns = [
'id',
'visit',
'physical_filter',
'band',
'ra',
'decl',
'zenithDistance',
1440 'zeroPoint',
'psfSigma',
'skyBg',
'skyNoise',
1441 'astromOffsetMean',
'astromOffsetStd',
'nPsfStar',
1442 'psfStarDeltaE1Median',
'psfStarDeltaE2Median',
1443 'psfStarDeltaE1Scatter',
'psfStarDeltaE2Scatter',
1444 'psfStarDeltaSizeMedian',
'psfStarDeltaSizeScatter',
1445 'psfStarScaledDeltaSizeScatter',
1446 'psfTraceRadiusDelta',
'maxDistToNearestPsf']
1447 ccdEntry = summaryTable[selectColumns].to_pandas().set_index(
'id')
1452 ccdEntry = ccdEntry.rename(columns={
"visit":
"visitId"})
1453 dataIds = [DataCoordinate.standardize(visitSummaryRef.dataId, detector=id)
for id
in
1455 packer = visitSummaryRef.dataId.universe.makePacker(
'visit_detector', visitSummaryRef.dataId)
1456 ccdVisitIds = [packer.pack(dataId)
for dataId
in dataIds]
1457 ccdEntry[
'ccdVisitId'] = ccdVisitIds
1458 ccdEntry[
'detector'] = summaryTable[
'id']
1459 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds()
if vR.getWcs()
1460 else np.nan
for vR
in visitSummary])
1461 ccdEntry[
"seeing"] = visitSummary[
'psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1463 ccdEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1464 ccdEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1465 ccdEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1466 expTime = visitInfo.getExposureTime()
1467 ccdEntry[
'expTime'] = expTime
1468 ccdEntry[
"obsStart"] = ccdEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1469 expTime_days = expTime / (60*60*24)
1470 ccdEntry[
"obsStartMJD"] = ccdEntry[
"expMidptMJD"] - 0.5 * expTime_days
1471 ccdEntry[
'darkTime'] = visitInfo.getDarkTime()
1472 ccdEntry[
'xSize'] = summaryTable[
'bbox_max_x'] - summaryTable[
'bbox_min_x']
1473 ccdEntry[
'ySize'] = summaryTable[
'bbox_max_y'] - summaryTable[
'bbox_min_y']
1474 ccdEntry[
'llcra'] = summaryTable[
'raCorners'][:, 0]
1475 ccdEntry[
'llcdec'] = summaryTable[
'decCorners'][:, 0]
1476 ccdEntry[
'ulcra'] = summaryTable[
'raCorners'][:, 1]
1477 ccdEntry[
'ulcdec'] = summaryTable[
'decCorners'][:, 1]
1478 ccdEntry[
'urcra'] = summaryTable[
'raCorners'][:, 2]
1479 ccdEntry[
'urcdec'] = summaryTable[
'decCorners'][:, 2]
1480 ccdEntry[
'lrcra'] = summaryTable[
'raCorners'][:, 3]
1481 ccdEntry[
'lrcdec'] = summaryTable[
'decCorners'][:, 3]
1485 ccdEntries.append(ccdEntry)
1487 outputCatalog = pd.concat(ccdEntries)
1488 outputCatalog.set_index(
'ccdVisitId', inplace=
True, verify_integrity=
True)
1489 return pipeBase.Struct(outputCatalog=outputCatalog)
1492class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1493 dimensions=(
"instrument",),
1494 defaultTemplates={
"calexpType":
""}):
1495 visitSummaries = connectionTypes.Input(
1496 doc=
"Per-visit consolidated exposure metadata",
1497 name=
"finalVisitSummary",
1498 storageClass=
"ExposureCatalog",
1499 dimensions=(
"instrument",
"visit",),
1503 outputCatalog = connectionTypes.Output(
1504 doc=
"Visit metadata table",
1506 storageClass=
"DataFrame",
1507 dimensions=(
"instrument",)
1511class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1512 pipelineConnections=MakeVisitTableConnections):
1516class MakeVisitTableTask(pipeBase.PipelineTask):
1517 """Produce a `visitTable` from the visit summary exposure catalogs.
1519 _DefaultName = 'makeVisitTable'
1520 ConfigClass = MakeVisitTableConfig
1522 def run(self, visitSummaries):
1523 """Make a table of visit information from the visit summary catalogs.
1528 List of exposure catalogs with per-detector summary information.
1531 result : `lsst.pipe.Base.Struct`
1532 Results struct
with attribute:
1535 Catalog of visit information.
1538 for visitSummary
in visitSummaries:
1539 visitSummary = visitSummary.get()
1540 visitRow = visitSummary[0]
1541 visitInfo = visitRow.getVisitInfo()
1544 visitEntry[
"visitId"] = visitRow[
'visit']
1545 visitEntry[
"visit"] = visitRow[
'visit']
1546 visitEntry[
"physical_filter"] = visitRow[
'physical_filter']
1547 visitEntry[
"band"] = visitRow[
'band']
1548 raDec = visitInfo.getBoresightRaDec()
1549 visitEntry[
"ra"] = raDec.getRa().asDegrees()
1550 visitEntry[
"decl"] = raDec.getDec().asDegrees()
1551 visitEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1552 azAlt = visitInfo.getBoresightAzAlt()
1553 visitEntry[
"azimuth"] = azAlt.getLongitude().asDegrees()
1554 visitEntry[
"altitude"] = azAlt.getLatitude().asDegrees()
1555 visitEntry[
"zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1556 visitEntry[
"airmass"] = visitInfo.getBoresightAirmass()
1557 expTime = visitInfo.getExposureTime()
1558 visitEntry[
"expTime"] = expTime
1559 visitEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1560 visitEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1561 visitEntry[
"obsStart"] = visitEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1562 expTime_days = expTime / (60*60*24)
1563 visitEntry[
"obsStartMJD"] = visitEntry[
"expMidptMJD"] - 0.5 * expTime_days
1564 visitEntries.append(visitEntry)
1570 outputCatalog = pd.DataFrame(data=visitEntries)
1571 outputCatalog.set_index(
'visitId', inplace=
True, verify_integrity=
True)
1572 return pipeBase.Struct(outputCatalog=outputCatalog)
1575class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1576 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")):
1578 inputCatalog = connectionTypes.Input(
1579 doc=
"Primary per-detector, single-epoch forced-photometry catalog. "
1580 "By default, it is the output of ForcedPhotCcdTask on calexps",
1582 storageClass=
"SourceCatalog",
1583 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1585 inputCatalogDiff = connectionTypes.Input(
1586 doc=
"Secondary multi-epoch, per-detector, forced photometry catalog. "
1587 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1589 storageClass=
"SourceCatalog",
1590 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1592 outputCatalog = connectionTypes.Output(
1593 doc=
"InputCatalogs horizonatally joined on `objectId` in DataFrame parquet format",
1594 name=
"mergedForcedSource",
1595 storageClass=
"DataFrame",
1596 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1600class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1601 pipelineConnections=WriteForcedSourceTableConnections):
1602 key = lsst.pex.config.Field(
1603 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1609class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1610 """Merge and convert per-detector forced source catalogs to DataFrame Parquet format.
1612 Because the predecessor ForcedPhotCcdTask operates per-detector,
1613 per-tract, (i.e., it has tract in its dimensions), detectors
1614 on the tract boundary may have multiple forced source catalogs.
1616 The successor task TransformForcedSourceTable runs per-patch
1617 and temporally-aggregates overlapping mergedForcedSource catalogs
from all
1618 available multiple epochs.
1620 _DefaultName = "writeForcedSourceTable"
1621 ConfigClass = WriteForcedSourceTableConfig
1623 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1624 inputs = butlerQC.get(inputRefs)
1626 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
1627 inputs[
'band'] = butlerQC.quantum.dataId.full[
'band']
1628 outputs = self.run(**inputs)
1629 butlerQC.put(outputs, outputRefs)
1631 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1633 for table, dataset,
in zip((inputCatalog, inputCatalogDiff), (
'calexp',
'diff')):
1634 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=
False)
1635 df = df.reindex(sorted(df.columns), axis=1)
1636 df[
'ccdVisitId'] = ccdVisitId
if ccdVisitId
else pd.NA
1637 df[
'band'] = band
if band
else pd.NA
1638 df.columns = pd.MultiIndex.from_tuples([(dataset, c)
for c
in df.columns],
1639 names=(
'dataset',
'column'))
1643 outputCatalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
1644 return pipeBase.Struct(outputCatalog=outputCatalog)
1647class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1648 dimensions=(
"instrument",
"skymap",
"patch",
"tract")):
1650 inputCatalogs = connectionTypes.Input(
1651 doc=
"DataFrames of merged ForcedSources produced by WriteForcedSourceTableTask",
1652 name=
"mergedForcedSource",
1653 storageClass=
"DataFrame",
1654 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
1658 referenceCatalog = connectionTypes.Input(
1659 doc=
"Reference catalog which was used to seed the forcedPhot. Columns "
1660 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1663 storageClass=
"DataFrame",
1664 dimensions=(
"tract",
"patch",
"skymap"),
1667 outputCatalog = connectionTypes.Output(
1668 doc=
"Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1669 "specified set of functors",
1670 name=
"forcedSourceTable",
1671 storageClass=
"DataFrame",
1672 dimensions=(
"tract",
"patch",
"skymap")
1677 pipelineConnections=TransformForcedSourceTableConnections):
1678 referenceColumns = pexConfig.ListField(
1680 default=[
"detect_isPrimary",
"detect_isTractInner",
"detect_isPatchInner"],
1682 doc=
"Columns to pull from reference catalog",
1684 keyRef = lsst.pex.config.Field(
1685 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1689 key = lsst.pex.config.Field(
1690 doc=
"Rename the output DataFrame index to this name",
1692 default=
"forcedSourceId",
1695 def setDefaults(self):
1696 super().setDefaults()
1697 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'ForcedSource.yaml')
1698 self.columnsFromDataId = [
'tract',
'patch']
1702 """Transform/standardize a ForcedSource catalog
1704 Transforms each wide, per-detector forcedSource DataFrame per the
1705 specification file (per-camera defaults found in ForcedSource.yaml).
1706 All epochs that overlap the patch are aggregated into one per-patch
1707 narrow-DataFrame file.
1709 No de-duplication of rows
is performed. Duplicate resolutions flags are
1710 pulled
in from the referenceCatalog: `detect_isPrimary`,
1711 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
1712 for analysis
or compare duplicates
for QA.
1714 The resulting table includes multiple bands. Epochs (MJDs)
and other useful
1715 per-visit rows can be retreived by joining
with the CcdVisitTable on
1718 _DefaultName = "transformForcedSourceTable"
1719 ConfigClass = TransformForcedSourceTableConfig
1721 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1722 inputs = butlerQC.get(inputRefs)
1723 if self.funcs
is None:
1724 raise ValueError(
"config.functorFile is None. "
1725 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
1726 outputs = self.run(inputs[
'inputCatalogs'], inputs[
'referenceCatalog'], funcs=self.funcs,
1727 dataId=outputRefs.outputCatalog.dataId.full)
1729 butlerQC.put(outputs, outputRefs)
1731 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
1733 ref = referenceCatalog.get(parameters={
"columns": self.config.referenceColumns})
1734 self.log.info(
"Aggregating %s input catalogs" % (len(inputCatalogs)))
1735 for handle
in inputCatalogs:
1736 result = self.transform(
None, handle, funcs, dataId)
1738 dfs.append(result.df.join(ref, how=
'inner'))
1740 outputCatalog = pd.concat(dfs)
1744 outputCatalog.index.rename(self.config.keyRef, inplace=
True)
1746 outputCatalog.reset_index(inplace=
True)
1749 outputCatalog.set_index(
"forcedSourceId", inplace=
True, verify_integrity=
True)
1751 outputCatalog.index.rename(self.config.key, inplace=
True)
1753 self.log.info(
"Made a table of %d columns and %d rows",
1754 len(outputCatalog.columns), len(outputCatalog))
1755 return pipeBase.Struct(outputCatalog=outputCatalog)
1758class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
1759 defaultTemplates={
"catalogType":
""},
1760 dimensions=(
"instrument",
"tract")):
1761 inputCatalogs = connectionTypes.Input(
1762 doc=
"Input per-patch DataFrame Tables to be concatenated",
1763 name=
"{catalogType}ForcedSourceTable",
1764 storageClass=
"DataFrame",
1765 dimensions=(
"tract",
"patch",
"skymap"),
1769 outputCatalog = connectionTypes.Output(
1770 doc=
"Output per-tract concatenation of DataFrame Tables",
1771 name=
"{catalogType}ForcedSourceTable_tract",
1772 storageClass=
"DataFrame",
1773 dimensions=(
"tract",
"skymap"),
1777class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
1778 pipelineConnections=ConsolidateTractConnections):
1782class ConsolidateTractTask(pipeBase.PipelineTask):
1783 """Concatenate any per-patch, dataframe list into a single
1784 per-tract DataFrame.
1786 _DefaultName = 'ConsolidateTract'
1787 ConfigClass = ConsolidateTractConfig
1789 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1790 inputs = butlerQC.get(inputRefs)
1793 self.log.info(
"Concatenating %s per-patch %s Tables",
1794 len(inputs[
'inputCatalogs']),
1795 inputRefs.inputCatalogs[0].datasetType.name)
1796 df = pd.concat(inputs[
'inputCatalogs'])
1797 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
def __init__(self, handles, functors, filt=None, flags=None, refFlags=None, forcedFlags=None)
def compute(self, dropna=False, pool=None)
def __init__(self, *args, **kwargs)
def getAnalysis(self, handles, funcs=None, band=None)
def run(self, handle, funcs=None, dataId=None, band=None)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def transform(self, band, handles, funcs, dataId)
def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None)