24from collections
import defaultdict
34from lsst.obs.base
import ExposureIdInfo
38from lsst.pipe.base import CmdLineTask, ArgumentParser, DataIdContainer
40from lsst.daf.butler
import DeferredDatasetHandle, DataCoordinate
43from .parquetTable
import ParquetTable
44from .multiBandUtils
import makeMergeArgumentParser, MergeSourcesRunner
45from .functors
import CompositeFunctor, Column
47log = logging.getLogger(__name__)
50def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None):
51 """Flattens a dataframe with multilevel column index
53 newDf = pd.DataFrame()
55 dfBands = df.columns.unique(level=0).values
58 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
59 newColumns = {c: columnFormat.format(band, c)
60 for c
in subdf.columns
if c
not in noDupCols}
61 cols = list(newColumns.keys())
62 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
65 presentBands = dfBands
if inputBands
is None else list(set(inputBands).intersection(dfBands))
67 noDupDf = df[presentBands[0]][noDupCols]
68 newDf = pd.concat([noDupDf, newDf], axis=1)
73 defaultTemplates={
"coaddName":
"deep"},
74 dimensions=(
"tract",
"patch",
"skymap")):
75 inputCatalogMeas = connectionTypes.Input(
76 doc=
"Catalog of source measurements on the deepCoadd.",
77 dimensions=(
"tract",
"patch",
"band",
"skymap"),
78 storageClass=
"SourceCatalog",
79 name=
"{coaddName}Coadd_meas",
82 inputCatalogForcedSrc = connectionTypes.Input(
83 doc=
"Catalog of forced measurements (shape and position parameters held fixed) on the deepCoadd.",
84 dimensions=(
"tract",
"patch",
"band",
"skymap"),
85 storageClass=
"SourceCatalog",
86 name=
"{coaddName}Coadd_forced_src",
89 inputCatalogRef = connectionTypes.Input(
90 doc=
"Catalog marking the primary detection (which band provides a good shape and position)"
91 "for each detection in deepCoadd_mergeDet.",
92 dimensions=(
"tract",
"patch",
"skymap"),
93 storageClass=
"SourceCatalog",
94 name=
"{coaddName}Coadd_ref"
96 outputCatalog = connectionTypes.Output(
97 doc=
"A vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
98 "stored as a DataFrame with a multi-level column index per-patch.",
99 dimensions=(
"tract",
"patch",
"skymap"),
100 storageClass=
"DataFrame",
101 name=
"{coaddName}Coadd_obj"
105class WriteObjectTableConfig(pipeBase.PipelineTaskConfig,
106 pipelineConnections=WriteObjectTableConnections):
107 engine = pexConfig.Field(
110 doc=
"Parquet engine for writing (pyarrow or fastparquet)"
112 coaddName = pexConfig.Field(
119class WriteObjectTableTask(CmdLineTask, pipeBase.PipelineTask):
120 """Write filter-merged source tables to parquet
122 _DefaultName = "writeObjectTable"
123 ConfigClass = WriteObjectTableConfig
124 RunnerClass = MergeSourcesRunner
127 inputDatasets = (
'forced_src',
'meas',
'ref')
130 outputDataset =
'obj'
132 def __init__(self, butler=None, schema=None, **kwargs):
136 super().__init__(**kwargs)
138 def runDataRef(self, patchRefList):
140 @brief Merge coadd sources
from multiple bands. Calls
@ref `run` which must be defined
in
141 subclasses that inherit
from MergeSourcesTask.
142 @param[
in] patchRefList list of data references
for each filter
144 catalogs = dict(self.readCatalog(patchRef) for patchRef
in patchRefList)
145 dataId = patchRefList[0].dataId
146 mergedCatalog = self.run(catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
147 self.write(patchRefList[0],
ParquetTable(dataFrame=mergedCatalog))
149 def runQuantum(self, butlerQC, inputRefs, outputRefs):
150 inputs = butlerQC.get(inputRefs)
152 measDict = {ref.dataId[
'band']: {
'meas': cat}
for ref, cat
in
153 zip(inputRefs.inputCatalogMeas, inputs[
'inputCatalogMeas'])}
154 forcedSourceDict = {ref.dataId[
'band']: {
'forced_src': cat}
for ref, cat
in
155 zip(inputRefs.inputCatalogForcedSrc, inputs[
'inputCatalogForcedSrc'])}
158 for band
in measDict.keys():
159 catalogs[band] = {
'meas': measDict[band][
'meas'],
160 'forced_src': forcedSourceDict[band][
'forced_src'],
161 'ref': inputs[
'inputCatalogRef']}
162 dataId = butlerQC.quantum.dataId
163 df = self.run(catalogs=catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
164 outputs = pipeBase.Struct(outputCatalog=df)
165 butlerQC.put(outputs, outputRefs)
168 def _makeArgumentParser(cls):
169 """Create a suitable ArgumentParser.
171 We will use the ArgumentParser to get a list of data
172 references for patches; the RunnerClass will sort them into lists
173 of data references
for the same patch.
175 References first of self.inputDatasets, rather than
181 """Read input catalogs
183 Read all the input datasets given by the 'inputDatasets'
188 patchRef : `lsst.daf.persistence.ButlerDataRef`
189 Data reference
for patch
193 Tuple consisting of band name
and a dict of catalogs, keyed by
196 band = patchRef.get(self.config.coaddName + "Coadd_filterLabel", immediate=
True).bandLabel
198 for dataset
in self.inputDatasets:
199 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + dataset, immediate=
True)
200 self.log.info(
"Read %d sources from %s for band %s: %s",
201 len(catalog), dataset, band, patchRef.dataId)
202 catalogDict[dataset] = catalog
203 return band, catalogDict
205 def run(self, catalogs, tract, patch):
206 """Merge multiple catalogs.
211 Mapping from filter names to dict of catalogs.
213 tractId to use
for the tractId column
215 patchId to use
for the patchId column
219 catalog : `pandas.DataFrame`
224 for filt, tableDict
in catalogs.items():
225 for dataset, table
in tableDict.items():
227 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
230 df = df.reindex(sorted(df.columns), axis=1)
231 df[
'tractId'] = tract
232 df[
'patchId'] = patch
235 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
236 names=(
'dataset',
'band',
'column'))
239 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
242 def write(self, patchRef, catalog):
247 catalog : `ParquetTable`
249 patchRef : `lsst.daf.persistence.ButlerDataRef`
250 Data reference for patch
252 patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
255 mergeDataId = patchRef.dataId.copy()
256 del mergeDataId[
"filter"]
257 self.log.info(
"Wrote merged catalog: %s", mergeDataId)
260 """No metadata to write, and not sure how to write it for a list of dataRefs.
265class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
266 defaultTemplates={
"catalogType":
""},
267 dimensions=(
"instrument",
"visit",
"detector")):
269 catalog = connectionTypes.Input(
270 doc=
"Input full-depth catalog of sources produced by CalibrateTask",
271 name=
"{catalogType}src",
272 storageClass=
"SourceCatalog",
273 dimensions=(
"instrument",
"visit",
"detector")
275 outputCatalog = connectionTypes.Output(
276 doc=
"Catalog of sources, `src` in Parquet format. The 'id' column is "
277 "replaced with an index; all other columns are unchanged.",
278 name=
"{catalogType}source",
279 storageClass=
"DataFrame",
280 dimensions=(
"instrument",
"visit",
"detector")
284class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
285 pipelineConnections=WriteSourceTableConnections):
289class WriteSourceTableTask(CmdLineTask, pipeBase.PipelineTask):
290 """Write source table to parquet
292 _DefaultName = "writeSourceTable"
293 ConfigClass = WriteSourceTableConfig
295 def runQuantum(self, butlerQC, inputRefs, outputRefs):
296 inputs = butlerQC.get(inputRefs)
297 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
298 result = self.run(**inputs).table
299 outputs = pipeBase.Struct(outputCatalog=result.toDataFrame())
300 butlerQC.put(outputs, outputRefs)
302 def run(self, catalog, ccdVisitId=None, **kwargs):
303 """Convert `src` catalog to parquet
307 catalog: `afwTable.SourceCatalog`
308 catalog to be converted
310 ccdVisitId to be added as a column
314 result : `lsst.pipe.base.Struct`
316 `ParquetTable` version of the input catalog
318 self.log.info("Generating parquet table from src catalog ccdVisitId=%s", ccdVisitId)
319 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
320 df[
'ccdVisitId'] = ccdVisitId
321 return pipeBase.Struct(table=
ParquetTable(dataFrame=df))
324class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
325 defaultTemplates={
"catalogType":
"",
326 "skyWcsName":
"jointcal",
327 "photoCalibName":
"fgcm"},
328 dimensions=(
"instrument",
"visit",
"detector",
"skymap")):
329 skyMap = connectionTypes.Input(
330 doc=
"skyMap needed to choose which tract-level calibrations to use when multiple available",
331 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
332 storageClass=
"SkyMap",
333 dimensions=(
"skymap",),
335 exposure = connectionTypes.Input(
336 doc=
"Input exposure to perform photometry on.",
338 storageClass=
"ExposureF",
339 dimensions=[
"instrument",
"visit",
"detector"],
341 externalSkyWcsTractCatalog = connectionTypes.Input(
342 doc=(
"Per-tract, per-visit wcs calibrations. These catalogs use the detector "
343 "id for the catalog id, sorted on id for fast lookup."),
344 name=
"{skyWcsName}SkyWcsCatalog",
345 storageClass=
"ExposureCatalog",
346 dimensions=[
"instrument",
"visit",
"tract"],
349 externalSkyWcsGlobalCatalog = connectionTypes.Input(
350 doc=(
"Per-visit wcs calibrations computed globally (with no tract information). "
351 "These catalogs use the detector id for the catalog id, sorted on id for "
353 name=
"{skyWcsName}SkyWcsCatalog",
354 storageClass=
"ExposureCatalog",
355 dimensions=[
"instrument",
"visit"],
357 externalPhotoCalibTractCatalog = connectionTypes.Input(
358 doc=(
"Per-tract, per-visit photometric calibrations. These catalogs use the "
359 "detector id for the catalog id, sorted on id for fast lookup."),
360 name=
"{photoCalibName}PhotoCalibCatalog",
361 storageClass=
"ExposureCatalog",
362 dimensions=[
"instrument",
"visit",
"tract"],
365 externalPhotoCalibGlobalCatalog = connectionTypes.Input(
366 doc=(
"Per-visit photometric calibrations computed globally (with no tract "
367 "information). These catalogs use the detector id for the catalog id, "
368 "sorted on id for fast lookup."),
369 name=
"{photoCalibName}PhotoCalibCatalog",
370 storageClass=
"ExposureCatalog",
371 dimensions=[
"instrument",
"visit"],
374 def __init__(self, *, config=None):
375 super().__init__(config=config)
378 if config.doApplyExternalSkyWcs
and config.doReevaluateSkyWcs:
379 if config.useGlobalExternalSkyWcs:
380 self.inputs.remove(
"externalSkyWcsTractCatalog")
382 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
384 self.inputs.remove(
"externalSkyWcsTractCatalog")
385 self.inputs.remove(
"externalSkyWcsGlobalCatalog")
386 if config.doApplyExternalPhotoCalib
and config.doReevaluatePhotoCalib:
387 if config.useGlobalExternalPhotoCalib:
388 self.inputs.remove(
"externalPhotoCalibTractCatalog")
390 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
392 self.inputs.remove(
"externalPhotoCalibTractCatalog")
393 self.inputs.remove(
"externalPhotoCalibGlobalCatalog")
396class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
397 pipelineConnections=WriteRecalibratedSourceTableConnections):
399 doReevaluatePhotoCalib = pexConfig.Field(
402 doc=(
"Add or replace local photoCalib columns from either the calexp.photoCalib or jointcal/FGCM")
404 doReevaluateSkyWcs = pexConfig.Field(
407 doc=(
"Add or replace local WCS columns from either the calexp.wcs or or jointcal")
409 doApplyExternalPhotoCalib = pexConfig.Field(
412 doc=(
"Whether to apply external photometric calibration via an "
413 "`lsst.afw.image.PhotoCalib` object. Uses the "
414 "``externalPhotoCalibName`` field to determine which calibration "
417 doApplyExternalSkyWcs = pexConfig.Field(
420 doc=(
"Whether to apply external astrometric calibration via an "
421 "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
422 "field to determine which calibration to load."),
424 useGlobalExternalPhotoCalib = pexConfig.Field(
427 doc=(
"When using doApplyExternalPhotoCalib, use 'global' calibrations "
428 "that are not run per-tract. When False, use per-tract photometric "
429 "calibration files.")
431 useGlobalExternalSkyWcs = pexConfig.Field(
434 doc=(
"When using doApplyExternalSkyWcs, use 'global' calibrations "
435 "that are not run per-tract. When False, use per-tract wcs "
441 if self.doApplyExternalSkyWcs
and not self.doReevaluateSkyWcs:
442 log.warning(
"doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
443 "External SkyWcs will not be read or evaluated.")
444 if self.doApplyExternalPhotoCalib
and not self.doReevaluatePhotoCalib:
445 log.warning(
"doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
446 "External PhotoCalib will not be read or evaluated.")
449class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
450 """Write source table to parquet
452 _DefaultName = "writeRecalibratedSourceTable"
453 ConfigClass = WriteRecalibratedSourceTableConfig
455 def runQuantum(self, butlerQC, inputRefs, outputRefs):
456 inputs = butlerQC.get(inputRefs)
457 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
458 inputs[
'exposureIdInfo'] = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"visit_detector")
460 if self.config.doReevaluatePhotoCalib
or self.config.doReevaluateSkyWcs:
461 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
462 inputs[
'exposure'] = self.attachCalibs(inputRefs, **inputs)
464 inputs[
'catalog'] = self.addCalibColumns(**inputs)
466 result = self.run(**inputs).table
467 outputs = pipeBase.Struct(outputCatalog=result.toDataFrame())
468 butlerQC.put(outputs, outputRefs)
470 def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
471 externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
472 externalPhotoCalibTractCatalog=None, **kwargs):
473 """Apply external calibrations to exposure per configuration
475 When multiple tract-level calibrations overlap, select the one with the
476 center closest to detector.
480 inputRefs : `lsst.pipe.base.InputQuantizedConnection`,
for dataIds of
482 skyMap : `lsst.skymap.SkyMap`
483 exposure : `lsst.afw.image.exposure.Exposure`
484 Input exposure to adjust calibrations.
486 Exposure catalog
with external skyWcs to be applied per config
488 Exposure catalog
with external skyWcs to be applied per config
490 Exposure catalog
with external photoCalib to be applied per config
496 exposure : `lsst.afw.image.exposure.Exposure`
497 Exposure
with adjusted calibrations.
499 if not self.config.doApplyExternalSkyWcs:
501 externalSkyWcsCatalog =
None
502 elif self.config.useGlobalExternalSkyWcs:
504 externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
505 self.log.info(
'Applying global SkyWcs')
508 inputRef = getattr(inputRefs,
'externalSkyWcsTractCatalog')
509 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
512 self.log.info(
'Applying tract-level SkyWcs from tract %s', tracts[ind])
514 ind = self.getClosestTract(tracts, skyMap,
515 exposure.getBBox(), exposure.getWcs())
516 self.log.info(
'Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
517 'Applying closest to detector center: tract=%s',
str(tracts), tracts[ind])
519 externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
521 if not self.config.doApplyExternalPhotoCalib:
523 externalPhotoCalibCatalog =
None
524 elif self.config.useGlobalExternalPhotoCalib:
526 externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
527 self.log.info(
'Applying global PhotoCalib')
530 inputRef = getattr(inputRefs,
'externalPhotoCalibTractCatalog')
531 tracts = [ref.dataId[
'tract']
for ref
in inputRef]
534 self.log.info(
'Applying tract-level PhotoCalib from tract %s', tracts[ind])
536 ind = self.getClosestTract(tracts, skyMap,
537 exposure.getBBox(), exposure.getWcs())
538 self.log.info(
'Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
539 'Applying closest to detector center: tract=%s',
str(tracts), tracts[ind])
541 externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
543 return self.prepareCalibratedExposure(exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog)
545 def getClosestTract(self, tracts, skyMap, bbox, wcs):
546 """Find the index of the tract closest to detector from list of tractIds
550 tracts: `list` [`int`]
551 Iterable of integer tractIds
552 skyMap : `lsst.skymap.SkyMap`
553 skyMap to lookup tract geometry and wcs
555 Detector bbox, center of which will compared to tract centers
557 Detector Wcs object to map the detector center to SkyCoord
566 center = wcs.pixelToSky(bbox.getCenter())
568 for tractId
in tracts:
569 tract = skyMap[tractId]
570 tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
571 sep.append(center.separation(tractCenter))
573 return np.argmin(sep)
575 def prepareCalibratedExposure(self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None):
576 """Prepare a calibrated exposure and apply external calibrations
581 exposure : `lsst.afw.image.exposure.Exposure`
582 Input exposure to adjust calibrations.
584 Exposure catalog
with external skyWcs to be applied
585 if config.doApplyExternalSkyWcs=
True. Catalog uses the detector id
586 for the catalog id, sorted on id
for fast lookup.
588 Exposure catalog
with external photoCalib to be applied
589 if config.doApplyExternalPhotoCalib=
True. Catalog uses the detector
590 id
for the catalog id, sorted on id
for fast lookup.
594 exposure : `lsst.afw.image.exposure.Exposure`
595 Exposure
with adjusted calibrations.
597 detectorId = exposure.getInfo().getDetector().getId()
599 if externalPhotoCalibCatalog
is not None:
600 row = externalPhotoCalibCatalog.find(detectorId)
602 self.log.warning(
"Detector id %s not found in externalPhotoCalibCatalog; "
603 "Using original photoCalib.", detectorId)
605 photoCalib = row.getPhotoCalib()
606 if photoCalib
is None:
607 self.log.warning(
"Detector id %s has None for photoCalib in externalPhotoCalibCatalog; "
608 "Using original photoCalib.", detectorId)
610 exposure.setPhotoCalib(photoCalib)
612 if externalSkyWcsCatalog
is not None:
613 row = externalSkyWcsCatalog.find(detectorId)
615 self.log.warning(
"Detector id %s not found in externalSkyWcsCatalog; "
616 "Using original skyWcs.", detectorId)
618 skyWcs = row.getWcs()
620 self.log.warning(
"Detector id %s has None for skyWcs in externalSkyWcsCatalog; "
621 "Using original skyWcs.", detectorId)
623 exposure.setWcs(skyWcs)
627 def addCalibColumns(self, catalog, exposure, exposureIdInfo, **kwargs):
628 """Add replace columns with calibs evaluated at each centroid
630 Add or replace
'base_LocalWcs' `base_LocalPhotoCalib
' columns in a
631 a source catalog, by rerunning the plugins.
636 catalog to which calib columns will be added
637 exposure : `lsst.afw.image.exposure.Exposure`
638 Exposure with attached PhotoCalibs
and SkyWcs attributes to be
639 reevaluated at local centroids. Pixels are
not required.
640 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
645 Source Catalog
with requested local calib columns
647 measureConfig = SingleFrameMeasurementTask.ConfigClass()
648 measureConfig.doReplaceWithNoise = False
650 measureConfig.plugins.names = []
651 if self.config.doReevaluateSkyWcs:
652 measureConfig.plugins.names.add(
'base_LocalWcs')
653 self.log.info(
"Re-evaluating base_LocalWcs plugin")
654 if self.config.doReevaluatePhotoCalib:
655 measureConfig.plugins.names.add(
'base_LocalPhotoCalib')
656 self.log.info(
"Re-evaluating base_LocalPhotoCalib plugin")
657 pluginsNotToCopy = tuple(measureConfig.plugins.names)
661 aliasMap = catalog.schema.getAliasMap()
662 mapper = afwTable.SchemaMapper(catalog.schema)
663 for item
in catalog.schema:
664 if not item.field.getName().startswith(pluginsNotToCopy):
665 mapper.addMapping(item.key)
667 schema = mapper.getOutputSchema()
668 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
669 schema.setAliasMap(aliasMap)
670 newCat = afwTable.SourceCatalog(schema)
671 newCat.extend(catalog, mapper=mapper)
673 measurement.run(measCat=newCat, exposure=exposure, exposureId=exposureIdInfo.expId)
678class PostprocessAnalysis(object):
679 """Calculate columns from ParquetTable
681 This object manages and organizes an arbitrary set of computations
682 on a catalog. The catalog
is defined by a
684 `deepCoadd_obj` dataset,
and the computations are defined by a collection
685 of `lsst.pipe.tasks.functor.Functor` objects (
or, equivalently,
686 a `CompositeFunctor`).
688 After the object
is initialized, accessing the `.df` attribute (which
689 holds the `pandas.DataFrame` containing the results of the calculations) triggers
690 computation of said dataframe.
692 One of the conveniences of using this object
is the ability to define a desired common
693 filter
for all functors. This enables the same functor collection to be passed to
694 several different `PostprocessAnalysis` objects without having to change the original
695 functor collection, since the `filt` keyword argument of this object triggers an
696 overwrite of the `filt` property
for all functors
in the collection.
698 This object also allows a list of refFlags to be passed,
and defines a set of default
699 refFlags that are always included even
if not requested.
701 If a list of `ParquetTable` object
is passed, rather than a single one, then the
702 calculations will be mapped over all the input catalogs. In principle, it should
703 be straightforward to parallelize this activity, but initial tests have failed
704 (see TODO
in code comments).
708 parq : `lsst.pipe.tasks.ParquetTable` (
or list of such)
709 Source catalog(s)
for computation
712 Computations to do (functors that act on `parq`).
713 If a dict, the output
714 DataFrame will have columns keyed accordingly.
715 If a list, the column keys will come
from the
716 `.shortname` attribute of each functor.
718 filt : `str` (optional)
719 Filter
in which to calculate. If provided,
720 this will overwrite any existing `.filt` attribute
721 of the provided functors.
723 flags : `list` (optional)
724 List of flags (per-band) to include
in output table.
725 Taken
from the `meas` dataset
if applied to a multilevel Object Table.
727 refFlags : `list` (optional)
728 List of refFlags (only reference band) to include
in output table.
730 forcedFlags : `list` (optional)
731 List of flags (per-band) to include
in output table.
732 Taken
from the ``forced_src`` dataset
if applied to a
733 multilevel Object Table. Intended
for flags
from measurement plugins
734 only run during multi-band forced-photometry.
736 _defaultRefFlags = []
739 def __init__(self, parq, functors, filt=None, flags=None, refFlags=None, forcedFlags=None):
741 self.functors = functors
744 self.flags = list(flags)
if flags
is not None else []
745 self.forcedFlags = list(forcedFlags)
if forcedFlags
is not None else []
746 self.refFlags = list(self._defaultRefFlags)
747 if refFlags
is not None:
748 self.refFlags += list(refFlags)
753 def defaultFuncs(self):
754 funcs = dict(self._defaultFuncs)
759 additionalFuncs = self.defaultFuncs
760 additionalFuncs.update({flag:
Column(flag, dataset=
'forced_src')
for flag
in self.forcedFlags})
761 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.refFlags})
762 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.flags})
764 if isinstance(self.functors, CompositeFunctor):
769 func.funcDict.update(additionalFuncs)
770 func.filt = self.filt
776 return [name
for name, func
in self.func.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
784 def compute(self, dropna=False, pool=None):
786 if type(self.parq)
in (list, tuple):
788 dflist = [self.func(parq, dropna=dropna)
for parq
in self.parq]
791 dflist = pool.map(functools.partial(self.func, dropna=dropna), self.parq)
792 self._df = pd.concat(dflist)
794 self._df = self.func(self.parq, dropna=dropna)
801 """Expected Connections for subclasses of TransformCatalogBaseTask.
805 inputCatalog = connectionTypes.Input(
807 storageClass=
"DataFrame",
809 outputCatalog = connectionTypes.Output(
811 storageClass=
"DataFrame",
816 pipelineConnections=TransformCatalogBaseConnections):
817 functorFile = pexConfig.Field(
819 doc=
"Path to YAML file specifying Science Data Model functors to use "
820 "when copying columns and computing calibrated values.",
824 primaryKey = pexConfig.Field(
826 doc=
"Name of column to be set as the DataFrame index. If None, the index"
827 "will be named `id`",
834 """Base class for transforming/standardizing a catalog
836 by applying functors that convert units and apply calibrations.
837 The purpose of this task
is to perform a set of computations on
838 an input `ParquetTable` dataset (such
as `deepCoadd_obj`)
and write the
839 results to a new dataset (which needs to be declared
in an `outputDataset`
842 The calculations to be performed are defined
in a YAML file that specifies
843 a set of functors to be computed, provided
as
844 a `--functorFile` config parameter. An example of such a YAML file
869 - base_InputCount_value
872 functor: DeconvolvedMoments
877 - merge_measurement_i
878 - merge_measurement_r
879 - merge_measurement_z
880 - merge_measurement_y
881 - merge_measurement_g
882 - base_PixelFlags_flag_inexact_psfCenter
885 The names
for each entry under
"func" will become the names of columns
in the
887 Positional arguments to be passed to each functor are
in the `args` list,
888 and any additional entries
for each column other than
"functor" or "args" (e.g., `
'filt'`,
889 `
'dataset'`) are treated
as keyword arguments to be passed to the functor initialization.
891 The
"flags" entry
is the default shortcut
for `Column` functors.
892 All columns listed under
"flags" will be copied to the output table
893 untransformed. They can be of any datatype.
894 In the special case of transforming a multi-level oject table
with
895 band
and dataset indices (deepCoadd_obj), these will be taked
from the
896 `meas` dataset
and exploded out per band.
898 There are two special shortcuts that only apply when transforming
899 multi-level Object (deepCoadd_obj) tables:
900 - The
"refFlags" entry
is shortcut
for `Column` functor
901 taken
from the `
'ref'` dataset
if transforming an ObjectTable.
902 - The
"forcedFlags" entry
is shortcut
for `Column` functors.
903 taken
from the ``forced_src`` dataset
if transforming an ObjectTable.
904 These are expanded out per band.
907 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
908 to organize
and excecute the calculations.
912 def _DefaultName(self):
913 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
917 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
921 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
925 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
929 if self.config.functorFile:
930 self.log.info(
'Loading tranform functor definitions from %s',
931 self.config.functorFile)
932 self.
funcsfuncs = CompositeFunctor.from_file(self.config.functorFile)
933 self.
funcsfuncs.update(dict(PostprocessAnalysis._defaultFuncs))
935 self.
funcsfuncs =
None
938 inputs = butlerQC.get(inputRefs)
939 if self.
funcsfuncs
is None:
940 raise ValueError(
"config.functorFile is None. "
941 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
942 result = self.
runrun(parq=inputs[
'inputCatalog'], funcs=self.
funcsfuncs,
943 dataId=outputRefs.outputCatalog.dataId.full)
944 outputs = pipeBase.Struct(outputCatalog=result)
945 butlerQC.put(outputs, outputRefs)
949 if self.
funcsfuncs
is None:
950 raise ValueError(
"config.functorFile is None. "
951 "Must be a valid path to yaml in order to run as a CommandlineTask.")
952 df = self.
runrun(parq, funcs=self.
funcsfuncs, dataId=dataRef.dataId)
953 self.
writewrite(df, dataRef)
956 def run(self, parq, funcs=None, dataId=None, band=None):
957 """Do postprocessing calculations
959 Takes a `ParquetTable` object and dataId,
960 returns a dataframe
with results of postprocessing calculations.
965 ParquetTable
from which calculations are done.
966 funcs : `lsst.pipe.tasks.functors.Functors`
967 Functors to apply to the table
's columns
968 dataId : dict, optional
969 Used to add a `patchId` column to the output dataframe.
970 band : `str`, optional
971 Filter band that is being processed.
978 self.log.info("Transforming/standardizing the source table dataId: %s", dataId)
980 df = self.
transformtransform(band, parq, funcs, dataId).df
981 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
985 return self.
funcsfuncs
989 funcs = self.
funcsfuncs
990 analysis = PostprocessAnalysis(parq, funcs, filt=band)
994 analysis = self.
getAnalysisgetAnalysis(parq, funcs=funcs, band=band)
996 if dataId
is not None:
997 for key, value
in dataId.items():
1000 if self.config.primaryKey:
1001 if df.index.name != self.config.primaryKey
and self.config.primaryKey
in df:
1002 df.reset_index(inplace=
True, drop=
True)
1003 df.set_index(self.config.primaryKey, inplace=
True)
1005 return pipeBase.Struct(
1014 """No metadata to write.
1020 defaultTemplates={
"coaddName":
"deep"},
1021 dimensions=(
"tract",
"patch",
"skymap")):
1022 inputCatalog = connectionTypes.Input(
1023 doc=
"The vertical concatenation of the deepCoadd_{ref|meas|forced_src} catalogs, "
1024 "stored as a DataFrame with a multi-level column index per-patch.",
1025 dimensions=(
"tract",
"patch",
"skymap"),
1026 storageClass=
"DataFrame",
1027 name=
"{coaddName}Coadd_obj",
1030 outputCatalog = connectionTypes.Output(
1031 doc=
"Per-Patch Object Table of columns transformed from the deepCoadd_obj table per the standard "
1033 dimensions=(
"tract",
"patch",
"skymap"),
1034 storageClass=
"DataFrame",
1040 pipelineConnections=TransformObjectCatalogConnections):
1041 coaddName = pexConfig.Field(
1047 filterMap = pexConfig.DictField(
1051 doc=(
"Dictionary mapping full filter name to short one for column name munging."
1052 "These filters determine the output columns no matter what filters the "
1053 "input data actually contain."),
1054 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
1055 "Will be removed after v22.")
1057 outputBands = pexConfig.ListField(
1061 doc=(
"These bands and only these bands will appear in the output,"
1062 " NaN-filled if the input does not include them."
1063 " If None, then use all bands found in the input.")
1065 camelCase = pexConfig.Field(
1068 doc=(
"Write per-band columns names with camelCase, else underscore "
1069 "For example: gPsFlux instead of g_PsFlux.")
1071 multilevelOutput = pexConfig.Field(
1074 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
1075 "and name-munged (False).")
1077 goodFlags = pexConfig.ListField(
1080 doc=(
"List of 'good' flags that should be set False when populating empty tables. "
1081 "All other flags are considered to be 'bad' flags and will be set to True.")
1083 floatFillValue = pexConfig.Field(
1086 doc=
"Fill value for float fields when populating empty tables."
1088 integerFillValue = pexConfig.Field(
1091 doc=
"Fill value for integer fields when populating empty tables."
1094 def setDefaults(self):
1095 super().setDefaults()
1096 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Object.yaml')
1097 self.primaryKey =
'objectId'
1098 self.goodFlags = [
'calib_astrometry_used',
1099 'calib_photometry_reserved',
1100 'calib_photometry_used',
1101 'calib_psf_candidate',
1102 'calib_psf_reserved',
1107 """Produce a flattened Object Table to match the format specified in
1110 Do the same set of postprocessing calculations on all bands
1112 This is identical to `TransformCatalogBaseTask`,
except for that it does the
1113 specified functor calculations
for all filters present
in the
1114 input `deepCoadd_obj` table. Any specific `
"filt"` keywords specified
1115 by the YAML file will be superceded.
1117 _DefaultName = "transformObjectCatalog"
1118 ConfigClass = TransformObjectCatalogConfig
1121 inputDataset =
'deepCoadd_obj'
1122 outputDataset =
'objectTable'
1125 def _makeArgumentParser(cls):
1126 parser = ArgumentParser(name=cls._DefaultName)
1127 parser.add_id_argument(
"--id", cls.inputDataset,
1128 ContainerClass=CoaddDataIdContainer,
1129 help=
"data ID, e.g. --id tract=12345 patch=1,2")
1132 def run(self, parq, funcs=None, dataId=None, band=None):
1136 templateDf = pd.DataFrame()
1138 if isinstance(parq, DeferredDatasetHandle):
1139 columns = parq.get(component=
'columns')
1140 inputBands = columns.unique(level=1).values
1142 inputBands = parq.columnLevelNames[
'band']
1144 outputBands = self.config.outputBands
if self.config.outputBands
else inputBands
1147 for inputBand
in inputBands:
1148 if inputBand
not in outputBands:
1149 self.log.info(
"Ignoring %s band data in the input", inputBand)
1151 self.log.info(
"Transforming the catalog of band %s", inputBand)
1152 result = self.transform(inputBand, parq, funcs, dataId)
1153 dfDict[inputBand] = result.df
1154 analysisDict[inputBand] = result.analysis
1155 if templateDf.empty:
1156 templateDf = result.df
1159 for filt
in outputBands:
1160 if filt
not in dfDict:
1161 self.log.info(
"Adding empty columns for band %s", filt)
1162 dfTemp = templateDf.copy()
1163 for col
in dfTemp.columns:
1164 testValue = dfTemp[col].values[0]
1165 if isinstance(testValue, (np.bool_, pd.BooleanDtype)):
1167 if col
in self.config.goodFlags:
1171 elif isinstance(testValue, numbers.Integral):
1175 if isinstance(testValue, np.unsignedinteger):
1176 raise ValueError(
"Parquet tables may not have unsigned integer columns.")
1178 fillValue = self.config.integerFillValue
1180 fillValue = self.config.floatFillValue
1181 dfTemp[col].values[:] = fillValue
1182 dfDict[filt] = dfTemp
1185 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
1187 if not self.config.multilevelOutput:
1188 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
1189 if self.config.primaryKey
in noDupCols:
1190 noDupCols.remove(self.config.primaryKey)
1191 if dataId
is not None:
1192 noDupCols += list(dataId.keys())
1193 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase,
1194 inputBands=inputBands)
1196 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
1203 def makeDataRefList(self, namespace):
1204 """Make self.refList from self.idList
1206 Generate a list of data references given tract and/
or patch.
1207 This was adapted
from `TractQADataIdContainer`, which was
1208 `TractDataIdContainer` modifie to
not require
"filter".
1209 Only existing dataRefs are returned.
1211 def getPatchRefList(tract):
1212 return [namespace.butler.dataRef(datasetType=self.datasetType,
1213 tract=tract.getId(),
1214 patch=
"%d,%d" % patch.getIndex())
for patch
in tract]
1216 tractRefs = defaultdict(list)
1217 for dataId
in self.idList:
1218 skymap = self.getSkymap(namespace)
1220 if "tract" in dataId:
1221 tractId = dataId[
"tract"]
1222 if "patch" in dataId:
1223 tractRefs[tractId].append(namespace.butler.dataRef(datasetType=self.datasetType,
1225 patch=dataId[
'patch']))
1227 tractRefs[tractId] += getPatchRefList(skymap[tractId])
1229 tractRefs = dict((tract.getId(), tractRefs.get(tract.getId(), []) + getPatchRefList(tract))
1230 for tract
in skymap)
1232 for tractRefList
in tractRefs.values():
1233 existingRefs = [ref
for ref
in tractRefList
if ref.datasetExists()]
1234 outputRefList.append(existingRefs)
1236 self.refList = outputRefList
1239class ConsolidateObjectTableConnections(pipeBase.PipelineTaskConnections,
1240 dimensions=(
"tract",
"skymap")):
1241 inputCatalogs = connectionTypes.Input(
1242 doc=
"Per-Patch objectTables conforming to the standard data model.",
1244 storageClass=
"DataFrame",
1245 dimensions=(
"tract",
"patch",
"skymap"),
1248 outputCatalog = connectionTypes.Output(
1249 doc=
"Pre-tract horizontal concatenation of the input objectTables",
1250 name=
"objectTable_tract",
1251 storageClass=
"DataFrame",
1252 dimensions=(
"tract",
"skymap"),
1256class ConsolidateObjectTableConfig(pipeBase.PipelineTaskConfig,
1257 pipelineConnections=ConsolidateObjectTableConnections):
1258 coaddName = pexConfig.Field(
1265class ConsolidateObjectTableTask(CmdLineTask, pipeBase.PipelineTask):
1266 """Write patch-merged source tables to a tract-level parquet file
1268 Concatenates `objectTable` list into a per-visit `objectTable_tract`
1270 _DefaultName = "consolidateObjectTable"
1271 ConfigClass = ConsolidateObjectTableConfig
1273 inputDataset =
'objectTable'
1274 outputDataset =
'objectTable_tract'
1276 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1277 inputs = butlerQC.get(inputRefs)
1278 self.log.info(
"Concatenating %s per-patch Object Tables",
1279 len(inputs[
'inputCatalogs']))
1280 df = pd.concat(inputs[
'inputCatalogs'])
1281 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1284 def _makeArgumentParser(cls):
1285 parser = ArgumentParser(name=cls._DefaultName)
1287 parser.add_id_argument(
"--id", cls.inputDataset,
1288 help=
"data ID, e.g. --id tract=12345",
1289 ContainerClass=TractObjectDataIdContainer)
1292 def runDataRef(self, patchRefList):
1293 df = pd.concat([patchRef.get().toDataFrame()
for patchRef
in patchRefList])
1294 patchRefList[0].put(
ParquetTable(dataFrame=df), self.outputDataset)
1297 """No metadata to write.
1302class TransformSourceTableConnections(pipeBase.PipelineTaskConnections,
1303 defaultTemplates={
"catalogType":
""},
1304 dimensions=(
"instrument",
"visit",
"detector")):
1306 inputCatalog = connectionTypes.Input(
1307 doc=
"Wide input catalog of sources produced by WriteSourceTableTask",
1308 name=
"{catalogType}source",
1309 storageClass=
"DataFrame",
1310 dimensions=(
"instrument",
"visit",
"detector"),
1313 outputCatalog = connectionTypes.Output(
1314 doc=
"Narrower, per-detector Source Table transformed and converted per a "
1315 "specified set of functors",
1316 name=
"{catalogType}sourceTable",
1317 storageClass=
"DataFrame",
1318 dimensions=(
"instrument",
"visit",
"detector")
1323 pipelineConnections=TransformSourceTableConnections):
1325 def setDefaults(self):
1326 super().setDefaults()
1327 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'Source.yaml')
1328 self.primaryKey =
'sourceId'
1332 """Transform/standardize a source catalog
1334 _DefaultName = "transformSourceTable"
1335 ConfigClass = TransformSourceTableConfig
1337 inputDataset =
'source'
1338 outputDataset =
'sourceTable'
1341 def _makeArgumentParser(cls):
1342 parser = ArgumentParser(name=cls._DefaultName)
1343 parser.add_id_argument(
"--id", datasetType=cls.inputDataset,
1345 help=
"data ID, e.g. --id visit=12345 ccd=0")
1348 def runDataRef(self, dataRef):
1349 """Override to specify band label to run()."""
1350 parq = dataRef.get()
1351 funcs = self.getFunctors()
1352 band = dataRef.get(
"calexp_filterLabel", immediate=
True).bandLabel
1353 df = self.run(parq, funcs=funcs, dataId=dataRef.dataId, band=band)
1354 self.write(df, dataRef)
1358class ConsolidateVisitSummaryConnections(pipeBase.PipelineTaskConnections,
1359 dimensions=(
"instrument",
"visit",),
1360 defaultTemplates={
"calexpType":
""}):
1361 calexp = connectionTypes.Input(
1362 doc=
"Processed exposures used for metadata",
1363 name=
"{calexpType}calexp",
1364 storageClass=
"ExposureF",
1365 dimensions=(
"instrument",
"visit",
"detector"),
1369 visitSummary = connectionTypes.Output(
1370 doc=(
"Per-visit consolidated exposure metadata. These catalogs use "
1371 "detector id for the id and are sorted for fast lookups of a "
1373 name=
"{calexpType}visitSummary",
1374 storageClass=
"ExposureCatalog",
1375 dimensions=(
"instrument",
"visit"),
1379class ConsolidateVisitSummaryConfig(pipeBase.PipelineTaskConfig,
1380 pipelineConnections=ConsolidateVisitSummaryConnections):
1381 """Config for ConsolidateVisitSummaryTask"""
1385class ConsolidateVisitSummaryTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
1386 """Task to consolidate per-detector visit metadata.
1388 This task aggregates the following metadata from all the detectors
in a
1389 single visit into an exposure catalog:
1393 - The physical_filter
and band (
if available).
1394 - The psf size, shape,
and effective area at the center of the detector.
1395 - The corners of the bounding box
in right ascension/declination.
1397 Other quantities such
as Detector, Psf, ApCorrMap,
and TransmissionCurve
1398 are
not persisted here because of storage concerns,
and because of their
1399 limited utility
as summary statistics.
1401 Tests
for this task are performed
in ci_hsc_gen3.
1403 _DefaultName = "consolidateVisitSummary"
1404 ConfigClass = ConsolidateVisitSummaryConfig
1407 def _makeArgumentParser(cls):
1408 parser = ArgumentParser(name=cls._DefaultName)
1410 parser.add_id_argument(
"--id",
"calexp",
1411 help=
"data ID, e.g. --id visit=12345",
1412 ContainerClass=VisitDataIdContainer)
1416 """No metadata to persist, so override to remove metadata persistance.
1420 def writeConfig(self, butler, clobber=False, doBackup=True):
1421 """No config to persist, so override to remove config persistance.
1425 def runDataRef(self, dataRefList):
1426 visit = dataRefList[0].dataId[
'visit']
1428 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)",
1429 len(dataRefList), visit)
1431 expCatalog = self._combineExposureMetadata(visit, dataRefList, isGen3=
False)
1433 dataRefList[0].put(expCatalog,
'visitSummary', visit=visit)
1435 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1436 dataRefs = butlerQC.get(inputRefs.calexp)
1437 visit = dataRefs[0].dataId.byName()[
'visit']
1439 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)",
1440 len(dataRefs), visit)
1442 expCatalog = self._combineExposureMetadata(visit, dataRefs)
1444 butlerQC.put(expCatalog, outputRefs.visitSummary)
1446 def _combineExposureMetadata(self, visit, dataRefs, isGen3=True):
1447 """Make a combined exposure catalog from a list of dataRefs.
1448 These dataRefs must point to exposures with wcs, summaryStats,
1449 and other visit metadata.
1454 Visit identification number.
1456 List of dataRefs
in visit. May be list of
1457 `lsst.daf.persistence.ButlerDataRef` (Gen2)
or
1458 `lsst.daf.butler.DeferredDatasetHandle` (Gen3).
1459 isGen3 : `bool`, optional
1460 Specifies
if this
is a Gen3 list of datarefs.
1465 Exposure catalog
with per-detector summary information.
1467 schema = self._makeVisitSummarySchema()
1468 cat = afwTable.ExposureCatalog(schema)
1469 cat.resize(len(dataRefs))
1471 cat['visit'] = visit
1473 for i, dataRef
in enumerate(dataRefs):
1475 visitInfo = dataRef.get(component=
'visitInfo')
1476 filterLabel = dataRef.get(component=
'filterLabel')
1477 summaryStats = dataRef.get(component=
'summaryStats')
1478 detector = dataRef.get(component=
'detector')
1479 wcs = dataRef.get(component=
'wcs')
1480 photoCalib = dataRef.get(component=
'photoCalib')
1481 detector = dataRef.get(component=
'detector')
1482 bbox = dataRef.get(component=
'bbox')
1483 validPolygon = dataRef.get(component=
'validPolygon')
1488 exp = dataRef.get(datasetType=
'calexp_sub', bbox=gen2_read_bbox)
1489 visitInfo = exp.getInfo().getVisitInfo()
1490 filterLabel = dataRef.get(
"calexp_filterLabel")
1491 summaryStats = exp.getInfo().getSummaryStats()
1493 photoCalib = exp.getPhotoCalib()
1494 detector = exp.getDetector()
1495 bbox = dataRef.get(datasetType=
'calexp_bbox')
1496 validPolygon = exp.getInfo().getValidPolygon()
1500 rec.setVisitInfo(visitInfo)
1502 rec.setPhotoCalib(photoCalib)
1503 rec.setValidPolygon(validPolygon)
1505 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
1506 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
1507 rec.setId(detector.getId())
1508 rec[
'psfSigma'] = summaryStats.psfSigma
1509 rec[
'psfIxx'] = summaryStats.psfIxx
1510 rec[
'psfIyy'] = summaryStats.psfIyy
1511 rec[
'psfIxy'] = summaryStats.psfIxy
1512 rec[
'psfArea'] = summaryStats.psfArea
1513 rec[
'raCorners'][:] = summaryStats.raCorners
1514 rec[
'decCorners'][:] = summaryStats.decCorners
1515 rec[
'ra'] = summaryStats.ra
1516 rec[
'decl'] = summaryStats.decl
1517 rec[
'zenithDistance'] = summaryStats.zenithDistance
1518 rec[
'zeroPoint'] = summaryStats.zeroPoint
1519 rec[
'skyBg'] = summaryStats.skyBg
1520 rec[
'skyNoise'] = summaryStats.skyNoise
1521 rec[
'meanVar'] = summaryStats.meanVar
1522 rec[
'astromOffsetMean'] = summaryStats.astromOffsetMean
1523 rec[
'astromOffsetStd'] = summaryStats.astromOffsetStd
1524 rec[
'nPsfStar'] = summaryStats.nPsfStar
1525 rec[
'psfStarDeltaE1Median'] = summaryStats.psfStarDeltaE1Median
1526 rec[
'psfStarDeltaE2Median'] = summaryStats.psfStarDeltaE2Median
1527 rec[
'psfStarDeltaE1Scatter'] = summaryStats.psfStarDeltaE1Scatter
1528 rec[
'psfStarDeltaE2Scatter'] = summaryStats.psfStarDeltaE2Scatter
1529 rec[
'psfStarDeltaSizeMedian'] = summaryStats.psfStarDeltaSizeMedian
1530 rec[
'psfStarDeltaSizeScatter'] = summaryStats.psfStarDeltaSizeScatter
1531 rec[
'psfStarScaledDeltaSizeScatter'] = summaryStats.psfStarScaledDeltaSizeScatter
1533 metadata = dafBase.PropertyList()
1534 metadata.add(
"COMMENT",
"Catalog id is detector id, sorted.")
1536 metadata.add(
"COMMENT",
"Only detectors with data have entries.")
1537 cat.setMetadata(metadata)
1542 def _makeVisitSummarySchema(self):
1543 """Make the schema for the visitSummary catalog."""
1544 schema = afwTable.ExposureTable.makeMinimalSchema()
1545 schema.addField(
'visit', type=
'L', doc=
'Visit number')
1546 schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
1547 schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
1548 schema.addField(
'psfSigma', type=
'F',
1549 doc=
'PSF model second-moments determinant radius (center of chip) (pixel)')
1550 schema.addField(
'psfArea', type=
'F',
1551 doc=
'PSF model effective area (center of chip) (pixel**2)')
1552 schema.addField(
'psfIxx', type=
'F',
1553 doc=
'PSF model Ixx (center of chip) (pixel**2)')
1554 schema.addField(
'psfIyy', type=
'F',
1555 doc=
'PSF model Iyy (center of chip) (pixel**2)')
1556 schema.addField(
'psfIxy', type=
'F',
1557 doc=
'PSF model Ixy (center of chip) (pixel**2)')
1558 schema.addField(
'raCorners', type=
'ArrayD', size=4,
1559 doc=
'Right Ascension of bounding box corners (degrees)')
1560 schema.addField(
'decCorners', type=
'ArrayD', size=4,
1561 doc=
'Declination of bounding box corners (degrees)')
1562 schema.addField(
'ra', type=
'D',
1563 doc=
'Right Ascension of bounding box center (degrees)')
1564 schema.addField(
'decl', type=
'D',
1565 doc=
'Declination of bounding box center (degrees)')
1566 schema.addField(
'zenithDistance', type=
'F',
1567 doc=
'Zenith distance of bounding box center (degrees)')
1568 schema.addField(
'zeroPoint', type=
'F',
1569 doc=
'Mean zeropoint in detector (mag)')
1570 schema.addField(
'skyBg', type=
'F',
1571 doc=
'Average sky background (ADU)')
1572 schema.addField(
'skyNoise', type=
'F',
1573 doc=
'Average sky noise (ADU)')
1574 schema.addField(
'meanVar', type=
'F',
1575 doc=
'Mean variance of the weight plane (ADU**2)')
1576 schema.addField(
'astromOffsetMean', type=
'F',
1577 doc=
'Mean offset of astrometric calibration matches (arcsec)')
1578 schema.addField(
'astromOffsetStd', type=
'F',
1579 doc=
'Standard deviation of offsets of astrometric calibration matches (arcsec)')
1580 schema.addField(
'nPsfStar', type=
'I', doc=
'Number of stars used for PSF model')
1581 schema.addField(
'psfStarDeltaE1Median', type=
'F',
1582 doc=
'Median E1 residual (starE1 - psfE1) for psf stars')
1583 schema.addField(
'psfStarDeltaE2Median', type=
'F',
1584 doc=
'Median E2 residual (starE2 - psfE2) for psf stars')
1585 schema.addField(
'psfStarDeltaE1Scatter', type=
'F',
1586 doc=
'Scatter (via MAD) of E1 residual (starE1 - psfE1) for psf stars')
1587 schema.addField(
'psfStarDeltaE2Scatter', type=
'F',
1588 doc=
'Scatter (via MAD) of E2 residual (starE2 - psfE2) for psf stars')
1589 schema.addField(
'psfStarDeltaSizeMedian', type=
'F',
1590 doc=
'Median size residual (starSize - psfSize) for psf stars (pixel)')
1591 schema.addField(
'psfStarDeltaSizeScatter', type=
'F',
1592 doc=
'Scatter (via MAD) of size residual (starSize - psfSize) for psf stars (pixel)')
1593 schema.addField(
'psfStarScaledDeltaSizeScatter', type=
'F',
1594 doc=
'Scatter (via MAD) of size residual scaled by median size squared')
1599class VisitDataIdContainer(DataIdContainer):
1600 """DataIdContainer that groups sensor-level id's by visit
1603 def makeDataRefList(self, namespace):
1604 """Make self.refList from self.idList
1606 Generate a list of data references grouped by visit.
1610 namespace : `argparse.Namespace`
1611 Namespace used by `lsst.pipe.base.CmdLineTask` to parse command line arguments
1614 visitRefs = defaultdict(list)
1615 for dataId
in self.idList:
1616 if "visit" in dataId:
1617 visitId = dataId[
"visit"]
1619 subset = namespace.butler.subset(self.datasetType, dataId=dataId)
1620 visitRefs[visitId].extend([dataRef
for dataRef
in subset])
1623 for refList
in visitRefs.values():
1624 existingRefs = [ref
for ref
in refList
if ref.datasetExists()]
1626 outputRefList.append(existingRefs)
1628 self.refList = outputRefList
1631class ConsolidateSourceTableConnections(pipeBase.PipelineTaskConnections,
1632 defaultTemplates={
"catalogType":
""},
1633 dimensions=(
"instrument",
"visit")):
1634 inputCatalogs = connectionTypes.Input(
1635 doc=
"Input per-detector Source Tables",
1636 name=
"{catalogType}sourceTable",
1637 storageClass=
"DataFrame",
1638 dimensions=(
"instrument",
"visit",
"detector"),
1641 outputCatalog = connectionTypes.Output(
1642 doc=
"Per-visit concatenation of Source Table",
1643 name=
"{catalogType}sourceTable_visit",
1644 storageClass=
"DataFrame",
1645 dimensions=(
"instrument",
"visit")
1649class ConsolidateSourceTableConfig(pipeBase.PipelineTaskConfig,
1650 pipelineConnections=ConsolidateSourceTableConnections):
1654class ConsolidateSourceTableTask(CmdLineTask, pipeBase.PipelineTask):
1655 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1657 _DefaultName = 'consolidateSourceTable'
1658 ConfigClass = ConsolidateSourceTableConfig
1660 inputDataset =
'sourceTable'
1661 outputDataset =
'sourceTable_visit'
1663 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1664 from .makeCoaddTempExp
import reorderRefs
1666 detectorOrder = [ref.dataId[
'detector']
for ref
in inputRefs.inputCatalogs]
1667 detectorOrder.sort()
1668 inputRefs = reorderRefs(inputRefs, detectorOrder, dataIdKey=
'detector')
1669 inputs = butlerQC.get(inputRefs)
1670 self.log.info(
"Concatenating %s per-detector Source Tables",
1671 len(inputs[
'inputCatalogs']))
1672 df = pd.concat(inputs[
'inputCatalogs'])
1673 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
1675 def runDataRef(self, dataRefList):
1676 self.log.info(
"Concatenating %s per-detector Source Tables", len(dataRefList))
1677 df = pd.concat([dataRef.get().toDataFrame()
for dataRef
in dataRefList])
1678 dataRefList[0].put(
ParquetTable(dataFrame=df), self.outputDataset)
1681 def _makeArgumentParser(cls):
1682 parser = ArgumentParser(name=cls._DefaultName)
1684 parser.add_id_argument(
"--id", cls.inputDataset,
1685 help=
"data ID, e.g. --id visit=12345",
1686 ContainerClass=VisitDataIdContainer)
1690 """No metadata to write.
1694 def writeConfig(self, butler, clobber=False, doBackup=True):
1695 """No config to write.
1700class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
1701 dimensions=(
"instrument",),
1702 defaultTemplates={
"calexpType":
""}):
1703 visitSummaryRefs = connectionTypes.Input(
1704 doc=
"Data references for per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
1705 name=
"{calexpType}visitSummary",
1706 storageClass=
"ExposureCatalog",
1707 dimensions=(
"instrument",
"visit"),
1711 outputCatalog = connectionTypes.Output(
1712 doc=
"CCD and Visit metadata table",
1713 name=
"ccdVisitTable",
1714 storageClass=
"DataFrame",
1715 dimensions=(
"instrument",)
1719class MakeCcdVisitTableConfig(pipeBase.PipelineTaskConfig,
1720 pipelineConnections=MakeCcdVisitTableConnections):
1724class MakeCcdVisitTableTask(CmdLineTask, pipeBase.PipelineTask):
1725 """Produce a `ccdVisitTable` from the `visitSummary` exposure catalogs.
1727 _DefaultName = 'makeCcdVisitTable'
1728 ConfigClass = MakeCcdVisitTableConfig
1730 def run(self, visitSummaryRefs):
1731 """ Make a table of ccd information from the `visitSummary` catalogs.
1734 visitSummaryRefs : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1735 List of DeferredDatasetHandles pointing to exposure catalogs with
1736 per-detector summary information.
1739 result : `lsst.pipe.Base.Struct`
1740 Results struct
with attribute:
1742 Catalog of ccd
and visit information.
1745 for visitSummaryRef
in visitSummaryRefs:
1746 visitSummary = visitSummaryRef.get()
1747 visitInfo = visitSummary[0].getVisitInfo()
1750 summaryTable = visitSummary.asAstropy()
1751 selectColumns = [
'id',
'visit',
'physical_filter',
'band',
'ra',
'decl',
'zenithDistance',
1752 'zeroPoint',
'psfSigma',
'skyBg',
'skyNoise']
1753 ccdEntry = summaryTable[selectColumns].to_pandas().set_index(
'id')
1757 ccdEntry = ccdEntry.rename(columns={
"visit":
"visitId"})
1758 dataIds = [DataCoordinate.standardize(visitSummaryRef.dataId, detector=id)
for id
in
1760 packer = visitSummaryRef.dataId.universe.makePacker(
'visit_detector', visitSummaryRef.dataId)
1761 ccdVisitIds = [packer.pack(dataId)
for dataId
in dataIds]
1762 ccdEntry[
'ccdVisitId'] = ccdVisitIds
1763 ccdEntry[
'detector'] = summaryTable[
'id']
1764 pixToArcseconds = np.array([vR.getWcs().getPixelScale().asArcseconds()
for vR
in visitSummary])
1765 ccdEntry[
"seeing"] = visitSummary[
'psfSigma'] * np.sqrt(8 * np.log(2)) * pixToArcseconds
1767 ccdEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1768 ccdEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1769 ccdEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1770 expTime = visitInfo.getExposureTime()
1771 ccdEntry[
'expTime'] = expTime
1772 ccdEntry[
"obsStart"] = ccdEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1773 expTime_days = expTime / (60*60*24)
1774 ccdEntry[
"obsStartMJD"] = ccdEntry[
"expMidptMJD"] - 0.5 * expTime_days
1775 ccdEntry[
'darkTime'] = visitInfo.getDarkTime()
1776 ccdEntry[
'xSize'] = summaryTable[
'bbox_max_x'] - summaryTable[
'bbox_min_x']
1777 ccdEntry[
'ySize'] = summaryTable[
'bbox_max_y'] - summaryTable[
'bbox_min_y']
1778 ccdEntry[
'llcra'] = summaryTable[
'raCorners'][:, 0]
1779 ccdEntry[
'llcdec'] = summaryTable[
'decCorners'][:, 0]
1780 ccdEntry[
'ulcra'] = summaryTable[
'raCorners'][:, 1]
1781 ccdEntry[
'ulcdec'] = summaryTable[
'decCorners'][:, 1]
1782 ccdEntry[
'urcra'] = summaryTable[
'raCorners'][:, 2]
1783 ccdEntry[
'urcdec'] = summaryTable[
'decCorners'][:, 2]
1784 ccdEntry[
'lrcra'] = summaryTable[
'raCorners'][:, 3]
1785 ccdEntry[
'lrcdec'] = summaryTable[
'decCorners'][:, 3]
1788 ccdEntries.append(ccdEntry)
1790 outputCatalog = pd.concat(ccdEntries)
1791 outputCatalog.set_index(
'ccdVisitId', inplace=
True, verify_integrity=
True)
1792 return pipeBase.Struct(outputCatalog=outputCatalog)
1795class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
1796 dimensions=(
"instrument",),
1797 defaultTemplates={
"calexpType":
""}):
1798 visitSummaries = connectionTypes.Input(
1799 doc=
"Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
1800 name=
"{calexpType}visitSummary",
1801 storageClass=
"ExposureCatalog",
1802 dimensions=(
"instrument",
"visit",),
1806 outputCatalog = connectionTypes.Output(
1807 doc=
"Visit metadata table",
1809 storageClass=
"DataFrame",
1810 dimensions=(
"instrument",)
1814class MakeVisitTableConfig(pipeBase.PipelineTaskConfig,
1815 pipelineConnections=MakeVisitTableConnections):
1819class MakeVisitTableTask(CmdLineTask, pipeBase.PipelineTask):
1820 """Produce a `visitTable` from the `visitSummary` exposure catalogs.
1822 _DefaultName = 'makeVisitTable'
1823 ConfigClass = MakeVisitTableConfig
1825 def run(self, visitSummaries):
1826 """ Make a table of visit information from the `visitSummary` catalogs
1831 List of exposure catalogs with per-detector summary information.
1834 result : `lsst.pipe.Base.Struct`
1835 Results struct
with attribute:
1837 Catalog of visit information.
1840 for visitSummary
in visitSummaries:
1841 visitSummary = visitSummary.get()
1842 visitRow = visitSummary[0]
1843 visitInfo = visitRow.getVisitInfo()
1846 visitEntry[
"visitId"] = visitRow[
'visit']
1847 visitEntry[
"visit"] = visitRow[
'visit']
1848 visitEntry[
"physical_filter"] = visitRow[
'physical_filter']
1849 visitEntry[
"band"] = visitRow[
'band']
1850 raDec = visitInfo.getBoresightRaDec()
1851 visitEntry[
"ra"] = raDec.getRa().asDegrees()
1852 visitEntry[
"decl"] = raDec.getDec().asDegrees()
1853 visitEntry[
"skyRotation"] = visitInfo.getBoresightRotAngle().asDegrees()
1854 azAlt = visitInfo.getBoresightAzAlt()
1855 visitEntry[
"azimuth"] = azAlt.getLongitude().asDegrees()
1856 visitEntry[
"altitude"] = azAlt.getLatitude().asDegrees()
1857 visitEntry[
"zenithDistance"] = 90 - azAlt.getLatitude().asDegrees()
1858 visitEntry[
"airmass"] = visitInfo.getBoresightAirmass()
1859 expTime = visitInfo.getExposureTime()
1860 visitEntry[
"expTime"] = expTime
1861 visitEntry[
"expMidpt"] = visitInfo.getDate().toPython()
1862 visitEntry[
"expMidptMJD"] = visitInfo.getDate().get(dafBase.DateTime.MJD)
1863 visitEntry[
"obsStart"] = visitEntry[
"expMidpt"] - 0.5 * pd.Timedelta(seconds=expTime)
1864 expTime_days = expTime / (60*60*24)
1865 visitEntry[
"obsStartMJD"] = visitEntry[
"expMidptMJD"] - 0.5 * expTime_days
1866 visitEntries.append(visitEntry)
1871 outputCatalog = pd.DataFrame(data=visitEntries)
1872 outputCatalog.set_index(
'visitId', inplace=
True, verify_integrity=
True)
1873 return pipeBase.Struct(outputCatalog=outputCatalog)
1876class WriteForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1877 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")):
1879 inputCatalog = connectionTypes.Input(
1880 doc=
"Primary per-detector, single-epoch forced-photometry catalog. "
1881 "By default, it is the output of ForcedPhotCcdTask on calexps",
1883 storageClass=
"SourceCatalog",
1884 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1886 inputCatalogDiff = connectionTypes.Input(
1887 doc=
"Secondary multi-epoch, per-detector, forced photometry catalog. "
1888 "By default, it is the output of ForcedPhotCcdTask run on image differences.",
1890 storageClass=
"SourceCatalog",
1891 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1893 outputCatalog = connectionTypes.Output(
1894 doc=
"InputCatalogs horizonatally joined on `objectId` in Parquet format",
1895 name=
"mergedForcedSource",
1896 storageClass=
"DataFrame",
1897 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract")
1901class WriteForcedSourceTableConfig(pipeBase.PipelineTaskConfig,
1902 pipelineConnections=WriteForcedSourceTableConnections):
1903 key = lsst.pex.config.Field(
1904 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1910class WriteForcedSourceTableTask(pipeBase.PipelineTask):
1911 """Merge and convert per-detector forced source catalogs to parquet
1913 Because the predecessor ForcedPhotCcdTask operates per-detector,
1914 per-tract, (i.e., it has tract in its dimensions), detectors
1915 on the tract boundary may have multiple forced source catalogs.
1917 The successor task TransformForcedSourceTable runs per-patch
1918 and temporally-aggregates overlapping mergedForcedSource catalogs
from all
1919 available multiple epochs.
1921 _DefaultName = "writeForcedSourceTable"
1922 ConfigClass = WriteForcedSourceTableConfig
1924 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1925 inputs = butlerQC.get(inputRefs)
1927 inputs[
'ccdVisitId'] = butlerQC.quantum.dataId.pack(
"visit_detector")
1928 inputs[
'band'] = butlerQC.quantum.dataId.full[
'band']
1929 outputs = self.run(**inputs)
1930 butlerQC.put(outputs, outputRefs)
1932 def run(self, inputCatalog, inputCatalogDiff, ccdVisitId=None, band=None):
1934 for table, dataset,
in zip((inputCatalog, inputCatalogDiff), (
'calexp',
'diff')):
1935 df = table.asAstropy().to_pandas().set_index(self.config.key, drop=
False)
1936 df = df.reindex(sorted(df.columns), axis=1)
1937 df[
'ccdVisitId'] = ccdVisitId
if ccdVisitId
else pd.NA
1938 df[
'band'] = band
if band
else pd.NA
1939 df.columns = pd.MultiIndex.from_tuples([(dataset, c)
for c
in df.columns],
1940 names=(
'dataset',
'column'))
1944 outputCatalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
1945 return pipeBase.Struct(outputCatalog=outputCatalog)
1948class TransformForcedSourceTableConnections(pipeBase.PipelineTaskConnections,
1949 dimensions=(
"instrument",
"skymap",
"patch",
"tract")):
1951 inputCatalogs = connectionTypes.Input(
1952 doc=
"Parquet table of merged ForcedSources produced by WriteForcedSourceTableTask",
1953 name=
"mergedForcedSource",
1954 storageClass=
"DataFrame",
1955 dimensions=(
"instrument",
"visit",
"detector",
"skymap",
"tract"),
1959 referenceCatalog = connectionTypes.Input(
1960 doc=
"Reference catalog which was used to seed the forcedPhot. Columns "
1961 "objectId, detect_isPrimary, detect_isTractInner, detect_isPatchInner "
1964 storageClass=
"DataFrame",
1965 dimensions=(
"tract",
"patch",
"skymap"),
1968 outputCatalog = connectionTypes.Output(
1969 doc=
"Narrower, temporally-aggregated, per-patch ForcedSource Table transformed and converted per a "
1970 "specified set of functors",
1971 name=
"forcedSourceTable",
1972 storageClass=
"DataFrame",
1973 dimensions=(
"tract",
"patch",
"skymap")
1978 pipelineConnections=TransformForcedSourceTableConnections):
1979 referenceColumns = pexConfig.ListField(
1981 default=[
"detect_isPrimary",
"detect_isTractInner",
"detect_isPatchInner"],
1983 doc=
"Columns to pull from reference catalog",
1985 keyRef = lsst.pex.config.Field(
1986 doc=
"Column on which to join the two input tables on and make the primary key of the output",
1990 key = lsst.pex.config.Field(
1991 doc=
"Rename the output DataFrame index to this name",
1993 default=
"forcedSourceId",
1996 def setDefaults(self):
1997 super().setDefaults()
1998 self.functorFile = os.path.join(
'$PIPE_TASKS_DIR',
'schemas',
'ForcedSource.yaml')
2002 """Transform/standardize a ForcedSource catalog
2004 Transforms each wide, per-detector forcedSource parquet table per the
2005 specification file (per-camera defaults found in ForcedSource.yaml).
2006 All epochs that overlap the patch are aggregated into one per-patch
2007 narrow-parquet file.
2009 No de-duplication of rows
is performed. Duplicate resolutions flags are
2010 pulled
in from the referenceCatalog: `detect_isPrimary`,
2011 `detect_isTractInner`,`detect_isPatchInner`, so that user may de-duplicate
2012 for analysis
or compare duplicates
for QA.
2014 The resulting table includes multiple bands. Epochs (MJDs)
and other useful
2015 per-visit rows can be retreived by joining
with the CcdVisitTable on
2018 _DefaultName = "transformForcedSourceTable"
2019 ConfigClass = TransformForcedSourceTableConfig
2021 def runQuantum(self, butlerQC, inputRefs, outputRefs):
2022 inputs = butlerQC.get(inputRefs)
2023 if self.funcs
is None:
2024 raise ValueError(
"config.functorFile is None. "
2025 "Must be a valid path to yaml in order to run Task as a PipelineTask.")
2026 outputs = self.run(inputs[
'inputCatalogs'], inputs[
'referenceCatalog'], funcs=self.funcs,
2027 dataId=outputRefs.outputCatalog.dataId.full)
2029 butlerQC.put(outputs, outputRefs)
2031 def run(self, inputCatalogs, referenceCatalog, funcs=None, dataId=None, band=None):
2033 ref = referenceCatalog.get(parameters={
"columns": self.config.referenceColumns})
2034 self.log.info(
"Aggregating %s input catalogs" % (len(inputCatalogs)))
2035 for handle
in inputCatalogs:
2036 result = self.transform(
None, handle, funcs, dataId)
2038 dfs.append(result.df.join(ref, how=
'inner'))
2040 outputCatalog = pd.concat(dfs)
2044 outputCatalog.index.rename(self.config.keyRef, inplace=
True)
2046 outputCatalog.reset_index(inplace=
True)
2048 outputCatalog.set_index(
"forcedSourceId", inplace=
True, verify_integrity=
True)
2050 outputCatalog.index.rename(self.config.key, inplace=
True)
2052 self.log.info(
"Made a table of %d columns and %d rows",
2053 len(outputCatalog.columns), len(outputCatalog))
2054 return pipeBase.Struct(outputCatalog=outputCatalog)
2057class ConsolidateTractConnections(pipeBase.PipelineTaskConnections,
2058 defaultTemplates={
"catalogType":
""},
2059 dimensions=(
"instrument",
"tract")):
2060 inputCatalogs = connectionTypes.Input(
2061 doc=
"Input per-patch DataFrame Tables to be concatenated",
2062 name=
"{catalogType}ForcedSourceTable",
2063 storageClass=
"DataFrame",
2064 dimensions=(
"tract",
"patch",
"skymap"),
2068 outputCatalog = connectionTypes.Output(
2069 doc=
"Output per-tract concatenation of DataFrame Tables",
2070 name=
"{catalogType}ForcedSourceTable_tract",
2071 storageClass=
"DataFrame",
2072 dimensions=(
"tract",
"skymap"),
2076class ConsolidateTractConfig(pipeBase.PipelineTaskConfig,
2077 pipelineConnections=ConsolidateTractConnections):
2081class ConsolidateTractTask(CmdLineTask, pipeBase.PipelineTask):
2082 """Concatenate any per-patch, dataframe list into a single
2085 _DefaultName = 'ConsolidateTract'
2086 ConfigClass = ConsolidateTractConfig
2088 def runQuantum(self, butlerQC, inputRefs, outputRefs):
2089 inputs = butlerQC.get(inputRefs)
2091 self.log.info(
"Concatenating %s per-patch %s Tables",
2092 len(inputs[
'inputCatalogs']),
2093 inputRefs.inputCatalogs[0].datasetType.name)
2094 df = pd.concat(inputs[
'inputCatalogs'])
2095 butlerQC.put(pipeBase.Struct(outputCatalog=df), outputRefs)
def runDataRef(self, dataRef)
def getAnalysis(self, parq, funcs=None, band=None)
def write(self, df, parqRef)
def __init__(self, *args, **kwargs)
def transform(self, band, parq, funcs, dataId)
def run(self, parq, funcs=None, dataId=None, band=None)
def writeMetadata(self, dataRef)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
def makeMergeArgumentParser(name, dataset)
Create a suitable ArgumentParser.
def readCatalog(task, patchRef)
Read input catalog.
def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False, inputBands=None)