25 from collections
import defaultdict
33 from lsst.pipe.base import CmdLineTask, ArgumentParser, DataIdContainer
36 from .parquetTable
import ParquetTable
37 from .multiBandUtils
import makeMergeArgumentParser, MergeSourcesRunner
38 from .functors
import CompositeFunctor, RAColumn, DecColumn, Column
41 def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False):
42 """Flattens a dataframe with multilevel column index
44 newDf = pd.DataFrame()
45 for band
in set(df.columns.to_frame()[
'band']):
47 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
48 newColumns = {c: columnFormat.format(band, c)
49 for c
in subdf.columns
if c
not in noDupCols}
50 cols = list(newColumns.keys())
51 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
53 newDf = pd.concat([subdf[noDupCols], newDf], axis=1)
58 engine = pexConfig.Field(
61 doc=
"Parquet engine for writing (pyarrow or fastparquet)"
63 coaddName = pexConfig.Field(
71 """Write filter-merged source tables to parquet
73 _DefaultName =
"writeObjectTable"
74 ConfigClass = WriteObjectTableConfig
75 RunnerClass = MergeSourcesRunner
78 inputDatasets = (
'forced_src',
'meas',
'ref')
83 def __init__(self, butler=None, schema=None, **kwargs):
87 CmdLineTask.__init__(self, **kwargs)
91 @brief Merge coadd sources from multiple bands. Calls @ref `run` which must be defined in
92 subclasses that inherit from MergeSourcesTask.
93 @param[in] patchRefList list of data references for each filter
95 catalogs = dict(self.
readCatalogreadCatalog(patchRef)
for patchRef
in patchRefList)
96 dataId = patchRefList[0].dataId
97 mergedCatalog = self.
runrun(catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
98 self.
writewrite(patchRefList[0], mergedCatalog)
101 def _makeArgumentParser(cls):
102 """Create a suitable ArgumentParser.
104 We will use the ArgumentParser to get a list of data
105 references for patches; the RunnerClass will sort them into lists
106 of data references for the same patch.
108 References first of self.inputDatasets, rather than
114 """Read input catalogs
116 Read all the input datasets given by the 'inputDatasets'
121 patchRef : `lsst.daf.persistence.ButlerDataRef`
122 Data reference for patch
126 Tuple consisting of band name and a dict of catalogs, keyed by
129 band = patchRef.get(self.config.coaddName +
"Coadd_filterLabel", immediate=
True).bandLabel
132 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + dataset, immediate=
True)
133 self.log.info(
"Read %d sources from %s for band %s: %s" %
134 (len(catalog), dataset, band, patchRef.dataId))
135 catalogDict[dataset] = catalog
136 return band, catalogDict
138 def run(self, catalogs, tract, patch):
139 """Merge multiple catalogs.
144 Mapping from filter names to dict of catalogs.
146 tractId to use for the tractId column
148 patchId to use for the patchId column
152 catalog : `lsst.pipe.tasks.parquetTable.ParquetTable`
153 Merged dataframe, with each column prefixed by
154 `filter_tag(filt)`, wrapped in the parquet writer shim class.
158 for filt, tableDict
in catalogs.items():
159 for dataset, table
in tableDict.items():
161 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
164 df = df.reindex(sorted(df.columns), axis=1)
165 df[
'tractId'] = tract
166 df[
'patchId'] = patch
169 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
170 names=(
'dataset',
'band',
'column'))
173 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
181 catalog : `ParquetTable`
183 patchRef : `lsst.daf.persistence.ButlerDataRef`
184 Data reference for patch
186 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDatasetoutputDataset)
189 mergeDataId = patchRef.dataId.copy()
190 del mergeDataId[
"filter"]
191 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
194 """No metadata to write, and not sure how to write it for a list of dataRefs.
199 class WriteSourceTableConfig(pexConfig.Config):
200 doApplyExternalPhotoCalib = pexConfig.Field(
203 doc=(
"Add local photoCalib columns from the calexp.photoCalib? Should only set True if "
204 "generating Source Tables from older src tables which do not already have local calib columns")
206 doApplyExternalSkyWcs = pexConfig.Field(
209 doc=(
"Add local WCS columns from the calexp.wcs? Should only set True if "
210 "generating Source Tables from older src tables which do not already have local calib columns")
215 """Write source table to parquet
217 _DefaultName =
"writeSourceTable"
218 ConfigClass = WriteSourceTableConfig
221 src = dataRef.get(
'src')
222 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
225 ccdVisitId = dataRef.get(
'ccdExposureId')
226 result = self.
runrun(src, ccdVisitId=ccdVisitId)
227 dataRef.put(result.table,
'source')
229 def run(self, catalog, ccdVisitId=None):
230 """Convert `src` catalog to parquet
234 catalog: `afwTable.SourceCatalog`
235 catalog to be converted
237 ccdVisitId to be added as a column
241 result : `lsst.pipe.base.Struct`
243 `ParquetTable` version of the input catalog
245 self.log.info(
"Generating parquet table from src catalog")
246 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
247 df[
'ccdVisitId'] = ccdVisitId
248 return pipeBase.Struct(table=
ParquetTable(dataFrame=df))
251 """Add columns with local calibration evaluated at each centroid
253 for backwards compatibility with old repos.
254 This exists for the purpose of converting old src catalogs
255 (which don't have the expected local calib columns) to Source Tables.
259 catalog: `afwTable.SourceCatalog`
260 catalog to which calib columns will be added
261 dataRef: `lsst.daf.persistence.ButlerDataRef
262 for fetching the calibs from disk.
266 newCat: `afwTable.SourceCatalog`
267 Source Catalog with requested local calib columns
269 mapper = afwTable.SchemaMapper(catalog.schema)
270 measureConfig = SingleFrameMeasurementTask.ConfigClass()
271 measureConfig.doReplaceWithNoise =
False
274 exposure = dataRef.get(
'calexp_sub',
277 mapper = afwTable.SchemaMapper(catalog.schema)
278 mapper.addMinimalSchema(catalog.schema,
True)
279 schema = mapper.getOutputSchema()
281 exposureIdInfo = dataRef.get(
"expIdInfo")
282 measureConfig.plugins.names = []
283 if self.config.doApplyExternalSkyWcs:
284 plugin =
'base_LocalWcs'
286 raise RuntimeError(f
"{plugin} already in src catalog. Set doApplyExternalSkyWcs=False")
288 measureConfig.plugins.names.add(plugin)
290 if self.config.doApplyExternalPhotoCalib:
291 plugin =
'base_LocalPhotoCalib'
293 raise RuntimeError(f
"{plugin} already in src catalog. Set doApplyExternalPhotoCalib=False")
295 measureConfig.plugins.names.add(plugin)
297 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
298 newCat = afwTable.SourceCatalog(schema)
299 newCat.extend(catalog, mapper=mapper)
300 measurement.run(measCat=newCat, exposure=exposure, exposureId=exposureIdInfo.expId)
304 """No metadata to write.
309 def _makeArgumentParser(cls):
310 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
311 parser.add_id_argument(
"--id",
'src',
312 help=
"data ID, e.g. --id visit=12345 ccd=0")
317 """Calculate columns from ParquetTable
319 This object manages and organizes an arbitrary set of computations
320 on a catalog. The catalog is defined by a
321 `lsst.pipe.tasks.parquetTable.ParquetTable` object (or list thereof), such as a
322 `deepCoadd_obj` dataset, and the computations are defined by a collection
323 of `lsst.pipe.tasks.functor.Functor` objects (or, equivalently,
324 a `CompositeFunctor`).
326 After the object is initialized, accessing the `.df` attribute (which
327 holds the `pandas.DataFrame` containing the results of the calculations) triggers
328 computation of said dataframe.
330 One of the conveniences of using this object is the ability to define a desired common
331 filter for all functors. This enables the same functor collection to be passed to
332 several different `PostprocessAnalysis` objects without having to change the original
333 functor collection, since the `filt` keyword argument of this object triggers an
334 overwrite of the `filt` property for all functors in the collection.
336 This object also allows a list of refFlags to be passed, and defines a set of default
337 refFlags that are always included even if not requested.
339 If a list of `ParquetTable` object is passed, rather than a single one, then the
340 calculations will be mapped over all the input catalogs. In principle, it should
341 be straightforward to parallelize this activity, but initial tests have failed
342 (see TODO in code comments).
346 parq : `lsst.pipe.tasks.ParquetTable` (or list of such)
347 Source catalog(s) for computation
349 functors : `list`, `dict`, or `lsst.pipe.tasks.functors.CompositeFunctor`
350 Computations to do (functors that act on `parq`).
351 If a dict, the output
352 DataFrame will have columns keyed accordingly.
353 If a list, the column keys will come from the
354 `.shortname` attribute of each functor.
356 filt : `str` (optional)
357 Filter in which to calculate. If provided,
358 this will overwrite any existing `.filt` attribute
359 of the provided functors.
361 flags : `list` (optional)
362 List of flags (per-band) to include in output table.
364 refFlags : `list` (optional)
365 List of refFlags (only reference band) to include in output table.
369 _defaultRefFlags = []
370 _defaultFuncs = ((
'coord_ra',
RAColumn()),
373 def __init__(self, parq, functors, filt=None, flags=None, refFlags=None):
378 self.
flagsflags = list(flags)
if flags
is not None else []
380 if refFlags
is not None:
381 self.
refFlagsrefFlags += list(refFlags)
393 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.
refFlagsrefFlags})
394 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.
flagsflags})
396 if isinstance(self.
functorsfunctors, CompositeFunctor):
401 func.funcDict.update(additionalFuncs)
402 func.filt = self.
filtfilt
408 return [name
for name, func
in self.
funcfunc.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
412 if self.
_df_df
is None:
418 if type(self.
parqparq)
in (list, tuple):
420 dflist = [self.
funcfunc(parq, dropna=dropna)
for parq
in self.
parqparq]
423 dflist = pool.map(functools.partial(self.
funcfunc, dropna=dropna), self.
parqparq)
424 self.
_df_df = pd.concat(dflist)
426 self.
_df_df = self.
funcfunc(self.
parqparq, dropna=dropna)
432 functorFile = pexConfig.Field(
434 doc=
'Path to YAML file specifying functors to be computed',
441 """Base class for transforming/standardizing a catalog
443 by applying functors that convert units and apply calibrations.
444 The purpose of this task is to perform a set of computations on
445 an input `ParquetTable` dataset (such as `deepCoadd_obj`) and write the
446 results to a new dataset (which needs to be declared in an `outputDataset`
449 The calculations to be performed are defined in a YAML file that specifies
450 a set of functors to be computed, provided as
451 a `--functorFile` config parameter. An example of such a YAML file
476 - base_InputCount_value
479 functor: DeconvolvedMoments
484 - merge_measurement_i
485 - merge_measurement_r
486 - merge_measurement_z
487 - merge_measurement_y
488 - merge_measurement_g
489 - base_PixelFlags_flag_inexact_psfCenter
492 The names for each entry under "func" will become the names of columns in the
493 output dataset. All the functors referenced are defined in `lsst.pipe.tasks.functors`.
494 Positional arguments to be passed to each functor are in the `args` list,
495 and any additional entries for each column other than "functor" or "args" (e.g., `'filt'`,
496 `'dataset'`) are treated as keyword arguments to be passed to the functor initialization.
498 The "refFlags" entry is shortcut for a bunch of `Column` functors with the original column and
499 taken from the `'ref'` dataset.
501 The "flags" entry will be expanded out per band.
503 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
504 to organize and excecute the calculations.
508 def _DefaultName(self):
509 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
513 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
517 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
521 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
526 df = self.
runrun(parq, funcs=funcs, dataId=dataRef.dataId)
527 self.
writewrite(df, dataRef)
530 def run(self, parq, funcs=None, dataId=None, band=None):
531 """Do postprocessing calculations
533 Takes a `ParquetTable` object and dataId,
534 returns a dataframe with results of postprocessing calculations.
538 parq : `lsst.pipe.tasks.parquetTable.ParquetTable`
539 ParquetTable from which calculations are done.
540 funcs : `lsst.pipe.tasks.functors.Functors`
541 Functors to apply to the table's columns
542 dataId : dict, optional
543 Used to add a `patchId` column to the output dataframe.
544 band : `str`, optional
545 Filter band that is being processed.
552 self.log.info(
"Transforming/standardizing the source table dataId: %s", dataId)
554 df = self.
transformtransform(band, parq, funcs, dataId).df
555 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
559 funcs = CompositeFunctor.from_file(self.config.functorFile)
560 funcs.update(dict(PostprocessAnalysis._defaultFuncs))
571 analysis = self.
getAnalysisgetAnalysis(parq, funcs=funcs, band=band)
573 if dataId
is not None:
574 for key, value
in dataId.items():
577 return pipeBase.Struct(
586 """No metadata to write.
591 class TransformObjectCatalogConfig(TransformCatalogBaseConfig):
592 coaddName = pexConfig.Field(
598 filterMap = pexConfig.DictField(
602 doc=(
"Dictionary mapping full filter name to short one for column name munging."
603 "These filters determine the output columns no matter what filters the "
604 "input data actually contain."),
605 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
606 "Will be removed after v22.")
608 outputBands = pexConfig.ListField(
612 doc=(
"These bands and only these bands will appear in the output,"
613 " NaN-filled if the input does not include them."
614 " If None, then use all bands found in the input.")
616 camelCase = pexConfig.Field(
619 doc=(
"Write per-band columns names with camelCase, else underscore "
620 "For example: gPsFlux instead of g_PsFlux.")
622 multilevelOutput = pexConfig.Field(
625 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
626 "and name-munged (False).")
631 """Produce a flattened Object Table to match the format specified in
634 Do the same set of postprocessing calculations on all bands
636 This is identical to `TransformCatalogBaseTask`, except for that it does the
637 specified functor calculations for all filters present in the
638 input `deepCoadd_obj` table. Any specific `"filt"` keywords specified
639 by the YAML file will be superceded.
641 _DefaultName =
"transformObjectCatalog"
642 ConfigClass = TransformObjectCatalogConfig
644 inputDataset =
'deepCoadd_obj'
645 outputDataset =
'objectTable'
648 def _makeArgumentParser(cls):
651 ContainerClass=CoaddDataIdContainer,
652 help=
"data ID, e.g. --id tract=12345 patch=1,2")
655 def run(self, parq, funcs=None, dataId=None, band=None):
659 templateDf = pd.DataFrame()
660 outputBands = parq.columnLevelNames[
'band']
if self.config.outputBands
is None else \
661 self.config.outputBands
664 for inputBand
in parq.columnLevelNames[
'band']:
665 if inputBand
not in outputBands:
666 self.log.info(
"Ignoring %s band data in the input", inputBand)
668 self.log.info(
"Transforming the catalog of band %s", inputBand)
669 result = self.
transformtransform(inputBand, parq, funcs, dataId)
670 dfDict[inputBand] = result.df
671 analysisDict[inputBand] = result.analysis
673 templateDf = result.df
676 for filt
in outputBands:
677 if filt
not in dfDict:
678 self.log.info(
"Adding empty columns for band %s", filt)
679 dfDict[filt] = pd.DataFrame().reindex_like(templateDf)
682 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
684 if not self.config.multilevelOutput:
685 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
686 if dataId
is not None:
687 noDupCols += list(dataId.keys())
688 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase)
690 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
697 """Make self.refList from self.idList
699 Generate a list of data references given tract and/or patch.
700 This was adapted from `TractQADataIdContainer`, which was
701 `TractDataIdContainer` modifie to not require "filter".
702 Only existing dataRefs are returned.
704 def getPatchRefList(tract):
705 return [namespace.butler.dataRef(datasetType=self.datasetType,
707 patch=
"%d,%d" % patch.getIndex())
for patch
in tract]
709 tractRefs = defaultdict(list)
710 for dataId
in self.idList:
711 skymap = self.
getSkymapgetSkymap(namespace)
713 if "tract" in dataId:
714 tractId = dataId[
"tract"]
715 if "patch" in dataId:
716 tractRefs[tractId].append(namespace.butler.dataRef(datasetType=self.datasetType,
718 patch=dataId[
'patch']))
720 tractRefs[tractId] += getPatchRefList(skymap[tractId])
722 tractRefs = dict((tract.getId(), tractRefs.get(tract.getId(), []) + getPatchRefList(tract))
725 for tractRefList
in tractRefs.values():
726 existingRefs = [ref
for ref
in tractRefList
if ref.datasetExists()]
727 outputRefList.append(existingRefs)
733 coaddName = pexConfig.Field(
741 """Write patch-merged source tables to a tract-level parquet file
743 _DefaultName =
"consolidateObjectTable"
744 ConfigClass = ConsolidateObjectTableConfig
746 inputDataset =
'objectTable'
747 outputDataset =
'objectTable_tract'
750 def _makeArgumentParser(cls):
751 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
753 parser.add_id_argument(
"--id", cls.
inputDatasetinputDataset,
754 help=
"data ID, e.g. --id tract=12345",
755 ContainerClass=TractObjectDataIdContainer)
759 df = pd.concat([patchRef.get().toDataFrame()
for patchRef
in patchRefList])
763 """No metadata to write.
768 class TransformSourceTableConfig(TransformCatalogBaseConfig):
773 """Transform/standardize a source catalog
775 _DefaultName =
"transformSourceTable"
776 ConfigClass = TransformSourceTableConfig
778 inputDataset =
'source'
779 outputDataset =
'sourceTable'
782 """No metadata to write.
787 def _makeArgumentParser(cls):
791 help=
"data ID, e.g. --id visit=12345 ccd=0")
795 """Override to specify band label to run()."""
798 band = dataRef.get(
"calexp_filterLabel", immediate=
True).bandLabel
799 df = self.
runrun(parq, funcs=funcs, dataId=dataRef.dataId, band=band)
800 self.
writewrite(df, dataRef)
805 dimensions=(
"instrument",
"visit",),
806 defaultTemplates={}):
807 calexp = connectionTypes.Input(
808 doc=
"Processed exposures used for metadata",
810 storageClass=
"ExposureF",
811 dimensions=(
"instrument",
"visit",
"detector"),
815 visitSummary = connectionTypes.Output(
816 doc=
"Consolidated visit-level exposure metadata",
818 storageClass=
"ExposureCatalog",
819 dimensions=(
"instrument",
"visit"),
824 pipelineConnections=ConsolidateVisitSummaryConnections):
825 """Config for ConsolidateVisitSummaryTask"""
830 """Task to consolidate per-detector visit metadata.
832 This task aggregates the following metadata from all the detectors in a
833 single visit into an exposure catalog:
837 - The physical_filter and band (if available).
838 - The psf size, shape, and effective area at the center of the detector.
839 - The corners of the bounding box in right ascension/declination.
841 Other quantities such as Psf, ApCorrMap, and TransmissionCurve are not
842 persisted here because of storage concerns, and because of their limited
843 utility as summary statistics.
845 Tests for this task are performed in ci_hsc_gen3.
847 _DefaultName =
"consolidateVisitSummary"
848 ConfigClass = ConsolidateVisitSummaryConfig
851 def _makeArgumentParser(cls):
852 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
854 parser.add_id_argument(
"--id",
"calexp",
855 help=
"data ID, e.g. --id visit=12345",
856 ContainerClass=VisitDataIdContainer)
860 """No metadata to persist, so override to remove metadata persistance.
865 """No config to persist, so override to remove config persistance.
870 visit = dataRefList[0].dataId[
'visit']
872 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)" %
873 (len(dataRefList), visit))
875 expCatalog = self._combineExposureMetadata(visit, dataRefList, isGen3=
False)
877 dataRefList[0].put(expCatalog,
'visitSummary', visit=visit)
880 dataRefs = butlerQC.get(inputRefs.calexp)
881 visit = dataRefs[0].dataId.byName()[
'visit']
883 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)" %
884 (len(dataRefs), visit))
888 butlerQC.put(expCatalog, outputRefs.visitSummary)
890 def _combineExposureMetadata(self, visit, dataRefs, isGen3=True):
891 """Make a combined exposure catalog from a list of dataRefs.
896 Visit identification number
898 List of calexp dataRefs in visit. May be list of
899 `lsst.daf.persistence.ButlerDataRef` (Gen2) or
900 `lsst.daf.butler.DeferredDatasetHandle` (Gen3).
901 isGen3 : `bool`, optional
902 Specifies if this is a Gen3 list of datarefs.
906 visitSummary : `lsst.afw.table.ExposureCatalog`
907 Exposure catalog with per-detector summary information.
909 schema = afwTable.ExposureTable.makeMinimalSchema()
910 schema.addField(
'visit', type=
'I', doc=
'Visit number')
911 schema.addField(
'detector_id', type=
'I', doc=
'Detector number')
912 schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
913 schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
914 schema.addField(
'psfSigma', type=
'F',
915 doc=
'PSF model second-moments determinant radius (center of chip) (pixel)')
916 schema.addField(
'psfArea', type=
'F',
917 doc=
'PSF model effective area (center of chip) (pixel**2)')
918 schema.addField(
'psfIxx', type=
'F',
919 doc=
'PSF model Ixx (center of chip) (pixel**2)')
920 schema.addField(
'psfIyy', type=
'F',
921 doc=
'PSF model Iyy (center of chip) (pixel**2)')
922 schema.addField(
'psfIxy', type=
'F',
923 doc=
'PSF model Ixy (center of chip) (pixel**2)')
924 schema.addField(
'raCorners', type=
'ArrayD', size=4,
925 doc=
'Right Ascension of bounding box corners (degrees)')
926 schema.addField(
'decCorners', type=
'ArrayD', size=4,
927 doc=
'Declination of bounding box corners (degrees)')
929 cat = afwTable.ExposureCatalog(schema)
930 cat.resize(len(dataRefs))
934 for i, dataRef
in enumerate(dataRefs):
936 visitInfo = dataRef.get(component=
'visitInfo')
937 filterLabel = dataRef.get(component=
'filterLabel')
938 psf = dataRef.get(component=
'psf')
939 wcs = dataRef.get(component=
'wcs')
940 photoCalib = dataRef.get(component=
'photoCalib')
941 detector = dataRef.get(component=
'detector')
942 bbox = dataRef.get(component=
'bbox')
943 validPolygon = dataRef.get(component=
'validPolygon')
948 exp = dataRef.get(datasetType=
'calexp_sub', bbox=gen2_read_bbox)
949 visitInfo = exp.getInfo().getVisitInfo()
950 filterLabel = dataRef.get(
"calexp_filterLabel")
953 photoCalib = exp.getPhotoCalib()
954 detector = exp.getDetector()
955 bbox = dataRef.get(datasetType=
'calexp_bbox')
956 validPolygon = exp.getInfo().getValidPolygon()
960 rec.setVisitInfo(visitInfo)
962 rec.setPhotoCalib(photoCalib)
963 rec.setDetector(detector)
964 rec.setValidPolygon(validPolygon)
966 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
967 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
968 rec[
'detector_id'] = detector.getId()
969 shape = psf.computeShape(bbox.getCenter())
970 rec[
'psfSigma'] = shape.getDeterminantRadius()
971 rec[
'psfIxx'] = shape.getIxx()
972 rec[
'psfIyy'] = shape.getIyy()
973 rec[
'psfIxy'] = shape.getIxy()
974 im = psf.computeKernelImage(bbox.getCenter())
979 rec[
'psfArea'] = np.sum(im.array)/np.sum(im.array**2.)
982 rec[
'raCorners'][:] = [sph.getRa().asDegrees()
for sph
in sph_pts]
983 rec[
'decCorners'][:] = [sph.getDec().asDegrees()
for sph
in sph_pts]
989 """DataIdContainer that groups sensor-level id's by visit
993 """Make self.refList from self.idList
995 Generate a list of data references grouped by visit.
999 namespace : `argparse.Namespace`
1000 Namespace used by `lsst.pipe.base.CmdLineTask` to parse command line arguments
1003 visitRefs = defaultdict(list)
1004 for dataId
in self.idList:
1005 if "visit" in dataId:
1006 visitId = dataId[
"visit"]
1008 subset = namespace.butler.subset(self.datasetType, dataId=dataId)
1009 visitRefs[visitId].extend([dataRef
for dataRef
in subset])
1012 for refList
in visitRefs.values():
1013 existingRefs = [ref
for ref
in refList
if ref.datasetExists()]
1015 outputRefList.append(existingRefs)
1024 class ConsolidateSourceTableTask(CmdLineTask):
1025 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1027 _DefaultName =
'consolidateSourceTable'
1028 ConfigClass = ConsolidateSourceTableConfig
1030 inputDataset =
'sourceTable'
1031 outputDataset =
'sourceTable_visit'
1034 self.log.info(
"Concatenating %s per-detector Source Tables", len(dataRefList))
1035 df = pd.concat([dataRef.get().toDataFrame()
for dataRef
in dataRefList])
1039 def _makeArgumentParser(cls):
1040 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
1042 parser.add_id_argument(
"--id", cls.
inputDatasetinputDataset,
1043 help=
"data ID, e.g. --id visit=12345",
1044 ContainerClass=VisitDataIdContainer)
1048 """No metadata to write.
1053 """No config to write.
def getSkymap(self, namespace)
def runDataRef(self, patchRefList)
def writeMetadata(self, dataRef)
def writeConfig(self, butler, clobber=False, doBackup=True)
def runDataRef(self, dataRefList)
def writeMetadata(self, dataRef)
def runDataRef(self, dataRefList)
def writeConfig(self, butler, clobber=False, doBackup=True)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def writeMetadata(self, dataRef)
def _combineExposureMetadata(self, visit, dataRefs, isGen3=True)
def __init__(self, parq, functors, filt=None, flags=None, refFlags=None)
def compute(self, dropna=False, pool=None)
def makeDataRefList(self, namespace)
def runDataRef(self, dataRef)
def getAnalysis(self, parq, funcs=None, band=None)
def write(self, df, parqRef)
def transform(self, band, parq, funcs, dataId)
def run(self, parq, funcs=None, dataId=None, band=None)
def writeMetadata(self, dataRef)
def run(self, parq, funcs=None, dataId=None, band=None)
def writeMetadata(self, dataRef)
def runDataRef(self, dataRef)
def makeDataRefList(self, namespace)
def __init__(self, butler=None, schema=None, **kwargs)
def write(self, patchRef, catalog)
def runDataRef(self, patchRefList)
Merge coadd sources from multiple bands.
def run(self, catalogs, tract, patch)
def writeMetadata(self, dataRefList)
def readCatalog(self, patchRef)
def run(self, catalog, ccdVisitId=None)
def runDataRef(self, dataRef)
def writeMetadata(self, dataRef)
def addCalibColumns(self, catalog, dataRef)
def makeMergeArgumentParser(name, dataset)
Create a suitable ArgumentParser.
def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False)