25 from collections
import defaultdict
33 from lsst.pipe.base import CmdLineTask, ArgumentParser, DataIdContainer
36 from .parquetTable
import ParquetTable
37 from .multiBandUtils
import makeMergeArgumentParser, MergeSourcesRunner
38 from .functors
import CompositeFunctor, RAColumn, DecColumn, Column
41 def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False):
42 """Flattens a dataframe with multilevel column index
44 newDf = pd.DataFrame()
45 for band
in set(df.columns.to_frame()[
'band']):
47 columnFormat =
'{0}{1}' if camelCase
else '{0}_{1}'
48 newColumns = {c: columnFormat.format(band, c)
49 for c
in subdf.columns
if c
not in noDupCols}
50 cols = list(newColumns.keys())
51 newDf = pd.concat([newDf, subdf[cols].rename(columns=newColumns)], axis=1)
53 newDf = pd.concat([subdf[noDupCols], newDf], axis=1)
58 priorityList = pexConfig.ListField(
61 doc=
"Priority-ordered list of bands for the merge."
63 engine = pexConfig.Field(
66 doc=
"Parquet engine for writing (pyarrow or fastparquet)"
68 coaddName = pexConfig.Field(
75 pexConfig.Config.validate(self)
77 raise RuntimeError(
"No priority list provided")
81 """Write filter-merged source tables to parquet
83 _DefaultName =
"writeObjectTable"
84 ConfigClass = WriteObjectTableConfig
85 RunnerClass = MergeSourcesRunner
88 inputDatasets = (
'forced_src',
'meas',
'ref')
93 def __init__(self, butler=None, schema=None, **kwargs):
97 CmdLineTask.__init__(self, **kwargs)
101 @brief Merge coadd sources from multiple bands. Calls @ref `run` which must be defined in
102 subclasses that inherit from MergeSourcesTask.
103 @param[in] patchRefList list of data references for each filter
105 catalogs = dict(self.
readCatalogreadCatalog(patchRef)
for patchRef
in patchRefList)
106 dataId = patchRefList[0].dataId
107 mergedCatalog = self.
runrun(catalogs, tract=dataId[
'tract'], patch=dataId[
'patch'])
108 self.
writewrite(patchRefList[0], mergedCatalog)
111 def _makeArgumentParser(cls):
112 """Create a suitable ArgumentParser.
114 We will use the ArgumentParser to get a list of data
115 references for patches; the RunnerClass will sort them into lists
116 of data references for the same patch.
118 References first of self.inputDatasets, rather than
124 """Read input catalogs
126 Read all the input datasets given by the 'inputDatasets'
131 patchRef : `lsst.daf.persistence.ButlerDataRef`
132 Data reference for patch
136 Tuple consisting of band name and a dict of catalogs, keyed by
139 band = patchRef.get(self.config.coaddName +
"Coadd_filterLabel", immediate=
True).bandLabel
142 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + dataset, immediate=
True)
143 self.log.info(
"Read %d sources from %s for band %s: %s" %
144 (len(catalog), dataset, band, patchRef.dataId))
145 catalogDict[dataset] = catalog
146 return band, catalogDict
148 def run(self, catalogs, tract, patch):
149 """Merge multiple catalogs.
154 Mapping from filter names to dict of catalogs.
156 tractId to use for the tractId column
158 patchId to use for the patchId column
162 catalog : `lsst.pipe.tasks.parquetTable.ParquetTable`
163 Merged dataframe, with each column prefixed by
164 `filter_tag(filt)`, wrapped in the parquet writer shim class.
168 for filt, tableDict
in catalogs.items():
169 for dataset, table
in tableDict.items():
171 df = table.asAstropy().to_pandas().set_index(
'id', drop=
True)
174 df = df.reindex(sorted(df.columns), axis=1)
175 df[
'tractId'] = tract
176 df[
'patchId'] = patch
179 df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c)
for c
in df.columns],
180 names=(
'dataset',
'band',
'column'))
183 catalog = functools.reduce(
lambda d1, d2: d1.join(d2), dfs)
191 catalog : `ParquetTable`
193 patchRef : `lsst.daf.persistence.ButlerDataRef`
194 Data reference for patch
196 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDatasetoutputDataset)
199 mergeDataId = patchRef.dataId.copy()
200 del mergeDataId[
"filter"]
201 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
204 """No metadata to write, and not sure how to write it for a list of dataRefs.
209 class WriteSourceTableConfig(pexConfig.Config):
210 doApplyExternalPhotoCalib = pexConfig.Field(
213 doc=(
"Add local photoCalib columns from the calexp.photoCalib? Should only set True if "
214 "generating Source Tables from older src tables which do not already have local calib columns")
216 doApplyExternalSkyWcs = pexConfig.Field(
219 doc=(
"Add local WCS columns from the calexp.wcs? Should only set True if "
220 "generating Source Tables from older src tables which do not already have local calib columns")
225 """Write source table to parquet
227 _DefaultName =
"writeSourceTable"
228 ConfigClass = WriteSourceTableConfig
231 src = dataRef.get(
'src')
232 if self.config.doApplyExternalPhotoCalib
or self.config.doApplyExternalSkyWcs:
235 ccdVisitId = dataRef.get(
'ccdExposureId')
236 result = self.
runrun(src, ccdVisitId=ccdVisitId)
237 dataRef.put(result.table,
'source')
239 def run(self, catalog, ccdVisitId=None):
240 """Convert `src` catalog to parquet
244 catalog: `afwTable.SourceCatalog`
245 catalog to be converted
247 ccdVisitId to be added as a column
251 result : `lsst.pipe.base.Struct`
253 `ParquetTable` version of the input catalog
255 self.log.info(
"Generating parquet table from src catalog")
256 df = catalog.asAstropy().to_pandas().set_index(
'id', drop=
True)
257 df[
'ccdVisitId'] = ccdVisitId
258 return pipeBase.Struct(table=
ParquetTable(dataFrame=df))
261 """Add columns with local calibration evaluated at each centroid
263 for backwards compatibility with old repos.
264 This exists for the purpose of converting old src catalogs
265 (which don't have the expected local calib columns) to Source Tables.
269 catalog: `afwTable.SourceCatalog`
270 catalog to which calib columns will be added
271 dataRef: `lsst.daf.persistence.ButlerDataRef
272 for fetching the calibs from disk.
276 newCat: `afwTable.SourceCatalog`
277 Source Catalog with requested local calib columns
279 mapper = afwTable.SchemaMapper(catalog.schema)
280 measureConfig = SingleFrameMeasurementTask.ConfigClass()
281 measureConfig.doReplaceWithNoise =
False
284 exposure = dataRef.get(
'calexp_sub',
287 mapper = afwTable.SchemaMapper(catalog.schema)
288 mapper.addMinimalSchema(catalog.schema,
True)
289 schema = mapper.getOutputSchema()
291 exposureIdInfo = dataRef.get(
"expIdInfo")
292 measureConfig.plugins.names = []
293 if self.config.doApplyExternalSkyWcs:
294 plugin =
'base_LocalWcs'
296 raise RuntimeError(f
"{plugin} already in src catalog. Set doApplyExternalSkyWcs=False")
298 measureConfig.plugins.names.add(plugin)
300 if self.config.doApplyExternalPhotoCalib:
301 plugin =
'base_LocalPhotoCalib'
303 raise RuntimeError(f
"{plugin} already in src catalog. Set doApplyExternalPhotoCalib=False")
305 measureConfig.plugins.names.add(plugin)
307 measurement = SingleFrameMeasurementTask(config=measureConfig, schema=schema)
308 newCat = afwTable.SourceCatalog(schema)
309 newCat.extend(catalog, mapper=mapper)
310 measurement.run(measCat=newCat, exposure=exposure, exposureId=exposureIdInfo.expId)
314 """No metadata to write.
319 def _makeArgumentParser(cls):
320 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
321 parser.add_id_argument(
"--id",
'src',
322 help=
"data ID, e.g. --id visit=12345 ccd=0")
327 """Calculate columns from ParquetTable
329 This object manages and organizes an arbitrary set of computations
330 on a catalog. The catalog is defined by a
331 `lsst.pipe.tasks.parquetTable.ParquetTable` object (or list thereof), such as a
332 `deepCoadd_obj` dataset, and the computations are defined by a collection
333 of `lsst.pipe.tasks.functor.Functor` objects (or, equivalently,
334 a `CompositeFunctor`).
336 After the object is initialized, accessing the `.df` attribute (which
337 holds the `pandas.DataFrame` containing the results of the calculations) triggers
338 computation of said dataframe.
340 One of the conveniences of using this object is the ability to define a desired common
341 filter for all functors. This enables the same functor collection to be passed to
342 several different `PostprocessAnalysis` objects without having to change the original
343 functor collection, since the `filt` keyword argument of this object triggers an
344 overwrite of the `filt` property for all functors in the collection.
346 This object also allows a list of refFlags to be passed, and defines a set of default
347 refFlags that are always included even if not requested.
349 If a list of `ParquetTable` object is passed, rather than a single one, then the
350 calculations will be mapped over all the input catalogs. In principle, it should
351 be straightforward to parallelize this activity, but initial tests have failed
352 (see TODO in code comments).
356 parq : `lsst.pipe.tasks.ParquetTable` (or list of such)
357 Source catalog(s) for computation
359 functors : `list`, `dict`, or `lsst.pipe.tasks.functors.CompositeFunctor`
360 Computations to do (functors that act on `parq`).
361 If a dict, the output
362 DataFrame will have columns keyed accordingly.
363 If a list, the column keys will come from the
364 `.shortname` attribute of each functor.
366 filt : `str` (optional)
367 Filter in which to calculate. If provided,
368 this will overwrite any existing `.filt` attribute
369 of the provided functors.
371 flags : `list` (optional)
372 List of flags (per-band) to include in output table.
374 refFlags : `list` (optional)
375 List of refFlags (only reference band) to include in output table.
379 _defaultRefFlags = []
380 _defaultFuncs = ((
'coord_ra',
RAColumn()),
383 def __init__(self, parq, functors, filt=None, flags=None, refFlags=None):
388 self.
flagsflags = list(flags)
if flags
is not None else []
390 if refFlags
is not None:
391 self.
refFlagsrefFlags += list(refFlags)
403 additionalFuncs.update({flag:
Column(flag, dataset=
'ref')
for flag
in self.
refFlagsrefFlags})
404 additionalFuncs.update({flag:
Column(flag, dataset=
'meas')
for flag
in self.
flagsflags})
406 if isinstance(self.
functorsfunctors, CompositeFunctor):
411 func.funcDict.update(additionalFuncs)
412 func.filt = self.
filtfilt
418 return [name
for name, func
in self.
funcfunc.funcDict.items()
if func.noDup
or func.dataset ==
'ref']
422 if self.
_df_df
is None:
428 if type(self.
parqparq)
in (list, tuple):
430 dflist = [self.
funcfunc(parq, dropna=dropna)
for parq
in self.
parqparq]
433 dflist = pool.map(functools.partial(self.
funcfunc, dropna=dropna), self.
parqparq)
434 self.
_df_df = pd.concat(dflist)
436 self.
_df_df = self.
funcfunc(self.
parqparq, dropna=dropna)
442 functorFile = pexConfig.Field(
444 doc=
'Path to YAML file specifying functors to be computed',
451 """Base class for transforming/standardizing a catalog
453 by applying functors that convert units and apply calibrations.
454 The purpose of this task is to perform a set of computations on
455 an input `ParquetTable` dataset (such as `deepCoadd_obj`) and write the
456 results to a new dataset (which needs to be declared in an `outputDataset`
459 The calculations to be performed are defined in a YAML file that specifies
460 a set of functors to be computed, provided as
461 a `--functorFile` config parameter. An example of such a YAML file
486 - base_InputCount_value
489 functor: DeconvolvedMoments
494 - merge_measurement_i
495 - merge_measurement_r
496 - merge_measurement_z
497 - merge_measurement_y
498 - merge_measurement_g
499 - base_PixelFlags_flag_inexact_psfCenter
502 The names for each entry under "func" will become the names of columns in the
503 output dataset. All the functors referenced are defined in `lsst.pipe.tasks.functors`.
504 Positional arguments to be passed to each functor are in the `args` list,
505 and any additional entries for each column other than "functor" or "args" (e.g., `'filt'`,
506 `'dataset'`) are treated as keyword arguments to be passed to the functor initialization.
508 The "refFlags" entry is shortcut for a bunch of `Column` functors with the original column and
509 taken from the `'ref'` dataset.
511 The "flags" entry will be expanded out per band.
513 This task uses the `lsst.pipe.tasks.postprocess.PostprocessAnalysis` object
514 to organize and excecute the calculations.
518 def _DefaultName(self):
519 raise NotImplementedError(
'Subclass must define "_DefaultName" attribute')
523 raise NotImplementedError(
'Subclass must define "outputDataset" attribute')
527 raise NotImplementedError(
'Subclass must define "inputDataset" attribute')
531 raise NotImplementedError(
'Subclass must define "ConfigClass" attribute')
536 df = self.
runrun(parq, funcs=funcs, dataId=dataRef.dataId)
537 self.
writewrite(df, dataRef)
540 def run(self, parq, funcs=None, dataId=None, band=None):
541 """Do postprocessing calculations
543 Takes a `ParquetTable` object and dataId,
544 returns a dataframe with results of postprocessing calculations.
548 parq : `lsst.pipe.tasks.parquetTable.ParquetTable`
549 ParquetTable from which calculations are done.
550 funcs : `lsst.pipe.tasks.functors.Functors`
551 Functors to apply to the table's columns
552 dataId : dict, optional
553 Used to add a `patchId` column to the output dataframe.
554 band : `str`, optional
555 Filter band that is being processed.
562 self.log.info(
"Transforming/standardizing the source table dataId: %s", dataId)
564 df = self.
transformtransform(band, parq, funcs, dataId).df
565 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
569 funcs = CompositeFunctor.from_file(self.config.functorFile)
570 funcs.update(dict(PostprocessAnalysis._defaultFuncs))
581 analysis = self.
getAnalysisgetAnalysis(parq, funcs=funcs, band=band)
583 if dataId
is not None:
584 for key, value
in dataId.items():
587 return pipeBase.Struct(
596 """No metadata to write.
601 class TransformObjectCatalogConfig(TransformCatalogBaseConfig):
602 coaddName = pexConfig.Field(
608 filterMap = pexConfig.DictField(
612 doc=(
"Dictionary mapping full filter name to short one for column name munging."
613 "These filters determine the output columns no matter what filters the "
614 "input data actually contain."),
615 deprecated=(
"Coadds are now identified by the band, so this transform is unused."
616 "Will be removed after v22.")
618 outputBands = pexConfig.ListField(
622 doc=(
"These bands and only these bands will appear in the output,"
623 " NaN-filled if the input does not include them."
624 " If None, then use all bands found in the input.")
626 camelCase = pexConfig.Field(
629 doc=(
"Write per-band columns names with camelCase, else underscore "
630 "For example: gPsFlux instead of g_PsFlux.")
632 multilevelOutput = pexConfig.Field(
635 doc=(
"Whether results dataframe should have a multilevel column index (True) or be flat "
636 "and name-munged (False).")
641 """Produce a flattened Object Table to match the format specified in
644 Do the same set of postprocessing calculations on all bands
646 This is identical to `TransformCatalogBaseTask`, except for that it does the
647 specified functor calculations for all filters present in the
648 input `deepCoadd_obj` table. Any specific `"filt"` keywords specified
649 by the YAML file will be superceded.
651 _DefaultName =
"transformObjectCatalog"
652 ConfigClass = TransformObjectCatalogConfig
654 inputDataset =
'deepCoadd_obj'
655 outputDataset =
'objectTable'
658 def _makeArgumentParser(cls):
661 ContainerClass=CoaddDataIdContainer,
662 help=
"data ID, e.g. --id tract=12345 patch=1,2")
665 def run(self, parq, funcs=None, dataId=None, band=None):
669 templateDf = pd.DataFrame()
670 outputBands = parq.columnLevelNames[
'band']
if self.config.outputBands
is None else \
671 self.config.outputBands
674 for inputBand
in parq.columnLevelNames[
'band']:
675 if inputBand
not in outputBands:
676 self.log.info(
"Ignoring %s band data in the input", inputBand)
678 self.log.info(
"Transforming the catalog of band %s", inputBand)
679 result = self.
transformtransform(inputBand, parq, funcs, dataId)
680 dfDict[inputBand] = result.df
681 analysisDict[inputBand] = result.analysis
683 templateDf = result.df
686 for filt
in outputBands:
687 if filt
not in dfDict:
688 self.log.info(
"Adding empty columns for band %s", filt)
689 dfDict[filt] = pd.DataFrame().reindex_like(templateDf)
692 df = pd.concat(dfDict, axis=1, names=[
'band',
'column'])
694 if not self.config.multilevelOutput:
695 noDupCols = list(set.union(*[set(v.noDupCols)
for v
in analysisDict.values()]))
696 if dataId
is not None:
697 noDupCols += list(dataId.keys())
698 df =
flattenFilters(df, noDupCols=noDupCols, camelCase=self.config.camelCase)
700 self.log.info(
"Made a table of %d columns and %d rows", len(df.columns), len(df))
707 """Make self.refList from self.idList
709 Generate a list of data references given tract and/or patch.
710 This was adapted from `TractQADataIdContainer`, which was
711 `TractDataIdContainer` modifie to not require "filter".
712 Only existing dataRefs are returned.
714 def getPatchRefList(tract):
715 return [namespace.butler.dataRef(datasetType=self.datasetType,
717 patch=
"%d,%d" % patch.getIndex())
for patch
in tract]
719 tractRefs = defaultdict(list)
720 for dataId
in self.idList:
721 skymap = self.
getSkymapgetSkymap(namespace)
723 if "tract" in dataId:
724 tractId = dataId[
"tract"]
725 if "patch" in dataId:
726 tractRefs[tractId].append(namespace.butler.dataRef(datasetType=self.datasetType,
728 patch=dataId[
'patch']))
730 tractRefs[tractId] += getPatchRefList(skymap[tractId])
732 tractRefs = dict((tract.getId(), tractRefs.get(tract.getId(), []) + getPatchRefList(tract))
735 for tractRefList
in tractRefs.values():
736 existingRefs = [ref
for ref
in tractRefList
if ref.datasetExists()]
737 outputRefList.append(existingRefs)
743 coaddName = pexConfig.Field(
751 """Write patch-merged source tables to a tract-level parquet file
753 _DefaultName =
"consolidateObjectTable"
754 ConfigClass = ConsolidateObjectTableConfig
756 inputDataset =
'objectTable'
757 outputDataset =
'objectTable_tract'
760 def _makeArgumentParser(cls):
761 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
763 parser.add_id_argument(
"--id", cls.
inputDatasetinputDataset,
764 help=
"data ID, e.g. --id tract=12345",
765 ContainerClass=TractObjectDataIdContainer)
769 df = pd.concat([patchRef.get().toDataFrame()
for patchRef
in patchRefList])
773 """No metadata to write.
778 class TransformSourceTableConfig(TransformCatalogBaseConfig):
783 """Transform/standardize a source catalog
785 _DefaultName =
"transformSourceTable"
786 ConfigClass = TransformSourceTableConfig
788 inputDataset =
'source'
789 outputDataset =
'sourceTable'
792 """No metadata to write.
797 def _makeArgumentParser(cls):
801 help=
"data ID, e.g. --id visit=12345 ccd=0")
805 """Override to specify band label to run()."""
808 band = dataRef.get(
"calexp_filterLabel", immediate=
True).bandLabel
809 df = self.
runrun(parq, funcs=funcs, dataId=dataRef.dataId, band=band)
810 self.
writewrite(df, dataRef)
815 dimensions=(
"instrument",
"visit",),
816 defaultTemplates={}):
817 calexp = connectionTypes.Input(
818 doc=
"Processed exposures used for metadata",
820 storageClass=
"ExposureF",
821 dimensions=(
"instrument",
"visit",
"detector"),
825 visitSummary = connectionTypes.Output(
826 doc=
"Consolidated visit-level exposure metadata",
828 storageClass=
"ExposureCatalog",
829 dimensions=(
"instrument",
"visit"),
834 pipelineConnections=ConsolidateVisitSummaryConnections):
835 """Config for ConsolidateVisitSummaryTask"""
840 """Task to consolidate per-detector visit metadata.
842 This task aggregates the following metadata from all the detectors in a
843 single visit into an exposure catalog:
847 - The physical_filter and band (if available).
848 - The psf size, shape, and effective area at the center of the detector.
849 - The corners of the bounding box in right ascension/declination.
851 Other quantities such as Psf, ApCorrMap, and TransmissionCurve are not
852 persisted here because of storage concerns, and because of their limited
853 utility as summary statistics.
855 Tests for this task are performed in ci_hsc_gen3.
857 _DefaultName =
"consolidateVisitSummary"
858 ConfigClass = ConsolidateVisitSummaryConfig
861 def _makeArgumentParser(cls):
862 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
864 parser.add_id_argument(
"--id",
"calexp",
865 help=
"data ID, e.g. --id visit=12345",
866 ContainerClass=VisitDataIdContainer)
870 """No metadata to persist, so override to remove metadata persistance.
875 """No config to persist, so override to remove config persistance.
880 visit = dataRefList[0].dataId[
'visit']
882 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)" %
883 (len(dataRefList), visit))
885 expCatalog = self._combineExposureMetadata(visit, dataRefList, isGen3=
False)
887 dataRefList[0].put(expCatalog,
'visitSummary', visit=visit)
890 dataRefs = butlerQC.get(inputRefs.calexp)
891 visit = dataRefs[0].dataId.byName()[
'visit']
893 self.log.debug(
"Concatenating metadata from %d per-detector calexps (visit %d)" %
894 (len(dataRefs), visit))
898 butlerQC.put(expCatalog, outputRefs.visitSummary)
900 def _combineExposureMetadata(self, visit, dataRefs, isGen3=True):
901 """Make a combined exposure catalog from a list of dataRefs.
906 Visit identification number
908 List of calexp dataRefs in visit. May be list of
909 `lsst.daf.persistence.ButlerDataRef` (Gen2) or
910 `lsst.daf.butler.DeferredDatasetHandle` (Gen3).
911 isGen3 : `bool`, optional
912 Specifies if this is a Gen3 list of datarefs.
916 visitSummary : `lsst.afw.table.ExposureCatalog`
917 Exposure catalog with per-detector summary information.
919 schema = afwTable.ExposureTable.makeMinimalSchema()
920 schema.addField(
'visit', type=
'I', doc=
'Visit number')
921 schema.addField(
'detector_id', type=
'I', doc=
'Detector number')
922 schema.addField(
'physical_filter', type=
'String', size=32, doc=
'Physical filter')
923 schema.addField(
'band', type=
'String', size=32, doc=
'Name of band')
924 schema.addField(
'psfSigma', type=
'F',
925 doc=
'PSF model second-moments determinant radius (center of chip) (pixel)')
926 schema.addField(
'psfArea', type=
'F',
927 doc=
'PSF model effective area (center of chip) (pixel**2)')
928 schema.addField(
'psfIxx', type=
'F',
929 doc=
'PSF model Ixx (center of chip) (pixel**2)')
930 schema.addField(
'psfIyy', type=
'F',
931 doc=
'PSF model Iyy (center of chip) (pixel**2)')
932 schema.addField(
'psfIxy', type=
'F',
933 doc=
'PSF model Ixy (center of chip) (pixel**2)')
934 schema.addField(
'raCorners', type=
'ArrayD', size=4,
935 doc=
'Right Ascension of bounding box corners (degrees)')
936 schema.addField(
'decCorners', type=
'ArrayD', size=4,
937 doc=
'Declination of bounding box corners (degrees)')
939 cat = afwTable.ExposureCatalog(schema)
940 cat.resize(len(dataRefs))
944 for i, dataRef
in enumerate(dataRefs):
946 visitInfo = dataRef.get(component=
'visitInfo')
947 filterLabel = dataRef.get(component=
'filterLabel')
948 psf = dataRef.get(component=
'psf')
949 wcs = dataRef.get(component=
'wcs')
950 photoCalib = dataRef.get(component=
'photoCalib')
951 detector = dataRef.get(component=
'detector')
952 bbox = dataRef.get(component=
'bbox')
953 validPolygon = dataRef.get(component=
'validPolygon')
958 exp = dataRef.get(datasetType=
'calexp_sub', bbox=gen2_read_bbox)
959 visitInfo = exp.getInfo().getVisitInfo()
960 filterLabel = exp.getFilterLabel()
963 photoCalib = exp.getPhotoCalib()
964 detector = exp.getDetector()
965 bbox = dataRef.get(datasetType=
'calexp_bbox')
966 validPolygon = exp.getInfo().getValidPolygon()
970 rec.setVisitInfo(visitInfo)
972 rec.setPhotoCalib(photoCalib)
973 rec.setDetector(detector)
974 rec.setValidPolygon(validPolygon)
976 rec[
'physical_filter'] = filterLabel.physicalLabel
if filterLabel.hasPhysicalLabel()
else ""
977 rec[
'band'] = filterLabel.bandLabel
if filterLabel.hasBandLabel()
else ""
978 rec[
'detector_id'] = detector.getId()
979 shape = psf.computeShape(bbox.getCenter())
980 rec[
'psfSigma'] = shape.getDeterminantRadius()
981 rec[
'psfIxx'] = shape.getIxx()
982 rec[
'psfIyy'] = shape.getIyy()
983 rec[
'psfIxy'] = shape.getIxy()
984 im = psf.computeKernelImage(bbox.getCenter())
989 rec[
'psfArea'] = np.sum(im.array)/np.sum(im.array**2.)
992 rec[
'raCorners'][:] = [sph.getRa().asDegrees()
for sph
in sph_pts]
993 rec[
'decCorners'][:] = [sph.getDec().asDegrees()
for sph
in sph_pts]
999 """DataIdContainer that groups sensor-level id's by visit
1003 """Make self.refList from self.idList
1005 Generate a list of data references grouped by visit.
1009 namespace : `argparse.Namespace`
1010 Namespace used by `lsst.pipe.base.CmdLineTask` to parse command line arguments
1013 visitRefs = defaultdict(list)
1014 for dataId
in self.idList:
1015 if "visit" in dataId:
1016 visitId = dataId[
"visit"]
1018 subset = namespace.butler.subset(self.datasetType, dataId=dataId)
1019 visitRefs[visitId].extend([dataRef
for dataRef
in subset])
1022 for refList
in visitRefs.values():
1023 existingRefs = [ref
for ref
in refList
if ref.datasetExists()]
1025 outputRefList.append(existingRefs)
1034 class ConsolidateSourceTableTask(CmdLineTask):
1035 """Concatenate `sourceTable` list into a per-visit `sourceTable_visit`
1037 _DefaultName =
'consolidateSourceTable'
1038 ConfigClass = ConsolidateSourceTableConfig
1040 inputDataset =
'sourceTable'
1041 outputDataset =
'sourceTable_visit'
1044 self.log.info(
"Concatenating %s per-detector Source Tables", len(dataRefList))
1045 df = pd.concat([dataRef.get().toDataFrame()
for dataRef
in dataRefList])
1049 def _makeArgumentParser(cls):
1050 parser = ArgumentParser(name=cls.
_DefaultName_DefaultName)
1052 parser.add_id_argument(
"--id", cls.
inputDatasetinputDataset,
1053 help=
"data ID, e.g. --id visit=12345",
1054 ContainerClass=VisitDataIdContainer)
1058 """No metadata to write.
1063 """No config to write.
def getSkymap(self, namespace)
def runDataRef(self, patchRefList)
def writeMetadata(self, dataRef)
def writeConfig(self, butler, clobber=False, doBackup=True)
def runDataRef(self, dataRefList)
def writeMetadata(self, dataRef)
def runDataRef(self, dataRefList)
def writeConfig(self, butler, clobber=False, doBackup=True)
def runQuantum(self, butlerQC, inputRefs, outputRefs)
def writeMetadata(self, dataRef)
def _combineExposureMetadata(self, visit, dataRefs, isGen3=True)
def __init__(self, parq, functors, filt=None, flags=None, refFlags=None)
def compute(self, dropna=False, pool=None)
def makeDataRefList(self, namespace)
def runDataRef(self, dataRef)
def getAnalysis(self, parq, funcs=None, band=None)
def write(self, df, parqRef)
def transform(self, band, parq, funcs, dataId)
def run(self, parq, funcs=None, dataId=None, band=None)
def writeMetadata(self, dataRef)
def run(self, parq, funcs=None, dataId=None, band=None)
def writeMetadata(self, dataRef)
def runDataRef(self, dataRef)
def makeDataRefList(self, namespace)
def __init__(self, butler=None, schema=None, **kwargs)
def write(self, patchRef, catalog)
def runDataRef(self, patchRefList)
Merge coadd sources from multiple bands.
def run(self, catalogs, tract, patch)
def writeMetadata(self, dataRefList)
def readCatalog(self, patchRef)
def run(self, catalog, ccdVisitId=None)
def runDataRef(self, dataRef)
def writeMetadata(self, dataRef)
def addCalibColumns(self, catalog, dataRef)
def makeMergeArgumentParser(name, dataset)
Create a suitable ArgumentParser.
def flattenFilters(df, noDupCols=['coord_ra', 'coord_dec'], camelCase=False)