_DefaultName = "writeObjectTable"
ConfigClass = WriteObjectTableConfig
# Names of table datasets to be merged
inputDatasets = ('forced_src', 'meas', 'ref')
# Tag of output dataset written by `MergeSourcesTask.write`
outputDataset = 'obj'
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
measDict = {ref.dataId['band']: {'meas': cat} for ref, cat in
zip(inputRefs.inputCatalogMeas, inputs['inputCatalogMeas'])}
forcedSourceDict = {ref.dataId['band']: {'forced_src': cat} for ref, cat in
zip(inputRefs.inputCatalogForcedSrc, inputs['inputCatalogForcedSrc'])}
catalogs = {}
for band in measDict.keys():
catalogs[band] = {'meas': measDict[band]['meas'],
'forced_src': forcedSourceDict[band]['forced_src'],
'ref': inputs['inputCatalogRef']}
dataId = butlerQC.quantum.dataId
df = self.run(catalogs=catalogs, tract=dataId['tract'], patch=dataId['patch'])
outputs = pipeBase.Struct(outputCatalog=df)
butlerQC.put(outputs, outputRefs)
def run(self, catalogs, tract, patch):
dfs = []
for filt, tableDict in catalogs.items():
for dataset, table in tableDict.items():
# Convert afwTable to pandas DataFrame
df = table.asAstropy().to_pandas().set_index('id', drop=True)
# Sort columns by name, to ensure matching schema among patches
df = df.reindex(sorted(df.columns), axis=1)
df = df.assign(tractId=tract, patchId=patch)
# Make columns a 3-level MultiIndex
df.columns = pd.MultiIndex.from_tuples([(dataset, filt, c) for c in df.columns],
names=('dataset', 'band', 'column'))
dfs.append(df)
# We do this dance and not `pd.concat(dfs)` because the pandas
# concatenation uses infinite memory.
catalog = functools.reduce(lambda d1, d2: d1.join(d2), dfs)
return catalog
class WriteSourceTableConnections(pipeBase.PipelineTaskConnections,
defaultTemplates={"catalogType": ""},
dimensions=("instrument", "visit", "detector")):
catalog = connectionTypes.Input(
doc="Input full-depth catalog of sources produced by CalibrateTask",
name="{catalogType}src",
storageClass="SourceCatalog",
dimensions=("instrument", "visit", "detector")
)
outputCatalog = connectionTypes.Output(
doc="Catalog of sources, `src` in DataFrame/Parquet format. The 'id' column is "
"replaced with an index; all other columns are unchanged.",
name="{catalogType}source",
storageClass="DataFrame",
dimensions=("instrument", "visit", "detector")
)
class WriteSourceTableConfig(pipeBase.PipelineTaskConfig,
pipelineConnections=WriteSourceTableConnections):
idGenerator = DetectorVisitIdGeneratorConfig.make_field()
class WriteSourceTableTask(pipeBase.PipelineTask):
_DefaultName = "writeSourceTable"
ConfigClass = WriteSourceTableConfig
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
inputs['ccdVisitId'] = self.config.idGenerator.apply(butlerQC.quantum.dataId).catalog_id
result = self.run(**inputs)
outputs = pipeBase.Struct(outputCatalog=result.table)
butlerQC.put(outputs, outputRefs)
def run(self, catalog, ccdVisitId=None, **kwargs):
self.log.info("Generating DataFrame from src catalog ccdVisitId=%s", ccdVisitId)
df = catalog.asAstropy().to_pandas().set_index('id', drop=True)
df['ccdVisitId'] = ccdVisitId
return pipeBase.Struct(table=df)
class WriteRecalibratedSourceTableConnections(WriteSourceTableConnections,
defaultTemplates={"catalogType": "",
"skyWcsName": "gbdesAstrometricFit",
"photoCalibName": "fgcm"},
dimensions=("instrument", "visit", "detector", "skymap")):
skyMap = connectionTypes.Input(
doc="skyMap needed to choose which tract-level calibrations to use when multiple available",
name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
storageClass="SkyMap",
dimensions=("skymap",),
)
exposure = connectionTypes.Input(
doc="Input exposure to perform photometry on.",
name="calexp",
storageClass="ExposureF",
dimensions=["instrument", "visit", "detector"],
)
externalSkyWcsTractCatalog = connectionTypes.Input(
doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
"id for the catalog id, sorted on id for fast lookup."),
name="{skyWcsName}SkyWcsCatalog",
storageClass="ExposureCatalog",
dimensions=["instrument", "visit", "tract"],
multiple=True
)
externalSkyWcsGlobalCatalog = connectionTypes.Input(
doc=("Per-visit wcs calibrations computed globally (with no tract information). "
"These catalogs use the detector id for the catalog id, sorted on id for "
"fast lookup."),
name="finalVisitSummary",
storageClass="ExposureCatalog",
dimensions=["instrument", "visit"],
)
externalPhotoCalibTractCatalog = connectionTypes.Input(
doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
"detector id for the catalog id, sorted on id for fast lookup."),
name="{photoCalibName}PhotoCalibCatalog",
storageClass="ExposureCatalog",
dimensions=["instrument", "visit", "tract"],
multiple=True
)
externalPhotoCalibGlobalCatalog = connectionTypes.Input(
doc=("Per-visit photometric calibrations computed globally (with no tract "
"information). These catalogs use the detector id for the catalog id, "
"sorted on id for fast lookup."),
name="finalVisitSummary",
storageClass="ExposureCatalog",
dimensions=["instrument", "visit"],
)
def __init__(self, *, config=None):
super().__init__(config=config)
# Same connection boilerplate as all other applications of
# Global/Tract calibrations
if config.doApplyExternalSkyWcs and config.doReevaluateSkyWcs:
if config.useGlobalExternalSkyWcs:
self.inputs.remove("externalSkyWcsTractCatalog")
else:
self.inputs.remove("externalSkyWcsGlobalCatalog")
else:
self.inputs.remove("externalSkyWcsTractCatalog")
self.inputs.remove("externalSkyWcsGlobalCatalog")
if config.doApplyExternalPhotoCalib and config.doReevaluatePhotoCalib:
if config.useGlobalExternalPhotoCalib:
self.inputs.remove("externalPhotoCalibTractCatalog")
else:
self.inputs.remove("externalPhotoCalibGlobalCatalog")
else:
self.inputs.remove("externalPhotoCalibTractCatalog")
self.inputs.remove("externalPhotoCalibGlobalCatalog")
class WriteRecalibratedSourceTableConfig(WriteSourceTableConfig,
pipelineConnections=WriteRecalibratedSourceTableConnections):
doReevaluatePhotoCalib = pexConfig.Field(
dtype=bool,
default=True,
doc=("Add or replace local photoCalib columns")
)
doReevaluateSkyWcs = pexConfig.Field(
dtype=bool,
default=True,
doc=("Add or replace local WCS columns and update the coord columns, coord_ra and coord_dec")
)
doApplyExternalPhotoCalib = pexConfig.Field(
dtype=bool,
default=True,
doc=("If and only if doReevaluatePhotoCalib, apply the photometric calibrations from an external ",
"algorithm such as FGCM or jointcal, else use the photoCalib already attached to the exposure."),
)
doApplyExternalSkyWcs = pexConfig.Field(
dtype=bool,
default=True,
doc=("if and only if doReevaluateSkyWcs, apply the WCS from an external algorithm such as jointcal, ",
"else use the wcs already attached to the exposure."),
)
useGlobalExternalPhotoCalib = pexConfig.Field(
dtype=bool,
default=True,
doc=("When using doApplyExternalPhotoCalib, use 'global' calibrations "
"that are not run per-tract. When False, use per-tract photometric "
"calibration files.")
)
useGlobalExternalSkyWcs = pexConfig.Field(
dtype=bool,
default=True,
doc=("When using doApplyExternalSkyWcs, use 'global' calibrations "
"that are not run per-tract. When False, use per-tract wcs "
"files.")
)
idGenerator = DetectorVisitIdGeneratorConfig.make_field()
def validate(self):
super().validate()
if self.doApplyExternalSkyWcs and not self.doReevaluateSkyWcs:
log.warning("doApplyExternalSkyWcs=True but doReevaluateSkyWcs=False"
"External SkyWcs will not be read or evaluated.")
if self.doApplyExternalPhotoCalib and not self.doReevaluatePhotoCalib:
log.warning("doApplyExternalPhotoCalib=True but doReevaluatePhotoCalib=False."
"External PhotoCalib will not be read or evaluated.")
class WriteRecalibratedSourceTableTask(WriteSourceTableTask):
_DefaultName = "writeRecalibratedSourceTable"
ConfigClass = WriteRecalibratedSourceTableConfig
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
inputs['idGenerator'] = idGenerator
inputs['ccdVisitId'] = idGenerator.catalog_id
if self.config.doReevaluatePhotoCalib or self.config.doReevaluateSkyWcs:
if self.config.doApplyExternalPhotoCalib or self.config.doApplyExternalSkyWcs:
inputs['exposure'] = self.attachCalibs(inputRefs, **inputs)
inputs['catalog'] = self.addCalibColumns(**inputs)
result = self.run(**inputs)
outputs = pipeBase.Struct(outputCatalog=result.table)
butlerQC.put(outputs, outputRefs)
def attachCalibs(self, inputRefs, skyMap, exposure, externalSkyWcsGlobalCatalog=None,
externalSkyWcsTractCatalog=None, externalPhotoCalibGlobalCatalog=None,
externalPhotoCalibTractCatalog=None, **kwargs):
if not self.config.doApplyExternalSkyWcs:
# Do not modify the exposure's SkyWcs
externalSkyWcsCatalog = None
elif self.config.useGlobalExternalSkyWcs:
# Use the global external SkyWcs
externalSkyWcsCatalog = externalSkyWcsGlobalCatalog
self.log.info('Applying global SkyWcs')
else:
# use tract-level external SkyWcs from the closest overlapping tract
inputRef = getattr(inputRefs, 'externalSkyWcsTractCatalog')
tracts = [ref.dataId['tract'] for ref in inputRef]
if len(tracts) == 1:
ind = 0
self.log.info('Applying tract-level SkyWcs from tract %s', tracts[ind])
else:
if exposure.getWcs() is None: # TODO: could this look-up use the externalPhotoCalib?
raise ValueError("Trying to locate nearest tract, but exposure.wcs is None.")
ind = self.getClosestTract(tracts, skyMap,
exposure.getBBox(), exposure.getWcs())
self.log.info('Multiple overlapping externalSkyWcsTractCatalogs found (%s). '
'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
externalSkyWcsCatalog = externalSkyWcsTractCatalog[ind]
if not self.config.doApplyExternalPhotoCalib:
# Do not modify the exposure's PhotoCalib
externalPhotoCalibCatalog = None
elif self.config.useGlobalExternalPhotoCalib:
# Use the global external PhotoCalib
externalPhotoCalibCatalog = externalPhotoCalibGlobalCatalog
self.log.info('Applying global PhotoCalib')
else:
# use tract-level external PhotoCalib from the closest overlapping tract
inputRef = getattr(inputRefs, 'externalPhotoCalibTractCatalog')
tracts = [ref.dataId['tract'] for ref in inputRef]
if len(tracts) == 1:
ind = 0
self.log.info('Applying tract-level PhotoCalib from tract %s', tracts[ind])
else:
ind = self.getClosestTract(tracts, skyMap,
exposure.getBBox(), exposure.getWcs())
self.log.info('Multiple overlapping externalPhotoCalibTractCatalogs found (%s). '
'Applying closest to detector center: tract=%s', str(tracts), tracts[ind])
externalPhotoCalibCatalog = externalPhotoCalibTractCatalog[ind]
return self.prepareCalibratedExposure(exposure, externalSkyWcsCatalog, externalPhotoCalibCatalog)
def getClosestTract(self, tracts, skyMap, bbox, wcs):
if len(tracts) == 1:
return 0
center = wcs.pixelToSky(bbox.getCenter())
sep = []
for tractId in tracts:
tract = skyMap[tractId]
tractCenter = tract.getWcs().pixelToSky(tract.getBBox().getCenter())
sep.append(center.separation(tractCenter))
return np.argmin(sep)
def prepareCalibratedExposure(self, exposure, externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None):
Definition at line 529 of file postprocess.py.