lsst.pipe.tasks g59f6dce098+83b7f37e51
Loading...
Searching...
No Matches
Classes | Variables
lsst.pipe.tasks.multiBand Namespace Reference

Classes

class  DetectCoaddSourcesConnections
 

Variables

lsst schema : `lsst.afw.table.Schema`, optional
 
lsst exposure : `lsst.afw.image.Exposure`
 
lsst idFactory : `lsst.afw.table.IdFactory`
 
int expId : `int`
 
lsst result : `lsst.pipe.base.Struct`
 
lsst butler : `lsst.daf.butler.Butler` or `None`, optional
 
lsst peakSchema : ``lsst.afw.table.Schema`, optional
 
lsst refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional
 
dict initInputs : `dict`, optional
 
lsst sources : `lsst.afw.table.SourceCatalog`
 
lsst skyInfo : `lsst.pipe.base.Struct`
 
int exposureId : `int` or `bytes`
 
lsst ccdInputs : `lsst.afw.table.ExposureCatalog`, optional
 
list visitCatalogs : `list` of `lsst.afw.table.SourceCatalogs`, optional
 
list wcsUpdates : `list` of `lsst.afw.geom.SkyWcs`, optional
 
dict sourceTableHandleDict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`], optional
 
dict finalizedSourceTableHandleDict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`], optional
 
lsst results : `lsst.pipe.base.Struct`
 

Variable Documentation

◆ butler

None lsst.pipe.tasks.multiBand.butler : `lsst.daf.butler.Butler` or `None`, optional
if self.config.doScaleVariance:
    varScale = self.scaleVariance.run(exposure.maskedImage)
    exposure.getMetadata().add("VARIANCE_SCALE", varScale)
backgrounds = afwMath.BackgroundList()
table = afwTable.SourceTable.make(self.schema, idFactory)
detections = self.detection.run(table, exposure, expId=expId)
sources = detections.sources
if hasattr(detections, "background") and detections.background:
    for bg in detections.background:
        backgrounds.append(bg)
return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)


##############################################################################################################


class DeblendCoaddSourcesConfig(Config):
singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
                                      doc="Deblend sources separately in each band")
multiBandDeblend = ConfigurableField(target=ScarletDeblendTask,
                                     doc="Deblend sources simultaneously across bands")
simultaneous = Field(dtype=bool,
                     default=True,
                     doc="Simultaneously deblend all bands? "
                         "True uses `multibandDeblend` while False uses `singleBandDeblend`")
coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
hasFakes = Field(dtype=bool,
                 default=False,
                 doc="Should be set to True if fake sources have been inserted into the input data.")

def setDefaults(self):
    Config.setDefaults(self)
    self.singleBandDeblend.propagateAllPeaks = True


class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections,
                                       dimensions=("tract", "patch", "band", "skymap"),
                                       defaultTemplates={"inputCoaddName": "deep",
                                                         "outputCoaddName": "deep",
                                                         "deblendedCatalog": "deblendedFlux"}):
inputSchema = cT.InitInput(
    doc="Input schema for measure merged task produced by a deblender or detection task",
    name="{inputCoaddName}Coadd_deblendedFlux_schema",
    storageClass="SourceCatalog"
)
outputSchema = cT.InitOutput(
    doc="Output schema after all new fields are added by task",
    name="{inputCoaddName}Coadd_meas_schema",
    storageClass="SourceCatalog"
)
refCat = cT.PrerequisiteInput(
    doc="Reference catalog used to match measured sources against known sources",
    name="ref_cat",
    storageClass="SimpleCatalog",
    dimensions=("skypix",),
    deferLoad=True,
    multiple=True
)
exposure = cT.Input(
    doc="Input coadd image",
    name="{inputCoaddName}Coadd_calexp",
    storageClass="ExposureF",
    dimensions=("tract", "patch", "band", "skymap")
)
skyMap = cT.Input(
    doc="SkyMap to use in processing",
    name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
    storageClass="SkyMap",
    dimensions=("skymap",),
)
visitCatalogs = cT.Input(
    doc="Source catalogs for visits which overlap input tract, patch, band. Will be "
        "further filtered in the task for the purpose of propagating flags from image calibration "
        "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.",
    name="src",
    dimensions=("instrument", "visit", "detector"),
    storageClass="SourceCatalog",
    multiple=True
)
sourceTableHandles = cT.Input(
    doc=("Source tables that are derived from the ``CalibrateTask`` sources. "
         "These tables contain astrometry and photometry flags, and optionally "
         "PSF flags."),
    name="sourceTable_visit",
    storageClass="DataFrame",
    dimensions=("instrument", "visit"),
    multiple=True,
    deferLoad=True,
)
finalizedSourceTableHandles = cT.Input(
    doc=("Finalized source tables from ``FinalizeCalibrationTask``. These "
         "tables contain PSF flags from the finalized PSF estimation."),
    name="finalized_src_table",
    storageClass="DataFrame",
    dimensions=("instrument", "visit"),
    multiple=True,
    deferLoad=True,
)
inputCatalog = cT.Input(
    doc=("Name of the input catalog to use."
         "If the single band deblender was used this should be 'deblendedFlux."
         "If the multi-band deblender was used this should be 'deblendedModel, "
         "or deblendedFlux if the multiband deblender was configured to output "
         "deblended flux catalogs. If no deblending was performed this should "
         "be 'mergeDet'"),
    name="{inputCoaddName}Coadd_{deblendedCatalog}",
    storageClass="SourceCatalog",
    dimensions=("tract", "patch", "band", "skymap"),
)
scarletCatalog = cT.Input(
    doc="Catalogs produced by multiband deblending",
    name="{inputCoaddName}Coadd_deblendedCatalog",
    storageClass="SourceCatalog",
    dimensions=("tract", "patch", "skymap"),
)
scarletModels = cT.Input(
    doc="Multiband scarlet models produced by the deblender",
    name="{inputCoaddName}Coadd_scarletModelData",
    storageClass="ScarletModelData",
    dimensions=("tract", "patch", "skymap"),
)
outputSources = cT.Output(
    doc="Source catalog containing all the measurement information generated in this task",
    name="{outputCoaddName}Coadd_meas",
    dimensions=("tract", "patch", "band", "skymap"),
    storageClass="SourceCatalog",
)
matchResult = cT.Output(
    doc="Match catalog produced by configured matcher, optional on doMatchSources",
    name="{outputCoaddName}Coadd_measMatch",
    dimensions=("tract", "patch", "band", "skymap"),
    storageClass="Catalog",
)
denormMatches = cT.Output(
    doc="Denormalized Match catalog produced by configured matcher, optional on "
        "doWriteMatchesDenormalized",
    name="{outputCoaddName}Coadd_measMatchFull",
    dimensions=("tract", "patch", "band", "skymap"),
    storageClass="Catalog",
)

def __init__(self, *, config=None):
    super().__init__(config=config)
    if config.doPropagateFlags is False:
        self.inputs -= set(("visitCatalogs",))
        self.inputs -= set(("sourceTableHandles",))
        self.inputs -= set(("finalizedSourceTableHandles",))
    elif config.propagateFlags.target == PropagateSourceFlagsTask:
        # New PropagateSourceFlagsTask does not use visitCatalogs.
        self.inputs -= set(("visitCatalogs",))
        # Check for types of flags required.
        if not config.propagateFlags.source_flags:
            self.inputs -= set(("sourceTableHandles",))
        if not config.propagateFlags.finalized_source_flags:
            self.inputs -= set(("finalizedSourceTableHandles",))
    else:
        # Deprecated PropagateVisitFlagsTask uses visitCatalogs.
        self.inputs -= set(("sourceTableHandles",))
        self.inputs -= set(("finalizedSourceTableHandles",))

    if config.inputCatalog == "deblendedCatalog":
        self.inputs -= set(("inputCatalog",))

        if not config.doAddFootprints:
            self.inputs -= set(("scarletModels",))
    else:
        self.inputs -= set(("deblendedCatalog"))
        self.inputs -= set(("scarletModels",))

    if config.doMatchSources is False:
        self.outputs -= set(("matchResult",))

    if config.doWriteMatchesDenormalized is False:
        self.outputs -= set(("denormMatches",))


class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
                                  pipelineConnections=MeasureMergedCoaddSourcesConnections):
inputCatalog = ChoiceField(
    dtype=str,
    default="deblendedCatalog",
    allowed={
        "deblendedCatalog": "Output catalog from ScarletDeblendTask",
        "deblendedFlux": "Output catalog from SourceDeblendTask",
        "mergeDet": "The merged detections before deblending."
    },
    doc="The name of the input catalog.",
)
doAddFootprints = Field(dtype=bool,
                        default=True,
                        doc="Whether or not to add footprints to the input catalog from scarlet models. "
                            "This should be true whenever using the multi-band deblender, "
                            "otherwise this should be False.")
doConserveFlux = Field(dtype=bool, default=True,
                       doc="Whether to use the deblender models as templates to re-distribute the flux "
                           "from the 'exposure' (True), or to perform measurements on the deblender "
                           "model footprints.")
doStripFootprints = Field(dtype=bool, default=True,
                          doc="Whether to strip footprints from the output catalog before "
                              "saving to disk. "
                              "This is usually done when using scarlet models to save disk space.")
measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
doPropagateFlags = Field(
    dtype=bool, default=True,
    doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
)
propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd")
doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
doWriteMatchesDenormalized = Field(
    dtype=bool,
    default=False,
    doc=("Write reference matches in denormalized format? "
         "This format uses more disk space, but is more convenient to read."),
)
coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
checkUnitsParseStrict = Field(
    doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
    dtype=str,
    default="raise",
)
doApCorr = Field(
    dtype=bool,
    default=True,
    doc="Apply aperture corrections"
)
applyApCorr = ConfigurableField(
    target=ApplyApCorrTask,
    doc="Subtask to apply aperture corrections"
)
doRunCatalogCalculation = Field(
    dtype=bool,
    default=True,
    doc='Run catalogCalculation task'
)
catalogCalculation = ConfigurableField(
    target=CatalogCalculationTask,
    doc="Subtask to run catalogCalculation plugins on catalog"
)

hasFakes = Field(
    dtype=bool,
    default=False,
    doc="Should be set to True if fake sources have been inserted into the input data."
)

@property
def refObjLoader(self):
    return self.match.refObjLoader

def setDefaults(self):
    super().setDefaults()
    self.measurement.plugins.names |= ['base_InputCount',
                                       'base_Variance',
                                       'base_LocalPhotoCalib',
                                       'base_LocalWcs']
    self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
                                                                   'INEXACT_PSF']
    self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
                                                                 'INEXACT_PSF']


class MeasureMergedCoaddSourcesTask(PipelineTask):

Definition at line 511 of file multiBand.py.

◆ ccdInputs

lsst lsst.pipe.tasks.multiBand.ccdInputs : `lsst.afw.table.ExposureCatalog`, optional

Definition at line 684 of file multiBand.py.

◆ expId

int lsst.pipe.tasks.multiBand.expId : `int`

Definition at line 203 of file multiBand.py.

◆ exposure

lsst lsst.pipe.tasks.multiBand.exposure : `lsst.afw.image.Exposure`
_DefaultName = "detectCoaddSources"
ConfigClass = DetectCoaddSourcesConfig

def __init__(self, schema=None, **kwargs):
    # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree
    # call structure has been reviewed carefully to be sure super will work as intended.
    super().__init__(**kwargs)
    if schema is None:
        schema = afwTable.SourceTable.makeMinimalSchema()
    self.schema = schema
    self.makeSubtask("detection", schema=self.schema)
    if self.config.doScaleVariance:
        self.makeSubtask("scaleVariance")

    self.detectionSchema = afwTable.SourceCatalog(self.schema)

def runQuantum(self, butlerQC, inputRefs, outputRefs):
    inputs = butlerQC.get(inputRefs)
    exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch_band")
    inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
    inputs["expId"] = exposureIdInfo.expId
    outputs = self.run(**inputs)
    butlerQC.put(outputs, outputRefs)

def run(self, exposure, idFactory, expId):
_DefaultName = "measureCoaddSources"
ConfigClass = MeasureMergedCoaddSourcesConfig

def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
             **kwargs):
    super().__init__(**kwargs)
    self.deblended = self.config.inputCatalog.startswith("deblended")
    self.inputCatalog = "Coadd_" + self.config.inputCatalog
    if initInputs is not None:
        schema = initInputs['inputSchema'].schema
    if schema is None:
        assert butler is not None, "Neither butler nor schema is defined"
        schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema").schema
    self.schemaMapper = afwTable.SchemaMapper(schema)
    self.schemaMapper.addMinimalSchema(schema)
    self.schema = self.schemaMapper.getOutputSchema()
    self.algMetadata = PropertyList()
    self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
    self.makeSubtask("setPrimaryFlags", schema=self.schema)
    if self.config.doMatchSources:
        self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
    if self.config.doPropagateFlags:
        self.makeSubtask("propagateFlags", schema=self.schema)
    self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
    if self.config.doApCorr:
        self.makeSubtask("applyApCorr", schema=self.schema)
    if self.config.doRunCatalogCalculation:
        self.makeSubtask("catalogCalculation", schema=self.schema)

    self.outputSchema = afwTable.SourceCatalog(self.schema)

def runQuantum(self, butlerQC, inputRefs, outputRefs):
    inputs = butlerQC.get(inputRefs)

    refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat],
                                         inputs.pop('refCat'),
                                         name=self.config.connections.refCat,
                                         config=self.config.refObjLoader,
                                         log=self.log)
    self.match.setRefObjLoader(refObjLoader)

    # Set psfcache
    # move this to run after gen2 deprecation
    inputs['exposure'].getPsf().setCacheCapacity(self.config.psfCache)

    # Get unique integer ID for IdFactory and RNG seeds
    exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch")
    inputs['exposureId'] = exposureIdInfo.expId
    idFactory = exposureIdInfo.makeSourceIdFactory()
    # Transform inputCatalog
    table = afwTable.SourceTable.make(self.schema, idFactory)
    sources = afwTable.SourceCatalog(table)
    # Load the correct input catalog
    if "scarletCatalog" in inputs:
        inputCatalog = inputs.pop("scarletCatalog")
        catalogRef = inputRefs.scarletCatalog
    else:
        inputCatalog = inputs.pop("inputCatalog")
        catalogRef = inputRefs.inputCatalog
    sources.extend(inputCatalog, self.schemaMapper)
    del inputCatalog
    # Add the HeavyFootprints to the deblended sources
    if self.config.doAddFootprints:
        modelData = inputs.pop('scarletModels')
        if self.config.doConserveFlux:
            redistributeImage = inputs['exposure'].image
        else:
            redistributeImage = None
        modelData.updateCatalogFootprints(
            catalog=sources,
            band=inputRefs.exposure.dataId["band"],
            psfModel=inputs['exposure'].getPsf(),
            redistributeImage=redistributeImage,
            removeScarletData=True,
        )
    table = sources.getTable()
    table.setMetadata(self.algMetadata)  # Capture algorithm metadata to write out to the source catalog.
    inputs['sources'] = sources

    skyMap = inputs.pop('skyMap')
    tractNumber = catalogRef.dataId['tract']
    tractInfo = skyMap[tractNumber]
    patchInfo = tractInfo.getPatchInfo(catalogRef.dataId['patch'])
    skyInfo = Struct(
        skyMap=skyMap,
        tractInfo=tractInfo,
        patchInfo=patchInfo,
        wcs=tractInfo.getWcs(),
        bbox=patchInfo.getOuterBBox()
    )
    inputs['skyInfo'] = skyInfo

    if self.config.doPropagateFlags:
        if self.config.propagateFlags.target == PropagateSourceFlagsTask:
            # New version
            ccdInputs = inputs["exposure"].getInfo().getCoaddInputs().ccds
            inputs["ccdInputs"] = ccdInputs

            if "sourceTableHandles" in inputs:
                sourceTableHandles = inputs.pop("sourceTableHandles")
                sourceTableHandleDict = {handle.dataId["visit"]: handle
                                         for handle in sourceTableHandles}
                inputs["sourceTableHandleDict"] = sourceTableHandleDict
            if "finalizedSourceTableHandles" in inputs:
                finalizedSourceTableHandles = inputs.pop("finalizedSourceTableHandles")
                finalizedSourceTableHandleDict = {handle.dataId["visit"]: handle
                                                  for handle in finalizedSourceTableHandles}
                inputs["finalizedSourceTableHandleDict"] = finalizedSourceTableHandleDict
        else:
            # Deprecated legacy version
            # Filter out any visit catalog that is not coadd inputs
            ccdInputs = inputs['exposure'].getInfo().getCoaddInputs().ccds
            visitKey = ccdInputs.schema.find("visit").key
            ccdKey = ccdInputs.schema.find("ccd").key
            inputVisitIds = set()
            ccdRecordsWcs = {}
            for ccdRecord in ccdInputs:
                visit = ccdRecord.get(visitKey)
                ccd = ccdRecord.get(ccdKey)
                inputVisitIds.add((visit, ccd))
                ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()

            inputCatalogsToKeep = []
            inputCatalogWcsUpdate = []
            for i, dataRef in enumerate(inputRefs.visitCatalogs):
                key = (dataRef.dataId['visit'], dataRef.dataId['detector'])
                if key in inputVisitIds:
                    inputCatalogsToKeep.append(inputs['visitCatalogs'][i])
                    inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
            inputs['visitCatalogs'] = inputCatalogsToKeep
            inputs['wcsUpdates'] = inputCatalogWcsUpdate
            inputs['ccdInputs'] = ccdInputs

    outputs = self.run(**inputs)
    # Strip HeavyFootprints to save space on disk
    sources = outputs.outputSources
    butlerQC.put(outputs, outputRefs)

def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
        butler=None, sourceTableHandleDict=None, finalizedSourceTableHandleDict=None):

Definition at line 198 of file multiBand.py.

◆ exposureId

int lsst.pipe.tasks.multiBand.exposureId : `int` or `bytes`

Definition at line 682 of file multiBand.py.

◆ finalizedSourceTableHandleDict

dict lsst.pipe.tasks.multiBand.finalizedSourceTableHandleDict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`], optional

Definition at line 705 of file multiBand.py.

◆ idFactory

lsst lsst.pipe.tasks.multiBand.idFactory : `lsst.afw.table.IdFactory`

Definition at line 201 of file multiBand.py.

◆ initInputs

dict lsst.pipe.tasks.multiBand.initInputs : `dict`, optional

Definition at line 522 of file multiBand.py.

◆ peakSchema

lsst lsst.pipe.tasks.multiBand.peakSchema : ``lsst.afw.table.Schema`, optional

Definition at line 516 of file multiBand.py.

◆ refObjLoader

lsst lsst.pipe.tasks.multiBand.refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`, optional

Definition at line 518 of file multiBand.py.

◆ result

lsst lsst.pipe.tasks.multiBand.result : `lsst.pipe.base.Struct`

Definition at line 208 of file multiBand.py.

◆ results

lsst lsst.pipe.tasks.multiBand.results : `lsst.pipe.base.Struct`

Definition at line 712 of file multiBand.py.

◆ schema

lsst lsst.pipe.tasks.multiBand.schema : `lsst.afw.table.Schema`, optional
doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
doInsertFakes = Field(dtype=bool, default=False,
                      doc="Run fake sources injection task",
                      deprecated=("doInsertFakes is no longer supported. This config will be removed "
                                  "after v24."))
insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
                                doc="Injection of fake sources for testing "
                                "purposes (must be retargeted)",
                                deprecated=("insertFakes is no longer supported. This config will "
                                            "be removed after v24."))
hasFakes = Field(
    dtype=bool,
    default=False,
    doc="Should be set to True if fake sources have been inserted into the input data.",
)

def setDefaults(self):
    super().setDefaults()
    self.detection.thresholdType = "pixel_stdev"
    self.detection.isotropicGrow = True
    # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
    self.detection.reEstimateBackground = False
    self.detection.background.useApprox = False
    self.detection.background.binSize = 4096
    self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
    self.detection.doTempWideBackground = True  # Suppress large footprints that overwhelm the deblender


class DetectCoaddSourcesTask(PipelineTask):

Definition at line 158 of file multiBand.py.

◆ skyInfo

lsst lsst.pipe.tasks.multiBand.skyInfo : `lsst.pipe.base.Struct`

Definition at line 679 of file multiBand.py.

◆ sources

lsst lsst.pipe.tasks.multiBand.sources : `lsst.afw.table.SourceCatalog`

Definition at line 676 of file multiBand.py.

◆ sourceTableHandleDict

dict lsst.pipe.tasks.multiBand.sourceTableHandleDict : `dict` [`int`, `lsst.daf.butler.DeferredDatasetHandle`], optional

Definition at line 701 of file multiBand.py.

◆ visitCatalogs

list lsst.pipe.tasks.multiBand.visitCatalogs : `list` of `lsst.afw.table.SourceCatalogs`, optional

Definition at line 687 of file multiBand.py.

◆ wcsUpdates

list lsst.pipe.tasks.multiBand.wcsUpdates : `list` of `lsst.afw.geom.SkyWcs`, optional

Definition at line 692 of file multiBand.py.