lsst.pipe.tasks g6a99470703+701c574c97
multiBand.py
Go to the documentation of this file.
1#!/usr/bin/env python
2#
3# LSST Data Management System
4# Copyright 2008-2015 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23import warnings
24
25from lsst.pipe.base import (Struct, PipelineTask, PipelineTaskConfig, PipelineTaskConnections)
26import lsst.pipe.base.connectionTypes as cT
27from lsst.pex.config import Config, Field, ConfigurableField, ChoiceField
28from lsst.meas.algorithms import DynamicDetectionTask, ReferenceObjectLoader, ScaleVarianceTask
29from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
30from lsst.meas.deblender import SourceDeblendTask
31from lsst.meas.extensions.scarlet import ScarletDeblendTask
32from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
33from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
34from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
35from lsst.pipe.tasks.propagateSourceFlags import PropagateSourceFlagsTask
36import lsst.afw.table as afwTable
37import lsst.afw.math as afwMath
38from lsst.daf.base import PropertyList
39from lsst.skymap import BaseSkyMap
40from lsst.obs.base import ExposureIdInfo
41
42# NOTE: these imports are a convenience so multiband users only have to import this file.
43from .mergeDetections import MergeDetectionsConfig, MergeDetectionsTask # noqa: F401
44from .mergeMeasurements import MergeMeasurementsConfig, MergeMeasurementsTask # noqa: F401
45from .multiBandUtils import CullPeaksConfig, _makeGetSchemaCatalogs # noqa: F401
46from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesSingleConfig # noqa: F401
47from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesSingleTask # noqa: F401
48from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiConfig # noqa: F401
49from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiTask # noqa: F401
50
51
52"""
53New set types:
54* deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
55* deepCoadd_mergeDet: merged detections (tract, patch)
56* deepCoadd_meas: measurements of merged detections (tract, patch, filter)
57* deepCoadd_ref: reference sources (tract, patch)
58All of these have associated *_schema catalogs that require no data ID and hold no records.
59
60In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
61the mergeDet, meas, and ref dataset Footprints:
62* deepCoadd_peak_schema
63"""
64
65
66
67class DetectCoaddSourcesConnections(PipelineTaskConnections,
68 dimensions=("tract", "patch", "band", "skymap"),
69 defaultTemplates={"inputCoaddName": "deep", "outputCoaddName": "deep"}):
70 detectionSchema = cT.InitOutput(
71 doc="Schema of the detection catalog",
72 name="{outputCoaddName}Coadd_det_schema",
73 storageClass="SourceCatalog",
74 )
75 exposure = cT.Input(
76 doc="Exposure on which detections are to be performed",
77 name="{inputCoaddName}Coadd",
78 storageClass="ExposureF",
79 dimensions=("tract", "patch", "band", "skymap")
80 )
81 outputBackgrounds = cT.Output(
82 doc="Output Backgrounds used in detection",
83 name="{outputCoaddName}Coadd_calexp_background",
84 storageClass="Background",
85 dimensions=("tract", "patch", "band", "skymap")
86 )
87 outputSources = cT.Output(
88 doc="Detected sources catalog",
89 name="{outputCoaddName}Coadd_det",
90 storageClass="SourceCatalog",
91 dimensions=("tract", "patch", "band", "skymap")
92 )
93 outputExposure = cT.Output(
94 doc="Exposure post detection",
95 name="{outputCoaddName}Coadd_calexp",
96 storageClass="ExposureF",
97 dimensions=("tract", "patch", "band", "skymap")
98 )
99
100
101class DetectCoaddSourcesConfig(PipelineTaskConfig, pipelineConnections=DetectCoaddSourcesConnections):
102 """!
103 @anchor DetectCoaddSourcesConfig_
104
105 @brief Configuration parameters for the DetectCoaddSourcesTask
106 """
107 doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
108 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
109 detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
110 coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
111 doInsertFakes = Field(dtype=bool, default=False,
112 doc="Run fake sources injection task",
113 deprecated=("doInsertFakes is no longer supported. This config will be removed "
114 "after v24."))
115 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
116 doc="Injection of fake sources for testing "
117 "purposes (must be retargeted)",
118 deprecated=("insertFakes is no longer supported. This config will "
119 "be removed after v24."))
120 hasFakes = Field(
121 dtype=bool,
122 default=False,
123 doc="Should be set to True if fake sources have been inserted into the input data.",
124 )
125
126 def setDefaults(self):
127 super().setDefaults()
128 self.detection.thresholdType = "pixel_stdev"
129 self.detection.isotropicGrow = True
130 # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131 self.detection.reEstimateBackground = False
132 self.detection.background.useApprox = False
133 self.detection.background.binSize = 4096
134 self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135 self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
136
137
143
144
145class DetectCoaddSourcesTask(PipelineTask):
146 """Detect sources on a coadd."""
147 _DefaultName = "detectCoaddSources"
148 ConfigClass = DetectCoaddSourcesConfig
149 getSchemaCatalogs = _makeGetSchemaCatalogs("det")
150
151 def __init__(self, schema=None, **kwargs):
152 """!
153 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask.
154
155 Keyword arguments (in addition to those forwarded to PipelineTask.__init__):
156
157 @param[in] schema: initial schema for the output catalog, modified-in place to include all
158 fields set by this task. If None, the source minimal schema will be used.
159 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
160 """
161 # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree
162 # call structure has been reviewed carefully to be sure super will work as intended.
163 super().__init__(**kwargs)
164 if schema is None:
165 schema = afwTable.SourceTable.makeMinimalSchema()
166 self.schema = schema
167 self.makeSubtask("detection", schema=self.schema)
168 if self.config.doScaleVariance:
169 self.makeSubtask("scaleVariance")
170
171 self.detectionSchema = afwTable.SourceCatalog(self.schema)
172
173 def runQuantum(self, butlerQC, inputRefs, outputRefs):
174 inputs = butlerQC.get(inputRefs)
175 exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch_band")
176 inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
177 inputs["expId"] = exposureIdInfo.expId
178 outputs = self.run(**inputs)
179 butlerQC.put(outputs, outputRefs)
180
181 def run(self, exposure, idFactory, expId):
182 """!
183 @brief Run detection on an exposure.
184
185 First scale the variance plane to match the observed variance
186 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to
187 detect sources.
188
189 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
190 depending on configuration).
191 @param[in] idFactory: IdFactory to set source identifiers
192 @param[in] expId: Exposure identifier (integer) for RNG seed
193
194 @return a pipe.base.Struct with fields
195 - sources: catalog of detections
196 - backgrounds: list of backgrounds
197 """
198 if self.config.doScaleVariance:
199 varScale = self.scaleVariance.run(exposure.maskedImage)
200 exposure.getMetadata().add("VARIANCE_SCALE", varScale)
201 backgrounds = afwMath.BackgroundList()
202 table = afwTable.SourceTable.make(self.schema, idFactory)
203 detections = self.detection.run(table, exposure, expId=expId)
204 sources = detections.sources
205 fpSets = detections.fpSets
206 if hasattr(fpSets, "background") and fpSets.background:
207 for bg in fpSets.background:
208 backgrounds.append(bg)
209 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
210
211
212
213
214
215class DeblendCoaddSourcesConfig(Config):
216 """DeblendCoaddSourcesConfig
217
218 Configuration parameters for the `DeblendCoaddSourcesTask`.
219 """
220 singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
221 doc="Deblend sources separately in each band")
222 multiBandDeblend = ConfigurableField(target=ScarletDeblendTask,
223 doc="Deblend sources simultaneously across bands")
224 simultaneous = Field(dtype=bool,
225 default=True,
226 doc="Simultaneously deblend all bands? "
227 "True uses `multibandDeblend` while False uses `singleBandDeblend`")
228 coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
229 hasFakes = Field(dtype=bool,
230 default=False,
231 doc="Should be set to True if fake sources have been inserted into the input data.")
232
233 def setDefaults(self):
234 Config.setDefaults(self)
235 self.singleBandDeblend.propagateAllPeaks = True
236
237
238class MeasureMergedCoaddSourcesConnections(PipelineTaskConnections, dimensions=("tract", "patch", "band", "skymap"),
239 defaultTemplates={"inputCoaddName": "deep",
240 "outputCoaddName": "deep",
241 "deblendedCatalog": "deblendedFlux"}):
242 warnings.warn("MeasureMergedCoaddSourcesConnections.defaultTemplates is deprecated and no longer used. "
243 "Use MeasureMergedCoaddSourcesConfig.inputCatalog.")
244 inputSchema = cT.InitInput(
245 doc="Input schema for measure merged task produced by a deblender or detection task",
246 name="{inputCoaddName}Coadd_deblendedFlux_schema",
247 storageClass="SourceCatalog"
248 )
249 outputSchema = cT.InitOutput(
250 doc="Output schema after all new fields are added by task",
251 name="{inputCoaddName}Coadd_meas_schema",
252 storageClass="SourceCatalog"
253 )
254 refCat = cT.PrerequisiteInput(
255 doc="Reference catalog used to match measured sources against known sources",
256 name="ref_cat",
257 storageClass="SimpleCatalog",
258 dimensions=("skypix",),
259 deferLoad=True,
260 multiple=True
261 )
262 exposure = cT.Input(
263 doc="Input coadd image",
264 name="{inputCoaddName}Coadd_calexp",
265 storageClass="ExposureF",
266 dimensions=("tract", "patch", "band", "skymap")
267 )
268 skyMap = cT.Input(
269 doc="SkyMap to use in processing",
270 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
271 storageClass="SkyMap",
272 dimensions=("skymap",),
273 )
274 visitCatalogs = cT.Input(
275 doc="Source catalogs for visits which overlap input tract, patch, band. Will be "
276 "further filtered in the task for the purpose of propagating flags from image calibration "
277 "and characterization to coadd objects. Only used in legacy PropagateVisitFlagsTask.",
278 name="src",
279 dimensions=("instrument", "visit", "detector"),
280 storageClass="SourceCatalog",
281 multiple=True
282 )
283 sourceTableHandles = cT.Input(
284 doc=("Source tables that are derived from the ``CalibrateTask`` sources. "
285 "These tables contain astrometry and photometry flags, and optionally "
286 "PSF flags."),
287 name="sourceTable_visit",
288 storageClass="DataFrame",
289 dimensions=("instrument", "visit"),
290 multiple=True,
291 deferLoad=True,
292 )
293 finalizedSourceTableHandles = cT.Input(
294 doc=("Finalized source tables from ``FinalizeCalibrationTask``. These "
295 "tables contain PSF flags from the finalized PSF estimation."),
296 name="finalized_src_table",
297 storageClass="DataFrame",
298 dimensions=("instrument", "visit"),
299 multiple=True,
300 deferLoad=True,
301 )
302 inputCatalog = cT.Input(
303 doc=("Name of the input catalog to use."
304 "If the single band deblender was used this should be 'deblendedFlux."
305 "If the multi-band deblender was used this should be 'deblendedModel, "
306 "or deblendedFlux if the multiband deblender was configured to output "
307 "deblended flux catalogs. If no deblending was performed this should "
308 "be 'mergeDet'"),
309 name="{inputCoaddName}Coadd_{deblendedCatalog}",
310 storageClass="SourceCatalog",
311 dimensions=("tract", "patch", "band", "skymap"),
312 )
313 scarletCatalog = cT.Input(
314 doc="Catalogs produced by multiband deblending",
315 name="{inputCoaddName}Coadd_deblendedCatalog",
316 storageClass="SourceCatalog",
317 dimensions=("tract", "patch", "skymap"),
318 )
319 scarletModels = cT.Input(
320 doc="Multiband scarlet models produced by the deblender",
321 name="{inputCoaddName}Coadd_scarletModelData",
322 storageClass="ScarletModelData",
323 dimensions=("tract", "patch", "skymap"),
324 )
325 outputSources = cT.Output(
326 doc="Source catalog containing all the measurement information generated in this task",
327 name="{outputCoaddName}Coadd_meas",
328 dimensions=("tract", "patch", "band", "skymap"),
329 storageClass="SourceCatalog",
330 )
331 matchResult = cT.Output(
332 doc="Match catalog produced by configured matcher, optional on doMatchSources",
333 name="{outputCoaddName}Coadd_measMatch",
334 dimensions=("tract", "patch", "band", "skymap"),
335 storageClass="Catalog",
336 )
337 denormMatches = cT.Output(
338 doc="Denormalized Match catalog produced by configured matcher, optional on "
339 "doWriteMatchesDenormalized",
340 name="{outputCoaddName}Coadd_measMatchFull",
341 dimensions=("tract", "patch", "band", "skymap"),
342 storageClass="Catalog",
343 )
344
345 def __init__(self, *, config=None):
346 super().__init__(config=config)
347 if config.doPropagateFlags is False:
348 self.inputs -= set(("visitCatalogs",))
349 self.inputs -= set(("sourceTableHandles",))
350 self.inputs -= set(("finalizedSourceTableHandles",))
351 elif config.propagateFlags.target == PropagateSourceFlagsTask:
352 # New PropagateSourceFlagsTask does not use visitCatalogs.
353 self.inputs -= set(("visitCatalogs",))
354 # Check for types of flags required.
355 if not config.propagateFlags.source_flags:
356 self.inputs -= set(("sourceTableHandles",))
357 if not config.propagateFlags.finalized_source_flags:
358 self.inputs -= set(("finalizedSourceTableHandles",))
359 else:
360 # Deprecated PropagateVisitFlagsTask uses visitCatalogs.
361 self.inputs -= set(("sourceTableHandles",))
362 self.inputs -= set(("finalizedSourceTableHandles",))
363
364 if config.inputCatalog == "deblendedCatalog":
365 self.inputs -= set(("inputCatalog",))
366
367 if not config.doAddFootprints:
368 self.inputs -= set(("scarletModels",))
369 else:
370 self.inputs -= set(("deblendedCatalog"))
371 self.inputs -= set(("scarletModels",))
372
373 if config.doMatchSources is False:
374 self.outputs -= set(("matchResult",))
375
376 if config.doWriteMatchesDenormalized is False:
377 self.outputs -= set(("denormMatches",))
378
379
380class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig,
381 pipelineConnections=MeasureMergedCoaddSourcesConnections):
382 """!
383 @anchor MeasureMergedCoaddSourcesConfig_
384
385 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask
386 """
387 inputCatalog = ChoiceField(
388 dtype=str,
389 default="deblendedCatalog",
390 allowed={
391 "deblendedCatalog": "Output catalog from ScarletDeblendTask",
392 "deblendedFlux": "Output catalog from SourceDeblendTask",
393 "mergeDet": "The merged detections before deblending."
394 },
395 doc="The name of the input catalog.",
396 )
397 doAddFootprints = Field(dtype=bool,
398 default=True,
399 doc="Whether or not to add footprints to the input catalog from scarlet models. "
400 "This should be true whenever using the multi-band deblender, "
401 "otherwise this should be False.")
402 doConserveFlux = Field(dtype=bool, default=True,
403 doc="Whether to use the deblender models as templates to re-distribute the flux "
404 "from the 'exposure' (True), or to perform measurements on the deblender "
405 "model footprints.")
406 doStripFootprints = Field(dtype=bool, default=True,
407 doc="Whether to strip footprints from the output catalog before "
408 "saving to disk. "
409 "This is usually done when using scarlet models to save disk space.")
410 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
411 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
412 doPropagateFlags = Field(
413 dtype=bool, default=True,
414 doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
415 )
416 propagateFlags = ConfigurableField(target=PropagateSourceFlagsTask, doc="Propagate source flags to coadd")
417 doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
418 match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
419 doWriteMatchesDenormalized = Field(
420 dtype=bool,
421 default=False,
422 doc=("Write reference matches in denormalized format? "
423 "This format uses more disk space, but is more convenient to read."),
424 )
425 coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
426 psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
427 checkUnitsParseStrict = Field(
428 doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
429 dtype=str,
430 default="raise",
431 )
432 doApCorr = Field(
433 dtype=bool,
434 default=True,
435 doc="Apply aperture corrections"
436 )
437 applyApCorr = ConfigurableField(
438 target=ApplyApCorrTask,
439 doc="Subtask to apply aperture corrections"
440 )
441 doRunCatalogCalculation = Field(
442 dtype=bool,
443 default=True,
444 doc='Run catalogCalculation task'
445 )
446 catalogCalculation = ConfigurableField(
447 target=CatalogCalculationTask,
448 doc="Subtask to run catalogCalculation plugins on catalog"
449 )
450
451 hasFakes = Field(
452 dtype=bool,
453 default=False,
454 doc="Should be set to True if fake sources have been inserted into the input data."
455 )
456
457 @property
458 def refObjLoader(self):
459 return self.match.refObjLoader
460
461 def setDefaults(self):
462 super().setDefaults()
463 self.measurement.plugins.names |= ['base_InputCount',
464 'base_Variance',
465 'base_LocalPhotoCalib',
466 'base_LocalWcs']
467 self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
468 'INEXACT_PSF']
469 self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
470 'INEXACT_PSF']
471
472
473
479
480class MeasureMergedCoaddSourcesTask(PipelineTask):
481 """Deblend sources from main catalog in each coadd seperately and measure."""
482 _DefaultName = "measureCoaddSources"
483 ConfigClass = MeasureMergedCoaddSourcesConfig
484 getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
485
486 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
487 **kwargs):
488 """!
489 @brief Initialize the task.
490
491 Keyword arguments (in addition to those forwarded to PipelineTask.__init__):
492 @param[in] schema: the schema of the merged detection catalog used as input to this one
493 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
494 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
495 catalog. May be None if the loader can be constructed from the butler argument or all steps
496 requiring a reference catalog are disabled.
497 @param[in] butler: a butler used to read the input schemas from disk or construct the reference
498 catalog loader, if schema or peakSchema or refObjLoader is None
499
500 The task will set its own self.schema attribute to the schema of the output measurement catalog.
501 This will include all fields from the input schema, as well as additional fields for all the
502 measurements.
503 """
504 super().__init__(**kwargs)
505 self.deblended = self.config.inputCatalog.startswith("deblended")
506 self.inputCatalog = "Coadd_" + self.config.inputCatalog
507 if initInputs is not None:
508 schema = initInputs['inputSchema'].schema
509 if schema is None:
510 assert butler is not None, "Neither butler nor schema is defined"
511 schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema", immediate=True).schema
512 self.schemaMapper = afwTable.SchemaMapper(schema)
513 self.schemaMapper.addMinimalSchema(schema)
514 self.schema = self.schemaMapper.getOutputSchema()
515 self.algMetadata = PropertyList()
516 self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
517 self.makeSubtask("setPrimaryFlags", schema=self.schema)
518 if self.config.doMatchSources:
519 self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
520 if self.config.doPropagateFlags:
521 self.makeSubtask("propagateFlags", schema=self.schema)
522 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
523 if self.config.doApCorr:
524 self.makeSubtask("applyApCorr", schema=self.schema)
525 if self.config.doRunCatalogCalculation:
526 self.makeSubtask("catalogCalculation", schema=self.schema)
527
528 self.outputSchema = afwTable.SourceCatalog(self.schema)
529
530 def runQuantum(self, butlerQC, inputRefs, outputRefs):
531 inputs = butlerQC.get(inputRefs)
532
533 refObjLoader = ReferenceObjectLoader([ref.datasetRef.dataId for ref in inputRefs.refCat],
534 inputs.pop('refCat'),
535 name=self.config.connections.refCat,
536 config=self.config.refObjLoader,
537 log=self.log)
538 self.match.setRefObjLoader(refObjLoader)
539
540 # Set psfcache
541 # move this to run after gen2 deprecation
542 inputs['exposure'].getPsf().setCacheCapacity(self.config.psfCache)
543
544 # Get unique integer ID for IdFactory and RNG seeds
545 exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId, "tract_patch")
546 inputs['exposureId'] = exposureIdInfo.expId
547 idFactory = exposureIdInfo.makeSourceIdFactory()
548 # Transform inputCatalog
549 table = afwTable.SourceTable.make(self.schema, idFactory)
550 sources = afwTable.SourceCatalog(table)
551 # Load the correct input catalog
552 if "scarletCatalog" in inputs:
553 inputCatalog = inputs.pop("scarletCatalog")
554 catalogRef = inputRefs.scarletCatalog
555 else:
556 inputCatalog = inputs.pop("inputCatalog")
557 catalogRef = inputRefs.inputCatalog
558 sources.extend(inputCatalog, self.schemaMapper)
559 del inputCatalog
560 # Add the HeavyFootprints to the deblended sources
561 if self.config.doAddFootprints:
562 modelData = inputs.pop('scarletModels')
563 if self.config.doConserveFlux:
564 redistributeImage = inputs['exposure'].image
565 else:
566 redistributeImage = None
567 modelData.updateCatalogFootprints(
568 catalog=sources,
569 band=inputRefs.exposure.dataId["band"],
570 psfModel=inputs['exposure'].getPsf(),
571 redistributeImage=redistributeImage,
572 removeScarletData=True,
573 )
574 table = sources.getTable()
575 table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
576 inputs['sources'] = sources
577
578 skyMap = inputs.pop('skyMap')
579 tractNumber = catalogRef.dataId['tract']
580 tractInfo = skyMap[tractNumber]
581 patchInfo = tractInfo.getPatchInfo(catalogRef.dataId['patch'])
582 skyInfo = Struct(
583 skyMap=skyMap,
584 tractInfo=tractInfo,
585 patchInfo=patchInfo,
586 wcs=tractInfo.getWcs(),
587 bbox=patchInfo.getOuterBBox()
588 )
589 inputs['skyInfo'] = skyInfo
590
591 if self.config.doPropagateFlags:
592 if self.config.propagateFlags.target == PropagateSourceFlagsTask:
593 # New version
594 ccdInputs = inputs["exposure"].getInfo().getCoaddInputs().ccds
595 inputs["ccdInputs"] = ccdInputs
596
597 if "sourceTableHandles" in inputs:
598 sourceTableHandles = inputs.pop("sourceTableHandles")
599 sourceTableHandleDict = {handle.dataId["visit"]: handle
600 for handle in sourceTableHandles}
601 inputs["sourceTableHandleDict"] = sourceTableHandleDict
602 if "finalizedSourceTableHandles" in inputs:
603 finalizedSourceTableHandles = inputs.pop("finalizedSourceTableHandles")
604 finalizedSourceTableHandleDict = {handle.dataId["visit"]: handle
605 for handle in finalizedSourceTableHandles}
606 inputs["finalizedSourceTableHandleDict"] = finalizedSourceTableHandleDict
607 else:
608 # Deprecated legacy version
609 # Filter out any visit catalog that is not coadd inputs
610 ccdInputs = inputs['exposure'].getInfo().getCoaddInputs().ccds
611 visitKey = ccdInputs.schema.find("visit").key
612 ccdKey = ccdInputs.schema.find("ccd").key
613 inputVisitIds = set()
614 ccdRecordsWcs = {}
615 for ccdRecord in ccdInputs:
616 visit = ccdRecord.get(visitKey)
617 ccd = ccdRecord.get(ccdKey)
618 inputVisitIds.add((visit, ccd))
619 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
620
621 inputCatalogsToKeep = []
622 inputCatalogWcsUpdate = []
623 for i, dataRef in enumerate(inputRefs.visitCatalogs):
624 key = (dataRef.dataId['visit'], dataRef.dataId['detector'])
625 if key in inputVisitIds:
626 inputCatalogsToKeep.append(inputs['visitCatalogs'][i])
627 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
628 inputs['visitCatalogs'] = inputCatalogsToKeep
629 inputs['wcsUpdates'] = inputCatalogWcsUpdate
630 inputs['ccdInputs'] = ccdInputs
631
632 outputs = self.run(**inputs)
633 # Strip HeavyFootprints to save space on disk
634 sources = outputs.outputSources
635 butlerQC.put(outputs, outputRefs)
636
637 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
638 butler=None, sourceTableHandleDict=None, finalizedSourceTableHandleDict=None):
639 """Run measurement algorithms on the input exposure, and optionally populate the
640 resulting catalog with extra information.
641
642 Parameters
643 ----------
644 exposure : `lsst.afw.exposure.Exposure`
645 The input exposure on which measurements are to be performed
647 A catalog built from the results of merged detections, or
648 deblender outputs.
649 skyInfo : `lsst.pipe.base.Struct`
650 A struct containing information about the position of the input exposure within
651 a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box
652 exposureId : `int` or `bytes`
653 packed unique number or bytes unique to the input exposure
655 Catalog containing information on the individual visits which went into making
656 the coadd.
657 sourceTableHandleDict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
658 Dict for sourceTable_visit handles (key is visit) for propagating flags.
659 These tables are derived from the ``CalibrateTask`` sources, and contain
660 astrometry and photometry flags, and optionally PSF flags.
661 finalizedSourceTableHandleDict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`], optional
662 Dict for finalized_src_table handles (key is visit) for propagating flags.
663 These tables are derived from ``FinalizeCalibrationTask`` and contain
664 PSF flags from the finalized PSF estimation.
665 visitCatalogs : list of `lsst.afw.table.SourceCatalogs`
666 A list of source catalogs corresponding to measurements made on the individual
667 visits which went into the input exposure. If None and butler is `None` then
668 the task cannot propagate visit flags to the output catalog.
669 Deprecated, to be removed with PropagateVisitFlagsTask.
670 wcsUpdates : list of `lsst.afw.geom.SkyWcs`
671 If visitCatalogs is not `None` this should be a list of wcs objects which correspond
672 to the input visits. Used to put all coordinates to common system. If `None` and
673 butler is `None` then the task cannot propagate visit flags to the output catalog.
674 Deprecated, to be removed with PropagateVisitFlagsTask.
675 butler : `None`
676 This was a Gen2 butler used to load visit catalogs.
677 No longer used and should not be set. Will be removed in the
678 future.
679
680 Returns
681 -------
682 results : `lsst.pipe.base.Struct`
683 Results of running measurement task. Will contain the catalog in the
684 sources attribute. Optionally will have results of matching to a
685 reference catalog in the matchResults attribute, and denormalized
686 matches in the denormMatches attribute.
687 """
688 if butler is not None:
689 warnings.warn("The 'butler' parameter is no longer used and can be safely removed.",
690 category=FutureWarning, stacklevel=2)
691 butler = None
692
693 self.measurement.run(sources, exposure, exposureId=exposureId)
694
695 if self.config.doApCorr:
696 self.applyApCorr.run(
697 catalog=sources,
698 apCorrMap=exposure.getInfo().getApCorrMap()
699 )
700
701 # TODO DM-11568: this contiguous check-and-copy could go away if we
702 # reserve enough space during SourceDetection and/or SourceDeblend.
703 # NOTE: sourceSelectors require contiguous catalogs, so ensure
704 # contiguity now, so views are preserved from here on.
705 if not sources.isContiguous():
706 sources = sources.copy(deep=True)
707
708 if self.config.doRunCatalogCalculation:
709 self.catalogCalculation.run(sources)
710
711 self.setPrimaryFlags.run(sources, skyMap=skyInfo.skyMap, tractInfo=skyInfo.tractInfo,
712 patchInfo=skyInfo.patchInfo)
713 if self.config.doPropagateFlags:
714 if self.config.propagateFlags.target == PropagateSourceFlagsTask:
715 # New version
716 self.propagateFlags.run(
717 sources,
718 ccdInputs,
719 sourceTableHandleDict,
720 finalizedSourceTableHandleDict
721 )
722 else:
723 # Legacy deprecated version
724 self.propagateFlags.run(
725 butler,
726 sources,
727 ccdInputs,
728 exposure.getWcs(),
729 visitCatalogs,
730 wcsUpdates
731 )
732
733 results = Struct()
734
735 if self.config.doMatchSources:
736 matchResult = self.match.run(sources, exposure.getInfo().getFilter().bandLabel)
737 matches = afwTable.packMatches(matchResult.matches)
738 matches.table.setMetadata(matchResult.matchMeta)
739 results.matchResult = matches
740 if self.config.doWriteMatchesDenormalized:
741 if matchResult.matches:
742 denormMatches = denormalizeMatches(matchResult.matches, matchResult.matchMeta)
743 else:
744 self.log.warning("No matches, so generating dummy denormalized matches file")
745 denormMatches = afwTable.BaseCatalog(afwTable.Schema())
746 denormMatches.setMetadata(PropertyList())
747 denormMatches.getMetadata().add("COMMENT",
748 "This catalog is empty because no matches were found.")
749 results.denormMatches = denormMatches
750 results.denormMatches = denormMatches
751
752 results.outputSources = sources
753 return results
754