24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25 PipelineTask, PipelineTaskConfig, InitInputDatasetField,
26 InitOutputDatasetField, InputDatasetField, OutputDatasetField,
30 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
43 from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
44 from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
45 from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
46 from .multiBandUtils
import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory
47 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleConfig
48 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleTask
49 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiConfig
50 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiTask
55 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 56 * deepCoadd_mergeDet: merged detections (tract, patch) 57 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 58 * deepCoadd_ref: reference sources (tract, patch) 59 All of these have associated *_schema catalogs that require no data ID and hold no records. 61 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 62 the mergeDet, meas, and ref dataset Footprints: 63 * deepCoadd_peak_schema 71 @anchor DetectCoaddSourcesConfig_ 73 @brief Configuration parameters for the DetectCoaddSourcesTask 75 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
76 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc=
"Variance rescaling")
77 detection = ConfigurableField(target=DynamicDetectionTask, doc=
"Source detection")
78 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
79 doInsertFakes = Field(dtype=bool, default=
False,
80 doc=
"Run fake sources injection task")
81 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
82 doc=
"Injection of fake sources for testing " 83 "purposes (must be retargeted)")
84 detectionSchema = InitOutputDatasetField(
85 doc=
"Schema of the detection catalog",
86 name=
"{}Coadd_det_schema",
87 storageClass=
"SourceCatalog",
89 exposure = InputDatasetField(
90 doc=
"Exposure on which detections are to be performed",
93 storageClass=
"ExposureF",
94 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
96 outputBackgrounds = OutputDatasetField(
97 doc=
"Output Backgrounds used in detection",
98 name=
"{}Coadd_calexp_background",
100 storageClass=
"Background",
101 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
103 outputSources = OutputDatasetField(
104 doc=
"Detected sources catalog",
107 storageClass=
"SourceCatalog",
108 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
110 outputExposure = OutputDatasetField(
111 doc=
"Exposure post detection",
112 name=
"{}Coadd_calexp",
114 storageClass=
"ExposureF",
115 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
117 quantum = QuantumConfig(
118 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
122 Config.setDefaults(self)
123 self.
detection.thresholdType =
"pixel_stdev" 126 self.
detection.reEstimateBackground =
False 127 self.
detection.background.useApprox =
False 129 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 130 self.
detection.doTempWideBackground =
True 142 @anchor DetectCoaddSourcesTask_ 144 @brief Detect sources on a coadd 146 @section pipe_tasks_multiBand_Contents Contents 148 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 149 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 150 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 151 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 152 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 153 - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 155 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 157 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 159 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 160 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 161 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 162 propagate the full covariance matrix -- but it is simple and works well in practice. 164 After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref 165 SourceDetectionTask_ "detection" subtask. 168 deepCoadd{tract,patch,filter}: ExposureF 170 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 171 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 173 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 177 DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. 178 You can retarget this subtask if you wish. 180 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 182 @copydoc \_\_init\_\_ 184 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 188 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 190 See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 192 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 194 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 195 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 198 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 199 @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 200 @ref SourceDetectionTask_ "SourceDetectionTask" for further information. 202 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example 203 of using DetectCoaddSourcesTask 205 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 206 the task is to update the background, detect all sources in a single band and generate a set of parent 207 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 208 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 209 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 210 calling detectCoaddSources.py with the `--help` command line argument: 212 detectCoaddSources.py --help 215 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 216 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 217 steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 219 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 221 that will process the HSC-I band data. The results are written to 222 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 224 It is also necessary to run: 226 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 228 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 229 processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". 231 _DefaultName =
"detectCoaddSources" 232 ConfigClass = DetectCoaddSourcesConfig
233 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
234 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
237 def _makeArgumentParser(cls):
239 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
240 ContainerClass=ExistingCoaddDataIdContainer)
245 coaddName = config.coaddName
246 for name
in (
"outputBackgrounds",
"outputSources",
"outputExposure"):
247 attr = getattr(config, name)
248 setattr(attr,
"name", attr.name.format(coaddName))
250 return outputTypeDict
254 coaddName = config.coaddName
255 attr = config.detectionSchema
256 setattr(attr,
"name", attr.name.format(coaddName))
262 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. 264 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 266 @param[in] schema: initial schema for the output catalog, modified-in place to include all 267 fields set by this task. If None, the source minimal schema will be used. 268 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 274 schema = afwTable.SourceTable.makeMinimalSchema()
275 if self.config.doInsertFakes:
276 self.makeSubtask(
"insertFakes")
278 self.makeSubtask(
"detection", schema=self.
schema)
279 if self.config.doScaleVariance:
280 self.makeSubtask(
"scaleVariance")
283 return {
"detectionSchema": afwTable.SourceCatalog(self.
schema)}
287 @brief Run detection on a coadd. 289 Invokes @ref run and then uses @ref write to output the 292 @param[in] patchRef: data reference for patch 294 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
295 expId = int(patchRef.get(self.config.coaddName +
"CoaddId"))
297 self.
write(results, patchRef)
302 inputData[
"idFactory"] = afwTable.IdFactory.makeSimple()
303 inputData[
"expId"] = 0
304 return self.
run(**inputData)
306 def run(self, exposure, idFactory, expId):
308 @brief Run detection on an exposure. 310 First scale the variance plane to match the observed variance 311 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to 314 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 315 depending on configuration). 316 @param[in] idFactory: IdFactory to set source identifiers 317 @param[in] expId: Exposure identifier (integer) for RNG seed 319 @return a pipe.base.Struct with fields 320 - sources: catalog of detections 321 - backgrounds: list of backgrounds 323 if self.config.doScaleVariance:
324 varScale = self.scaleVariance.
run(exposure.maskedImage)
325 exposure.getMetadata().add(
"variance_scale", varScale)
326 backgrounds = afwMath.BackgroundList()
327 if self.config.doInsertFakes:
328 self.insertFakes.
run(exposure, background=backgrounds)
329 table = afwTable.SourceTable.make(self.
schema, idFactory)
330 detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
331 sources = detections.sources
332 fpSets = detections.fpSets
333 if hasattr(fpSets,
"background")
and fpSets.background:
334 for bg
in fpSets.background:
335 backgrounds.append(bg)
336 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
340 @brief Write out results from runDetection. 342 @param[in] exposure: Exposure to write out 343 @param[in] results: Struct returned from runDetection 344 @param[in] patchRef: data reference for patch 346 coaddName = self.config.coaddName +
"Coadd" 347 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
348 patchRef.put(results.outputSources, coaddName +
"_det")
349 patchRef.put(results.outputExposure, coaddName +
"_calexp")
355 """DeblendCoaddSourcesConfig 357 Configuration parameters for the `DeblendCoaddSourcesTask`. 359 singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
360 doc=
"Deblend sources separately in each band")
361 multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
362 doc=
"Deblend sources simultaneously across bands")
363 simultaneous = Field(dtype=bool, default=
False, doc=
"Simultaneously deblend all bands?")
364 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
367 Config.setDefaults(self)
372 """Task runner for the `MergeSourcesTask` 374 Required because the run method requires a list of 375 dataRefs rather than a single dataRef. 379 """Provide a list of patch references for each patch, tract, filter combo. 386 Keyword arguments passed to the task 391 List of tuples, where each tuple is a (dataRef, kwargs) pair. 393 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
394 kwargs[
"psfCache"] = parsedCmd.psfCache
395 return [(list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
399 """Deblend the sources in a merged catalog 401 Deblend sources from master catalog in each coadd. 402 This can either be done separately in each band using the HSC-SDSS deblender 403 (`DeblendCoaddSourcesTask.config.simultaneous==False`) 404 or use SCARLET to simultaneously fit the blend in all bands 405 (`DeblendCoaddSourcesTask.config.simultaneous==True`). 406 The task will set its own `self.schema` atribute to the `Schema` of the 407 output deblended catalog. 408 This will include all fields from the input `Schema`, as well as additional fields 411 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description 412 --------------------------------------------------------- 418 Butler used to read the input schemas from disk or 419 construct the reference catalog loader, if `schema` or `peakSchema` or 421 The schema of the merged detection catalog as an input to this task. 423 The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog 425 ConfigClass = DeblendCoaddSourcesConfig
426 RunnerClass = DeblendCoaddSourcesRunner
427 _DefaultName =
"deblendCoaddSources" 428 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
431 def _makeArgumentParser(cls):
433 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
434 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
435 ContainerClass=ExistingCoaddDataIdContainer)
436 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
439 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
440 CmdLineTask.__init__(self, **kwargs)
442 assert butler
is not None,
"Neither butler nor schema is defined" 443 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
447 if peakSchema
is None:
448 assert butler
is not None,
"Neither butler nor peakSchema is defined" 449 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
451 if self.config.simultaneous:
452 self.makeSubtask(
"multiBandDeblend", schema=self.
schema, peakSchema=peakSchema)
454 self.makeSubtask(
"singleBandDeblend", schema=self.
schema, peakSchema=peakSchema)
457 """Return a dict of empty catalogs for each catalog dataset produced by this task. 462 Dictionary of empty catalogs, with catalog names as keys. 464 catalog = afwTable.SourceCatalog(self.
schema)
465 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
466 self.config.coaddName +
"Coadd_deblendedModel": catalog}
471 Deblend each source simultaneously or separately 472 (depending on `DeblendCoaddSourcesTask.config.simultaneous`). 473 Set `is-primary` and related flags. 474 Propagate flags from individual visits. 475 Write the deblended sources out. 480 List of data references for each filter 482 if self.config.simultaneous:
486 for patchRef
in patchRefList:
487 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
488 filters.append(patchRef.dataId[
"filter"])
489 exposures.append(exposure)
492 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
493 fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
494 for n
in range(len(patchRefList)):
495 self.
write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
498 for patchRef
in patchRefList:
499 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
500 exposure.getPsf().setCacheCapacity(psfCache)
502 self.singleBandDeblend.run(exposure, sources)
503 self.
write(patchRef, sources)
506 """Read merged catalog 508 Read the catalog of merged detections and create a catalog 513 dataRef: data reference 514 Data reference for catalog of merged detections 518 sources: `SourceCatalog` 519 List of sources in merged catalog 521 We also need to add columns to hold the measurements we're about to make 522 so we can measure in-place. 524 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
525 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
528 idFactory.notify(s.getId())
529 table = afwTable.SourceTable.make(self.
schema, idFactory)
530 sources = afwTable.SourceCatalog(table)
534 def write(self, dataRef, flux_sources, template_sources=None):
535 """Write the source catalog(s) 539 dataRef: Data Reference 540 Reference to the output catalog. 541 flux_sources: `SourceCatalog` 542 Flux conserved sources to write to file. 543 If using the single band deblender, this is the catalog 545 template_sources: `SourceCatalog` 546 Source catalog using the multiband template models 551 if flux_sources
is not None:
552 assert not self.config.simultaneous
or self.config.multiBandDeblend.conserveFlux
553 dataRef.put(flux_sources, self.config.coaddName +
"Coadd_deblendedFlux")
557 if template_sources
is not None:
558 assert self.config.multiBandDeblend.saveTemplates
559 dataRef.put(template_sources, self.config.coaddName +
"Coadd_deblendedModel")
560 self.log.info(
"Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
563 """Write the metadata produced from processing the data. 567 List of Butler data references used to write the metadata. 568 The metadata is written to dataset type `CmdLineTask._getMetadataName`. 570 for dataRef
in dataRefList:
572 metadataName = self._getMetadataName()
573 if metadataName
is not None:
574 dataRef.put(self.getFullMetadata(), metadataName)
575 except Exception
as e:
576 self.log.warn(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
579 """Get the ExposureId from a data reference 581 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
586 @anchor MeasureMergedCoaddSourcesConfig_ 588 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask 590 inputCatalog = Field(dtype=str, default=
"deblendedFlux",
591 doc=(
"Name of the input catalog to use." 592 "If the single band deblender was used this should be 'deblendedFlux." 593 "If the multi-band deblender was used this should be 'deblendedModel." 594 "If no deblending was performed this should be 'mergeDet'"))
595 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
596 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
597 doPropagateFlags = Field(
598 dtype=bool, default=
True,
599 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 601 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
602 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
603 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
604 doWriteMatchesDenormalized = Field(
607 doc=(
"Write reference matches in denormalized format? " 608 "This format uses more disk space, but is more convenient to read."),
610 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
611 psfCache = Field(dtype=int, default=100, doc=
"Size of psfCache")
612 checkUnitsParseStrict = Field(
613 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
620 doc=
"Apply aperture corrections" 622 applyApCorr = ConfigurableField(
623 target=ApplyApCorrTask,
624 doc=
"Subtask to apply aperture corrections" 626 doRunCatalogCalculation = Field(
629 doc=
'Run catalogCalculation task' 631 catalogCalculation = ConfigurableField(
632 target=CatalogCalculationTask,
633 doc=
"Subtask to run catalogCalculation plugins on catalog" 635 inputSchema = InitInputDatasetField(
636 doc=
"Input schema for measure merged task produced by a deblender or detection task",
637 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux_schema",
638 storageClass=
"SourceCatalog" 640 outputSchema = InitOutputDatasetField(
641 doc=
"Output schema after all new fields are added by task",
642 nameTemplate=
"{inputCoaddName}Coadd_meas_schema",
643 storageClass=
"SourceCatalog" 645 refCat = InputDatasetField(
646 doc=
"Reference catalog used to match measured sources against known sources",
648 storageClass=
"SimpleCatalog",
649 dimensions=(
"SkyPix",),
652 exposure = InputDatasetField(
653 doc=
"Input coadd image",
654 nameTemplate=
"{inputCoaddName}Coadd_calexp",
656 storageClass=
"ExposureF",
657 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
659 skyMap = InputDatasetField(
660 doc=
"SkyMap to use in processing",
661 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
662 storageClass=
"SkyMap",
663 dimensions=(
"SkyMap",),
666 visitCatalogs = InputDatasetField(
667 doc=
"Source catalogs for visits which overlap input tract, patch, abstract_filter. Will be " 668 "further filtered in the task for the purpose of propagating flags from image calibration " 669 "and characterization to codd objects",
671 dimensions=(
"Instrument",
"Visit",
"Detector"),
672 storageClass=
"SourceCatalog" 674 intakeCatalog = InputDatasetField(
675 doc=(
"Name of the input catalog to use." 676 "If the single band deblender was used this should be 'deblendedFlux." 677 "If the multi-band deblender was used this should be 'deblendedModel, " 678 "or deblendedFlux if the multiband deblender was configured to output " 679 "deblended flux catalogs. If no deblending was performed this should " 681 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux",
682 storageClass=
"SourceCatalog",
683 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
686 outputSources = OutputDatasetField(
687 doc=
"Source catalog containing all the measurement information generated in this task",
688 nameTemplate=
"{outputCoaddName}Coadd_meas",
689 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
690 storageClass=
"SourceCatalog",
693 matchResult = OutputDatasetField(
694 doc=
"Match catalog produced by configured matcher, optional on doMatchSources",
695 nameTemplate=
"{outputCoaddName}Coadd_measMatch",
696 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
697 storageClass=
"Catalog",
700 denormMatches = OutputDatasetField(
701 doc=
"Denormalized Match catalog produced by configured matcher, optional on " 702 "doWriteMatchesDenormalized",
703 nameTemplate=
"{outputCoaddName}Coadd_measMatchFull",
704 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
705 storageClass=
"Catalog",
711 return self.
match.refObjLoader
715 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
716 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
717 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
718 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
720 self.
measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
732 """Get the psfCache setting into MeasureMergedCoaddSourcesTask""" 735 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
740 @anchor MeasureMergedCoaddSourcesTask_ 742 @brief Deblend sources from master catalog in each coadd seperately and measure. 744 @section pipe_tasks_multiBand_Contents Contents 746 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 747 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 748 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 749 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 750 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 751 - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 753 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 755 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 756 measurement in each coadd. 758 Given a master input catalog of sources (peaks and footprints) or deblender outputs 759 (including a HeavyFootprint in each band), measure each source on the 760 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 761 consistent set of child sources. 763 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 764 properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit 765 flags are propagated to the coadd sources. 767 Optionally, we can match the coadd sources to an external reference catalog. 770 deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog 771 @n deepCoadd_calexp{tract,patch,filter}: ExposureF 773 deepCoadd_meas{tract,patch,filter}: SourceCatalog 777 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 780 <DT> @ref SingleFrameMeasurementTask_ "measurement" 781 <DD> Measure source properties of deblended sources.</DD> 782 <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags" 783 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 784 not at the edge of the field and that have either not been deblended or are the children of deblended 786 <DT> @ref PropagateVisitFlagsTask_ "propagateFlags" 787 <DD> Propagate flags set in individual visits to the coadd.</DD> 788 <DT> @ref DirectMatchTask_ "match" 789 <DD> Match input sources to a reference catalog (optional). 792 These subtasks may be retargeted as required. 794 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 796 @copydoc \_\_init\_\_ 798 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 802 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 804 See @ref MeasureMergedCoaddSourcesConfig_ 806 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 808 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 809 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 812 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 813 the various sub-tasks. See the documetation for individual sub-tasks for more information. 815 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using 816 MeasureMergedCoaddSourcesTask 818 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 819 The next stage in the multi-band processing procedure will merge these measurements into a suitable 820 catalog for driving forced photometry. 822 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds 824 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 825 `--help` command line argument: 827 measureCoaddSources.py --help 830 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 831 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 832 step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 835 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 837 This will process the HSC-I band data. The results are written in 838 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 840 It is also necessary to run 842 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 844 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 845 procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". 847 _DefaultName =
"measureCoaddSources" 848 ConfigClass = MeasureMergedCoaddSourcesConfig
849 RunnerClass = MeasureMergedCoaddSourcesRunner
850 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
851 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
854 def _makeArgumentParser(cls):
856 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
857 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
858 ContainerClass=ExistingCoaddDataIdContainer)
859 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
862 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
865 @brief Initialize the task. 867 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 868 @param[in] schema: the schema of the merged detection catalog used as input to this one 869 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 870 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 871 catalog. May be None if the loader can be constructed from the butler argument or all steps 872 requiring a reference catalog are disabled. 873 @param[in] butler: a butler used to read the input schemas from disk or construct the reference 874 catalog loader, if schema or peakSchema or refObjLoader is None 876 The task will set its own self.schema attribute to the schema of the output measurement catalog. 877 This will include all fields from the input schema, as well as additional fields for all the 881 self.
deblended = self.config.inputCatalog.startswith(
"deblended")
883 if initInputs
is not None:
884 schema = initInputs[
'inputSchema'].schema
886 assert butler
is not None,
"Neither butler nor schema is defined" 887 schema = butler.get(self.config.coaddName + self.
inputCatalog +
"_schema", immediate=
True).schema
892 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
893 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
894 if self.config.doMatchSources:
895 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
896 if self.config.doPropagateFlags:
897 self.makeSubtask(
"propagateFlags", schema=self.
schema)
898 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
899 if self.config.doApCorr:
900 self.makeSubtask(
"applyApCorr", schema=self.
schema)
901 if self.config.doRunCatalogCalculation:
902 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
907 if not config.doPropagateFlags:
908 inputDatasetTypes.pop(
"visitCatalogs")
909 return inputDatasetTypes
914 if config.doMatchSources
is False:
915 outputDatasetTypes.pop(
"matchResult")
916 if config.doWriteMatchesDenormalized
is False:
917 outputDatasetTypes.pop(
"denormMatches")
918 return outputDatasetTypes
921 return {
"outputSchema": afwTable.SourceCatalog(self.
schema)}
924 refObjLoader = ReferenceObjectLoader(inputDataIds[
'refCat'], butler,
925 config=self.config.refObjLoader, log=self.log)
926 self.match.setRefObjLoader(refObjLoader)
930 inputData[
'exposure'].getPsf().setCacheCapacity(self.config.psfCache)
933 idFactory = afwTable.IdFactory.makeSimple()
934 table = afwTable.SourceTable.make(self.
schema, idFactory)
935 sources = afwTable.SourceCatalog(table)
936 sources.extend(inputData.pop(
'intakeCatalog'), self.
schemaMapper)
937 table = sources.getTable()
939 inputData[
'sources'] = sources
941 inputData[
'exposureId'] = 0
943 skyMap = inputData.pop(
'skyMap')
944 tractNumber = inputDataIds[
'intakeCatalog'][
'tract']
945 tractInfo = skyMap[tractNumber]
946 patchInfo = tractInfo.getPatchInfo(inputDataIds[
'intakeCatalog'][
'patch'])
951 wcs=tractInfo.getWcs(),
952 bbox=patchInfo.getOuterBBox()
954 inputData[
'skyInfo'] = skyInfo
956 if self.config.doPropagateFlags:
958 ccdInputs = inputData[
'exposure'].getInfo().getCoaddInputs().ccds
959 visitKey = ccdInputs.schema.find(
"visit").key
960 ccdKey = ccdInputs.schema.find(
"ccd").key
961 inputVisitIds = set()
963 for ccdRecord
in ccdInputs:
964 visit = ccdRecord.get(visitKey)
965 ccd = ccdRecord.get(ccdKey)
966 inputVisitIds.add((visit, ccd))
967 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
969 inputCatalogsToKeep = []
970 inputCatalogWcsUpdate = []
971 for i, dataId
in enumerate(inputDataIds[
'visitCatalogs']):
972 key = (dataId[
'visit'], dataId[
'detector'])
973 if key
in inputVisitIds:
974 inputCatalogsToKeep.append(inputData[
'visitCatalogs'][i])
975 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
976 inputData[
'visitCatalogs'] = inputCatalogsToKeep
977 inputData[
'wcsUpdates'] = inputCatalogWcsUpdate
978 inputData[
'ccdInputs'] = ccdInputs
980 return self.
run(**inputData)
984 @brief Deblend and measure. 986 @param[in] patchRef: Patch reference. 988 Set 'is-primary' and related flags. Propagate flags 989 from individual visits. Optionally match the sources to a reference catalog and write the matches. 990 Finally, write the deblended sources and measurements out. 992 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
993 exposure.getPsf().setCacheCapacity(psfCache)
995 table = sources.getTable()
997 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
999 results = self.
run(exposure=exposure, sources=sources,
1000 ccdInputs=self.propagateFlags.getCcdInputs(exposure),
1001 skyInfo=skyInfo, butler=patchRef.getButler(),
1004 if self.config.doMatchSources:
1006 self.
write(patchRef, results.outputSources)
1008 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
1010 """Run measurement algorithms on the input exposure, and optionally populate the 1011 resulting catalog with extra information. 1015 exposure : `lsst.afw.exposure.Exposure` 1016 The input exposure on which measurements are to be performed 1017 sources : `lsst.afw.table.SourceCatalog` 1018 A catalog built from the results of merged detections, or 1020 skyInfo : `lsst.pipe.base.Struct` 1021 A struct containing information about the position of the input exposure within 1022 a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box 1023 exposureId : `int` or `bytes` 1024 packed unique number or bytes unique to the input exposure 1025 ccdInputs : `lsst.afw.table.ExposureCatalog` 1026 Catalog containing information on the individual visits which went into making 1028 visitCatalogs : list of `lsst.afw.table.SourceCatalogs` or `None` 1029 A list of source catalogs corresponding to measurements made on the individual 1030 visits which went into the input exposure. If None and butler is `None` then 1031 the task cannot propagate visit flags to the output catalog. 1032 wcsUpdates : list of `lsst.afw.geom.SkyWcs` or `None` 1033 If visitCatalogs is not `None` this should be a list of wcs objects which correspond 1034 to the input visits. Used to put all coordinates to common system. If `None` and 1035 butler is `None` then the task cannot propagate visit flags to the output catalog. 1036 butler : `lsst.daf.butler.Butler` or `lsst.daf.persistence.Butler` 1037 Either a gen2 or gen3 butler used to load visit catalogs 1041 results : `lsst.pipe.base.Struct` 1042 Results of running measurement task. Will contain the catalog in the 1043 sources attribute. Optionally will have results of matching to a 1044 reference catalog in the matchResults attribute, and denormalized 1045 matches in the denormMatches attribute. 1047 self.measurement.
run(sources, exposure, exposureId=exposureId)
1049 if self.config.doApCorr:
1050 self.applyApCorr.
run(
1052 apCorrMap=exposure.getInfo().getApCorrMap()
1059 if not sources.isContiguous():
1060 sources = sources.copy(deep=
True)
1062 if self.config.doRunCatalogCalculation:
1063 self.catalogCalculation.
run(sources)
1065 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1067 if self.config.doPropagateFlags:
1068 self.propagateFlags.
run(butler, sources, ccdInputs, exposure.getWcs(), visitCatalogs, wcsUpdates)
1072 if self.config.doMatchSources:
1073 matchResult = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1074 matches = afwTable.packMatches(matchResult.matches)
1075 matches.table.setMetadata(matchResult.matchMeta)
1076 results.matchResult = matches
1077 if self.config.doWriteMatchesDenormalized:
1078 results.denormMatches = denormalizeMatches(matchResult.matches,
1079 matchResult.matchMeta)
1081 results.outputSources = sources
1086 @brief Read input sources. 1088 @param[in] dataRef: Data reference for catalog of merged detections 1089 @return List of sources in merged catalog 1091 We also need to add columns to hold the measurements we're about to make 1092 so we can measure in-place. 1094 merged = dataRef.get(self.config.coaddName + self.
inputCatalog, immediate=
True)
1095 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1098 idFactory.notify(s.getId())
1099 table = afwTable.SourceTable.make(self.
schema, idFactory)
1100 sources = afwTable.SourceCatalog(table)
1106 @brief Write matches of the sources to the astrometric reference catalog. 1108 @param[in] dataRef: data reference 1109 @param[in] results: results struct from run method 1111 if hasattr(results,
"matchResult"):
1112 dataRef.put(results.matchResult, self.config.coaddName +
"Coadd_measMatch")
1113 if hasattr(results,
"denormMatches"):
1114 dataRef.put(results.denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1118 @brief Write the source catalog. 1120 @param[in] dataRef: data reference 1121 @param[in] sources: source catalog 1123 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1124 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1127 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
def readSources(self, dataRef)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def getTargetList(parsedCmd, kwargs)
def runDataRef(self, patchRef)
Run detection on a coadd.
Configuration parameters for the DetectCoaddSourcesTask.
def runDataRef(self, patchRefList, psfCache=100)
def __init__(self, schema=None, kwargs)
Initialize the task.
def getInitOutputDatasets(self)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Deblend sources from master catalog in each coadd seperately and measure.
def getInitOutputDatasetTypes(cls, config)
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
def getInitOutputDatasets(self)
def getOutputDatasetTypes(cls, config)
def writeMatches(self, dataRef, results)
Write matches of the sources to the astrometric reference catalog.
def getExposureId(self, dataRef)
def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, butler=None)
def readSources(self, dataRef)
Read input sources.
def getSchemaCatalogs(self)
def write(self, results, patchRef)
Write out results from runDetection.
def getInputDatasetTypes(cls, config)
def getExposureId(self, dataRef)
def write(self, dataRef, flux_sources, template_sources=None)
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, kwargs)
Initialize the task.
def write(self, dataRef, sources)
Write the source catalog.
def getTargetList(parsedCmd, kwargs)
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Detect sources on a coadd.
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
def writeMetadata(self, dataRefList)
def getOutputDatasetTypes(cls, config)