24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25 PipelineTask, InitOutputDatasetField, InputDatasetField, OutputDatasetField,
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
42 from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
43 from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
44 from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
45 from .multiBandUtils
import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory
50 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 51 * deepCoadd_mergeDet: merged detections (tract, patch) 52 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 53 * deepCoadd_ref: reference sources (tract, patch) 54 All of these have associated *_schema catalogs that require no data ID and hold no records. 56 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 57 the mergeDet, meas, and ref dataset Footprints: 58 * deepCoadd_peak_schema 66 @anchor DetectCoaddSourcesConfig_ 68 @brief Configuration parameters for the DetectCoaddSourcesTask 70 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
71 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc=
"Variance rescaling")
72 detection = ConfigurableField(target=DynamicDetectionTask, doc=
"Source detection")
73 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
74 doInsertFakes = Field(dtype=bool, default=
False,
75 doc=
"Run fake sources injection task")
76 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
77 doc=
"Injection of fake sources for testing " 78 "purposes (must be retargeted)")
79 detectionSchema = InitOutputDatasetField(
80 doc=
"Schema of the detection catalog",
81 name=
"{}Coadd_det_schema",
82 storageClass=
"SourceCatalog",
84 exposure = InputDatasetField(
85 doc=
"Exposure on which detections are to be performed",
88 storageClass=
"Exposure",
89 units=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
91 outputBackgrounds = OutputDatasetField(
92 doc=
"Output Backgrounds used in detection",
93 name=
"{}Coadd_calexp_background",
95 storageClass=
"Background",
96 units=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
98 outputSources = OutputDatasetField(
99 doc=
"Detected sources catalog",
102 storageClass=
"SourceCatalog",
103 units=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
105 outputExposure = OutputDatasetField(
106 doc=
"Exposure post detection",
107 name=
"{}Coadd_calexp",
109 storageClass=
"Exposure",
110 units=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
112 quantum = QuantumConfig(
113 units=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
117 Config.setDefaults(self)
118 self.
detection.thresholdType =
"pixel_stdev" 121 self.
detection.reEstimateBackground =
False 122 self.
detection.background.useApprox =
False 124 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 125 self.
detection.doTempWideBackground =
True 137 @anchor DetectCoaddSourcesTask_ 139 @brief Detect sources on a coadd 141 @section pipe_tasks_multiBand_Contents Contents 143 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 144 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 145 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 146 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 147 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 148 - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 150 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 152 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 154 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 155 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 156 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 157 propagate the full covariance matrix -- but it is simple and works well in practice. 159 After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref 160 SourceDetectionTask_ "detection" subtask. 163 deepCoadd{tract,patch,filter}: ExposureF 165 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 166 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 168 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 172 DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. 173 You can retarget this subtask if you wish. 175 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 177 @copydoc \_\_init\_\_ 179 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 183 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 185 See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 187 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 189 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 190 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 193 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 194 @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 195 @ref SourceDetectionTask_ "SourceDetectionTask" for further information. 197 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example 198 of using DetectCoaddSourcesTask 200 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 201 the task is to update the background, detect all sources in a single band and generate a set of parent 202 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 203 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 204 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 205 calling detectCoaddSources.py with the `--help` command line argument: 207 detectCoaddSources.py --help 210 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 211 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 212 steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 214 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 216 that will process the HSC-I band data. The results are written to 217 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 219 It is also necessary to run: 221 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 223 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 224 processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". 226 _DefaultName =
"detectCoaddSources" 227 ConfigClass = DetectCoaddSourcesConfig
228 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
229 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
232 def _makeArgumentParser(cls):
234 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
235 ContainerClass=ExistingCoaddDataIdContainer)
240 coaddName = config.coaddName
241 for name
in (
"outputBackgrounds",
"outputSources",
"outputExposure"):
242 attr = getattr(config, name)
243 setattr(attr,
"name", attr.name.format(coaddName))
245 return outputTypeDict
249 coaddName = config.coaddName
250 attr = config.detectionSchema
251 setattr(attr,
"name", attr.name.format(coaddName))
258 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. 260 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 262 @param[in] schema: initial schema for the output catalog, modified-in place to include all 263 fields set by this task. If None, the source minimal schema will be used. 264 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 270 schema = afwTable.SourceTable.makeMinimalSchema()
271 if self.config.doInsertFakes:
272 self.makeSubtask(
"insertFakes")
274 self.makeSubtask(
"detection", schema=self.
schema)
275 if self.config.doScaleVariance:
276 self.makeSubtask(
"scaleVariance")
279 return {
"detectionSchema": afwTable.SourceCatalog(self.
schema)}
283 @brief Run detection on a coadd. 285 Invokes @ref run and then uses @ref write to output the 288 @param[in] patchRef: data reference for patch 290 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
291 expId = int(patchRef.get(self.config.coaddName +
"CoaddId"))
293 self.
write(results, patchRef)
298 inputData[
"idFactory"] = afwTable.IdFactory.makeSimple()
299 inputData[
"expId"] = 0
300 return self.
run(**inputData)
302 def run(self, exposure, idFactory, expId):
304 @brief Run detection on an exposure. 306 First scale the variance plane to match the observed variance 307 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to 310 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 311 depending on configuration). 312 @param[in] idFactory: IdFactory to set source identifiers 313 @param[in] expId: Exposure identifier (integer) for RNG seed 315 @return a pipe.base.Struct with fields 316 - sources: catalog of detections 317 - backgrounds: list of backgrounds 319 if self.config.doScaleVariance:
320 varScale = self.scaleVariance.
run(exposure.maskedImage)
321 exposure.getMetadata().add(
"variance_scale", varScale)
322 backgrounds = afwMath.BackgroundList()
323 if self.config.doInsertFakes:
324 self.insertFakes.
run(exposure, background=backgrounds)
325 table = afwTable.SourceTable.make(self.
schema, idFactory)
326 detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
327 sources = detections.sources
328 fpSets = detections.fpSets
329 if hasattr(fpSets,
"background")
and fpSets.background:
330 for bg
in fpSets.background:
331 backgrounds.append(bg)
332 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
336 @brief Write out results from runDetection. 338 @param[in] exposure: Exposure to write out 339 @param[in] results: Struct returned from runDetection 340 @param[in] patchRef: data reference for patch 342 coaddName = self.config.coaddName +
"Coadd" 343 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
344 patchRef.put(results.outputSources, coaddName +
"_det")
345 patchRef.put(results.outputExposure, coaddName +
"_calexp")
351 """DeblendCoaddSourcesConfig 353 Configuration parameters for the `DeblendCoaddSourcesTask`. 355 singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
356 doc=
"Deblend sources separately in each band")
357 multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
358 doc=
"Deblend sources simultaneously across bands")
359 simultaneous = Field(dtype=bool, default=
False, doc=
"Simultaneously deblend all bands?")
360 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
363 Config.setDefaults(self)
368 """Task runner for the `MergeSourcesTask` 370 Required because the run method requires a list of 371 dataRefs rather than a single dataRef. 375 """Provide a list of patch references for each patch, tract, filter combo. 382 Keyword arguments passed to the task 387 List of tuples, where each tuple is a (dataRef, kwargs) pair. 389 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
390 kwargs[
"psfCache"] = parsedCmd.psfCache
391 return [(list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
395 """Deblend the sources in a merged catalog 397 Deblend sources from master catalog in each coadd. 398 This can either be done separately in each band using the HSC-SDSS deblender 399 (`DeblendCoaddSourcesTask.config.simultaneous==False`) 400 or use SCARLET to simultaneously fit the blend in all bands 401 (`DeblendCoaddSourcesTask.config.simultaneous==True`). 402 The task will set its own `self.schema` atribute to the `Schema` of the 403 output deblended catalog. 404 This will include all fields from the input `Schema`, as well as additional fields 407 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description 408 --------------------------------------------------------- 414 Butler used to read the input schemas from disk or 415 construct the reference catalog loader, if `schema` or `peakSchema` or 417 The schema of the merged detection catalog as an input to this task. 419 The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog 421 ConfigClass = DeblendCoaddSourcesConfig
422 RunnerClass = DeblendCoaddSourcesRunner
423 _DefaultName =
"deblendCoaddSources" 424 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
427 def _makeArgumentParser(cls):
429 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
430 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
431 ContainerClass=ExistingCoaddDataIdContainer)
432 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
435 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
436 CmdLineTask.__init__(self, **kwargs)
438 assert butler
is not None,
"Neither butler nor schema is defined" 439 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
443 if peakSchema
is None:
444 assert butler
is not None,
"Neither butler nor peakSchema is defined" 445 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
447 if self.config.simultaneous:
448 self.makeSubtask(
"multiBandDeblend", schema=self.
schema, peakSchema=peakSchema)
450 self.makeSubtask(
"singleBandDeblend", schema=self.
schema, peakSchema=peakSchema)
453 """Return a dict of empty catalogs for each catalog dataset produced by this task. 458 Dictionary of empty catalogs, with catalog names as keys. 460 catalog = afwTable.SourceCatalog(self.
schema)
461 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
462 self.config.coaddName +
"Coadd_deblendedModel": catalog}
467 Deblend each source simultaneously or separately 468 (depending on `DeblendCoaddSourcesTask.config.simultaneous`). 469 Set `is-primary` and related flags. 470 Propagate flags from individual visits. 471 Write the deblended sources out. 476 List of data references for each filter 478 if self.config.simultaneous:
482 for patchRef
in patchRefList:
483 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
484 filters.append(patchRef.dataId[
"filter"])
485 exposures.append(exposure)
488 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
489 fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
490 for n
in range(len(patchRefList)):
491 self.
write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
494 for patchRef
in patchRefList:
495 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
496 exposure.getPsf().setCacheCapacity(psfCache)
498 self.singleBandDeblend.run(exposure, sources)
499 self.
write(patchRef, sources)
502 """Read merged catalog 504 Read the catalog of merged detections and create a catalog 509 dataRef: data reference 510 Data reference for catalog of merged detections 514 sources: `SourceCatalog` 515 List of sources in merged catalog 517 We also need to add columns to hold the measurements we're about to make 518 so we can measure in-place. 520 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
521 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
524 idFactory.notify(s.getId())
525 table = afwTable.SourceTable.make(self.
schema, idFactory)
526 sources = afwTable.SourceCatalog(table)
530 def write(self, dataRef, flux_sources, template_sources=None):
531 """Write the source catalog(s) 535 dataRef: Data Reference 536 Reference to the output catalog. 537 flux_sources: `SourceCatalog` 538 Flux conserved sources to write to file. 539 If using the single band deblender, this is the catalog 541 template_sources: `SourceCatalog` 542 Source catalog using the multiband template models 547 if flux_sources
is not None:
548 assert not self.config.simultaneous
or self.config.multiBandDeblend.conserveFlux
549 dataRef.put(flux_sources, self.config.coaddName +
"Coadd_deblendedFlux")
553 if template_sources
is not None:
554 assert self.config.multiBandDeblend.saveTemplates
555 dataRef.put(template_sources, self.config.coaddName +
"Coadd_deblendedModel")
556 self.log.info(
"Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
559 """Write the metadata produced from processing the data. 563 List of Butler data references used to write the metadata. 564 The metadata is written to dataset type `CmdLineTask._getMetadataName`. 566 for dataRef
in dataRefList:
568 metadataName = self._getMetadataName()
569 if metadataName
is not None:
570 dataRef.put(self.getFullMetadata(), metadataName)
571 except Exception
as e:
572 self.log.warn(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
575 """Get the ExposureId from a data reference 577 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
582 @anchor MeasureMergedCoaddSourcesConfig_ 584 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask 586 inputCatalog = Field(dtype=str, default=
"deblendedFlux",
587 doc=(
"Name of the input catalog to use." 588 "If the single band deblender was used this should be 'deblendedFlux." 589 "If the multi-band deblender was used this should be 'deblendedModel." 590 "If no deblending was performed this should be 'mergeDet'"))
591 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
592 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
593 doPropagateFlags = Field(
594 dtype=bool, default=
True,
595 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 597 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
598 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
599 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
600 doWriteMatchesDenormalized = Field(
603 doc=(
"Write reference matches in denormalized format? " 604 "This format uses more disk space, but is more convenient to read."),
606 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
607 checkUnitsParseStrict = Field(
608 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
615 doc=
"Apply aperture corrections" 617 applyApCorr = ConfigurableField(
618 target=ApplyApCorrTask,
619 doc=
"Subtask to apply aperture corrections" 621 doRunCatalogCalculation = Field(
624 doc=
'Run catalogCalculation task' 626 catalogCalculation = ConfigurableField(
627 target=CatalogCalculationTask,
628 doc=
"Subtask to run catalogCalculation plugins on catalog" 632 Config.setDefaults(self)
633 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
634 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
636 self.
measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
648 """Get the psfCache setting into MeasureMergedCoaddSourcesTask""" 651 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
656 @anchor MeasureMergedCoaddSourcesTask_ 658 @brief Deblend sources from master catalog in each coadd seperately and measure. 660 @section pipe_tasks_multiBand_Contents Contents 662 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 663 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 664 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 665 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 666 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 667 - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 669 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 671 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 672 measurement in each coadd. 674 Given a master input catalog of sources (peaks and footprints) or deblender outputs 675 (including a HeavyFootprint in each band), measure each source on the 676 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 677 consistent set of child sources. 679 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 680 properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit 681 flags are propagated to the coadd sources. 683 Optionally, we can match the coadd sources to an external reference catalog. 686 deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog 687 @n deepCoadd_calexp{tract,patch,filter}: ExposureF 689 deepCoadd_meas{tract,patch,filter}: SourceCatalog 693 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 696 <DT> @ref SingleFrameMeasurementTask_ "measurement" 697 <DD> Measure source properties of deblended sources.</DD> 698 <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags" 699 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 700 not at the edge of the field and that have either not been deblended or are the children of deblended 702 <DT> @ref PropagateVisitFlagsTask_ "propagateFlags" 703 <DD> Propagate flags set in individual visits to the coadd.</DD> 704 <DT> @ref DirectMatchTask_ "match" 705 <DD> Match input sources to a reference catalog (optional). 708 These subtasks may be retargeted as required. 710 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 712 @copydoc \_\_init\_\_ 714 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 718 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 720 See @ref MeasureMergedCoaddSourcesConfig_ 722 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 724 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 725 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 728 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 729 the various sub-tasks. See the documetation for individual sub-tasks for more information. 731 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using 732 MeasureMergedCoaddSourcesTask 734 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 735 The next stage in the multi-band processing procedure will merge these measurements into a suitable 736 catalog for driving forced photometry. 738 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds 740 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 741 `--help` command line argument: 743 measureCoaddSources.py --help 746 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 747 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 748 step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 751 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 753 This will process the HSC-I band data. The results are written in 754 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 756 It is also necessary to run 758 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 760 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 761 procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". 763 _DefaultName =
"measureCoaddSources" 764 ConfigClass = MeasureMergedCoaddSourcesConfig
765 RunnerClass = MeasureMergedCoaddSourcesRunner
766 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
767 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
770 def _makeArgumentParser(cls):
772 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
773 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
774 ContainerClass=ExistingCoaddDataIdContainer)
775 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
778 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
780 @brief Initialize the task. 782 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 783 @param[in] schema: the schema of the merged detection catalog used as input to this one 784 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 785 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 786 catalog. May be None if the loader can be constructed from the butler argument or all steps 787 requiring a reference catalog are disabled. 788 @param[in] butler: a butler used to read the input schemas from disk or construct the reference 789 catalog loader, if schema or peakSchema or refObjLoader is None 791 The task will set its own self.schema attribute to the schema of the output measurement catalog. 792 This will include all fields from the input schema, as well as additional fields for all the 795 CmdLineTask.__init__(self, **kwargs)
796 self.
deblended = self.config.inputCatalog.startswith(
"deblended")
799 assert butler
is not None,
"Neither butler nor schema is defined" 800 schema = butler.get(self.config.coaddName + self.
inputCatalog +
"_schema", immediate=
True).schema
805 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
806 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
807 if self.config.doMatchSources:
808 if refObjLoader
is None:
809 assert butler
is not None,
"Neither butler nor refObjLoader is defined" 810 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
811 if self.config.doPropagateFlags:
812 self.makeSubtask(
"propagateFlags", schema=self.
schema)
813 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
814 if self.config.doApCorr:
815 self.makeSubtask(
"applyApCorr", schema=self.
schema)
816 if self.config.doRunCatalogCalculation:
817 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
821 @brief Deblend and measure. 823 @param[in] patchRef: Patch reference. 825 Set 'is-primary' and related flags. Propagate flags 826 from individual visits. Optionally match the sources to a reference catalog and write the matches. 827 Finally, write the deblended sources and measurements out. 829 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
830 exposure.getPsf().setCacheCapacity(psfCache)
832 table = sources.getTable()
835 self.measurement.run(sources, exposure, exposureId=self.
getExposureId(patchRef))
837 if self.config.doApCorr:
838 self.applyApCorr.run(
840 apCorrMap=exposure.getInfo().getApCorrMap()
847 if not sources.isContiguous():
848 sources = sources.copy(deep=
True)
850 if self.config.doRunCatalogCalculation:
851 self.catalogCalculation.run(sources)
853 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
854 self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
856 if self.config.doPropagateFlags:
857 self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
859 if self.config.doMatchSources:
861 self.
write(patchRef, sources)
865 @brief Read input sources. 867 @param[in] dataRef: Data reference for catalog of merged detections 868 @return List of sources in merged catalog 870 We also need to add columns to hold the measurements we're about to make 871 so we can measure in-place. 873 merged = dataRef.get(self.config.coaddName + self.
inputCatalog, immediate=
True)
874 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
877 idFactory.notify(s.getId())
878 table = afwTable.SourceTable.make(self.
schema, idFactory)
879 sources = afwTable.SourceCatalog(table)
885 @brief Write matches of the sources to the astrometric reference catalog. 887 We use the Wcs in the exposure to match sources. 889 @param[in] dataRef: data reference 890 @param[in] exposure: exposure with Wcs 891 @param[in] sources: source catalog 893 result = self.match.run(sources, exposure.getInfo().getFilter().getName())
895 matches = afwTable.packMatches(result.matches)
896 matches.table.setMetadata(result.matchMeta)
897 dataRef.put(matches, self.config.coaddName +
"Coadd_measMatch")
898 if self.config.doWriteMatchesDenormalized:
899 denormMatches = denormalizeMatches(result.matches, result.matchMeta)
900 dataRef.put(denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
904 @brief Write the source catalog. 906 @param[in] dataRef: data reference 907 @param[in] sources: source catalog 909 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
910 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
913 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
def readSources(self, dataRef)
def getTargetList(parsedCmd, kwargs)
def runDataRef(self, patchRef)
Run detection on a coadd.
Configuration parameters for the DetectCoaddSourcesTask.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds)
def runDataRef(self, patchRefList, psfCache=100)
def __init__(self, schema=None, kwargs)
Initialize the task.
Deblend sources from master catalog in each coadd seperately and measure.
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
def getInitOutputDatasetTypes(cls, config)
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
def getInitOutputDatasets(self)
def getExposureId(self, dataRef)
def readSources(self, dataRef)
Read input sources.
def getSchemaCatalogs(self)
def write(self, results, patchRef)
Write out results from runDetection.
def getExposureId(self, dataRef)
def write(self, dataRef, flux_sources, template_sources=None)
def write(self, dataRef, sources)
Write the source catalog.
def getTargetList(parsedCmd, kwargs)
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Detect sources on a coadd.
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
def writeMetadata(self, dataRefList)
def getOutputDatasetTypes(cls, config)