24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25 PipelineTask, PipelineTaskConfig, InitInputDatasetField,
26 InitOutputDatasetField, InputDatasetField, OutputDatasetField)
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
42 from .mergeDetections
import MergeDetectionsConfig, MergeDetectionsTask
43 from .mergeMeasurements
import MergeMeasurementsConfig, MergeMeasurementsTask
44 from .multiBandUtils
import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs
45 from .multiBandUtils
import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory
46 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleConfig
47 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesSingleTask
48 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiConfig
49 from .deblendCoaddSourcesPipeline
import DeblendCoaddSourcesMultiTask
54 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 55 * deepCoadd_mergeDet: merged detections (tract, patch) 56 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 57 * deepCoadd_ref: reference sources (tract, patch) 58 All of these have associated *_schema catalogs that require no data ID and hold no records. 60 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 61 the mergeDet, meas, and ref dataset Footprints: 62 * deepCoadd_peak_schema 70 @anchor DetectCoaddSourcesConfig_ 72 @brief Configuration parameters for the DetectCoaddSourcesTask 74 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
75 scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc=
"Variance rescaling")
76 detection = ConfigurableField(target=DynamicDetectionTask, doc=
"Source detection")
77 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
78 doInsertFakes = Field(dtype=bool, default=
False,
79 doc=
"Run fake sources injection task")
80 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
81 doc=
"Injection of fake sources for testing " 82 "purposes (must be retargeted)")
83 detectionSchema = InitOutputDatasetField(
84 doc=
"Schema of the detection catalog",
85 nameTemplate=
"{outputCoaddName}Coadd_det_schema",
86 storageClass=
"SourceCatalog",
88 exposure = InputDatasetField(
89 doc=
"Exposure on which detections are to be performed",
90 nameTemplate=
"{inputCoaddName}Coadd",
92 storageClass=
"ExposureF",
93 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
95 outputBackgrounds = OutputDatasetField(
96 doc=
"Output Backgrounds used in detection",
97 nameTemplate=
"{outputCoaddName}Coadd_calexp_background",
99 storageClass=
"Background",
100 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
102 outputSources = OutputDatasetField(
103 doc=
"Detected sources catalog",
104 nameTemplate=
"{outputCoaddName}Coadd_det",
106 storageClass=
"SourceCatalog",
107 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
109 outputExposure = OutputDatasetField(
110 doc=
"Exposure post detection",
111 nameTemplate=
"{outputCoaddName}Coadd_calexp",
113 storageClass=
"ExposureF",
114 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
119 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
120 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
121 self.
detection.thresholdType =
"pixel_stdev" 124 self.
detection.reEstimateBackground =
False 125 self.
detection.background.useApprox =
False 127 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 128 self.
detection.doTempWideBackground =
True 140 @anchor DetectCoaddSourcesTask_ 142 @brief Detect sources on a coadd 144 @section pipe_tasks_multiBand_Contents Contents 146 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 147 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 148 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 149 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 150 - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 151 - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 153 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 155 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 157 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 158 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 159 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 160 propagate the full covariance matrix -- but it is simple and works well in practice. 162 After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref 163 SourceDetectionTask_ "detection" subtask. 166 deepCoadd{tract,patch,filter}: ExposureF 168 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 169 @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 171 @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 175 DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask. 176 You can retarget this subtask if you wish. 178 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 180 @copydoc \_\_init\_\_ 182 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 186 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 188 See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 190 @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 192 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 193 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 196 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 197 @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 198 @ref SourceDetectionTask_ "SourceDetectionTask" for further information. 200 @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example 201 of using DetectCoaddSourcesTask 203 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 204 the task is to update the background, detect all sources in a single band and generate a set of parent 205 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 206 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 207 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 208 calling detectCoaddSources.py with the `--help` command line argument: 210 detectCoaddSources.py --help 213 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 214 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 215 steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 217 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 219 that will process the HSC-I band data. The results are written to 220 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 222 It is also necessary to run: 224 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 226 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 227 processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask". 229 _DefaultName =
"detectCoaddSources" 230 ConfigClass = DetectCoaddSourcesConfig
231 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
232 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
235 def _makeArgumentParser(cls):
237 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
238 ContainerClass=ExistingCoaddDataIdContainer)
243 @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask. 245 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 247 @param[in] schema: initial schema for the output catalog, modified-in place to include all 248 fields set by this task. If None, the source minimal schema will be used. 249 @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 255 schema = afwTable.SourceTable.makeMinimalSchema()
256 if self.config.doInsertFakes:
257 self.makeSubtask(
"insertFakes")
259 self.makeSubtask(
"detection", schema=self.
schema)
260 if self.config.doScaleVariance:
261 self.makeSubtask(
"scaleVariance")
264 return {
"detectionSchema": afwTable.SourceCatalog(self.
schema)}
268 @brief Run detection on a coadd. 270 Invokes @ref run and then uses @ref write to output the 273 @param[in] patchRef: data reference for patch 275 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
276 expId = int(patchRef.get(self.config.coaddName +
"CoaddId"))
278 self.
write(results, patchRef)
283 inputData[
"idFactory"] = afwTable.IdFactory.makeSimple()
284 inputData[
"expId"] = 0
285 return self.
run(**inputData)
287 def run(self, exposure, idFactory, expId):
289 @brief Run detection on an exposure. 291 First scale the variance plane to match the observed variance 292 using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to 295 @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 296 depending on configuration). 297 @param[in] idFactory: IdFactory to set source identifiers 298 @param[in] expId: Exposure identifier (integer) for RNG seed 300 @return a pipe.base.Struct with fields 301 - sources: catalog of detections 302 - backgrounds: list of backgrounds 304 if self.config.doScaleVariance:
305 varScale = self.scaleVariance.
run(exposure.maskedImage)
306 exposure.getMetadata().add(
"variance_scale", varScale)
307 backgrounds = afwMath.BackgroundList()
308 if self.config.doInsertFakes:
309 self.insertFakes.
run(exposure, background=backgrounds)
310 table = afwTable.SourceTable.make(self.
schema, idFactory)
311 detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
312 sources = detections.sources
313 fpSets = detections.fpSets
314 if hasattr(fpSets,
"background")
and fpSets.background:
315 for bg
in fpSets.background:
316 backgrounds.append(bg)
317 return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
321 @brief Write out results from runDetection. 323 @param[in] exposure: Exposure to write out 324 @param[in] results: Struct returned from runDetection 325 @param[in] patchRef: data reference for patch 327 coaddName = self.config.coaddName +
"Coadd" 328 patchRef.put(results.outputBackgrounds, coaddName +
"_calexp_background")
329 patchRef.put(results.outputSources, coaddName +
"_det")
330 patchRef.put(results.outputExposure, coaddName +
"_calexp")
336 """DeblendCoaddSourcesConfig 338 Configuration parameters for the `DeblendCoaddSourcesTask`. 340 singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
341 doc=
"Deblend sources separately in each band")
342 multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
343 doc=
"Deblend sources simultaneously across bands")
344 simultaneous = Field(dtype=bool, default=
False, doc=
"Simultaneously deblend all bands?")
345 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
348 Config.setDefaults(self)
353 """Task runner for the `MergeSourcesTask` 355 Required because the run method requires a list of 356 dataRefs rather than a single dataRef. 360 """Provide a list of patch references for each patch, tract, filter combo. 367 Keyword arguments passed to the task 372 List of tuples, where each tuple is a (dataRef, kwargs) pair. 374 refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
375 kwargs[
"psfCache"] = parsedCmd.psfCache
376 return [(list(p.values()), kwargs)
for t
in refDict.values()
for p
in t.values()]
380 """Deblend the sources in a merged catalog 382 Deblend sources from master catalog in each coadd. 383 This can either be done separately in each band using the HSC-SDSS deblender 384 (`DeblendCoaddSourcesTask.config.simultaneous==False`) 385 or use SCARLET to simultaneously fit the blend in all bands 386 (`DeblendCoaddSourcesTask.config.simultaneous==True`). 387 The task will set its own `self.schema` atribute to the `Schema` of the 388 output deblended catalog. 389 This will include all fields from the input `Schema`, as well as additional fields 392 `pipe.tasks.multiband.DeblendCoaddSourcesTask Description 393 --------------------------------------------------------- 399 Butler used to read the input schemas from disk or 400 construct the reference catalog loader, if `schema` or `peakSchema` or 402 The schema of the merged detection catalog as an input to this task. 404 The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog 406 ConfigClass = DeblendCoaddSourcesConfig
407 RunnerClass = DeblendCoaddSourcesRunner
408 _DefaultName =
"deblendCoaddSources" 409 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
412 def _makeArgumentParser(cls):
414 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
415 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
416 ContainerClass=ExistingCoaddDataIdContainer)
417 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
420 def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
421 CmdLineTask.__init__(self, **kwargs)
423 assert butler
is not None,
"Neither butler nor schema is defined" 424 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
428 if peakSchema
is None:
429 assert butler
is not None,
"Neither butler nor peakSchema is defined" 430 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
432 if self.config.simultaneous:
433 self.makeSubtask(
"multiBandDeblend", schema=self.
schema, peakSchema=peakSchema)
435 self.makeSubtask(
"singleBandDeblend", schema=self.
schema, peakSchema=peakSchema)
438 """Return a dict of empty catalogs for each catalog dataset produced by this task. 443 Dictionary of empty catalogs, with catalog names as keys. 445 catalog = afwTable.SourceCatalog(self.
schema)
446 return {self.config.coaddName +
"Coadd_deblendedFlux": catalog,
447 self.config.coaddName +
"Coadd_deblendedModel": catalog}
452 Deblend each source simultaneously or separately 453 (depending on `DeblendCoaddSourcesTask.config.simultaneous`). 454 Set `is-primary` and related flags. 455 Propagate flags from individual visits. 456 Write the deblended sources out. 461 List of data references for each filter 463 if self.config.simultaneous:
467 for patchRef
in patchRefList:
468 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
469 filters.append(patchRef.dataId[
"filter"])
470 exposures.append(exposure)
473 exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
474 fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
475 for n
in range(len(patchRefList)):
476 self.
write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
479 for patchRef
in patchRefList:
480 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
481 exposure.getPsf().setCacheCapacity(psfCache)
483 self.singleBandDeblend.run(exposure, sources)
484 self.
write(patchRef, sources)
487 """Read merged catalog 489 Read the catalog of merged detections and create a catalog 494 dataRef: data reference 495 Data reference for catalog of merged detections 499 sources: `SourceCatalog` 500 List of sources in merged catalog 502 We also need to add columns to hold the measurements we're about to make 503 so we can measure in-place. 505 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
506 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
509 idFactory.notify(s.getId())
510 table = afwTable.SourceTable.make(self.
schema, idFactory)
511 sources = afwTable.SourceCatalog(table)
515 def write(self, dataRef, flux_sources, template_sources=None):
516 """Write the source catalog(s) 520 dataRef: Data Reference 521 Reference to the output catalog. 522 flux_sources: `SourceCatalog` 523 Flux conserved sources to write to file. 524 If using the single band deblender, this is the catalog 526 template_sources: `SourceCatalog` 527 Source catalog using the multiband template models 532 if flux_sources
is not None:
533 assert not self.config.simultaneous
or self.config.multiBandDeblend.conserveFlux
534 dataRef.put(flux_sources, self.config.coaddName +
"Coadd_deblendedFlux")
538 if template_sources
is not None:
539 assert self.config.multiBandDeblend.saveTemplates
540 dataRef.put(template_sources, self.config.coaddName +
"Coadd_deblendedModel")
541 self.log.info(
"Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
544 """Write the metadata produced from processing the data. 548 List of Butler data references used to write the metadata. 549 The metadata is written to dataset type `CmdLineTask._getMetadataName`. 551 for dataRef
in dataRefList:
553 metadataName = self._getMetadataName()
554 if metadataName
is not None:
555 dataRef.put(self.getFullMetadata(), metadataName)
556 except Exception
as e:
557 self.log.warn(
"Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
560 """Get the ExposureId from a data reference 562 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
567 @anchor MeasureMergedCoaddSourcesConfig_ 569 @brief Configuration parameters for the MeasureMergedCoaddSourcesTask 571 inputCatalog = Field(dtype=str, default=
"deblendedFlux",
572 doc=(
"Name of the input catalog to use." 573 "If the single band deblender was used this should be 'deblendedFlux." 574 "If the multi-band deblender was used this should be 'deblendedModel." 575 "If no deblending was performed this should be 'mergeDet'"))
576 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
577 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
578 doPropagateFlags = Field(
579 dtype=bool, default=
True,
580 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 582 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
583 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
584 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
585 doWriteMatchesDenormalized = Field(
588 doc=(
"Write reference matches in denormalized format? " 589 "This format uses more disk space, but is more convenient to read."),
591 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
592 psfCache = Field(dtype=int, default=100, doc=
"Size of psfCache")
593 checkUnitsParseStrict = Field(
594 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
601 doc=
"Apply aperture corrections" 603 applyApCorr = ConfigurableField(
604 target=ApplyApCorrTask,
605 doc=
"Subtask to apply aperture corrections" 607 doRunCatalogCalculation = Field(
610 doc=
'Run catalogCalculation task' 612 catalogCalculation = ConfigurableField(
613 target=CatalogCalculationTask,
614 doc=
"Subtask to run catalogCalculation plugins on catalog" 616 inputSchema = InitInputDatasetField(
617 doc=
"Input schema for measure merged task produced by a deblender or detection task",
618 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux_schema",
619 storageClass=
"SourceCatalog" 621 outputSchema = InitOutputDatasetField(
622 doc=
"Output schema after all new fields are added by task",
623 nameTemplate=
"{inputCoaddName}Coadd_meas_schema",
624 storageClass=
"SourceCatalog" 626 refCat = InputDatasetField(
627 doc=
"Reference catalog used to match measured sources against known sources",
629 storageClass=
"SimpleCatalog",
630 dimensions=(
"SkyPix",),
633 exposure = InputDatasetField(
634 doc=
"Input coadd image",
635 nameTemplate=
"{inputCoaddName}Coadd_calexp",
637 storageClass=
"ExposureF",
638 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
640 skyMap = InputDatasetField(
641 doc=
"SkyMap to use in processing",
642 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
643 storageClass=
"SkyMap",
644 dimensions=(
"SkyMap",),
647 visitCatalogs = InputDatasetField(
648 doc=
"Source catalogs for visits which overlap input tract, patch, abstract_filter. Will be " 649 "further filtered in the task for the purpose of propagating flags from image calibration " 650 "and characterization to codd objects",
652 dimensions=(
"Instrument",
"Visit",
"Detector"),
653 storageClass=
"SourceCatalog" 655 intakeCatalog = InputDatasetField(
656 doc=(
"Name of the input catalog to use." 657 "If the single band deblender was used this should be 'deblendedFlux." 658 "If the multi-band deblender was used this should be 'deblendedModel, " 659 "or deblendedFlux if the multiband deblender was configured to output " 660 "deblended flux catalogs. If no deblending was performed this should " 662 nameTemplate=
"{inputCoaddName}Coadd_deblendedFlux",
663 storageClass=
"SourceCatalog",
664 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
667 outputSources = OutputDatasetField(
668 doc=
"Source catalog containing all the measurement information generated in this task",
669 nameTemplate=
"{outputCoaddName}Coadd_meas",
670 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
671 storageClass=
"SourceCatalog",
674 matchResult = OutputDatasetField(
675 doc=
"Match catalog produced by configured matcher, optional on doMatchSources",
676 nameTemplate=
"{outputCoaddName}Coadd_measMatch",
677 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
678 storageClass=
"Catalog",
681 denormMatches = OutputDatasetField(
682 doc=
"Denormalized Match catalog produced by configured matcher, optional on " 683 "doWriteMatchesDenormalized",
684 nameTemplate=
"{outputCoaddName}Coadd_measMatchFull",
685 dimensions=(
"Tract",
"Patch",
"AbstractFilter",
"SkyMap"),
686 storageClass=
"Catalog",
692 return self.
match.refObjLoader
696 self.formatTemplateNames({
"inputCoaddName":
"deep",
"outputCoaddName":
"deep"})
697 self.quantum.dimensions = (
"Tract",
"Patch",
"AbstractFilter",
"SkyMap")
698 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
699 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
701 self.
measurement.plugins[
'base_PixelFlags'].masksFpCenter = [
'CLIPPED',
'SENSOR_EDGE',
713 """Get the psfCache setting into MeasureMergedCoaddSourcesTask""" 716 return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
721 @anchor MeasureMergedCoaddSourcesTask_ 723 @brief Deblend sources from master catalog in each coadd seperately and measure. 725 @section pipe_tasks_multiBand_Contents Contents 727 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 728 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 729 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 730 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 731 - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 732 - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 734 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 736 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 737 measurement in each coadd. 739 Given a master input catalog of sources (peaks and footprints) or deblender outputs 740 (including a HeavyFootprint in each band), measure each source on the 741 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 742 consistent set of child sources. 744 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 745 properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit 746 flags are propagated to the coadd sources. 748 Optionally, we can match the coadd sources to an external reference catalog. 751 deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog 752 @n deepCoadd_calexp{tract,patch,filter}: ExposureF 754 deepCoadd_meas{tract,patch,filter}: SourceCatalog 758 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 761 <DT> @ref SingleFrameMeasurementTask_ "measurement" 762 <DD> Measure source properties of deblended sources.</DD> 763 <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags" 764 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 765 not at the edge of the field and that have either not been deblended or are the children of deblended 767 <DT> @ref PropagateVisitFlagsTask_ "propagateFlags" 768 <DD> Propagate flags set in individual visits to the coadd.</DD> 769 <DT> @ref DirectMatchTask_ "match" 770 <DD> Match input sources to a reference catalog (optional). 773 These subtasks may be retargeted as required. 775 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 777 @copydoc \_\_init\_\_ 779 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 783 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 785 See @ref MeasureMergedCoaddSourcesConfig_ 787 @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 789 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 790 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 793 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 794 the various sub-tasks. See the documetation for individual sub-tasks for more information. 796 @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using 797 MeasureMergedCoaddSourcesTask 799 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 800 The next stage in the multi-band processing procedure will merge these measurements into a suitable 801 catalog for driving forced photometry. 803 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds 805 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 806 `--help` command line argument: 808 measureCoaddSources.py --help 811 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 812 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 813 step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 816 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 818 This will process the HSC-I band data. The results are written in 819 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 821 It is also necessary to run 823 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 825 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 826 procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask". 828 _DefaultName =
"measureCoaddSources" 829 ConfigClass = MeasureMergedCoaddSourcesConfig
830 RunnerClass = MeasureMergedCoaddSourcesRunner
831 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
832 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
835 def _makeArgumentParser(cls):
837 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
838 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
839 ContainerClass=ExistingCoaddDataIdContainer)
840 parser.add_argument(
"--psfCache", type=int, default=100, help=
"Size of CoaddPsf cache")
843 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
846 @brief Initialize the task. 848 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 849 @param[in] schema: the schema of the merged detection catalog used as input to this one 850 @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 851 @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 852 catalog. May be None if the loader can be constructed from the butler argument or all steps 853 requiring a reference catalog are disabled. 854 @param[in] butler: a butler used to read the input schemas from disk or construct the reference 855 catalog loader, if schema or peakSchema or refObjLoader is None 857 The task will set its own self.schema attribute to the schema of the output measurement catalog. 858 This will include all fields from the input schema, as well as additional fields for all the 862 self.
deblended = self.config.inputCatalog.startswith(
"deblended")
864 if initInputs
is not None:
865 schema = initInputs[
'inputSchema'].schema
867 assert butler
is not None,
"Neither butler nor schema is defined" 868 schema = butler.get(self.config.coaddName + self.
inputCatalog +
"_schema", immediate=
True).schema
873 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
874 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
875 if self.config.doMatchSources:
876 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
877 if self.config.doPropagateFlags:
878 self.makeSubtask(
"propagateFlags", schema=self.
schema)
879 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
880 if self.config.doApCorr:
881 self.makeSubtask(
"applyApCorr", schema=self.
schema)
882 if self.config.doRunCatalogCalculation:
883 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
888 if not config.doPropagateFlags:
889 inputDatasetTypes.pop(
"visitCatalogs")
890 return inputDatasetTypes
895 if config.doMatchSources
is False:
896 outputDatasetTypes.pop(
"matchResult")
897 if config.doWriteMatchesDenormalized
is False:
898 outputDatasetTypes.pop(
"denormMatches")
899 return outputDatasetTypes
902 return {
"outputSchema": afwTable.SourceCatalog(self.
schema)}
905 refObjLoader = ReferenceObjectLoader(inputDataIds[
'refCat'], butler,
906 config=self.config.refObjLoader, log=self.log)
907 self.match.setRefObjLoader(refObjLoader)
911 inputData[
'exposure'].getPsf().setCacheCapacity(self.config.psfCache)
914 idFactory = afwTable.IdFactory.makeSimple()
915 table = afwTable.SourceTable.make(self.
schema, idFactory)
916 sources = afwTable.SourceCatalog(table)
917 sources.extend(inputData.pop(
'intakeCatalog'), self.
schemaMapper)
918 table = sources.getTable()
920 inputData[
'sources'] = sources
922 inputData[
'exposureId'] = 0
924 skyMap = inputData.pop(
'skyMap')
925 tractNumber = inputDataIds[
'intakeCatalog'][
'tract']
926 tractInfo = skyMap[tractNumber]
927 patchInfo = tractInfo.getPatchInfo(inputDataIds[
'intakeCatalog'][
'patch'])
932 wcs=tractInfo.getWcs(),
933 bbox=patchInfo.getOuterBBox()
935 inputData[
'skyInfo'] = skyInfo
937 if self.config.doPropagateFlags:
939 ccdInputs = inputData[
'exposure'].getInfo().getCoaddInputs().ccds
940 visitKey = ccdInputs.schema.find(
"visit").key
941 ccdKey = ccdInputs.schema.find(
"ccd").key
942 inputVisitIds = set()
944 for ccdRecord
in ccdInputs:
945 visit = ccdRecord.get(visitKey)
946 ccd = ccdRecord.get(ccdKey)
947 inputVisitIds.add((visit, ccd))
948 ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
950 inputCatalogsToKeep = []
951 inputCatalogWcsUpdate = []
952 for i, dataId
in enumerate(inputDataIds[
'visitCatalogs']):
953 key = (dataId[
'visit'], dataId[
'detector'])
954 if key
in inputVisitIds:
955 inputCatalogsToKeep.append(inputData[
'visitCatalogs'][i])
956 inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
957 inputData[
'visitCatalogs'] = inputCatalogsToKeep
958 inputData[
'wcsUpdates'] = inputCatalogWcsUpdate
959 inputData[
'ccdInputs'] = ccdInputs
961 return self.
run(**inputData)
965 @brief Deblend and measure. 967 @param[in] patchRef: Patch reference. 969 Set 'is-primary' and related flags. Propagate flags 970 from individual visits. Optionally match the sources to a reference catalog and write the matches. 971 Finally, write the deblended sources and measurements out. 973 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
974 exposure.getPsf().setCacheCapacity(psfCache)
976 table = sources.getTable()
978 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
980 results = self.
run(exposure=exposure, sources=sources,
981 ccdInputs=self.propagateFlags.getCcdInputs(exposure),
982 skyInfo=skyInfo, butler=patchRef.getButler(),
985 if self.config.doMatchSources:
987 self.
write(patchRef, results.outputSources)
989 def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
991 """Run measurement algorithms on the input exposure, and optionally populate the 992 resulting catalog with extra information. 996 exposure : `lsst.afw.exposure.Exposure` 997 The input exposure on which measurements are to be performed 998 sources : `lsst.afw.table.SourceCatalog` 999 A catalog built from the results of merged detections, or 1001 skyInfo : `lsst.pipe.base.Struct` 1002 A struct containing information about the position of the input exposure within 1003 a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box 1004 exposureId : `int` or `bytes` 1005 packed unique number or bytes unique to the input exposure 1006 ccdInputs : `lsst.afw.table.ExposureCatalog` 1007 Catalog containing information on the individual visits which went into making 1009 visitCatalogs : list of `lsst.afw.table.SourceCatalogs` or `None` 1010 A list of source catalogs corresponding to measurements made on the individual 1011 visits which went into the input exposure. If None and butler is `None` then 1012 the task cannot propagate visit flags to the output catalog. 1013 wcsUpdates : list of `lsst.afw.geom.SkyWcs` or `None` 1014 If visitCatalogs is not `None` this should be a list of wcs objects which correspond 1015 to the input visits. Used to put all coordinates to common system. If `None` and 1016 butler is `None` then the task cannot propagate visit flags to the output catalog. 1017 butler : `lsst.daf.butler.Butler` or `lsst.daf.persistence.Butler` 1018 Either a gen2 or gen3 butler used to load visit catalogs 1022 results : `lsst.pipe.base.Struct` 1023 Results of running measurement task. Will contain the catalog in the 1024 sources attribute. Optionally will have results of matching to a 1025 reference catalog in the matchResults attribute, and denormalized 1026 matches in the denormMatches attribute. 1028 self.measurement.
run(sources, exposure, exposureId=exposureId)
1030 if self.config.doApCorr:
1031 self.applyApCorr.
run(
1033 apCorrMap=exposure.getInfo().getApCorrMap()
1040 if not sources.isContiguous():
1041 sources = sources.copy(deep=
True)
1043 if self.config.doRunCatalogCalculation:
1044 self.catalogCalculation.
run(sources)
1046 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1048 if self.config.doPropagateFlags:
1049 self.propagateFlags.
run(butler, sources, ccdInputs, exposure.getWcs(), visitCatalogs, wcsUpdates)
1053 if self.config.doMatchSources:
1054 matchResult = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1055 matches = afwTable.packMatches(matchResult.matches)
1056 matches.table.setMetadata(matchResult.matchMeta)
1057 results.matchResult = matches
1058 if self.config.doWriteMatchesDenormalized:
1059 results.denormMatches = denormalizeMatches(matchResult.matches,
1060 matchResult.matchMeta)
1062 results.outputSources = sources
1067 @brief Read input sources. 1069 @param[in] dataRef: Data reference for catalog of merged detections 1070 @return List of sources in merged catalog 1072 We also need to add columns to hold the measurements we're about to make 1073 so we can measure in-place. 1075 merged = dataRef.get(self.config.coaddName + self.
inputCatalog, immediate=
True)
1076 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1079 idFactory.notify(s.getId())
1080 table = afwTable.SourceTable.make(self.
schema, idFactory)
1081 sources = afwTable.SourceCatalog(table)
1087 @brief Write matches of the sources to the astrometric reference catalog. 1089 @param[in] dataRef: data reference 1090 @param[in] results: results struct from run method 1092 if hasattr(results,
"matchResult"):
1093 dataRef.put(results.matchResult, self.config.coaddName +
"Coadd_measMatch")
1094 if hasattr(results,
"denormMatches"):
1095 dataRef.put(results.denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1099 @brief Write the source catalog. 1101 @param[in] dataRef: data reference 1102 @param[in] sources: source catalog 1104 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1105 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1108 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
def readSources(self, dataRef)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def getTargetList(parsedCmd, kwargs)
def runDataRef(self, patchRef)
Run detection on a coadd.
Configuration parameters for the DetectCoaddSourcesTask.
def runDataRef(self, patchRefList, psfCache=100)
def __init__(self, schema=None, kwargs)
Initialize the task.
def getInitOutputDatasets(self)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Deblend sources from master catalog in each coadd seperately and measure.
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
def getInitOutputDatasets(self)
def getOutputDatasetTypes(cls, config)
def writeMatches(self, dataRef, results)
Write matches of the sources to the astrometric reference catalog.
def getExposureId(self, dataRef)
def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, butler=None)
def readSources(self, dataRef)
Read input sources.
def getSchemaCatalogs(self)
def write(self, results, patchRef)
Write out results from runDetection.
def getInputDatasetTypes(cls, config)
def getExposureId(self, dataRef)
def write(self, dataRef, flux_sources, template_sources=None)
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, kwargs)
Initialize the task.
def write(self, dataRef, sources)
Write the source catalog.
def getTargetList(parsedCmd, kwargs)
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Detect sources on a coadd.
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
def writeMetadata(self, dataRefList)