23 from __future__
import absolute_import, division, print_function
24 from builtins
import zip
25 from builtins
import range
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 49 * deepCoadd_mergeDet: merged detections (tract, patch) 50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 51 * deepCoadd_ref: reference sources (tract, patch) 52 All of these have associated *_schema catalogs that require no data ID and hold no records. 54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 55 the mergeDet, meas, and ref dataset Footprints: 56 * deepCoadd_peak_schema 60 def _makeGetSchemaCatalogs(datasetSuffix):
61 """Construct a getSchemaCatalogs instance method 63 These are identical for most of the classes here, so we'll consolidate 66 datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src" 69 def getSchemaCatalogs(self):
70 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 71 src = afwTable.SourceCatalog(self.schema)
72 if hasattr(self,
"algMetadata"):
73 src.getTable().setMetadata(self.algMetadata)
74 return {self.config.coaddName +
"Coadd_" + datasetSuffix: src}
75 return getSchemaCatalogs
78 def _makeMakeIdFactory(datasetName):
79 """Construct a makeIdFactory instance method 81 These are identical for all the classes here, so this consolidates 84 datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId" 87 def makeIdFactory(self, dataRef):
88 """Return an IdFactory for setting the detection identifiers 90 The actual parameters used in the IdFactory are provided by 91 the butler (through the provided data reference. 93 expBits = dataRef.get(self.config.coaddName + datasetName +
"_bits")
94 expId = int(dataRef.get(self.config.coaddName + datasetName))
95 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
100 """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i"). 104 return afwImage.Filter(name).getFilterProperty().getName()
111 \anchor DetectCoaddSourcesConfig_ 113 \brief Configuration parameters for the DetectCoaddSourcesTask 115 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
116 detection = ConfigurableField(target=SourceDetectionTask, doc=
"Source detection")
117 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
118 mask = ListField(dtype=str, default=[
"DETECTED",
"BAD",
"SAT",
"NO_DATA",
"INTRP"],
119 doc=
"Mask planes for pixels to ignore when scaling variance")
120 doInsertFakes = Field(dtype=bool, default=
False,
121 doc=
"Run fake sources injection task")
122 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123 doc=
"Injection of fake sources for testing " 124 "purposes (must be retargeted)")
127 Config.setDefaults(self)
128 self.
detection.thresholdType =
"pixel_stdev" 131 self.
detection.background.useApprox =
False 133 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 145 \anchor DetectCoaddSourcesTask_ 147 \brief Detect sources on a coadd 149 \section pipe_tasks_multiBand_Contents Contents 151 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 152 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 153 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 154 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 155 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 156 - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 158 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 160 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 162 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 163 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 164 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 165 propagate the full covariance matrix -- but it is simple and works well in practice. 167 After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref 168 SourceDetectionTask_ "detection" subtask. 171 deepCoadd{tract,patch,filter}: ExposureF 173 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 174 \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 176 \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 180 DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask. 181 You can retarget this subtask if you wish. 183 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 185 \copydoc \_\_init\_\_ 187 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 191 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 193 See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 195 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 197 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 198 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 201 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 202 \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 203 \ref SourceDetectionTask_ "SourceDetectionTask" for further information. 205 \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask 207 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 208 the task is to update the background, detect all sources in a single band and generate a set of parent 209 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 210 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 211 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 212 calling detectCoaddSources.py with the `--help` command line argument: 214 detectCoaddSources.py --help 217 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 218 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 219 steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 221 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 223 that will process the HSC-I band data. The results are written to 224 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 226 It is also necessary to run: 228 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 230 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 231 processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask". 233 _DefaultName =
"detectCoaddSources" 234 ConfigClass = DetectCoaddSourcesConfig
235 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
236 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
239 def _makeArgumentParser(cls):
241 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242 ContainerClass=ExistingCoaddDataIdContainer)
247 \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask. 249 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 251 \param[in] schema: initial schema for the output catalog, modified-in place to include all 252 fields set by this task. If None, the source minimal schema will be used. 253 \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 255 CmdLineTask.__init__(self, **kwargs)
257 schema = afwTable.SourceTable.makeMinimalSchema()
258 if self.config.doInsertFakes:
259 self.makeSubtask(
"insertFakes")
261 self.makeSubtask(
"detection", schema=self.
schema)
265 \brief Run detection on a coadd. 267 Invokes \ref runDetection and then uses \ref write to output the 270 \param[in] patchRef: data reference for patch 272 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
274 self.
write(exposure, results, patchRef)
279 \brief Run detection on an exposure. 281 First scale the variance plane to match the observed variance 282 using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to 285 \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled, 286 depending on configuration). 287 \param[in] idFactory: IdFactory to set source identifiers 289 \return a pipe.base.Struct with fields 290 - sources: catalog of detections 291 - backgrounds: list of backgrounds 293 if self.config.doScaleVariance:
294 varScale =
scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
295 exposure.getMetadata().add(
"variance_scale", varScale)
296 backgrounds = afwMath.BackgroundList()
297 if self.config.doInsertFakes:
298 self.insertFakes.
run(exposure, background=backgrounds)
299 table = afwTable.SourceTable.make(self.
schema, idFactory)
300 detections = self.detection.makeSourceCatalog(table, exposure)
301 sources = detections.sources
302 fpSets = detections.fpSets
303 if fpSets.background:
304 backgrounds.append(fpSets.background)
305 return Struct(sources=sources, backgrounds=backgrounds)
307 def write(self, exposure, results, patchRef):
309 \brief Write out results from runDetection. 311 \param[in] exposure: Exposure to write out 312 \param[in] results: Struct returned from runDetection 313 \param[in] patchRef: data reference for patch 315 coaddName = self.config.coaddName +
"Coadd" 316 patchRef.put(results.backgrounds, coaddName +
"_calexp_background")
317 patchRef.put(results.sources, coaddName +
"_det")
318 patchRef.put(exposure, coaddName +
"_calexp")
325 \anchor MergeSourcesRunner_ 327 \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method 328 requires a list of dataRefs rather than a single dataRef. 333 \brief Provide a butler to the Task constructor. 335 \param[in] parsedCmd the parsed command 336 \param[in] args tuple of a list of data references and kwargs (un-used) 337 \throws RuntimeError if both parsedCmd & args are None 339 if parsedCmd
is not None:
340 butler = parsedCmd.butler
341 elif args
is not None:
342 dataRefList, kwargs = args
343 butler = dataRefList[0].getButler()
345 raise RuntimeError(
"Neither parsedCmd or args specified")
346 return self.TaskClass(config=self.config, log=self.log, butler=butler)
351 \brief Provide a list of patch references for each patch. 353 The patch references within the list will have different filters. 355 \param[in] parsedCmd the parsed command 356 \param **kwargs key word arguments (unused) 357 \throws RuntimeError if multiple references are provided for the same combination of tract, patch and 361 for ref
in parsedCmd.id.refList:
362 tract = ref.dataId[
"tract"]
363 patch = ref.dataId[
"patch"]
364 filter = ref.dataId[
"filter"]
365 if not tract
in refList:
367 if not patch
in refList[tract]:
368 refList[tract][patch] = {}
369 if filter
in refList[tract][patch]:
370 raise RuntimeError(
"Multiple versions of %s" % (ref.dataId,))
371 refList[tract][patch][filter] = ref
372 return [(list(p.values()), kwargs)
for t
in refList.values()
for p
in t.values()]
377 \anchor MergeSourcesConfig_ 379 \brief Configuration for merging sources. 381 priorityList = ListField(dtype=str, default=[],
382 doc=
"Priority-ordered list of bands for the merge.")
383 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
386 Config.validate(self)
388 raise RuntimeError(
"No priority list provided")
393 \anchor MergeSourcesTask_ 395 \brief A base class for merging source catalogs. 397 Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are 398 so similar that it makes sense to re-use the code, in the form of this abstract base class. 400 NB: Do not use this class directly. Instead use one of the child classes that inherit from 401 MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_ 402 "MergeMeasurementsTask" 404 Sub-classes should set the following class variables: 405 * `_DefaultName`: name of Task 406 * `inputDataset`: name of dataset to read 407 * `outputDataset`: name of dataset to write 408 * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)` 410 In addition, sub-classes must implement the mergeCatalogs method. 413 ConfigClass = MergeSourcesConfig
414 RunnerClass = MergeSourcesRunner
417 getSchemaCatalogs =
None 420 def _makeArgumentParser(cls):
422 \brief Create a suitable ArgumentParser. 424 We will use the ArgumentParser to get a provide a list of data 425 references for patches; the RunnerClass will sort them into lists 426 of data references for the same patch 429 parser.add_id_argument(
"--id",
"deepCoadd_" + cls.
inputDataset,
430 ContainerClass=ExistingCoaddDataIdContainer,
431 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
436 \brief Obtain the input schema either directly or froma butler reference. 438 \param[in] butler butler reference to obtain the input schema from 439 \param[in] schema the input schema 442 assert butler
is not None,
"Neither butler nor schema specified" 443 schema = butler.get(self.config.coaddName +
"Coadd_" + self.
inputDataset +
"_schema",
444 immediate=
True).schema
447 def __init__(self, butler=None, schema=None, **kwargs):
449 \brief Initialize the task. 451 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 452 \param[in] schema the schema of the detection catalogs used as input to this one 453 \param[in] butler a butler used to read the input schema from disk, if schema is None 455 Derived classes should use the getInputSchema() method to handle the additional 456 arguments and retreive the actual input schema. 458 CmdLineTask.__init__(self, **kwargs)
460 def run(self, patchRefList):
462 \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in 463 subclasses that inherit from MergeSourcesTask. 465 \param[in] patchRefList list of data references for each filter 467 catalogs = dict(self.
readCatalog(patchRef)
for patchRef
in patchRefList)
468 mergedCatalog = self.
mergeCatalogs(catalogs, patchRefList[0])
469 self.
write(patchRefList[0], mergedCatalog)
473 \brief Read input catalog. 475 We read the input dataset provided by the 'inputDataset' 478 \param[in] patchRef data reference for patch 479 \return tuple consisting of the filter name and the catalog 481 filterName = patchRef.dataId[
"filter"]
482 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + self.
inputDataset, immediate=
True)
483 self.log.info(
"Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
484 return filterName, catalog
488 \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from 491 \param[in] catalogs dict mapping filter name to source catalog 493 \return merged catalog 495 raise NotImplementedError()
499 \brief Write the output. 501 \param[in] patchRef data reference for patch 502 \param[in] catalog catalog 504 We write as the dataset provided by the 'outputDataset' 507 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
510 mergeDataId = patchRef.dataId.copy()
511 del mergeDataId[
"filter"]
512 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
516 \brief No metadata to write, and not sure how to write it for a list of dataRefs. 521 class CullPeaksConfig(Config):
523 \anchor CullPeaksConfig_ 525 \brief Configuration for culling garbage peaks after merging footprints. 527 Peaks may also be culled after detection or during deblending; this configuration object 528 only deals with culling after merging Footprints. 530 These cuts are based on three quantities: 531 - nBands: the number of bands in which the peak was detected 532 - peakRank: the position of the peak within its family, sorted from brightest to faintest. 533 - peakRankNormalized: the peak rank divided by the total number of peaks in the family. 535 The formula that identifie peaks to cull is: 537 nBands < nBandsSufficient 538 AND (rank >= rankSufficient) 539 AND (rank >= rankConsider OR rank >= rankNormalizedConsider) 541 To disable peak culling, simply set nBandsSufficient=1. 544 nBandsSufficient = RangeField(dtype=int, default=2, min=1,
545 doc=
"Always keep peaks detected in this many bands")
546 rankSufficient = RangeField(dtype=int, default=20, min=1,
547 doc=
"Always keep this many peaks in each family")
548 rankConsidered = RangeField(dtype=int, default=30, min=1,
549 doc=(
"Keep peaks with less than this rank that also match the " 550 "rankNormalizedConsidered condition."))
551 rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
552 doc=(
"Keep peaks with less than this normalized rank that" 553 " also match the rankConsidered condition."))
558 \anchor MergeDetectionsConfig_ 560 \brief Configuration parameters for the MergeDetectionsTask. 562 minNewPeak = Field(dtype=float, default=1,
563 doc=
"Minimum distance from closest peak to create a new one (in arcsec).")
565 maxSamePeak = Field(dtype=float, default=0.3,
566 doc=
"When adding new catalogs to the merge, all peaks less than this distance " 567 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
568 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
570 skyFilterName = Field(dtype=str, default=
"sky",
571 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n" 572 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
573 skySourceRadius = Field(dtype=float, default=8,
574 doc=
"Radius, in pixels, of sky objects")
575 skyGrowDetectedFootprints = Field(dtype=int, default=0,
576 doc=
"Number of pixels to grow the detected footprint mask " 577 "when adding sky objects")
578 nSkySourcesPerPatch = Field(dtype=int, default=100,
579 doc=
"Try to add this many sky objects to the mergeDet list, which will\n" 580 "then be measured along with the detected objects in sourceMeasurementTask")
581 nTrialSkySourcesPerPatch = Field(dtype=int, default=
None, optional=
True,
582 doc=
"Maximum number of trial sky object positions\n" 583 "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
584 nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
585 doc=
"Set nTrialSkySourcesPerPatch to\n" 586 " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n" 587 "if nTrialSkySourcesPerPatch is None")
599 \anchor MergeDetectionsTask_ 601 \brief Merge coadd detections from multiple bands. 603 \section pipe_tasks_multiBand_Contents Contents 605 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose 606 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init 607 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run 608 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config 609 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug 610 - \ref pipe_tasks_multiband_MergeDetectionsTask_Example 612 \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description 614 Command-line task that merges sources detected in coadds of exposures obtained with different filters. 616 To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of 617 sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping 618 track of which band each source originates in. 620 The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright 621 objects are culled as described in \ref CullPeaksConfig_. 624 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 626 deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints) 630 MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask". 632 \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation 634 \copydoc \_\_init\_\_ 636 \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task 640 \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters 642 See \ref MergeDetectionsConfig_ 644 \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables 646 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d 647 to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 649 MergeDetectionsTask has no debug variables. 651 \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask 653 MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset 654 of the available bands. 655 The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the 656 chosen subset of filters. 657 Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources 658 and, eventually, perform forced photometry. 659 Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed. 660 A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the 661 `--help` command line argument: 663 mergeCoaddDetections.py --help 666 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 667 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 668 step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows: 670 mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 672 This will merge the HSC-I & -R band parent source catalogs and write the results to 673 `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`. 675 The next step in the multi-band processing procedure is 676 \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask" 678 ConfigClass = MergeDetectionsConfig
679 _DefaultName =
"mergeCoaddDetections" 681 outputDataset =
"mergeDet" 682 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
684 def __init__(self, butler=None, schema=None, **kwargs):
686 \brief Initialize the merge detections task. 688 A \ref FootprintMergeList_ "FootprintMergeList" will be used to 689 merge the source catalogs. 691 Additional keyword arguments (forwarded to MergeSourcesTask.__init__): 692 \param[in] schema the schema of the detection catalogs used as input to this one 693 \param[in] butler a butler used to read the input schema from disk, if schema is None 694 \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__ 696 The task will set its own self.schema attribute to the schema of the output merged catalog. 698 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
702 if self.config.nSkySourcesPerPatch > 0:
703 filterNames += [self.config.skyFilterName]
708 \brief Merge multiple catalogs. 710 After ordering the catalogs and filters in priority order, 711 \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by 712 \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove 713 garbage peaks detected around bright objects. 717 \param[out] mergedList 721 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
722 tractWcs = skyInfo.wcs
723 peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
724 samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
727 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
729 if band
in catalogs.keys()]
731 mergedList = self.
merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
739 mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
740 if skySourceFootprints:
741 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
743 for foot
in skySourceFootprints:
744 s = mergedList.addNew()
748 self.log.info(
"Added %d sky sources (%.0f%% of requested)",
749 len(skySourceFootprints),
750 100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
753 for record
in mergedList:
754 record.getFootprint().sortPeaks()
755 self.log.info(
"Merged to %d sources" % len(mergedList))
762 \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends). 764 \param[in] catalog Source catalog 766 keys = [item.key
for item
in self.
merged.getPeakSchema().extract(
"merge_peak_*").values()]
767 assert len(keys) > 0,
"Error finding flags that associate peaks with their detection bands." 770 for parentSource
in catalog:
773 keptPeaks = parentSource.getFootprint().getPeaks()
774 oldPeaks = list(keptPeaks)
776 familySize = len(oldPeaks)
777 totalPeaks += familySize
778 for rank, peak
in enumerate(oldPeaks):
779 if ((rank < self.config.cullPeaks.rankSufficient)
or 780 (sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
or 781 (rank < self.config.cullPeaks.rankConsidered
and 782 rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
783 keptPeaks.append(peak)
786 self.log.info(
"Culled %d of %d peaks" % (culledPeaks, totalPeaks))
790 Return a dict of empty catalogs for each catalog dataset produced by this task. 792 \param[out] dictionary of empty catalogs 794 mergeDet = afwTable.SourceCatalog(self.
schema)
795 peak = afwDetect.PeakCatalog(self.
merged.getPeakSchema())
796 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
797 self.config.coaddName +
"Coadd_peak": peak}
801 \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList 803 \param mergedList The merged Footprints from all the input bands 804 \param skyInfo A description of the patch 805 \param growDetectedFootprints The number of pixels to grow the detected footprints 808 if self.config.nSkySourcesPerPatch <= 0:
811 skySourceRadius = self.config.skySourceRadius
812 nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
813 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
814 if nTrialSkySourcesPerPatch
is None:
815 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
821 patchBBox = skyInfo.patchInfo.getOuterBBox()
822 mask = afwImage.Mask(patchBBox)
823 detectedMask = mask.getPlaneBitMask(
"DETECTED")
825 foot = s.getFootprint()
826 if growDetectedFootprints > 0:
827 foot.dilate(growDetectedFootprints)
828 foot.spans.setMask(mask, detectedMask)
830 xmin, ymin = patchBBox.getMin()
831 xmax, ymax = patchBBox.getMax()
833 xmin += skySourceRadius + 1
834 ymin += skySourceRadius + 1
835 xmax -= skySourceRadius + 1
836 ymax -= skySourceRadius + 1
838 skySourceFootprints = []
839 maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
840 for i
in range(nTrialSkySourcesPerPatch):
841 if len(skySourceFootprints) == nSkySourcesPerPatch:
844 x = int(numpy.random.uniform(xmin, xmax))
845 y = int(numpy.random.uniform(ymin, ymax))
846 spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
848 foot = afwDetect.Footprint(spans, patchBBox)
849 foot.setPeakSchema(self.
merged.getPeakSchema())
851 if not foot.spans.overlaps(maskToSpanSet):
852 foot.addPeak(x, y, 0)
853 foot.getPeaks()[0].set(
"merge_peak_%s" % self.config.skyFilterName,
True)
854 skySourceFootprints.append(foot)
856 return skySourceFootprints
861 \anchor MeasureMergedCoaddSourcesConfig_ 863 \brief Configuration parameters for the MeasureMergedCoaddSourcesTask 865 doDeblend = Field(dtype=bool, default=
True, doc=
"Deblend sources?")
866 deblend = ConfigurableField(target=SourceDeblendTask, doc=
"Deblend sources")
867 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
868 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
869 doPropagateFlags = Field(
870 dtype=bool, default=
True,
871 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 873 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
874 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
875 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
876 doWriteMatchesDenormalized = Field(
879 doc=(
"Write reference matches in denormalized format? " 880 "This format uses more disk space, but is more convenient to read."),
882 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
883 checkUnitsParseStrict = Field(
884 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
891 doc=
"Apply aperture corrections" 893 applyApCorr = ConfigurableField(
894 target=ApplyApCorrTask,
895 doc=
"Subtask to apply aperture corrections" 897 doRunCatalogCalculation = Field(
900 doc=
'Run catalogCalculation task' 902 catalogCalculation = ConfigurableField(
903 target=CatalogCalculationTask,
904 doc=
"Subtask to run catalogCalculation plugins on catalog" 908 Config.setDefaults(self)
909 self.
deblend.propagateAllPeaks =
True 910 self.
measurement.plugins.names |= [
'base_InputCount',
'base_Variance']
911 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED',
'SENSOR_EDGE',
924 \anchor MeasureMergedCoaddSourcesTask_ 926 \brief Deblend sources from master catalog in each coadd seperately and measure. 928 \section pipe_tasks_multiBand_Contents Contents 930 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 931 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 932 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 933 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 934 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 935 - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 937 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 939 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 940 measurement in each coadd. 942 Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the 943 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 944 consistent set of child sources. 946 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 947 properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit 948 flags are propagated to the coadd sources. 950 Optionally, we can match the coadd sources to an external reference catalog. 953 deepCoadd_mergeDet{tract,patch}: SourceCatalog 954 \n deepCoadd_calexp{tract,patch,filter}: ExposureF 956 deepCoadd_meas{tract,patch,filter}: SourceCatalog 960 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 963 <DT> \ref SourceDeblendTask_ "deblend" 964 <DD> Deblend all the sources from the master catalog.</DD> 965 <DT> \ref SingleFrameMeasurementTask_ "measurement" 966 <DD> Measure source properties of deblended sources.</DD> 967 <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags" 968 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 969 not at the edge of the field and that have either not been deblended or are the children of deblended 971 <DT> \ref PropagateVisitFlagsTask_ "propagateFlags" 972 <DD> Propagate flags set in individual visits to the coadd.</DD> 973 <DT> \ref DirectMatchTask_ "match" 974 <DD> Match input sources to a reference catalog (optional). 977 These subtasks may be retargeted as required. 979 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 981 \copydoc \_\_init\_\_ 983 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 987 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 989 See \ref MeasureMergedCoaddSourcesConfig_ 991 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 993 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 994 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 997 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 998 the various sub-tasks. See the documetation for individual sub-tasks for more information. 1000 \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask 1002 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 1003 The next stage in the multi-band processing procedure will merge these measurements into a suitable 1004 catalog for driving forced photometry. 1006 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed. 1007 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 1008 `--help` command line argument: 1010 measureCoaddSources.py --help 1013 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 1014 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 1015 step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 1018 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 1020 This will process the HSC-I band data. The results are written in 1021 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 1023 It is also necessary to run 1025 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 1027 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 1028 procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask". 1030 _DefaultName =
"measureCoaddSources" 1031 ConfigClass = MeasureMergedCoaddSourcesConfig
1032 RunnerClass = ButlerInitializedTaskRunner
1033 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
1034 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
1037 def _makeArgumentParser(cls):
1039 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
1040 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1041 ContainerClass=ExistingCoaddDataIdContainer)
1044 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1046 \brief Initialize the task. 1048 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 1049 \param[in] schema: the schema of the merged detection catalog used as input to this one 1050 \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 1051 \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 1052 catalog. May be None if the loader can be constructed from the butler argument or all steps 1053 requiring a reference catalog are disabled. 1054 \param[in] butler: a butler used to read the input schemas from disk or construct the reference 1055 catalog loader, if schema or peakSchema or refObjLoader is None 1057 The task will set its own self.schema attribute to the schema of the output measurement catalog. 1058 This will include all fields from the input schema, as well as additional fields for all the 1061 CmdLineTask.__init__(self, **kwargs)
1063 assert butler
is not None,
"Neither butler nor schema is defined" 1064 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
1069 if self.config.doDeblend:
1070 if peakSchema
is None:
1071 assert butler
is not None,
"Neither butler nor peakSchema is defined" 1072 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
1073 self.makeSubtask(
"deblend", schema=self.
schema, peakSchema=peakSchema)
1074 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
1075 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
1076 if self.config.doMatchSources:
1077 if refObjLoader
is None:
1078 assert butler
is not None,
"Neither butler nor refObjLoader is defined" 1079 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
1080 if self.config.doPropagateFlags:
1081 self.makeSubtask(
"propagateFlags", schema=self.
schema)
1082 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1083 if self.config.doApCorr:
1084 self.makeSubtask(
"applyApCorr", schema=self.
schema)
1085 if self.config.doRunCatalogCalculation:
1086 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
1090 \brief Deblend and measure. 1092 \param[in] patchRef: Patch reference. 1094 Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags 1095 from individual visits. Optionally match the sources to a reference catalog and write the matches. 1096 Finally, write the deblended sources and measurements out. 1098 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
1100 if self.config.doDeblend:
1101 self.deblend.
run(exposure, sources)
1103 bigKey = sources.schema[
"deblend_parentTooBig"].asKey()
1105 numBig = sum((s.get(bigKey)
for s
in sources))
1107 self.log.warn(
"Patch %s contains %d large footprints that were not deblended" %
1108 (patchRef.dataId, numBig))
1110 table = sources.getTable()
1113 self.measurement.
run(sources, exposure, exposureId=self.
getExposureId(patchRef))
1115 if self.config.doApCorr:
1116 self.applyApCorr.
run(
1118 apCorrMap=exposure.getInfo().getApCorrMap()
1121 if self.config.doRunCatalogCalculation:
1122 self.catalogCalculation.
run(sources)
1124 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1125 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1126 includeDeblend=self.config.doDeblend)
1127 if self.config.doPropagateFlags:
1128 self.propagateFlags.
run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1130 if self.config.doMatchSources:
1132 self.
write(patchRef, sources)
1136 \brief Read input sources. 1138 \param[in] dataRef: Data reference for catalog of merged detections 1139 \return List of sources in merged catalog 1141 We also need to add columns to hold the measurements we're about to make 1142 so we can measure in-place. 1144 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
1145 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1148 idFactory.notify(s.getId())
1149 table = afwTable.SourceTable.make(self.
schema, idFactory)
1150 sources = afwTable.SourceCatalog(table)
1156 \brief Write matches of the sources to the astrometric reference catalog. 1158 We use the Wcs in the exposure to match sources. 1160 \param[in] dataRef: data reference 1161 \param[in] exposure: exposure with Wcs 1162 \param[in] sources: source catalog 1164 result = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1166 matches = afwTable.packMatches(result.matches)
1167 matches.table.setMetadata(result.matchMeta)
1168 dataRef.put(matches, self.config.coaddName +
"Coadd_measMatch")
1169 if self.config.doWriteMatchesDenormalized:
1170 denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1171 dataRef.put(denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1176 \brief Write the source catalog. 1178 \param[in] dataRef: data reference 1179 \param[in] sources: source catalog 1181 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1182 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1185 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
1192 \anchor MergeMeasurementsConfig_ 1194 \brief Configuration parameters for the MergeMeasurementsTask 1196 pseudoFilterList = ListField(dtype=str, default=[
"sky"],
1197 doc=
"Names of filters which may have no associated detection\n" 1198 "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1199 snName = Field(dtype=str, default=
"base_PsfFlux",
1200 doc=
"Name of flux measurement for calculating the S/N when choosing the reference band.")
1201 minSN = Field(dtype=float, default=10.,
1202 doc=
"If the S/N from the priority band is below this value (and the S/N " 1203 "is larger than minSNDiff compared to the priority band), use the band with " 1204 "the largest S/N as the reference band.")
1205 minSNDiff = Field(dtype=float, default=3.,
1206 doc=
"If the difference in S/N between another band and the priority band is larger " 1207 "than this value (and the S/N in the priority band is less than minSN) " 1208 "use the band with the largest S/N as the reference band")
1209 flags = ListField(dtype=str, doc=
"Require that these flags, if available, are not set",
1210 default=[
"base_PixelFlags_flag_interpolatedCenter",
"base_PsfFlux_flag",
1211 "ext_photometryKron_KronFlux_flag",
"modelfit_CModel_flag", ])
1223 \anchor MergeMeasurementsTask_ 1225 \brief Merge measurements from multiple bands 1227 \section pipe_tasks_multiBand_Contents Contents 1229 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose 1230 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize 1231 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run 1232 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config 1233 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug 1234 - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example 1236 \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description 1238 Command-line task that merges measurements from multiple bands. 1240 Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter 1241 bands to construct a unified catalog that is suitable for driving forced photometry. Every source is 1242 required to have centroid, shape and flux measurements in each band. 1245 deepCoadd_meas{tract,patch,filter}: SourceCatalog 1247 deepCoadd_ref{tract,patch}: SourceCatalog 1251 MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask". 1253 \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization 1255 \copydoc \_\_init\_\_ 1257 \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task 1261 \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters 1263 See \ref MergeMeasurementsConfig_ 1265 \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables 1267 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1268 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1271 MergeMeasurementsTask has no debug variables. 1273 \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask 1275 MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band. 1276 The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in 1277 coadds and individual exposures. 1278 Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list 1279 of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help` 1280 command line argument: 1282 mergeCoaddMeasurements.py --help 1285 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 1286 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 1287 step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring 1290 mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 1292 This will merge the HSC-I & HSC-R band catalogs. The results are written in 1293 `$CI_HSC_DIR/DATA/deepCoadd-results/`. 1295 _DefaultName =
"mergeCoaddMeasurements" 1296 ConfigClass = MergeMeasurementsConfig
1297 inputDataset =
"meas" 1298 outputDataset =
"ref" 1299 getSchemaCatalogs = _makeGetSchemaCatalogs(
"ref")
1301 def __init__(self, butler=None, schema=None, **kwargs):
1303 Initialize the task. 1305 Additional keyword arguments (forwarded to MergeSourcesTask.__init__): 1306 \param[in] schema: the schema of the detection catalogs used as input to this one 1307 \param[in] butler: a butler used to read the input schema from disk, if schema is None 1309 The task will set its own self.schema attribute to the schema of the output merged catalog. 1311 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1315 self.
fluxKey = inputSchema.find(self.config.snName +
"_flux").getKey()
1316 self.
fluxErrKey = inputSchema.find(self.config.snName +
"_fluxSigma").getKey()
1317 self.
fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
1320 for band
in self.config.priorityList:
1322 outputKey = self.
schemaMapper.editOutputSchema().addField(
1323 "merge_measurement_%s" % short,
1325 doc=
"Flag field set if the measurements here are from the %s filter" % band
1327 peakKey = inputSchema.find(
"merge_peak_%s" % short).key
1328 footprintKey = inputSchema.find(
"merge_footprint_%s" % short).key
1329 self.
flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1333 for filt
in self.config.pseudoFilterList:
1337 self.log.warn(
"merge_peak is not set for pseudo-filter %s" % filt)
1340 for flag
in self.config.flags:
1343 except KeyError
as exc:
1344 self.log.warn(
"Can't find flag %s in schema: %s" % (flag, exc,))
1348 Merge measurement catalogs to create a single reference catalog for forced photometry 1350 \param[in] catalogs: the catalogs to be merged 1351 \param[in] patchRef: patch reference for data 1353 For parent sources, we choose the first band in config.priorityList for which the 1354 merge_footprint flag for that band is is True. 1356 For child sources, the logic is the same, except that we use the merge_peak flags. 1359 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1360 orderedKeys = [self.
flagKeys[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1362 mergedCatalog = afwTable.SourceCatalog(self.
schema)
1363 mergedCatalog.reserve(len(orderedCatalogs[0]))
1365 idKey = orderedCatalogs[0].table.getIdKey()
1366 for catalog
in orderedCatalogs[1:]:
1367 if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1368 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1372 for orderedRecords
in zip(*orderedCatalogs):
1375 maxSNFlagKeys =
None 1377 priorityRecord =
None 1378 priorityFlagKeys =
None 1380 hasPseudoFilter =
False 1384 for inputRecord, flagKeys
in zip(orderedRecords, orderedKeys):
1385 parent = (inputRecord.getParent() == 0
and inputRecord.get(flagKeys.footprint))
1386 child = (inputRecord.getParent() != 0
and inputRecord.get(flagKeys.peak))
1388 if not (parent
or child):
1390 if inputRecord.get(pseudoFilterKey):
1391 hasPseudoFilter =
True 1392 priorityRecord = inputRecord
1393 priorityFlagKeys = flagKeys
1398 isBad = any(inputRecord.get(flag)
for flag
in self.
badFlags)
1403 if numpy.isnan(sn)
or sn < 0.:
1405 if (parent
or child)
and priorityRecord
is None:
1406 priorityRecord = inputRecord
1407 priorityFlagKeys = flagKeys
1410 maxSNRecord = inputRecord
1411 maxSNFlagKeys = flagKeys
1424 bestRecord = priorityRecord
1425 bestFlagKeys = priorityFlagKeys
1426 elif (prioritySN < self.config.minSN
and (maxSN - prioritySN) > self.config.minSNDiff
and 1427 maxSNRecord
is not None):
1428 bestRecord = maxSNRecord
1429 bestFlagKeys = maxSNFlagKeys
1430 elif priorityRecord
is not None:
1431 bestRecord = priorityRecord
1432 bestFlagKeys = priorityFlagKeys
1434 if bestRecord
is not None and bestFlagKeys
is not None:
1435 outputRecord = mergedCatalog.addNew()
1437 outputRecord.set(bestFlagKeys.output,
True)
1439 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1440 inputRecord.getId())
1443 for inputCatalog
in orderedCatalogs:
1444 if len(mergedCatalog) != len(inputCatalog):
1445 raise ValueError(
"Mismatch between catalog sizes: %s != %s" %
1446 (len(mergedCatalog), len(orderedCatalogs)))
1448 return mergedCatalog
Merge coadd detections from multiple bands.
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0)
Return a list of Footprints of sky objects which don't overlap with anything in mergedList.
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
def run(self, patchRef)
Deblend and measure.
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
def write(self, exposure, results, patchRef)
Write out results from runDetection.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Configuration parameters for the DetectCoaddSourcesTask.
def __init__(self, schema=None, kwargs)
Initialize the task.
Merge measurements from multiple bands.
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Deblend sources from master catalog in each coadd seperately and measure.
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
def getShortFilterName(name)
def run(self, patchRef)
Run detection on a coadd.
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Configuration parameters for the MergeMeasurementsTask.
def readSources(self, dataRef)
Read input sources.
def getExposureId(self, dataRef)
Configuration parameters for the MergeDetectionsTask.
Configuration for merging sources.
def write(self, patchRef, catalog)
Write the output.
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
A base class for merging source catalogs.
def write(self, dataRef, sources)
Write the source catalog.
def readCatalog(self, patchRef)
Read input catalog.
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def runDetection(self, exposure, idFactory)
Run detection on an exposure.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Detect sources on a coadd.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.