23 from __future__
import absolute_import, division, print_function
24 from builtins
import zip
25 from builtins
import range
28 from lsst.coadd.utils.coaddDataIdContainer
import ExistingCoaddDataIdContainer
29 from lsst.pipe.base
import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config
import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms
import SourceDetectionTask
32 from lsst.meas.base
import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender
import SourceDeblendTask
35 from lsst.meas.astrom
import DirectMatchTask, denormalizeMatches
39 import lsst.afw.image
as afwImage
40 import lsst.afw.table
as afwTable
41 import lsst.afw.math
as afwMath
42 import lsst.afw.geom
as afwGeom
43 import lsst.afw.detection
as afwDetect
44 from lsst.daf.base
import PropertyList
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter) 49 * deepCoadd_mergeDet: merged detections (tract, patch) 50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter) 51 * deepCoadd_ref: reference sources (tract, patch) 52 All of these have associated *_schema catalogs that require no data ID and hold no records. 54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in 55 the mergeDet, meas, and ref dataset Footprints: 56 * deepCoadd_peak_schema 60 def _makeGetSchemaCatalogs(datasetSuffix):
61 """Construct a getSchemaCatalogs instance method 63 These are identical for most of the classes here, so we'll consolidate 66 datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src" 69 def getSchemaCatalogs(self):
70 """Return a dict of empty catalogs for each catalog dataset produced by this task.""" 71 src = afwTable.SourceCatalog(self.schema)
72 if hasattr(self,
"algMetadata"):
73 src.getTable().setMetadata(self.algMetadata)
74 return {self.config.coaddName +
"Coadd_" + datasetSuffix: src}
75 return getSchemaCatalogs
78 def _makeMakeIdFactory(datasetName):
79 """Construct a makeIdFactory instance method 81 These are identical for all the classes here, so this consolidates 84 datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId" 87 def makeIdFactory(self, dataRef):
88 """Return an IdFactory for setting the detection identifiers 90 The actual parameters used in the IdFactory are provided by 91 the butler (through the provided data reference. 93 expBits = dataRef.get(self.config.coaddName + datasetName +
"_bits")
94 expId = int(dataRef.get(self.config.coaddName + datasetName))
95 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
100 """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i"). 104 return afwImage.Filter(name).getFilterProperty().getName()
111 \anchor DetectCoaddSourcesConfig_ 113 \brief Configuration parameters for the DetectCoaddSourcesTask 115 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
116 detection = ConfigurableField(target=SourceDetectionTask, doc=
"Source detection")
117 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
118 mask = ListField(dtype=str, default=[
"DETECTED",
"BAD",
"SAT",
"NO_DATA",
"INTRP"],
119 doc=
"Mask planes for pixels to ignore when scaling variance")
120 doInsertFakes = Field(dtype=bool, default=
False,
121 doc=
"Run fake sources injection task")
122 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123 doc=
"Injection of fake sources for testing " 124 "purposes (must be retargeted)")
127 Config.setDefaults(self)
128 self.
detection.thresholdType =
"pixel_stdev" 131 self.
detection.background.useApprox =
False 133 self.
detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER' 145 \anchor DetectCoaddSourcesTask_ 147 \brief Detect sources on a coadd 149 \section pipe_tasks_multiBand_Contents Contents 151 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose 152 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize 153 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run 154 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config 155 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug 156 - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example 158 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description 160 Command-line task that detects sources on a coadd of exposures obtained with a single filter. 162 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise 163 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane 164 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should 165 propagate the full covariance matrix -- but it is simple and works well in practice. 167 After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref 168 SourceDetectionTask_ "detection" subtask. 171 deepCoadd{tract,patch,filter}: ExposureF 173 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 174 \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input 176 \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList 180 DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask. 181 You can retarget this subtask if you wish. 183 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization 185 \copydoc \_\_init\_\_ 187 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task 191 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters 193 See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig" 195 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables 197 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 198 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 201 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to 202 \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for 203 \ref SourceDetectionTask_ "SourceDetectionTask" for further information. 205 \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask 207 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of 208 the task is to update the background, detect all sources in a single band and generate a set of parent 209 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and, 210 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data 211 reference to the coadd to be processed. A list of the available optional arguments can be obtained by 212 calling detectCoaddSources.py with the `--help` command line argument: 214 detectCoaddSources.py --help 217 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 218 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed 219 steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows: 221 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 223 that will process the HSC-I band data. The results are written to 224 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`. 226 It is also necessary to run: 228 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 230 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 231 processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask". 233 _DefaultName =
"detectCoaddSources" 234 ConfigClass = DetectCoaddSourcesConfig
235 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
236 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
239 def _makeArgumentParser(cls):
241 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242 ContainerClass=ExistingCoaddDataIdContainer)
247 \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask. 249 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 251 \param[in] schema: initial schema for the output catalog, modified-in place to include all 252 fields set by this task. If None, the source minimal schema will be used. 253 \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__ 255 CmdLineTask.__init__(self, **kwargs)
257 schema = afwTable.SourceTable.makeMinimalSchema()
258 if self.config.doInsertFakes:
259 self.makeSubtask(
"insertFakes")
261 self.makeSubtask(
"detection", schema=self.
schema)
265 \brief Run detection on a coadd. 267 Invokes \ref runDetection and then uses \ref write to output the 270 \param[in] patchRef: data reference for patch 272 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
274 self.
write(exposure, results, patchRef)
279 \brief Run detection on an exposure. 281 First scale the variance plane to match the observed variance 282 using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to 285 \param[in] exposure: Exposure on which to detect 286 \param[in] idFactory: IdFactory to set source identifiers 288 \return a pipe.base.Struct with fields 289 - sources: catalog of detections 290 - backgrounds: list of backgrounds 292 if self.config.doScaleVariance:
293 scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
294 backgrounds = afwMath.BackgroundList()
295 if self.config.doInsertFakes:
296 self.insertFakes.
run(exposure, background=backgrounds)
297 table = afwTable.SourceTable.make(self.
schema, idFactory)
298 detections = self.detection.makeSourceCatalog(table, exposure)
299 sources = detections.sources
300 fpSets = detections.fpSets
301 if fpSets.background:
302 backgrounds.append(fpSets.background)
303 return Struct(sources=sources, backgrounds=backgrounds)
305 def write(self, exposure, results, patchRef):
307 \brief Write out results from runDetection. 309 \param[in] exposure: Exposure to write out 310 \param[in] results: Struct returned from runDetection 311 \param[in] patchRef: data reference for patch 313 coaddName = self.config.coaddName +
"Coadd" 314 patchRef.put(results.backgrounds, coaddName +
"_calexp_background")
315 patchRef.put(results.sources, coaddName +
"_det")
316 patchRef.put(exposure, coaddName +
"_calexp")
323 \anchor MergeSourcesRunner_ 325 \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method 326 requires a list of dataRefs rather than a single dataRef. 331 \brief Provide a butler to the Task constructor. 333 \param[in] parsedCmd the parsed command 334 \param[in] args tuple of a list of data references and kwargs (un-used) 335 \throws RuntimeError if both parsedCmd & args are None 337 if parsedCmd
is not None:
338 butler = parsedCmd.butler
339 elif args
is not None:
340 dataRefList, kwargs = args
341 butler = dataRefList[0].getButler()
343 raise RuntimeError(
"Neither parsedCmd or args specified")
344 return self.TaskClass(config=self.config, log=self.log, butler=butler)
349 \brief Provide a list of patch references for each patch. 351 The patch references within the list will have different filters. 353 \param[in] parsedCmd the parsed command 354 \param **kwargs key word arguments (unused) 355 \throws RuntimeError if multiple references are provided for the same combination of tract, patch and 359 for ref
in parsedCmd.id.refList:
360 tract = ref.dataId[
"tract"]
361 patch = ref.dataId[
"patch"]
362 filter = ref.dataId[
"filter"]
363 if not tract
in refList:
365 if not patch
in refList[tract]:
366 refList[tract][patch] = {}
367 if filter
in refList[tract][patch]:
368 raise RuntimeError(
"Multiple versions of %s" % (ref.dataId,))
369 refList[tract][patch][filter] = ref
370 return [(list(p.values()), kwargs)
for t
in refList.values()
for p
in t.values()]
375 \anchor MergeSourcesConfig_ 377 \brief Configuration for merging sources. 379 priorityList = ListField(dtype=str, default=[],
380 doc=
"Priority-ordered list of bands for the merge.")
381 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
384 Config.validate(self)
386 raise RuntimeError(
"No priority list provided")
391 \anchor MergeSourcesTask_ 393 \brief A base class for merging source catalogs. 395 Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are 396 so similar that it makes sense to re-use the code, in the form of this abstract base class. 398 NB: Do not use this class directly. Instead use one of the child classes that inherit from 399 MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_ 400 "MergeMeasurementsTask" 402 Sub-classes should set the following class variables: 403 * `_DefaultName`: name of Task 404 * `inputDataset`: name of dataset to read 405 * `outputDataset`: name of dataset to write 406 * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)` 408 In addition, sub-classes must implement the mergeCatalogs method. 411 ConfigClass = MergeSourcesConfig
412 RunnerClass = MergeSourcesRunner
415 getSchemaCatalogs =
None 418 def _makeArgumentParser(cls):
420 \brief Create a suitable ArgumentParser. 422 We will use the ArgumentParser to get a provide a list of data 423 references for patches; the RunnerClass will sort them into lists 424 of data references for the same patch 427 parser.add_id_argument(
"--id",
"deepCoadd_" + cls.
inputDataset,
428 ContainerClass=ExistingCoaddDataIdContainer,
429 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
434 \brief Obtain the input schema either directly or froma butler reference. 436 \param[in] butler butler reference to obtain the input schema from 437 \param[in] schema the input schema 440 assert butler
is not None,
"Neither butler nor schema specified" 441 schema = butler.get(self.config.coaddName +
"Coadd_" + self.
inputDataset +
"_schema",
442 immediate=
True).schema
445 def __init__(self, butler=None, schema=None, **kwargs):
447 \brief Initialize the task. 449 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 450 \param[in] schema the schema of the detection catalogs used as input to this one 451 \param[in] butler a butler used to read the input schema from disk, if schema is None 453 Derived classes should use the getInputSchema() method to handle the additional 454 arguments and retreive the actual input schema. 456 CmdLineTask.__init__(self, **kwargs)
458 def run(self, patchRefList):
460 \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in 461 subclasses that inherit from MergeSourcesTask. 463 \param[in] patchRefList list of data references for each filter 465 catalogs = dict(self.
readCatalog(patchRef)
for patchRef
in patchRefList)
466 mergedCatalog = self.
mergeCatalogs(catalogs, patchRefList[0])
467 self.
write(patchRefList[0], mergedCatalog)
471 \brief Read input catalog. 473 We read the input dataset provided by the 'inputDataset' 476 \param[in] patchRef data reference for patch 477 \return tuple consisting of the filter name and the catalog 479 filterName = patchRef.dataId[
"filter"]
480 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + self.
inputDataset, immediate=
True)
481 self.log.info(
"Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
482 return filterName, catalog
486 \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from 489 \param[in] catalogs dict mapping filter name to source catalog 491 \return merged catalog 493 raise NotImplementedError()
497 \brief Write the output. 499 \param[in] patchRef data reference for patch 500 \param[in] catalog catalog 502 We write as the dataset provided by the 'outputDataset' 505 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
508 mergeDataId = patchRef.dataId.copy()
509 del mergeDataId[
"filter"]
510 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
514 \brief No metadata to write, and not sure how to write it for a list of dataRefs. 519 class CullPeaksConfig(Config):
521 \anchor CullPeaksConfig_ 523 \brief Configuration for culling garbage peaks after merging footprints. 525 Peaks may also be culled after detection or during deblending; this configuration object 526 only deals with culling after merging Footprints. 528 These cuts are based on three quantities: 529 - nBands: the number of bands in which the peak was detected 530 - peakRank: the position of the peak within its family, sorted from brightest to faintest. 531 - peakRankNormalized: the peak rank divided by the total number of peaks in the family. 533 The formula that identifie peaks to cull is: 535 nBands < nBandsSufficient 536 AND (rank >= rankSufficient) 537 AND (rank >= rankConsider OR rank >= rankNormalizedConsider) 539 To disable peak culling, simply set nBandsSufficient=1. 542 nBandsSufficient = RangeField(dtype=int, default=2, min=1,
543 doc=
"Always keep peaks detected in this many bands")
544 rankSufficient = RangeField(dtype=int, default=20, min=1,
545 doc=
"Always keep this many peaks in each family")
546 rankConsidered = RangeField(dtype=int, default=30, min=1,
547 doc=(
"Keep peaks with less than this rank that also match the " 548 "rankNormalizedConsidered condition."))
549 rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
550 doc=(
"Keep peaks with less than this normalized rank that" 551 " also match the rankConsidered condition."))
556 \anchor MergeDetectionsConfig_ 558 \brief Configuration parameters for the MergeDetectionsTask. 560 minNewPeak = Field(dtype=float, default=1,
561 doc=
"Minimum distance from closest peak to create a new one (in arcsec).")
563 maxSamePeak = Field(dtype=float, default=0.3,
564 doc=
"When adding new catalogs to the merge, all peaks less than this distance " 565 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
566 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
568 skyFilterName = Field(dtype=str, default=
"sky",
569 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n" 570 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
571 skySourceRadius = Field(dtype=float, default=8,
572 doc=
"Radius, in pixels, of sky objects")
573 skyGrowDetectedFootprints = Field(dtype=int, default=0,
574 doc=
"Number of pixels to grow the detected footprint mask " 575 "when adding sky objects")
576 nSkySourcesPerPatch = Field(dtype=int, default=100,
577 doc=
"Try to add this many sky objects to the mergeDet list, which will\n" 578 "then be measured along with the detected objects in sourceMeasurementTask")
579 nTrialSkySourcesPerPatch = Field(dtype=int, default=
None, optional=
True,
580 doc=
"Maximum number of trial sky object positions\n" 581 "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
582 nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
583 doc=
"Set nTrialSkySourcesPerPatch to\n" 584 " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n" 585 "if nTrialSkySourcesPerPatch is None")
597 \anchor MergeDetectionsTask_ 599 \brief Merge coadd detections from multiple bands. 601 \section pipe_tasks_multiBand_Contents Contents 603 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose 604 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init 605 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run 606 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config 607 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug 608 - \ref pipe_tasks_multiband_MergeDetectionsTask_Example 610 \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description 612 Command-line task that merges sources detected in coadds of exposures obtained with different filters. 614 To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of 615 sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping 616 track of which band each source originates in. 618 The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright 619 objects are culled as described in \ref CullPeaksConfig_. 622 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 624 deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints) 628 MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask". 630 \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation 632 \copydoc \_\_init\_\_ 634 \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task 638 \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters 640 See \ref MergeDetectionsConfig_ 642 \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables 644 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d 645 to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files. 647 MergeDetectionsTask has no debug variables. 649 \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask 651 MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset 652 of the available bands. 653 The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the 654 chosen subset of filters. 655 Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources 656 and, eventually, perform forced photometry. 657 Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed. 658 A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the 659 `--help` command line argument: 661 mergeCoaddDetections.py --help 664 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 665 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 666 step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows: 668 mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 670 This will merge the HSC-I & -R band parent source catalogs and write the results to 671 `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`. 673 The next step in the multi-band processing procedure is 674 \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask" 676 ConfigClass = MergeDetectionsConfig
677 _DefaultName =
"mergeCoaddDetections" 679 outputDataset =
"mergeDet" 680 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
682 def __init__(self, butler=None, schema=None, **kwargs):
684 \brief Initialize the merge detections task. 686 A \ref FootprintMergeList_ "FootprintMergeList" will be used to 687 merge the source catalogs. 689 Additional keyword arguments (forwarded to MergeSourcesTask.__init__): 690 \param[in] schema the schema of the detection catalogs used as input to this one 691 \param[in] butler a butler used to read the input schema from disk, if schema is None 692 \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__ 694 The task will set its own self.schema attribute to the schema of the output merged catalog. 696 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
700 if self.config.nSkySourcesPerPatch > 0:
701 filterNames += [self.config.skyFilterName]
706 \brief Merge multiple catalogs. 708 After ordering the catalogs and filters in priority order, 709 \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by 710 \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove 711 garbage peaks detected around bright objects. 715 \param[out] mergedList 719 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
720 tractWcs = skyInfo.wcs
721 peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
722 samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
725 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
727 if band
in catalogs.keys()]
729 mergedList = self.
merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
737 mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
738 if skySourceFootprints:
739 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
741 for foot
in skySourceFootprints:
742 s = mergedList.addNew()
746 self.log.info(
"Added %d sky sources (%.0f%% of requested)",
747 len(skySourceFootprints),
748 100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
751 for record
in mergedList:
752 record.getFootprint().sortPeaks()
753 self.log.info(
"Merged to %d sources" % len(mergedList))
760 \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends). 762 \param[in] catalog Source catalog 764 keys = [item.key
for item
in self.
merged.getPeakSchema().extract(
"merge_peak_*").values()]
765 assert len(keys) > 0,
"Error finding flags that associate peaks with their detection bands." 768 for parentSource
in catalog:
771 keptPeaks = parentSource.getFootprint().getPeaks()
772 oldPeaks = list(keptPeaks)
774 familySize = len(oldPeaks)
775 totalPeaks += familySize
776 for rank, peak
in enumerate(oldPeaks):
777 if ((rank < self.config.cullPeaks.rankSufficient)
or 778 (sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
or 779 (rank < self.config.cullPeaks.rankConsidered
and 780 rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
781 keptPeaks.append(peak)
784 self.log.info(
"Culled %d of %d peaks" % (culledPeaks, totalPeaks))
788 Return a dict of empty catalogs for each catalog dataset produced by this task. 790 \param[out] dictionary of empty catalogs 792 mergeDet = afwTable.SourceCatalog(self.
schema)
793 peak = afwDetect.PeakCatalog(self.
merged.getPeakSchema())
794 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
795 self.config.coaddName +
"Coadd_peak": peak}
799 \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList 801 \param mergedList The merged Footprints from all the input bands 802 \param skyInfo A description of the patch 803 \param growDetectedFootprints The number of pixels to grow the detected footprints 806 if self.config.nSkySourcesPerPatch <= 0:
809 skySourceRadius = self.config.skySourceRadius
810 nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
811 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
812 if nTrialSkySourcesPerPatch
is None:
813 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
819 patchBBox = skyInfo.patchInfo.getOuterBBox()
820 mask = afwImage.Mask(patchBBox)
821 detectedMask = mask.getPlaneBitMask(
"DETECTED")
823 foot = s.getFootprint()
824 if growDetectedFootprints > 0:
825 foot.dilate(growDetectedFootprints)
826 foot.spans.setMask(mask, detectedMask)
828 xmin, ymin = patchBBox.getMin()
829 xmax, ymax = patchBBox.getMax()
831 xmin += skySourceRadius + 1
832 ymin += skySourceRadius + 1
833 xmax -= skySourceRadius + 1
834 ymax -= skySourceRadius + 1
836 skySourceFootprints = []
837 maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
838 for i
in range(nTrialSkySourcesPerPatch):
839 if len(skySourceFootprints) == nSkySourcesPerPatch:
842 x = int(numpy.random.uniform(xmin, xmax))
843 y = int(numpy.random.uniform(ymin, ymax))
844 spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
846 foot = afwDetect.Footprint(spans, patchBBox)
847 foot.setPeakSchema(self.
merged.getPeakSchema())
849 if not foot.spans.overlaps(maskToSpanSet):
850 foot.addPeak(x, y, 0)
851 foot.getPeaks()[0].set(
"merge_peak_%s" % self.config.skyFilterName,
True)
852 skySourceFootprints.append(foot)
854 return skySourceFootprints
859 \anchor MeasureMergedCoaddSourcesConfig_ 861 \brief Configuration parameters for the MeasureMergedCoaddSourcesTask 863 doDeblend = Field(dtype=bool, default=
True, doc=
"Deblend sources?")
864 deblend = ConfigurableField(target=SourceDeblendTask, doc=
"Deblend sources")
865 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
866 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
867 doPropagateFlags = Field(
868 dtype=bool, default=
True,
869 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)" 871 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
872 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
873 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
874 doWriteMatchesDenormalized = Field(
877 doc=(
"Write reference matches in denormalized format? " 878 "This format uses more disk space, but is more convenient to read."),
880 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
881 checkUnitsParseStrict = Field(
882 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
889 doc=
"Apply aperture corrections" 891 applyApCorr = ConfigurableField(
892 target=ApplyApCorrTask,
893 doc=
"Subtask to apply aperture corrections" 895 doRunCatalogCalculation = Field(
898 doc=
'Run catalogCalculation task' 900 catalogCalculation = ConfigurableField(
901 target=CatalogCalculationTask,
902 doc=
"Subtask to run catalogCalculation plugins on catalog" 906 Config.setDefaults(self)
907 self.
deblend.propagateAllPeaks =
True 908 self.
measurement.plugins.names |= [
'base_InputCount']
911 self.
measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED']
923 \anchor MeasureMergedCoaddSourcesTask_ 925 \brief Deblend sources from master catalog in each coadd seperately and measure. 927 \section pipe_tasks_multiBand_Contents Contents 929 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose 930 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize 931 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run 932 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config 933 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug 934 - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example 936 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description 938 Command-line task that uses peaks and footprints from a master catalog to perform deblending and 939 measurement in each coadd. 941 Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the 942 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a 943 consistent set of child sources. 945 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source 946 properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit 947 flags are propagated to the coadd sources. 949 Optionally, we can match the coadd sources to an external reference catalog. 952 deepCoadd_mergeDet{tract,patch}: SourceCatalog 953 \n deepCoadd_calexp{tract,patch,filter}: ExposureF 955 deepCoadd_meas{tract,patch,filter}: SourceCatalog 959 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks: 962 <DT> \ref SourceDeblendTask_ "deblend" 963 <DD> Deblend all the sources from the master catalog.</DD> 964 <DT> \ref SingleFrameMeasurementTask_ "measurement" 965 <DD> Measure source properties of deblended sources.</DD> 966 <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags" 967 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are 968 not at the edge of the field and that have either not been deblended or are the children of deblended 970 <DT> \ref PropagateVisitFlagsTask_ "propagateFlags" 971 <DD> Propagate flags set in individual visits to the coadd.</DD> 972 <DT> \ref DirectMatchTask_ "match" 973 <DD> Match input sources to a reference catalog (optional). 976 These subtasks may be retargeted as required. 978 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization 980 \copydoc \_\_init\_\_ 982 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task 986 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters 988 See \ref MeasureMergedCoaddSourcesConfig_ 990 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables 992 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 993 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 996 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to 997 the various sub-tasks. See the documetation for individual sub-tasks for more information. 999 \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask 1001 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs. 1002 The next stage in the multi-band processing procedure will merge these measurements into a suitable 1003 catalog for driving forced photometry. 1005 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed. 1006 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the 1007 `--help` command line argument: 1009 measureCoaddSources.py --help 1012 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 1013 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 1014 step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band 1017 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I 1019 This will process the HSC-I band data. The results are written in 1020 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits 1022 It is also necessary to run 1024 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R 1026 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band 1027 procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask". 1029 _DefaultName =
"measureCoaddSources" 1030 ConfigClass = MeasureMergedCoaddSourcesConfig
1031 RunnerClass = ButlerInitializedTaskRunner
1032 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
1033 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
1036 def _makeArgumentParser(cls):
1038 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
1039 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1040 ContainerClass=ExistingCoaddDataIdContainer)
1043 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1045 \brief Initialize the task. 1047 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__): 1048 \param[in] schema: the schema of the merged detection catalog used as input to this one 1049 \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog 1050 \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference 1051 catalog. May be None if the loader can be constructed from the butler argument or all steps 1052 requiring a reference catalog are disabled. 1053 \param[in] butler: a butler used to read the input schemas from disk or construct the reference 1054 catalog loader, if schema or peakSchema or refObjLoader is None 1056 The task will set its own self.schema attribute to the schema of the output measurement catalog. 1057 This will include all fields from the input schema, as well as additional fields for all the 1060 CmdLineTask.__init__(self, **kwargs)
1062 assert butler
is not None,
"Neither butler nor schema is defined" 1063 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
1068 if self.config.doDeblend:
1069 if peakSchema
is None:
1070 assert butler
is not None,
"Neither butler nor peakSchema is defined" 1071 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
1072 self.makeSubtask(
"deblend", schema=self.
schema, peakSchema=peakSchema)
1073 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
1074 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
1075 if self.config.doMatchSources:
1076 if refObjLoader
is None:
1077 assert butler
is not None,
"Neither butler nor refObjLoader is defined" 1078 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
1079 if self.config.doPropagateFlags:
1080 self.makeSubtask(
"propagateFlags", schema=self.
schema)
1081 self.
schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1082 if self.config.doApCorr:
1083 self.makeSubtask(
"applyApCorr", schema=self.
schema)
1084 if self.config.doRunCatalogCalculation:
1085 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
1089 \brief Deblend and measure. 1091 \param[in] patchRef: Patch reference. 1093 Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags 1094 from individual visits. Optionally match the sources to a reference catalog and write the matches. 1095 Finally, write the deblended sources and measurements out. 1097 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
1099 if self.config.doDeblend:
1100 self.deblend.
run(exposure, sources)
1102 bigKey = sources.schema[
"deblend_parentTooBig"].asKey()
1104 numBig = sum((s.get(bigKey)
for s
in sources))
1106 self.log.warn(
"Patch %s contains %d large footprints that were not deblended" %
1107 (patchRef.dataId, numBig))
1109 table = sources.getTable()
1112 self.measurement.
run(sources, exposure, exposureId=self.
getExposureId(patchRef))
1114 if self.config.doApCorr:
1115 self.applyApCorr.
run(
1117 apCorrMap=exposure.getInfo().getApCorrMap()
1120 if self.config.doRunCatalogCalculation:
1121 self.catalogCalculation.
run(sources)
1123 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1124 self.setPrimaryFlags.
run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1125 includeDeblend=self.config.doDeblend)
1126 if self.config.doPropagateFlags:
1127 self.propagateFlags.
run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1129 if self.config.doMatchSources:
1131 self.
write(patchRef, sources)
1135 \brief Read input sources. 1137 \param[in] dataRef: Data reference for catalog of merged detections 1138 \return List of sources in merged catalog 1140 We also need to add columns to hold the measurements we're about to make 1141 so we can measure in-place. 1143 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
1144 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1147 idFactory.notify(s.getId())
1148 table = afwTable.SourceTable.make(self.
schema, idFactory)
1149 sources = afwTable.SourceCatalog(table)
1155 \brief Write matches of the sources to the astrometric reference catalog. 1157 We use the Wcs in the exposure to match sources. 1159 \param[in] dataRef: data reference 1160 \param[in] exposure: exposure with Wcs 1161 \param[in] sources: source catalog 1163 result = self.match.
run(sources, exposure.getInfo().getFilter().getName())
1165 matches = afwTable.packMatches(result.matches)
1166 matches.table.setMetadata(result.matchMeta)
1167 dataRef.put(matches, self.config.coaddName +
"Coadd_measMatch")
1168 if self.config.doWriteMatchesDenormalized:
1169 denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1170 dataRef.put(denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1175 \brief Write the source catalog. 1177 \param[in] dataRef: data reference 1178 \param[in] sources: source catalog 1180 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1181 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1184 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
1191 \anchor MergeMeasurementsConfig_ 1193 \brief Configuration parameters for the MergeMeasurementsTask 1195 pseudoFilterList = ListField(dtype=str, default=[
"sky"],
1196 doc=
"Names of filters which may have no associated detection\n" 1197 "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1198 snName = Field(dtype=str, default=
"base_PsfFlux",
1199 doc=
"Name of flux measurement for calculating the S/N when choosing the reference band.")
1200 minSN = Field(dtype=float, default=10.,
1201 doc=
"If the S/N from the priority band is below this value (and the S/N " 1202 "is larger than minSNDiff compared to the priority band), use the band with " 1203 "the largest S/N as the reference band.")
1204 minSNDiff = Field(dtype=float, default=3.,
1205 doc=
"If the difference in S/N between another band and the priority band is larger " 1206 "than this value (and the S/N in the priority band is less than minSN) " 1207 "use the band with the largest S/N as the reference band")
1208 flags = ListField(dtype=str, doc=
"Require that these flags, if available, are not set",
1209 default=[
"base_PixelFlags_flag_interpolatedCenter",
"base_PsfFlux_flag",
1210 "ext_photometryKron_KronFlux_flag",
"modelfit_CModel_flag", ])
1222 \anchor MergeMeasurementsTask_ 1224 \brief Merge measurements from multiple bands 1226 \section pipe_tasks_multiBand_Contents Contents 1228 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose 1229 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize 1230 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run 1231 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config 1232 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug 1233 - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example 1235 \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description 1237 Command-line task that merges measurements from multiple bands. 1239 Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter 1240 bands to construct a unified catalog that is suitable for driving forced photometry. Every source is 1241 required to have centroid, shape and flux measurements in each band. 1244 deepCoadd_meas{tract,patch,filter}: SourceCatalog 1246 deepCoadd_ref{tract,patch}: SourceCatalog 1250 MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask". 1252 \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization 1254 \copydoc \_\_init\_\_ 1256 \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task 1260 \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters 1262 See \ref MergeMeasurementsConfig_ 1264 \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables 1266 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a 1267 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py 1270 MergeMeasurementsTask has no debug variables. 1272 \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask 1274 MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band. 1275 The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in 1276 coadds and individual exposures. 1277 Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list 1278 of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help` 1279 command line argument: 1281 mergeCoaddMeasurements.py --help 1284 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 1285 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 1286 step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring 1289 mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 1291 This will merge the HSC-I & HSC-R band catalogs. The results are written in 1292 `$CI_HSC_DIR/DATA/deepCoadd-results/`. 1294 _DefaultName =
"mergeCoaddMeasurements" 1295 ConfigClass = MergeMeasurementsConfig
1296 inputDataset =
"meas" 1297 outputDataset =
"ref" 1298 getSchemaCatalogs = _makeGetSchemaCatalogs(
"ref")
1300 def __init__(self, butler=None, schema=None, **kwargs):
1302 Initialize the task. 1304 Additional keyword arguments (forwarded to MergeSourcesTask.__init__): 1305 \param[in] schema: the schema of the detection catalogs used as input to this one 1306 \param[in] butler: a butler used to read the input schema from disk, if schema is None 1308 The task will set its own self.schema attribute to the schema of the output merged catalog. 1310 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1314 self.
fluxKey = inputSchema.find(self.config.snName +
"_flux").getKey()
1315 self.
fluxErrKey = inputSchema.find(self.config.snName +
"_fluxSigma").getKey()
1316 self.
fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
1319 for band
in self.config.priorityList:
1321 outputKey = self.
schemaMapper.editOutputSchema().addField(
1322 "merge_measurement_%s" % short,
1324 doc=
"Flag field set if the measurements here are from the %s filter" % band
1326 peakKey = inputSchema.find(
"merge_peak_%s" % short).key
1327 footprintKey = inputSchema.find(
"merge_footprint_%s" % short).key
1328 self.
flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1332 for filt
in self.config.pseudoFilterList:
1336 self.log.warn(
"merge_peak is not set for pseudo-filter %s" % filt)
1339 for flag
in self.config.flags:
1342 except KeyError
as exc:
1343 self.log.warn(
"Can't find flag %s in schema: %s" % (flag, exc,))
1347 Merge measurement catalogs to create a single reference catalog for forced photometry 1349 \param[in] catalogs: the catalogs to be merged 1350 \param[in] patchRef: patch reference for data 1352 For parent sources, we choose the first band in config.priorityList for which the 1353 merge_footprint flag for that band is is True. 1355 For child sources, the logic is the same, except that we use the merge_peak flags. 1358 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1359 orderedKeys = [self.
flagKeys[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1361 mergedCatalog = afwTable.SourceCatalog(self.
schema)
1362 mergedCatalog.reserve(len(orderedCatalogs[0]))
1364 idKey = orderedCatalogs[0].table.getIdKey()
1365 for catalog
in orderedCatalogs[1:]:
1366 if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1367 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1371 for orderedRecords
in zip(*orderedCatalogs):
1374 maxSNFlagKeys =
None 1376 priorityRecord =
None 1377 priorityFlagKeys =
None 1379 hasPseudoFilter =
False 1383 for inputRecord, flagKeys
in zip(orderedRecords, orderedKeys):
1384 parent = (inputRecord.getParent() == 0
and inputRecord.get(flagKeys.footprint))
1385 child = (inputRecord.getParent() != 0
and inputRecord.get(flagKeys.peak))
1387 if not (parent
or child):
1389 if inputRecord.get(pseudoFilterKey):
1390 hasPseudoFilter =
True 1391 priorityRecord = inputRecord
1392 priorityFlagKeys = flagKeys
1397 isBad = any(inputRecord.get(flag)
for flag
in self.
badFlags)
1402 if numpy.isnan(sn)
or sn < 0.:
1404 if (parent
or child)
and priorityRecord
is None:
1405 priorityRecord = inputRecord
1406 priorityFlagKeys = flagKeys
1409 maxSNRecord = inputRecord
1410 maxSNFlagKeys = flagKeys
1423 bestRecord = priorityRecord
1424 bestFlagKeys = priorityFlagKeys
1425 elif (prioritySN < self.config.minSN
and (maxSN - prioritySN) > self.config.minSNDiff
and 1426 maxSNRecord
is not None):
1427 bestRecord = maxSNRecord
1428 bestFlagKeys = maxSNFlagKeys
1429 elif priorityRecord
is not None:
1430 bestRecord = priorityRecord
1431 bestFlagKeys = priorityFlagKeys
1433 if bestRecord
is not None and bestFlagKeys
is not None:
1434 outputRecord = mergedCatalog.addNew()
1436 outputRecord.set(bestFlagKeys.output,
True)
1438 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1439 inputRecord.getId())
1442 for inputCatalog
in orderedCatalogs:
1443 if len(mergedCatalog) != len(inputCatalog):
1444 raise ValueError(
"Mismatch between catalog sizes: %s != %s" %
1445 (len(mergedCatalog), len(orderedCatalogs)))
1447 return mergedCatalog
Merge coadd detections from multiple bands.
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0)
Return a list of Footprints of sky objects which don't overlap with anything in mergedList.
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
def run(self, patchRef)
Deblend and measure.
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
def write(self, exposure, results, patchRef)
Write out results from runDetection.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Configuration parameters for the DetectCoaddSourcesTask.
def __init__(self, schema=None, kwargs)
Initialize the task.
Merge measurements from multiple bands.
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Deblend sources from master catalog in each coadd seperately and measure.
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
def getShortFilterName(name)
def run(self, patchRef)
Run detection on a coadd.
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Configuration parameters for the MergeMeasurementsTask.
def readSources(self, dataRef)
Read input sources.
def getExposureId(self, dataRef)
Configuration parameters for the MergeDetectionsTask.
Configuration for merging sources.
def write(self, patchRef, catalog)
Write the output.
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
A base class for merging source catalogs.
def write(self, dataRef, sources)
Write the source catalog.
def readCatalog(self, patchRef)
Read input catalog.
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def runDetection(self, exposure, idFactory)
Run detection on an exposure.
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Detect sources on a coadd.
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.