23 from __future__
import absolute_import, division, print_function
24 from builtins
import zip
25 from builtins
import range
28 from lsst.coadd.utils.coaddDataIdContainer
import ExistingCoaddDataIdContainer
29 from lsst.pipe.base
import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config
import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms
import SourceDetectionTask
32 from lsst.meas.base
import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender
import SourceDeblendTask
35 from lsst.meas.astrom
import DirectMatchTask, denormalizeMatches
39 import lsst.afw.image
as afwImage
40 import lsst.afw.table
as afwTable
41 import lsst.afw.math
as afwMath
42 import lsst.afw.geom
as afwGeom
43 import lsst.afw.detection
as afwDetect
44 from lsst.daf.base
import PropertyList
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
60 def _makeGetSchemaCatalogs(datasetSuffix):
61 """Construct a getSchemaCatalogs instance method
63 These are identical for most of the classes here, so we'll consolidate
66 datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
69 def getSchemaCatalogs(self):
70 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71 src = afwTable.SourceCatalog(self.schema)
72 if hasattr(self,
"algMetadata"):
73 src.getTable().setMetadata(self.algMetadata)
74 return {self.config.coaddName +
"Coadd_" + datasetSuffix: src}
75 return getSchemaCatalogs
78 def _makeMakeIdFactory(datasetName):
79 """Construct a makeIdFactory instance method
81 These are identical for all the classes here, so this consolidates
84 datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
87 def makeIdFactory(self, dataRef):
88 """Return an IdFactory for setting the detection identifiers
90 The actual parameters used in the IdFactory are provided by
91 the butler (through the provided data reference.
93 expBits = dataRef.get(self.config.coaddName + datasetName +
"_bits")
94 expId = int(dataRef.get(self.config.coaddName + datasetName))
95 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
100 """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
104 return afwImage.Filter(name).getFilterProperty().getName()
111 \anchor DetectCoaddSourcesConfig_
113 \brief Configuration parameters for the DetectCoaddSourcesTask
115 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
116 detection = ConfigurableField(target=SourceDetectionTask, doc=
"Source detection")
117 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
118 mask = ListField(dtype=str, default=[
"DETECTED",
"BAD",
"SAT",
"NO_DATA",
"INTRP"],
119 doc=
"Mask planes for pixels to ignore when scaling variance")
120 doInsertFakes = Field(dtype=bool, default=
False,
121 doc=
"Run fake sources injection task")
122 insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123 doc=
"Injection of fake sources for testing "
124 "purposes (must be retargeted)")
127 Config.setDefaults(self)
128 self.detection.thresholdType =
"pixel_stdev"
129 self.detection.isotropicGrow =
True
131 self.detection.background.useApprox =
False
132 self.detection.background.binSize = 4096
133 self.detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER'
145 \anchor DetectCoaddSourcesTask_
147 \brief Detect sources on a coadd
149 \section pipe_tasks_multiBand_Contents Contents
151 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
152 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
153 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
154 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
155 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
156 - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
158 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
160 Command-line task that detects sources on a coadd of exposures obtained with a single filter.
162 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
163 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
164 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
165 propagate the full covariance matrix -- but it is simple and works well in practice.
167 After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
168 SourceDetectionTask_ "detection" subtask.
171 deepCoadd{tract,patch,filter}: ExposureF
173 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
174 \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
176 \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
180 DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
181 You can retarget this subtask if you wish.
183 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
185 \copydoc \_\_init\_\_
187 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
191 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
193 See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
195 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
197 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
198 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
201 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
202 \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
203 \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
205 \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
207 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
208 the task is to update the background, detect all sources in a single band and generate a set of parent
209 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
210 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
211 reference to the coadd to be processed. A list of the available optional arguments can be obtained by
212 calling detectCoaddSources.py with the `--help` command line argument:
214 detectCoaddSources.py --help
217 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
218 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
219 steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
221 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
223 that will process the HSC-I band data. The results are written to
224 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
226 It is also necessary to run:
228 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
230 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
231 processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
233 _DefaultName =
"detectCoaddSources"
234 ConfigClass = DetectCoaddSourcesConfig
235 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
236 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
239 def _makeArgumentParser(cls):
240 parser = ArgumentParser(name=cls._DefaultName)
241 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242 ContainerClass=ExistingCoaddDataIdContainer)
247 \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
249 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
251 \param[in] schema: initial schema for the output catalog, modified-in place to include all
252 fields set by this task. If None, the source minimal schema will be used.
253 \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
255 CmdLineTask.__init__(self, **kwargs)
257 schema = afwTable.SourceTable.makeMinimalSchema()
258 if self.config.doInsertFakes:
259 self.makeSubtask(
"insertFakes")
261 self.makeSubtask(
"detection", schema=self.
schema)
265 \brief Run detection on a coadd.
267 Invokes \ref runDetection and then uses \ref write to output the
270 \param[in] patchRef: data reference for patch
272 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
274 self.
write(exposure, results, patchRef)
279 \brief Run detection on an exposure.
281 First scale the variance plane to match the observed variance
282 using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
285 \param[in] exposure: Exposure on which to detect
286 \param[in] idFactory: IdFactory to set source identifiers
288 \return a pipe.base.Struct with fields
289 - sources: catalog of detections
290 - backgrounds: list of backgrounds
292 if self.config.doScaleVariance:
293 scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
294 backgrounds = afwMath.BackgroundList()
295 if self.config.doInsertFakes:
296 self.insertFakes.run(exposure, background=backgrounds)
297 table = afwTable.SourceTable.make(self.
schema, idFactory)
298 detections = self.detection.makeSourceCatalog(table, exposure)
299 sources = detections.sources
300 fpSets = detections.fpSets
301 if fpSets.background:
302 backgrounds.append(fpSets.background)
303 return Struct(sources=sources, backgrounds=backgrounds)
305 def write(self, exposure, results, patchRef):
307 \brief Write out results from runDetection.
309 \param[in] exposure: Exposure to write out
310 \param[in] results: Struct returned from runDetection
311 \param[in] patchRef: data reference for patch
313 coaddName = self.config.coaddName +
"Coadd"
314 patchRef.put(results.backgrounds, coaddName +
"_calexp_background")
315 patchRef.put(results.sources, coaddName +
"_det")
316 patchRef.put(exposure, coaddName +
"_calexp")
323 \anchor MergeSourcesRunner_
325 \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
326 requires a list of dataRefs rather than a single dataRef.
331 \brief Provide a butler to the Task constructor.
333 \param[in] parsedCmd the parsed command
334 \param[in] args tuple of a list of data references and kwargs (un-used)
335 \throws RuntimeError if both parsedCmd & args are None
337 if parsedCmd
is not None:
338 butler = parsedCmd.butler
339 elif args
is not None:
340 dataRefList, kwargs = args
341 butler = dataRefList[0].getButler()
343 raise RuntimeError(
"Neither parsedCmd or args specified")
344 return self.TaskClass(config=self.config, log=self.log, butler=butler)
349 \brief Provide a list of patch references for each patch.
351 The patch references within the list will have different filters.
353 \param[in] parsedCmd the parsed command
354 \param **kwargs key word arguments (unused)
355 \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
359 for ref
in parsedCmd.id.refList:
360 tract = ref.dataId[
"tract"]
361 patch = ref.dataId[
"patch"]
362 filter = ref.dataId[
"filter"]
363 if not tract
in refList:
365 if not patch
in refList[tract]:
366 refList[tract][patch] = {}
367 if filter
in refList[tract][patch]:
368 raise RuntimeError(
"Multiple versions of %s" % (ref.dataId,))
369 refList[tract][patch][filter] = ref
370 return [(list(p.values()), kwargs)
for t
in refList.values()
for p
in t.values()]
375 \anchor MergeSourcesConfig_
377 \brief Configuration for merging sources.
379 priorityList = ListField(dtype=str, default=[],
380 doc=
"Priority-ordered list of bands for the merge.")
381 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
384 Config.validate(self)
386 raise RuntimeError(
"No priority list provided")
391 \anchor MergeSourcesTask_
393 \brief A base class for merging source catalogs.
395 Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
396 so similar that it makes sense to re-use the code, in the form of this abstract base class.
398 NB: Do not use this class directly. Instead use one of the child classes that inherit from
399 MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
400 "MergeMeasurementsTask"
402 Sub-classes should set the following class variables:
403 * `_DefaultName`: name of Task
404 * `inputDataset`: name of dataset to read
405 * `outputDataset`: name of dataset to write
406 * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
408 In addition, sub-classes must implement the mergeCatalogs method.
411 ConfigClass = MergeSourcesConfig
412 RunnerClass = MergeSourcesRunner
415 getSchemaCatalogs =
None
418 def _makeArgumentParser(cls):
420 \brief Create a suitable ArgumentParser.
422 We will use the ArgumentParser to get a provide a list of data
423 references for patches; the RunnerClass will sort them into lists
424 of data references for the same patch
426 parser = ArgumentParser(name=cls._DefaultName)
427 parser.add_id_argument(
"--id",
"deepCoadd_" + cls.inputDataset,
428 ContainerClass=ExistingCoaddDataIdContainer,
429 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
434 \brief Obtain the input schema either directly or froma butler reference.
436 \param[in] butler butler reference to obtain the input schema from
437 \param[in] schema the input schema
440 assert butler
is not None,
"Neither butler nor schema specified"
441 schema = butler.get(self.config.coaddName +
"Coadd_" + self.
inputDataset +
"_schema",
442 immediate=
True).schema
445 def __init__(self, butler=None, schema=None, **kwargs):
447 \brief Initialize the task.
449 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
450 \param[in] schema the schema of the detection catalogs used as input to this one
451 \param[in] butler a butler used to read the input schema from disk, if schema is None
453 Derived classes should use the getInputSchema() method to handle the additional
454 arguments and retreive the actual input schema.
456 CmdLineTask.__init__(self, **kwargs)
458 def run(self, patchRefList):
460 \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
461 subclasses that inherit from MergeSourcesTask.
463 \param[in] patchRefList list of data references for each filter
465 catalogs = dict(self.
readCatalog(patchRef)
for patchRef
in patchRefList)
466 mergedCatalog = self.
mergeCatalogs(catalogs, patchRefList[0])
467 self.
write(patchRefList[0], mergedCatalog)
471 \brief Read input catalog.
473 We read the input dataset provided by the 'inputDataset'
476 \param[in] patchRef data reference for patch
477 \return tuple consisting of the filter name and the catalog
479 filterName = patchRef.dataId[
"filter"]
480 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + self.
inputDataset, immediate=
True)
481 self.log.info(
"Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
482 return filterName, catalog
486 \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
489 \param[in] catalogs dict mapping filter name to source catalog
491 \return merged catalog
493 raise NotImplementedError()
497 \brief Write the output.
499 \param[in] patchRef data reference for patch
500 \param[in] catalog catalog
502 We write as the dataset provided by the 'outputDataset'
505 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
508 mergeDataId = patchRef.dataId.copy()
509 del mergeDataId[
"filter"]
510 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
514 \brief No metadata to write, and not sure how to write it for a list of dataRefs.
519 class CullPeaksConfig(Config):
521 \anchor CullPeaksConfig_
523 \brief Configuration for culling garbage peaks after merging footprints.
525 Peaks may also be culled after detection or during deblending; this configuration object
526 only deals with culling after merging Footprints.
528 These cuts are based on three quantities:
529 - nBands: the number of bands in which the peak was detected
530 - peakRank: the position of the peak within its family, sorted from brightest to faintest.
531 - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
533 The formula that identifie peaks to cull is:
535 nBands < nBandsSufficient
536 AND (rank >= rankSufficient)
537 AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
539 To disable peak culling, simply set nBandsSafe=1.
542 nBandsSufficient = RangeField(dtype=int, default=2, min=1,
543 doc=
"Always keep peaks detected in this many bands")
544 rankSufficient = RangeField(dtype=int, default=20, min=1,
545 doc=
"Always keep this many peaks in each family")
546 rankConsidered = RangeField(dtype=int, default=30, min=1,
547 doc=(
"Keep peaks with less than this rank that also match the "
548 "rankNormalizedConsidered condition."))
549 rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
550 doc=(
"Keep peaks with less than this normalized rank that"
551 " also match the rankConsidered condition."))
556 \anchor MergeDetectionsConfig_
558 \brief Configuration parameters for the MergeDetectionsTask.
560 minNewPeak = Field(dtype=float, default=1,
561 doc=
"Minimum distance from closest peak to create a new one (in arcsec).")
563 maxSamePeak = Field(dtype=float, default=0.3,
564 doc=
"When adding new catalogs to the merge, all peaks less than this distance "
565 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
566 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
568 skyFilterName = Field(dtype=str, default=
"sky",
569 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
570 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
571 skySourceRadius = Field(dtype=float, default=8,
572 doc=
"Radius, in pixels, of sky objects")
573 skyGrowDetectedFootprints = Field(dtype=int, default=0,
574 doc=
"Number of pixels to grow the detected footprint mask "
575 "when adding sky objects")
576 nSkySourcesPerPatch = Field(dtype=int, default=100,
577 doc=
"Try to add this many sky objects to the mergeDet list, which will\n"
578 "then be measured along with the detected objects in sourceMeasurementTask")
579 nTrialSkySourcesPerPatch = Field(dtype=int, default=
None, optional=
True,
580 doc=
"Maximum number of trial sky object positions\n"
581 "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
582 nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
583 doc=
"Set nTrialSkySourcesPerPatch to\n"
584 " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
585 "if nTrialSkySourcesPerPatch is None")
597 \anchor MergeDetectionsTask_
599 \brief Merge coadd detections from multiple bands.
601 \section pipe_tasks_multiBand_Contents Contents
603 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
604 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
605 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
606 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
607 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
608 - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
610 \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
612 Command-line task that merges sources detected in coadds of exposures obtained with different filters.
614 To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
615 sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
616 track of which band each source originates in.
618 The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
619 objects are culled as described in \ref CullPeaksConfig_.
622 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
624 deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
628 MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
630 \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
632 \copydoc \_\_init\_\_
634 \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
638 \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
640 See \ref MergeDetectionsConfig_
642 \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
644 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
645 to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
647 MergeDetectionsTask has no debug variables.
649 \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
651 MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
652 of the available bands.
653 The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
654 chosen subset of filters.
655 Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
656 and, eventually, perform forced photometry.
657 Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
658 A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
659 `--help` command line argument:
661 mergeCoaddDetections.py --help
664 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
665 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
666 step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
668 mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
670 This will merge the HSC-I & -R band parent source catalogs and write the results to
671 `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
673 The next step in the multi-band processing procedure is
674 \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
676 ConfigClass = MergeDetectionsConfig
677 _DefaultName =
"mergeCoaddDetections"
679 outputDataset =
"mergeDet"
680 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
682 def __init__(self, butler=None, schema=None, **kwargs):
684 \brief Initialize the merge detections task.
686 A \ref FootprintMergeList_ "FootprintMergeList" will be used to
687 merge the source catalogs.
689 Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
690 \param[in] schema the schema of the detection catalogs used as input to this one
691 \param[in] butler a butler used to read the input schema from disk, if schema is None
692 \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
694 The task will set its own self.schema attribute to the schema of the output merged catalog.
696 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
700 if self.config.nSkySourcesPerPatch > 0:
701 filterNames += [self.config.skyFilterName]
706 \brief Merge multiple catalogs.
708 After ordering the catalogs and filters in priority order,
709 \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
710 \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
711 garbage peaks detected around bright objects.
715 \param[out] mergedList
719 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
720 tractWcs = skyInfo.wcs
721 peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
722 samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
725 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
727 if band
in catalogs.keys()]
729 mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
737 mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
738 if skySourceFootprints:
739 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
741 for foot
in skySourceFootprints:
742 s = mergedList.addNew()
746 self.log.info(
"Added %d sky sources (%.0f%% of requested)",
747 len(skySourceFootprints),
748 100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
751 for record
in mergedList:
752 record.getFootprint().sortPeaks()
753 self.log.info(
"Merged to %d sources" % len(mergedList))
760 \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
762 \param[in] catalog Source catalog
764 keys = [item.key
for item
in self.merged.getPeakSchema().extract(
"merge.peak.*").values()]
767 for parentSource
in catalog:
770 keptPeaks = parentSource.getFootprint().getPeaks()
771 oldPeaks = list(keptPeaks)
773 familySize = len(oldPeaks)
774 totalPeaks += familySize
775 for rank, peak
in enumerate(oldPeaks):
776 if ((rank < self.config.cullPeaks.rankSufficient)
or
777 (self.config.cullPeaks.nBandsSufficient > 1
and
778 sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
or
779 (rank < self.config.cullPeaks.rankConsidered
and
780 rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
781 keptPeaks.append(peak)
784 self.log.info(
"Culled %d of %d peaks" % (culledPeaks, totalPeaks))
788 Return a dict of empty catalogs for each catalog dataset produced by this task.
790 \param[out] dictionary of empty catalogs
792 mergeDet = afwTable.SourceCatalog(self.
schema)
793 peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
794 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
795 self.config.coaddName +
"Coadd_peak": peak}
799 \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
801 \param mergedList The merged Footprints from all the input bands
802 \param skyInfo A description of the patch
803 \param growDetectedFootprints The number of pixels to grow the detected footprints
806 if self.config.nSkySourcesPerPatch <= 0:
809 skySourceRadius = self.config.skySourceRadius
810 nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
811 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
812 if nTrialSkySourcesPerPatch
is None:
813 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
819 patchBBox = skyInfo.patchInfo.getOuterBBox()
820 mask = afwImage.Mask(patchBBox)
821 detectedMask = mask.getPlaneBitMask(
"DETECTED")
823 foot = s.getFootprint()
824 if growDetectedFootprints > 0:
825 foot.dilate(growDetectedFootprints)
826 foot.spans.setMask(mask, detectedMask)
828 xmin, ymin = patchBBox.getMin()
829 xmax, ymax = patchBBox.getMax()
831 xmin += skySourceRadius + 1
832 ymin += skySourceRadius + 1
833 xmax -= skySourceRadius + 1
834 ymax -= skySourceRadius + 1
836 skySourceFootprints = []
837 maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
838 for i
in range(nTrialSkySourcesPerPatch):
839 if len(skySourceFootprints) == nSkySourcesPerPatch:
842 x = int(numpy.random.uniform(xmin, xmax))
843 y = int(numpy.random.uniform(ymin, ymax))
844 spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
846 foot = afwDetect.Footprint(spans, patchBBox)
847 foot.setPeakSchema(self.merged.getPeakSchema())
849 if not foot.spans.overlaps(maskToSpanSet):
850 foot.addPeak(x, y, 0)
851 foot.getPeaks()[0].set(
"merge_peak_%s" % self.config.skyFilterName,
True)
852 skySourceFootprints.append(foot)
854 return skySourceFootprints
859 \anchor MeasureMergedCoaddSourcesConfig_
861 \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
863 doDeblend = Field(dtype=bool, default=
True, doc=
"Deblend sources?")
864 deblend = ConfigurableField(target=SourceDeblendTask, doc=
"Deblend sources")
865 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
866 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
867 doPropagateFlags = Field(
868 dtype=bool, default=
True,
869 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
871 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
872 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
873 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
874 doWriteMatchesDenormalized = Field(
877 doc=(
"Write reference matches in denormalized format? "
878 "This format uses more disk space, but is more convenient to read."),
880 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
881 checkUnitsParseStrict = Field(
882 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
889 doc=
"Apply aperture corrections"
891 applyApCorr = ConfigurableField(
892 target=ApplyApCorrTask,
893 doc=
"Subtask to apply aperture corrections"
895 doRunCatalogCalculation = Field(
898 doc=
'Run catalogCalculation task'
900 catalogCalculation = ConfigurableField(
901 target=CatalogCalculationTask,
902 doc=
"Subtask to run catalogCalculation plugins on catalog"
906 Config.setDefaults(self)
907 self.deblend.propagateAllPeaks =
True
908 self.measurement.plugins.names |= [
'base_InputCount']
911 self.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED']
923 \anchor MeasureMergedCoaddSourcesTask_
925 \brief Deblend sources from master catalog in each coadd seperately and measure.
927 \section pipe_tasks_multiBand_Contents Contents
929 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
930 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
931 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
932 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
933 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
934 - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
936 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
938 Command-line task that uses peaks and footprints from a master catalog to perform deblending and
939 measurement in each coadd.
941 Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
942 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
943 consistent set of child sources.
945 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
946 properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
947 flags are propagated to the coadd sources.
949 Optionally, we can match the coadd sources to an external reference catalog.
952 deepCoadd_mergeDet{tract,patch}: SourceCatalog
953 \n deepCoadd_calexp{tract,patch,filter}: ExposureF
955 deepCoadd_meas{tract,patch,filter}: SourceCatalog
959 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
962 <DT> \ref SourceDeblendTask_ "deblend"
963 <DD> Deblend all the sources from the master catalog.</DD>
964 <DT> \ref SingleFrameMeasurementTask_ "measurement"
965 <DD> Measure source properties of deblended sources.</DD>
966 <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
967 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
968 not at the edge of the field and that have either not been deblended or are the children of deblended
970 <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
971 <DD> Propagate flags set in individual visits to the coadd.</DD>
972 <DT> \ref DirectMatchTask_ "match"
973 <DD> Match input sources to a reference catalog (optional).
976 These subtasks may be retargeted as required.
978 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
980 \copydoc \_\_init\_\_
982 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
986 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
988 See \ref MeasureMergedCoaddSourcesConfig_
990 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
992 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
993 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
996 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
997 the various sub-tasks. See the documetation for individual sub-tasks for more information.
999 \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
1001 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
1002 The next stage in the multi-band processing procedure will merge these measurements into a suitable
1003 catalog for driving forced photometry.
1005 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
1006 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
1007 `--help` command line argument:
1009 measureCoaddSources.py --help
1012 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1013 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1014 step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1017 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1019 This will process the HSC-I band data. The results are written in
1020 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1022 It is also necessary to run
1024 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1026 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1027 procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1029 _DefaultName =
"measureCoaddSources"
1030 ConfigClass = MeasureMergedCoaddSourcesConfig
1031 RunnerClass = ButlerInitializedTaskRunner
1032 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
1033 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
1036 def _makeArgumentParser(cls):
1037 parser = ArgumentParser(name=cls._DefaultName)
1038 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
1039 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1040 ContainerClass=ExistingCoaddDataIdContainer)
1043 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1045 \brief Initialize the task.
1047 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1048 \param[in] schema: the schema of the merged detection catalog used as input to this one
1049 \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1050 \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1051 catalog. May be None if the loader can be constructed from the butler argument or all steps
1052 requiring a reference catalog are disabled.
1053 \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1054 catalog loader, if schema or peakSchema or refObjLoader is None
1056 The task will set its own self.schema attribute to the schema of the output measurement catalog.
1057 This will include all fields from the input schema, as well as additional fields for all the
1060 CmdLineTask.__init__(self, **kwargs)
1062 assert butler
is not None,
"Neither butler nor schema is defined"
1063 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
1065 self.schemaMapper.addMinimalSchema(schema)
1066 self.
schema = self.schemaMapper.getOutputSchema()
1068 if self.config.doDeblend:
1069 if peakSchema
is None:
1070 assert butler
is not None,
"Neither butler nor peakSchema is defined"
1071 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
1072 self.makeSubtask(
"deblend", schema=self.
schema, peakSchema=peakSchema)
1073 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
1074 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
1075 if self.config.doMatchSources:
1076 if refObjLoader
is None:
1077 assert butler
is not None,
"Neither butler nor refObjLoader is defined"
1078 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
1079 if self.config.doPropagateFlags:
1080 self.makeSubtask(
"propagateFlags", schema=self.
schema)
1081 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1082 if self.config.doApCorr:
1083 self.makeSubtask(
"applyApCorr", schema=self.
schema)
1084 if self.config.doRunCatalogCalculation:
1085 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
1089 \brief Deblend and measure.
1091 \param[in] patchRef: Patch reference.
1093 Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1094 from individual visits. Optionally match the sources to a reference catalog and write the matches.
1095 Finally, write the deblended sources and measurements out.
1097 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
1099 if self.config.doDeblend:
1100 self.deblend.run(exposure, sources)
1102 bigKey = sources.schema[
"deblend_parentTooBig"].asKey()
1104 numBig = sum((s.get(bigKey)
for s
in sources))
1106 self.log.warn(
"Patch %s contains %d large footprints that were not deblended" %
1107 (patchRef.dataId, numBig))
1109 table = sources.getTable()
1112 self.measurement.run(sources, exposure, exposureId=self.
getExposureId(patchRef))
1114 if self.config.doApCorr:
1115 self.applyApCorr.run(
1117 apCorrMap=exposure.getInfo().getApCorrMap()
1120 if self.config.doRunCatalogCalculation:
1121 self.catalogCalculation.run(sources)
1123 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1124 self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1125 includeDeblend=self.config.doDeblend)
1126 if self.config.doPropagateFlags:
1127 self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1129 if self.config.doMatchSources:
1131 self.
write(patchRef, sources)
1135 \brief Read input sources.
1137 \param[in] dataRef: Data reference for catalog of merged detections
1138 \return List of sources in merged catalog
1140 We also need to add columns to hold the measurements we're about to make
1141 so we can measure in-place.
1143 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
1144 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1147 idFactory.notify(s.getId())
1148 table = afwTable.SourceTable.make(self.
schema, idFactory)
1149 sources = afwTable.SourceCatalog(table)
1155 \brief Write matches of the sources to the astrometric reference catalog.
1157 We use the Wcs in the exposure to match sources.
1159 \param[in] dataRef: data reference
1160 \param[in] exposure: exposure with Wcs
1161 \param[in] sources: source catalog
1163 result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1165 matches = afwTable.packMatches(result.matches)
1166 matches.table.setMetadata(result.matchMeta)
1167 dataRef.put(matches, self.config.coaddName +
"Coadd_measMatch")
1168 if self.config.doWriteMatchesDenormalized:
1169 denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1170 dataRef.put(denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1175 \brief Write the source catalog.
1177 \param[in] dataRef: data reference
1178 \param[in] sources: source catalog
1180 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1181 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1184 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
1191 \anchor MergeMeasurementsConfig_
1193 \brief Configuration parameters for the MergeMeasurementsTask
1195 pseudoFilterList = ListField(dtype=str, default=[
"sky"],
1196 doc=
"Names of filters which may have no associated detection\n"
1197 "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1198 snName = Field(dtype=str, default=
"base_PsfFlux",
1199 doc=
"Name of flux measurement for calculating the S/N when choosing the reference band.")
1200 minSN = Field(dtype=float, default=10.,
1201 doc=
"If the S/N from the priority band is below this value (and the S/N "
1202 "is larger than minSNDiff compared to the priority band), use the band with "
1203 "the largest S/N as the reference band.")
1204 minSNDiff = Field(dtype=float, default=3.,
1205 doc=
"If the difference in S/N between another band and the priority band is larger "
1206 "than this value (and the S/N in the priority band is less than minSN) "
1207 "use the band with the largest S/N as the reference band")
1208 flags = ListField(dtype=str, doc=
"Require that these flags, if available, are not set",
1209 default=[
"base_PixelFlags_flag_interpolatedCenter",
"base_PsfFlux_flag",
1210 "ext_photometryKron_KronFlux_flag",
"modelfit_CModel_flag", ])
1222 \anchor MergeMeasurementsTask_
1224 \brief Merge measurements from multiple bands
1226 \section pipe_tasks_multiBand_Contents Contents
1228 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1229 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1230 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1231 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1232 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1233 - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1235 \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1237 Command-line task that merges measurements from multiple bands.
1239 Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1240 bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1241 required to have centroid, shape and flux measurements in each band.
1244 deepCoadd_meas{tract,patch,filter}: SourceCatalog
1246 deepCoadd_ref{tract,patch}: SourceCatalog
1250 MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1252 \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1254 \copydoc \_\_init\_\_
1256 \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1260 \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1262 See \ref MergeMeasurementsConfig_
1264 \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1266 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1267 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1270 MergeMeasurementsTask has no debug variables.
1272 \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1274 MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1275 The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1276 coadds and individual exposures.
1277 Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1278 of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1279 command line argument:
1281 mergeCoaddMeasurements.py --help
1284 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1285 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1286 step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1289 mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1291 This will merge the HSC-I & HSC-R band catalogs. The results are written in
1292 `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1294 _DefaultName =
"mergeCoaddMeasurements"
1295 ConfigClass = MergeMeasurementsConfig
1296 inputDataset =
"meas"
1297 outputDataset =
"ref"
1298 getSchemaCatalogs = _makeGetSchemaCatalogs(
"ref")
1300 def __init__(self, butler=None, schema=None, **kwargs):
1302 Initialize the task.
1304 Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1305 \param[in] schema: the schema of the detection catalogs used as input to this one
1306 \param[in] butler: a butler used to read the input schema from disk, if schema is None
1308 The task will set its own self.schema attribute to the schema of the output merged catalog.
1310 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1313 self.schemaMapper.addMinimalSchema(inputSchema,
True)
1314 self.
fluxKey = inputSchema.find(self.config.snName +
"_flux").getKey()
1315 self.
fluxErrKey = inputSchema.find(self.config.snName +
"_fluxSigma").getKey()
1316 self.
fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
1319 for band
in self.config.priorityList:
1321 outputKey = self.schemaMapper.editOutputSchema().addField(
1322 "merge_measurement_%s" % short,
1324 doc=
"Flag field set if the measurements here are from the %s filter" % band
1326 peakKey = inputSchema.find(
"merge_peak_%s" % short).key
1327 footprintKey = inputSchema.find(
"merge_footprint_%s" % short).key
1328 self.
flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1329 self.
schema = self.schemaMapper.getOutputSchema()
1332 for filt
in self.config.pseudoFilterList:
1334 self.pseudoFilterKeys.append(self.schema.find(
"merge_peak_%s" % filt).getKey())
1336 self.log.warn(
"merge_peak is not set for pseudo-filter %s" % filt)
1339 for flag
in self.config.flags:
1341 self.
badFlags[flag] = self.schema.find(flag).getKey()
1342 except KeyError
as exc:
1343 self.log.warn(
"Can't find flag %s in schema: %s" % (flag, exc,))
1347 Merge measurement catalogs to create a single reference catalog for forced photometry
1349 \param[in] catalogs: the catalogs to be merged
1350 \param[in] patchRef: patch reference for data
1352 For parent sources, we choose the first band in config.priorityList for which the
1353 merge_footprint flag for that band is is True.
1355 For child sources, the logic is the same, except that we use the merge_peak flags.
1358 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1359 orderedKeys = [self.
flagKeys[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1361 mergedCatalog = afwTable.SourceCatalog(self.
schema)
1362 mergedCatalog.reserve(len(orderedCatalogs[0]))
1364 idKey = orderedCatalogs[0].table.getIdKey()
1365 for catalog
in orderedCatalogs[1:]:
1366 if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1367 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1371 for orderedRecords
in zip(*orderedCatalogs):
1374 maxSNFlagKeys =
None
1376 priorityRecord =
None
1377 priorityFlagKeys =
None
1379 hasPseudoFilter =
False
1383 for inputRecord, flagKeys
in zip(orderedRecords, orderedKeys):
1384 parent = (inputRecord.getParent() == 0
and inputRecord.get(flagKeys.footprint))
1385 child = (inputRecord.getParent() != 0
and inputRecord.get(flagKeys.peak))
1387 if not (parent
or child):
1389 if inputRecord.get(pseudoFilterKey):
1390 hasPseudoFilter =
True
1391 priorityRecord = inputRecord
1392 priorityFlagKeys = flagKeys
1397 isBad = any(inputRecord.get(flag)
for flag
in self.
badFlags)
1402 if numpy.isnan(sn)
or sn < 0.:
1404 if (parent
or child)
and priorityRecord
is None:
1405 priorityRecord = inputRecord
1406 priorityFlagKeys = flagKeys
1409 maxSNRecord = inputRecord
1410 maxSNFlagKeys = flagKeys
1423 bestRecord = priorityRecord
1424 bestFlagKeys = priorityFlagKeys
1425 elif (prioritySN < self.config.minSN
and (maxSN - prioritySN) > self.config.minSNDiff
and
1426 maxSNRecord
is not None):
1427 bestRecord = maxSNRecord
1428 bestFlagKeys = maxSNFlagKeys
1429 elif priorityRecord
is not None:
1430 bestRecord = priorityRecord
1431 bestFlagKeys = priorityFlagKeys
1433 if bestRecord
is not None and bestFlagKeys
is not None:
1434 outputRecord = mergedCatalog.addNew()
1436 outputRecord.set(bestFlagKeys.output,
True)
1438 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1439 inputRecord.getId())
1442 for inputCatalog
in orderedCatalogs:
1443 if len(mergedCatalog) != len(inputCatalog):
1444 raise ValueError(
"Mismatch between catalog sizes: %s != %s" %
1445 (len(mergedCatalog), len(orderedCatalogs)))
1447 return mergedCatalog
def writeMetadata
No metadata to write, and not sure how to write it for a list of dataRefs.
def cullPeaks
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
def run
Deblend and measure.
Merge coadd detections from multiple bands.
def __init__
Initialize the task.
def mergeCatalogs
Merge measurement catalogs to create a single reference catalog for forced photometry.
def runDetection
Run detection on an exposure.
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
def write
Write the source catalog.
def run
Merge coadd sources from multiple bands.
Configuration parameters for the DetectCoaddSourcesTask.
def run
Run detection on a coadd.
Merge measurements from multiple bands.
Deblend sources from master catalog in each coadd seperately and measure.
def getTargetList
Provide a list of patch references for each patch.
def __init__
Initialize the task.
def readSources
Read input sources.
def __init__
Initialize the task.
def scaleVariance
Scale the variance in a maskedImage.
def write
Write the output.
Configuration parameters for the MergeMeasurementsTask.
def mergeCatalogs
Merge multiple catalogs.
Configuration parameters for the MergeDetectionsTask.
Configuration for merging sources.
def mergeCatalogs
Merge multiple catalogs.
def makeTask
Provide a butler to the Task constructor.
def readCatalog
Read input catalog.
A base class for merging source catalogs.
def getSkyInfo
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def __init__
Initialize the task.
def getSkySourceFootprints
Return a list of Footprints of sky objects which don't overlap with anything in mergedList.
def __init__
Initialize the merge detections task.
def getInputSchema
Obtain the input schema either directly or froma butler reference.
Detect sources on a coadd.
def getSchemaCatalogs
Return a dict of empty catalogs for each catalog dataset produced by this task.
def write
Write out results from runDetection.
def writeMatches
Write matches of the sources to the astrometric reference catalog.