24 from __future__
import absolute_import, division, print_function
25 from builtins
import zip
26 from builtins
import range
29 from lsst.coadd.utils.coaddDataIdContainer
import ExistingCoaddDataIdContainer
30 from lsst.pipe.base
import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
31 from lsst.pex.config
import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
32 from lsst.meas.algorithms
import SourceDetectionTask
33 from lsst.meas.base
import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
34 from lsst.meas.deblender
import SourceDeblendTask
36 from lsst.meas.astrom
import DirectMatchTask, denormalizeMatches
39 import lsst.afw.image
as afwImage
40 import lsst.afw.table
as afwTable
41 import lsst.afw.math
as afwMath
42 import lsst.afw.geom
as afwGeom
43 import lsst.afw.detection
as afwDetect
44 from lsst.daf.base
import PropertyList
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
60 def _makeGetSchemaCatalogs(datasetSuffix):
61 """Construct a getSchemaCatalogs instance method
63 These are identical for most of the classes here, so we'll consolidate
66 datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
69 def getSchemaCatalogs(self):
70 """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71 src = afwTable.SourceCatalog(self.schema)
72 if hasattr(self,
"algMetadata"):
73 src.getTable().setMetadata(self.algMetadata)
74 return {self.config.coaddName +
"Coadd_" + datasetSuffix: src}
75 return getSchemaCatalogs
78 def _makeMakeIdFactory(datasetName):
79 """Construct a makeIdFactory instance method
81 These are identical for all the classes here, so this consolidates
84 datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
87 def makeIdFactory(self, dataRef):
88 """Return an IdFactory for setting the detection identifiers
90 The actual parameters used in the IdFactory are provided by
91 the butler (through the provided data reference.
93 expBits = dataRef.get(self.config.coaddName + datasetName +
"_bits")
94 expId = int(dataRef.get(self.config.coaddName + datasetName))
95 return afwTable.IdFactory.makeSource(expId, 64 - expBits)
100 """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
104 return afwImage.Filter(name).getFilterProperty().getName()
111 \anchor DetectCoaddSourcesConfig_
113 \brief Configuration parameters for the DetectCoaddSourcesTask
115 doScaleVariance = Field(dtype=bool, default=
True, doc=
"Scale variance plane using empirical noise?")
116 detection = ConfigurableField(target=SourceDetectionTask, doc=
"Source detection")
117 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
118 mask = ListField(dtype=str, default=[
"DETECTED",
"BAD",
"SAT",
"NO_DATA",
"INTRP"],
119 doc=
"Mask planes for pixels to ignore when scaling variance")
122 Config.setDefaults(self)
123 self.detection.thresholdType =
"pixel_stdev"
124 self.detection.isotropicGrow =
True
126 self.detection.background.useApprox =
False
127 self.detection.background.binSize = 4096
128 self.detection.background.undersampleStyle =
'REDUCE_INTERP_ORDER'
140 \anchor DetectCoaddSourcesTask_
142 \brief Detect sources on a coadd
144 \section pipe_tasks_multiBand_Contents Contents
146 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
147 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
148 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
149 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
150 - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
151 - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
153 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
155 Command-line task that detects sources on a coadd of exposures obtained with a single filter.
157 Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
158 properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
159 in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
160 propagate the full covariance matrix -- but it is simple and works well in practice.
162 After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
163 SourceDetectionTask_ "detection" subtask.
166 deepCoadd{tract,patch,filter}: ExposureF
168 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
169 \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
171 \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
175 DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
176 You can retarget this subtask if you wish.
178 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
180 \copydoc \_\_init\_\_
182 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
186 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
188 See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
190 \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
192 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
193 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
196 DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
197 \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
198 \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
200 \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
202 DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
203 the task is to update the background, detect all sources in a single band and generate a set of parent
204 footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
205 eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
206 reference to the coadd to be processed. A list of the available optional arguments can be obtained by
207 calling detectCoaddSources.py with the `--help` command line argument:
209 detectCoaddSources.py --help
212 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
213 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
214 steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
216 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
218 that will process the HSC-I band data. The results are written to
219 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
221 It is also necessary to run:
223 detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
225 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
226 processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
228 _DefaultName =
"detectCoaddSources"
229 ConfigClass = DetectCoaddSourcesConfig
230 getSchemaCatalogs = _makeGetSchemaCatalogs(
"det")
231 makeIdFactory = _makeMakeIdFactory(
"CoaddId")
234 def _makeArgumentParser(cls):
235 parser = ArgumentParser(name=cls._DefaultName)
236 parser.add_id_argument(
"--id",
"deepCoadd", help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
237 ContainerClass=ExistingCoaddDataIdContainer)
242 \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
244 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
246 \param[in] schema: initial schema for the output catalog, modified-in place to include all
247 fields set by this task. If None, the source minimal schema will be used.
248 \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
250 CmdLineTask.__init__(self, **kwargs)
252 schema = afwTable.SourceTable.makeMinimalSchema()
254 self.makeSubtask(
"detection", schema=self.
schema)
258 \brief Run detection on a coadd.
260 Invokes \ref runDetection and then uses \ref write to output the
263 \param[in] patchRef: data reference for patch
265 exposure = patchRef.get(self.config.coaddName +
"Coadd", immediate=
True)
267 self.
write(exposure, results, patchRef)
272 \brief Run detection on an exposure.
274 First scale the variance plane to match the observed variance
275 using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
278 \param[in] exposure: Exposure on which to detect
279 \param[in] idFactory: IdFactory to set source identifiers
281 \return a pipe.base.Struct with fields
282 - sources: catalog of detections
283 - backgrounds: list of backgrounds
285 if self.config.doScaleVariance:
286 scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
287 backgrounds = afwMath.BackgroundList()
288 table = afwTable.SourceTable.make(self.
schema, idFactory)
289 detections = self.detection.makeSourceCatalog(table, exposure)
290 sources = detections.sources
291 fpSets = detections.fpSets
292 if fpSets.background:
293 backgrounds.append(fpSets.background)
294 return Struct(sources=sources, backgrounds=backgrounds)
296 def write(self, exposure, results, patchRef):
298 \brief Write out results from runDetection.
300 \param[in] exposure: Exposure to write out
301 \param[in] results: Struct returned from runDetection
302 \param[in] patchRef: data reference for patch
304 coaddName = self.config.coaddName +
"Coadd"
305 patchRef.put(results.backgrounds, coaddName +
"_calexp_background")
306 patchRef.put(results.sources, coaddName +
"_det")
307 patchRef.put(exposure, coaddName +
"_calexp")
314 \anchor MergeSourcesRunner_
316 \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
317 requires a list of dataRefs rather than a single dataRef.
322 \brief Provide a butler to the Task constructor.
324 \param[in] parsedCmd the parsed command
325 \param[in] args tuple of a list of data references and kwargs (un-used)
326 \throws RuntimeError if both parsedCmd & args are None
328 if parsedCmd
is not None:
329 butler = parsedCmd.butler
330 elif args
is not None:
331 dataRefList, kwargs = args
332 butler = dataRefList[0].getButler()
334 raise RuntimeError(
"Neither parsedCmd or args specified")
335 return self.TaskClass(config=self.config, log=self.log, butler=butler)
340 \brief Provide a list of patch references for each patch.
342 The patch references within the list will have different filters.
344 \param[in] parsedCmd the parsed command
345 \param **kwargs key word arguments (unused)
346 \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
350 for ref
in parsedCmd.id.refList:
351 tract = ref.dataId[
"tract"]
352 patch = ref.dataId[
"patch"]
353 filter = ref.dataId[
"filter"]
354 if not tract
in refList:
356 if not patch
in refList[tract]:
357 refList[tract][patch] = {}
358 if filter
in refList[tract][patch]:
359 raise RuntimeError(
"Multiple versions of %s" % (ref.dataId,))
360 refList[tract][patch][filter] = ref
361 return [(list(p.values()), kwargs)
for t
in refList.values()
for p
in t.values()]
366 \anchor MergeSourcesConfig_
368 \brief Configuration for merging sources.
370 priorityList = ListField(dtype=str, default=[],
371 doc=
"Priority-ordered list of bands for the merge.")
372 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
375 Config.validate(self)
377 raise RuntimeError(
"No priority list provided")
382 \anchor MergeSourcesTask_
384 \brief A base class for merging source catalogs.
386 Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
387 so similar that it makes sense to re-use the code, in the form of this abstract base class.
389 NB: Do not use this class directly. Instead use one of the child classes that inherit from
390 MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
391 "MergeMeasurementsTask"
393 Sub-classes should set the following class variables:
394 * `_DefaultName`: name of Task
395 * `inputDataset`: name of dataset to read
396 * `outputDataset`: name of dataset to write
397 * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
399 In addition, sub-classes must implement the mergeCatalogs method.
402 ConfigClass = MergeSourcesConfig
403 RunnerClass = MergeSourcesRunner
406 getSchemaCatalogs =
None
409 def _makeArgumentParser(cls):
411 \brief Create a suitable ArgumentParser.
413 We will use the ArgumentParser to get a provide a list of data
414 references for patches; the RunnerClass will sort them into lists
415 of data references for the same patch
417 parser = ArgumentParser(name=cls._DefaultName)
418 parser.add_id_argument(
"--id",
"deepCoadd_" + cls.inputDataset,
419 ContainerClass=ExistingCoaddDataIdContainer,
420 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
425 \brief Obtain the input schema either directly or froma butler reference.
427 \param[in] butler butler reference to obtain the input schema from
428 \param[in] schema the input schema
431 assert butler
is not None,
"Neither butler nor schema specified"
432 schema = butler.get(self.config.coaddName +
"Coadd_" + self.
inputDataset +
"_schema",
433 immediate=
True).schema
436 def __init__(self, butler=None, schema=None, **kwargs):
438 \brief Initialize the task.
440 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
441 \param[in] schema the schema of the detection catalogs used as input to this one
442 \param[in] butler a butler used to read the input schema from disk, if schema is None
444 Derived classes should use the getInputSchema() method to handle the additional
445 arguments and retreive the actual input schema.
447 CmdLineTask.__init__(self, **kwargs)
449 def run(self, patchRefList):
451 \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
452 subclasses that inherit from MergeSourcesTask.
454 \param[in] patchRefList list of data references for each filter
456 catalogs = dict(self.
readCatalog(patchRef)
for patchRef
in patchRefList)
457 mergedCatalog = self.
mergeCatalogs(catalogs, patchRefList[0])
458 self.
write(patchRefList[0], mergedCatalog)
462 \brief Read input catalog.
464 We read the input dataset provided by the 'inputDataset'
467 \param[in] patchRef data reference for patch
468 \return tuple consisting of the filter name and the catalog
470 filterName = patchRef.dataId[
"filter"]
471 catalog = patchRef.get(self.config.coaddName +
"Coadd_" + self.
inputDataset, immediate=
True)
472 self.log.info(
"Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
473 return filterName, catalog
477 \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
480 \param[in] catalogs dict mapping filter name to source catalog
482 \return merged catalog
484 raise NotImplementedError()
488 \brief Write the output.
490 \param[in] patchRef data reference for patch
491 \param[in] catalog catalog
493 We write as the dataset provided by the 'outputDataset'
496 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
499 mergeDataId = patchRef.dataId.copy()
500 del mergeDataId[
"filter"]
501 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
505 \brief No metadata to write, and not sure how to write it for a list of dataRefs.
510 class CullPeaksConfig(Config):
512 \anchor CullPeaksConfig_
514 \brief Configuration for culling garbage peaks after merging footprints.
516 Peaks may also be culled after detection or during deblending; this configuration object
517 only deals with culling after merging Footprints.
519 These cuts are based on three quantities:
520 - nBands: the number of bands in which the peak was detected
521 - peakRank: the position of the peak within its family, sorted from brightest to faintest.
522 - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
524 The formula that identifie peaks to cull is:
526 nBands < nBandsSufficient
527 AND (rank >= rankSufficient)
528 AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
530 To disable peak culling, simply set nBandsSafe=1.
533 nBandsSufficient = RangeField(dtype=int, default=2, min=1,
534 doc=
"Always keep peaks detected in this many bands")
535 rankSufficient = RangeField(dtype=int, default=20, min=1,
536 doc=
"Always keep this many peaks in each family")
537 rankConsidered = RangeField(dtype=int, default=30, min=1,
538 doc=(
"Keep peaks with less than this rank that also match the "
539 "rankNormalizedConsidered condition."))
540 rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
541 doc=(
"Keep peaks with less than this normalized rank that"
542 " also match the rankConsidered condition."))
547 \anchor MergeDetectionsConfig_
549 \brief Configuration parameters for the MergeDetectionsTask.
551 minNewPeak = Field(dtype=float, default=1,
552 doc=
"Minimum distance from closest peak to create a new one (in arcsec).")
554 maxSamePeak = Field(dtype=float, default=0.3,
555 doc=
"When adding new catalogs to the merge, all peaks less than this distance "
556 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
557 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
559 skyFilterName = Field(dtype=str, default=
"sky",
560 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
561 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
562 skySourceRadius = Field(dtype=float, default=8,
563 doc=
"Radius, in pixels, of sky objects")
564 skyGrowDetectedFootprints = Field(dtype=int, default=0,
565 doc=
"Number of pixels to grow the detected footprint mask "
566 "when adding sky objects")
567 nSkySourcesPerPatch = Field(dtype=int, default=100,
568 doc=
"Try to add this many sky objects to the mergeDet list, which will\n"
569 "then be measured along with the detected objects in sourceMeasurementTask")
570 nTrialSkySourcesPerPatch = Field(dtype=int, default=
None, optional=
True,
571 doc=
"Maximum number of trial sky object positions\n"
572 "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
573 nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
574 doc=
"Set nTrialSkySourcesPerPatch to\n"
575 " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
576 "if nTrialSkySourcesPerPatch is None")
588 \anchor MergeDetectionsTask_
590 \brief Merge coadd detections from multiple bands.
592 \section pipe_tasks_multiBand_Contents Contents
594 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
595 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
596 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
597 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
598 - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
599 - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
601 \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
603 Command-line task that merges sources detected in coadds of exposures obtained with different filters.
605 To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
606 sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
607 track of which band each source originates in.
609 The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
610 objects are culled as described in \ref CullPeaksConfig_.
613 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
615 deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
619 MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
621 \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
623 \copydoc \_\_init\_\_
625 \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
629 \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
631 See \ref MergeDetectionsConfig_
633 \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
635 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
636 to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
638 MergeDetectionsTask has no debug variables.
640 \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
642 MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
643 of the available bands.
644 The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
645 chosen subset of filters.
646 Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
647 and, eventually, perform forced photometry.
648 Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
649 A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
650 `--help` command line argument:
652 mergeCoaddDetections.py --help
655 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
656 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
657 step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
659 mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
661 This will merge the HSC-I & -R band parent source catalogs and write the results to
662 `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
664 The next step in the multi-band processing procedure is
665 \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
667 ConfigClass = MergeDetectionsConfig
668 _DefaultName =
"mergeCoaddDetections"
670 outputDataset =
"mergeDet"
671 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
673 def __init__(self, butler=None, schema=None, **kwargs):
675 \brief Initialize the merge detections task.
677 A \ref FootprintMergeList_ "FootprintMergeList" will be used to
678 merge the source catalogs.
680 Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
681 \param[in] schema the schema of the detection catalogs used as input to this one
682 \param[in] butler a butler used to read the input schema from disk, if schema is None
683 \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
685 The task will set its own self.schema attribute to the schema of the output merged catalog.
687 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
691 if self.config.nSkySourcesPerPatch > 0:
692 filterNames += [self.config.skyFilterName]
697 \brief Merge multiple catalogs.
699 After ordering the catalogs and filters in priority order,
700 \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
701 \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
702 garbage peaks detected around bright objects.
706 \param[out] mergedList
710 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
711 tractWcs = skyInfo.wcs
712 peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
713 samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
716 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
718 if band
in catalogs.keys()]
720 mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
728 mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
729 if skySourceFootprints:
730 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
732 for foot
in skySourceFootprints:
733 s = mergedList.addNew()
737 self.log.info(
"Added %d sky sources (%.0f%% of requested)",
738 len(skySourceFootprints),
739 100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
742 for record
in mergedList:
743 record.getFootprint().sortPeaks()
744 self.log.info(
"Merged to %d sources" % len(mergedList))
751 \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
753 \param[in] catalog Source catalog
755 keys = [item.key
for item
in self.merged.getPeakSchema().extract(
"merge.peak.*").values()]
758 for parentSource
in catalog:
761 keptPeaks = parentSource.getFootprint().getPeaks()
762 oldPeaks = list(keptPeaks)
764 familySize = len(oldPeaks)
765 totalPeaks += familySize
766 for rank, peak
in enumerate(oldPeaks):
767 if ((rank < self.config.cullPeaks.rankSufficient)
or
768 (self.config.cullPeaks.nBandsSufficient > 1
and
769 sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
or
770 (rank < self.config.cullPeaks.rankConsidered
and
771 rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
772 keptPeaks.append(peak)
775 self.log.info(
"Culled %d of %d peaks" % (culledPeaks, totalPeaks))
779 Return a dict of empty catalogs for each catalog dataset produced by this task.
781 \param[out] dictionary of empty catalogs
783 mergeDet = afwTable.SourceCatalog(self.
schema)
784 peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
785 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
786 self.config.coaddName +
"Coadd_peak": peak}
790 \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
792 \param mergedList The merged Footprints from all the input bands
793 \param skyInfo A description of the patch
794 \param growDetectedFootprints The number of pixels to grow the detected footprints
797 if self.config.nSkySourcesPerPatch <= 0:
800 skySourceRadius = self.config.skySourceRadius
801 nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
802 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
803 if nTrialSkySourcesPerPatch
is None:
804 nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
810 patchBBox = skyInfo.patchInfo.getOuterBBox()
811 mask = afwImage.MaskU(patchBBox)
812 detectedMask = mask.getPlaneBitMask(
"DETECTED")
814 foot = s.getFootprint()
815 if growDetectedFootprints > 0:
816 foot.dilate(growDetectedFootprints)
817 foot.spans.setMask(mask, detectedMask)
819 xmin, ymin = patchBBox.getMin()
820 xmax, ymax = patchBBox.getMax()
822 xmin += skySourceRadius + 1
823 ymin += skySourceRadius + 1
824 xmax -= skySourceRadius + 1
825 ymax -= skySourceRadius + 1
827 skySourceFootprints = []
828 maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
829 for i
in range(nTrialSkySourcesPerPatch):
830 if len(skySourceFootprints) == nSkySourcesPerPatch:
833 x = int(numpy.random.uniform(xmin, xmax))
834 y = int(numpy.random.uniform(ymin, ymax))
835 spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
837 foot = afwDetect.Footprint(spans, patchBBox)
838 foot.setPeakSchema(self.merged.getPeakSchema())
840 if not foot.spans.overlaps(maskToSpanSet):
841 foot.addPeak(x, y, 0)
842 foot.getPeaks()[0].set(
"merge_peak_%s" % self.config.skyFilterName,
True)
843 skySourceFootprints.append(foot)
845 return skySourceFootprints
850 \anchor MeasureMergedCoaddSourcesConfig_
852 \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
854 doDeblend = Field(dtype=bool, default=
True, doc=
"Deblend sources?")
855 deblend = ConfigurableField(target=SourceDeblendTask, doc=
"Deblend sources")
856 measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc=
"Source measurement")
857 setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc=
"Set flags for primary tract/patch")
858 doPropagateFlags = Field(
859 dtype=bool, default=
True,
860 doc=
"Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
862 propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc=
"Propagate visit flags to coadd")
863 doMatchSources = Field(dtype=bool, default=
True, doc=
"Match sources to reference catalog?")
864 match = ConfigurableField(target=DirectMatchTask, doc=
"Matching to reference catalog")
865 doWriteMatchesDenormalized = Field(
868 doc=(
"Write reference matches in denormalized format? "
869 "This format uses more disk space, but is more convenient to read."),
871 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
872 checkUnitsParseStrict = Field(
873 doc=
"Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
880 doc=
"Apply aperture corrections"
882 applyApCorr = ConfigurableField(
883 target=ApplyApCorrTask,
884 doc=
"Subtask to apply aperture corrections"
886 doRunCatalogCalculation = Field(
889 doc=
'Run catalogCalculation task'
891 catalogCalculation = ConfigurableField(
892 target=CatalogCalculationTask,
893 doc=
"Subtask to run catalogCalculation plugins on catalog"
897 Config.setDefaults(self)
898 self.deblend.propagateAllPeaks =
True
899 self.measurement.plugins.names |= [
'base_InputCount']
902 self.measurement.plugins[
'base_PixelFlags'].masksFpAnywhere = [
'CLIPPED']
914 \anchor MeasureMergedCoaddSourcesTask_
916 \brief Deblend sources from master catalog in each coadd seperately and measure.
918 \section pipe_tasks_multiBand_Contents Contents
920 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
921 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
922 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
923 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
924 - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
925 - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
927 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
929 Command-line task that uses peaks and footprints from a master catalog to perform deblending and
930 measurement in each coadd.
932 Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
933 coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
934 consistent set of child sources.
936 The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
937 properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
938 flags are propagated to the coadd sources.
940 Optionally, we can match the coadd sources to an external reference catalog.
943 deepCoadd_mergeDet{tract,patch}: SourceCatalog
944 \n deepCoadd_calexp{tract,patch,filter}: ExposureF
946 deepCoadd_meas{tract,patch,filter}: SourceCatalog
950 MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
953 <DT> \ref SourceDeblendTask_ "deblend"
954 <DD> Deblend all the sources from the master catalog.</DD>
955 <DT> \ref SingleFrameMeasurementTask_ "measurement"
956 <DD> Measure source properties of deblended sources.</DD>
957 <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
958 <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
959 not at the edge of the field and that have either not been deblended or are the children of deblended
961 <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
962 <DD> Propagate flags set in individual visits to the coadd.</DD>
963 <DT> \ref DirectMatchTask_ "match"
964 <DD> Match input sources to a reference catalog (optional).
967 These subtasks may be retargeted as required.
969 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
971 \copydoc \_\_init\_\_
973 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
977 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
979 See \ref MeasureMergedCoaddSourcesConfig_
981 \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
983 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
984 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
987 MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
988 the various sub-tasks. See the documetation for individual sub-tasks for more information.
990 \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
992 After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
993 The next stage in the multi-band processing procedure will merge these measurements into a suitable
994 catalog for driving forced photometry.
996 Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
997 A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
998 `--help` command line argument:
1000 measureCoaddSources.py --help
1003 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1004 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1005 step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1008 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1010 This will process the HSC-I band data. The results are written in
1011 `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1013 It is also necessary to run
1015 measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1017 to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1018 procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1020 _DefaultName =
"measureCoaddSources"
1021 ConfigClass = MeasureMergedCoaddSourcesConfig
1022 RunnerClass = ButlerInitializedTaskRunner
1023 getSchemaCatalogs = _makeGetSchemaCatalogs(
"meas")
1024 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
1027 def _makeArgumentParser(cls):
1028 parser = ArgumentParser(name=cls._DefaultName)
1029 parser.add_id_argument(
"--id",
"deepCoadd_calexp",
1030 help=
"data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1031 ContainerClass=ExistingCoaddDataIdContainer)
1034 def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1036 \brief Initialize the task.
1038 Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1039 \param[in] schema: the schema of the merged detection catalog used as input to this one
1040 \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1041 \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1042 catalog. May be None if the loader can be constructed from the butler argument or all steps
1043 requiring a reference catalog are disabled.
1044 \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1045 catalog loader, if schema or peakSchema or refObjLoader is None
1047 The task will set its own self.schema attribute to the schema of the output measurement catalog.
1048 This will include all fields from the input schema, as well as additional fields for all the
1051 CmdLineTask.__init__(self, **kwargs)
1053 assert butler
is not None,
"Neither butler nor schema is defined"
1054 schema = butler.get(self.config.coaddName +
"Coadd_mergeDet_schema", immediate=
True).schema
1056 self.schemaMapper.addMinimalSchema(schema)
1057 self.
schema = self.schemaMapper.getOutputSchema()
1059 if self.config.doDeblend:
1060 if peakSchema
is None:
1061 assert butler
is not None,
"Neither butler nor peakSchema is defined"
1062 peakSchema = butler.get(self.config.coaddName +
"Coadd_peak_schema", immediate=
True).schema
1063 self.makeSubtask(
"deblend", schema=self.
schema, peakSchema=peakSchema)
1064 self.makeSubtask(
"measurement", schema=self.
schema, algMetadata=self.
algMetadata)
1065 self.makeSubtask(
"setPrimaryFlags", schema=self.
schema)
1066 if self.config.doMatchSources:
1067 if refObjLoader
is None:
1068 assert butler
is not None,
"Neither butler nor refObjLoader is defined"
1069 self.makeSubtask(
"match", butler=butler, refObjLoader=refObjLoader)
1070 if self.config.doPropagateFlags:
1071 self.makeSubtask(
"propagateFlags", schema=self.
schema)
1072 self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1073 if self.config.doApCorr:
1074 self.makeSubtask(
"applyApCorr", schema=self.
schema)
1075 if self.config.doRunCatalogCalculation:
1076 self.makeSubtask(
"catalogCalculation", schema=self.
schema)
1080 \brief Deblend and measure.
1082 \param[in] patchRef: Patch reference.
1084 Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1085 from individual visits. Optionally match the sources to a reference catalog and write the matches.
1086 Finally, write the deblended sources and measurements out.
1088 exposure = patchRef.get(self.config.coaddName +
"Coadd_calexp", immediate=
True)
1090 if self.config.doDeblend:
1091 self.deblend.run(exposure, sources)
1093 bigKey = sources.schema[
"deblend_parentTooBig"].asKey()
1095 numBig = sum((s.get(bigKey)
for s
in sources))
1097 self.log.warn(
"Patch %s contains %d large footprints that were not deblended" %
1098 (patchRef.dataId, numBig))
1100 table = sources.getTable()
1103 self.measurement.run(sources, exposure, exposureId=self.
getExposureId(patchRef))
1105 if self.config.doApCorr:
1106 self.applyApCorr.run(
1108 apCorrMap=exposure.getInfo().getApCorrMap()
1111 if self.config.doRunCatalogCalculation:
1112 self.catalogCalculation.run(sources)
1114 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1115 self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1116 includeDeblend=self.config.doDeblend)
1117 if self.config.doPropagateFlags:
1118 self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1120 if self.config.doMatchSources:
1122 self.
write(patchRef, sources)
1126 \brief Read input sources.
1128 \param[in] dataRef: Data reference for catalog of merged detections
1129 \return List of sources in merged catalog
1131 We also need to add columns to hold the measurements we're about to make
1132 so we can measure in-place.
1134 merged = dataRef.get(self.config.coaddName +
"Coadd_mergeDet", immediate=
True)
1135 self.log.info(
"Read %d detections: %s" % (len(merged), dataRef.dataId))
1138 idFactory.notify(s.getId())
1139 table = afwTable.SourceTable.make(self.
schema, idFactory)
1140 sources = afwTable.SourceCatalog(table)
1146 \brief Write matches of the sources to the astrometric reference catalog.
1148 We use the Wcs in the exposure to match sources.
1150 \param[in] dataRef: data reference
1151 \param[in] exposure: exposure with Wcs
1152 \param[in] sources: source catalog
1154 result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1156 matches = afwTable.packMatches(result.matches)
1157 matches.table.setMetadata(result.matchMeta)
1158 dataRef.put(matches, self.config.coaddName +
"Coadd_measMatch")
1159 if self.config.doWriteMatchesDenormalized:
1160 denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1161 dataRef.put(denormMatches, self.config.coaddName +
"Coadd_measMatchFull")
1166 \brief Write the source catalog.
1168 \param[in] dataRef: data reference
1169 \param[in] sources: source catalog
1171 dataRef.put(sources, self.config.coaddName +
"Coadd_meas")
1172 self.log.info(
"Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1175 return int(dataRef.get(self.config.coaddName +
"CoaddId"))
1182 \anchor MergeMeasurementsConfig_
1184 \brief Configuration parameters for the MergeMeasurementsTask
1186 pseudoFilterList = ListField(dtype=str, default=[
"sky"],
1187 doc=
"Names of filters which may have no associated detection\n"
1188 "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1189 snName = Field(dtype=str, default=
"base_PsfFlux",
1190 doc=
"Name of flux measurement for calculating the S/N when choosing the reference band.")
1191 minSN = Field(dtype=float, default=10.,
1192 doc=
"If the S/N from the priority band is below this value (and the S/N "
1193 "is larger than minSNDiff compared to the priority band), use the band with "
1194 "the largest S/N as the reference band.")
1195 minSNDiff = Field(dtype=float, default=3.,
1196 doc=
"If the difference in S/N between another band and the priority band is larger "
1197 "than this value (and the S/N in the priority band is less than minSN) "
1198 "use the band with the largest S/N as the reference band")
1199 flags = ListField(dtype=str, doc=
"Require that these flags, if available, are not set",
1200 default=[
"base_PixelFlags_flag_interpolatedCenter",
"base_PsfFlux_flag",
1201 "ext_photometryKron_KronFlux_flag",
"modelfit_CModel_flag", ])
1213 \anchor MergeMeasurementsTask_
1215 \brief Merge measurements from multiple bands
1217 \section pipe_tasks_multiBand_Contents Contents
1219 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1220 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1221 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1222 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1223 - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1224 - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1226 \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1228 Command-line task that merges measurements from multiple bands.
1230 Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1231 bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1232 required to have centroid, shape and flux measurements in each band.
1235 deepCoadd_meas{tract,patch,filter}: SourceCatalog
1237 deepCoadd_ref{tract,patch}: SourceCatalog
1241 MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1243 \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1245 \copydoc \_\_init\_\_
1247 \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1251 \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1253 See \ref MergeMeasurementsConfig_
1255 \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1257 The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1258 flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1261 MergeMeasurementsTask has no debug variables.
1263 \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1265 MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1266 The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1267 coadds and individual exposures.
1268 Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1269 of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1270 command line argument:
1272 mergeCoaddMeasurements.py --help
1275 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1276 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1277 step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1280 mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1282 This will merge the HSC-I & HSC-R band catalogs. The results are written in
1283 `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1285 _DefaultName =
"mergeCoaddMeasurements"
1286 ConfigClass = MergeMeasurementsConfig
1287 inputDataset =
"meas"
1288 outputDataset =
"ref"
1289 getSchemaCatalogs = _makeGetSchemaCatalogs(
"ref")
1291 def __init__(self, butler=None, schema=None, **kwargs):
1293 Initialize the task.
1295 Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1296 \param[in] schema: the schema of the detection catalogs used as input to this one
1297 \param[in] butler: a butler used to read the input schema from disk, if schema is None
1299 The task will set its own self.schema attribute to the schema of the output merged catalog.
1301 MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1304 self.schemaMapper.addMinimalSchema(inputSchema,
True)
1305 self.
fluxKey = inputSchema.find(self.config.snName +
"_flux").getKey()
1306 self.
fluxErrKey = inputSchema.find(self.config.snName +
"_fluxSigma").getKey()
1307 self.
fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
1310 for band
in self.config.priorityList:
1312 outputKey = self.schemaMapper.editOutputSchema().addField(
1313 "merge_measurement_%s" % short,
1315 doc=
"Flag field set if the measurements here are from the %s filter" % band
1317 peakKey = inputSchema.find(
"merge_peak_%s" % short).key
1318 footprintKey = inputSchema.find(
"merge_footprint_%s" % short).key
1319 self.
flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1320 self.
schema = self.schemaMapper.getOutputSchema()
1323 for filt
in self.config.pseudoFilterList:
1325 self.pseudoFilterKeys.append(self.schema.find(
"merge_peak_%s" % filt).getKey())
1327 self.log.warn(
"merge_peak is not set for pseudo-filter %s" % filt)
1330 for flag
in self.config.flags:
1332 self.
badFlags[flag] = self.schema.find(flag).getKey()
1333 except KeyError
as exc:
1334 self.log.warn(
"Can't find flag %s in schema: %s" % (flag, exc,))
1338 Merge measurement catalogs to create a single reference catalog for forced photometry
1340 \param[in] catalogs: the catalogs to be merged
1341 \param[in] patchRef: patch reference for data
1343 For parent sources, we choose the first band in config.priorityList for which the
1344 merge_footprint flag for that band is is True.
1346 For child sources, the logic is the same, except that we use the merge_peak flags.
1349 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1350 orderedKeys = [self.
flagKeys[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
1352 mergedCatalog = afwTable.SourceCatalog(self.
schema)
1353 mergedCatalog.reserve(len(orderedCatalogs[0]))
1355 idKey = orderedCatalogs[0].table.getIdKey()
1356 for catalog
in orderedCatalogs[1:]:
1357 if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1358 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1362 for orderedRecords
in zip(*orderedCatalogs):
1365 maxSNFlagKeys =
None
1367 priorityRecord =
None
1368 priorityFlagKeys =
None
1370 hasPseudoFilter =
False
1374 for inputRecord, flagKeys
in zip(orderedRecords, orderedKeys):
1375 parent = (inputRecord.getParent() == 0
and inputRecord.get(flagKeys.footprint))
1376 child = (inputRecord.getParent() != 0
and inputRecord.get(flagKeys.peak))
1378 if not (parent
or child):
1380 if inputRecord.get(pseudoFilterKey):
1381 hasPseudoFilter =
True
1382 priorityRecord = inputRecord
1383 priorityFlagKeys = flagKeys
1388 isBad = any(inputRecord.get(flag)
for flag
in self.
badFlags)
1393 if numpy.isnan(sn)
or sn < 0.:
1395 if (parent
or child)
and priorityRecord
is None:
1396 priorityRecord = inputRecord
1397 priorityFlagKeys = flagKeys
1400 maxSNRecord = inputRecord
1401 maxSNFlagKeys = flagKeys
1414 bestRecord = priorityRecord
1415 bestFlagKeys = priorityFlagKeys
1416 elif (prioritySN < self.config.minSN
and (maxSN - prioritySN) > self.config.minSNDiff
and
1417 maxSNRecord
is not None):
1418 bestRecord = maxSNRecord
1419 bestFlagKeys = maxSNFlagKeys
1420 elif priorityRecord
is not None:
1421 bestRecord = priorityRecord
1422 bestFlagKeys = priorityFlagKeys
1424 if bestRecord
is not None and bestFlagKeys
is not None:
1425 outputRecord = mergedCatalog.addNew()
1427 outputRecord.set(bestFlagKeys.output,
True)
1429 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1430 inputRecord.getId())
1433 for inputCatalog
in orderedCatalogs:
1434 if len(mergedCatalog) != len(inputCatalog):
1435 raise ValueError(
"Mismatch between catalog sizes: %s != %s" %
1436 (len(mergedCatalog), len(orderedCatalogs)))
1438 return mergedCatalog
def writeMetadata
No metadata to write, and not sure how to write it for a list of dataRefs.
def cullPeaks
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
def run
Deblend and measure.
Merge coadd detections from multiple bands.
def __init__
Initialize the task.
def mergeCatalogs
Merge measurement catalogs to create a single reference catalog for forced photometry.
def runDetection
Run detection on an exposure.
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
def write
Write the source catalog.
def run
Merge coadd sources from multiple bands.
Configuration parameters for the DetectCoaddSourcesTask.
def run
Run detection on a coadd.
Merge measurements from multiple bands.
Deblend sources from master catalog in each coadd seperately and measure.
def getTargetList
Provide a list of patch references for each patch.
def __init__
Initialize the task.
def readSources
Read input sources.
def __init__
Initialize the task.
def scaleVariance
Scale the variance in a maskedImage.
def write
Write the output.
Configuration parameters for the MergeMeasurementsTask.
def mergeCatalogs
Merge multiple catalogs.
Configuration parameters for the MergeDetectionsTask.
Configuration for merging sources.
def mergeCatalogs
Merge multiple catalogs.
def makeTask
Provide a butler to the Task constructor.
def readCatalog
Read input catalog.
A base class for merging source catalogs.
def getSkyInfo
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Configuration parameters for the MeasureMergedCoaddSourcesTask.
def __init__
Initialize the task.
def getSkySourceFootprints
Return a list of Footprints of sky objects which don't overlap with anything in mergedList.
def __init__
Initialize the merge detections task.
def getInputSchema
Obtain the input schema either directly or froma butler reference.
Detect sources on a coadd.
def getSchemaCatalogs
Return a dict of empty catalogs for each catalog dataset produced by this task.
def write
Write out results from runDetection.
def writeMatches
Write matches of the sources to the astrometric reference catalog.