24 from .multiBandUtils
import (CullPeaksConfig, MergeSourcesRunner, _makeMakeIdFactory, makeMergeArgumentParser,
25 getInputSchema, getShortFilterName, readCatalog)
33 from lsst.pex.config import Config, Field, ListField, ConfigurableField, ConfigField
34 from lsst.pipe.base import (CmdLineTask, PipelineTask, PipelineTaskConfig, InitOutputDatasetField,
35 InputDatasetField, InitInputDatasetField, OutputDatasetField, Struct)
41 @anchor MergeDetectionsConfig_ 43 @brief Configuration parameters for the MergeDetectionsTask. 45 minNewPeak = Field(dtype=float, default=1,
46 doc=
"Minimum distance from closest peak to create a new one (in arcsec).")
48 maxSamePeak = Field(dtype=float, default=0.3,
49 doc=
"When adding new catalogs to the merge, all peaks less than this distance " 50 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
51 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
53 skyFilterName = Field(dtype=str, default=
"sky",
54 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n" 55 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
56 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects")
57 priorityList = ListField(dtype=str, default=[],
58 doc=
"Priority-ordered list of bands for the merge.")
59 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
61 schema = InitInputDatasetField(
62 doc=
"Schema of the input detection catalog",
64 nameTemplate=
"{inputCoaddName}Coadd_det_schema",
65 storageClass=
"SourceCatalog" 68 outputSchema = InitOutputDatasetField(
69 doc=
"Schema of the merged detection catalog",
70 nameTemplate=
"{outputCoaddName}Coadd_mergeDet_schema",
71 storageClass=
"SourceCatalog" 74 catalogs = InputDatasetField(
75 doc=
"Detection Catalogs to be merged",
76 nameTemplate=
"{inputCoaddName}Coadd_det",
77 storageClass=
"SourceCatalog",
78 dimensions=(
"Tract",
"Patch",
"SkyMap",
"AbstractFilter")
81 skyMap = InputDatasetField(
82 doc=
"SkyMap to be used in merging",
83 nameTemplate=
"{inputCoaddName}Coadd_skyMap",
84 storageClass=
"SkyMap",
85 dimensions=(
"SkyMap",),
89 outputCatalog = OutputDatasetField(
90 doc=
"Merged Detection catalog",
91 nameTemplate=
"{outputCoaddName}Coadd_mergeDet",
92 storageClass=
"SourceCatalog",
93 dimensions=(
"Tract",
"Patch",
"SkyMap"),
98 Config.setDefaults(self)
99 self.formatTemplateNames({
"inputCoaddName":
'deep',
"outputCoaddName":
"deep"})
101 self.quantum.dimensions = (
"Tract",
"Patch",
"SkyMap")
106 raise RuntimeError(
"No priority list provided")
111 @anchor MergeDetectionsTask_ 113 @brief Merge coadd detections from multiple bands. 115 @section pipe_tasks_multiBand_Contents Contents 117 - @ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose 118 - @ref pipe_tasks_multiBand_MergeDetectionsTask_Init 119 - @ref pipe_tasks_multiBand_MergeDetectionsTask_Run 120 - @ref pipe_tasks_multiBand_MergeDetectionsTask_Config 121 - @ref pipe_tasks_multiBand_MergeDetectionsTask_Debug 122 - @ref pipe_tasks_multiband_MergeDetectionsTask_Example 124 @section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description 126 Command-line task that merges sources detected in coadds of exposures obtained with different filters. 128 To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of 129 sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping 130 track of which band each source originates in. 132 The catalog merge is performed by @ref getMergedSourceCatalog. Spurious peaks detected around bright 133 objects are culled as described in @ref CullPeaksConfig_. 136 deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints) 138 deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints) 142 @section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation 144 @copydoc \_\_init\_\_ 146 @section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task 150 @section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters 152 See @ref MergeDetectionsConfig_ 154 @section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables 156 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag @c -d 157 to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. 159 MergeDetectionsTask has no debug variables. 161 @section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask 163 MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset 164 of the available bands. 165 The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the 166 chosen subset of filters. 167 Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources 168 and, eventually, perform forced photometry. 169 Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed. 170 A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the 171 `--help` command line argument: 173 mergeCoaddDetections.py --help 176 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 177 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 178 step 5 at @ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows: 180 mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 182 This will merge the HSC-I & -R band parent source catalogs and write the results to 183 `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`. 185 The next step in the multi-band processing procedure is 186 @ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask" 188 ConfigClass = MergeDetectionsConfig
189 RunnerClass = MergeSourcesRunner
190 _DefaultName =
"mergeCoaddDetections" 192 outputDataset =
"mergeDet" 193 makeIdFactory = _makeMakeIdFactory(
"MergedCoaddId")
196 def _makeArgumentParser(cls):
202 def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
205 @brief Initialize the merge detections task. 207 A @ref FootprintMergeList_ "FootprintMergeList" will be used to 208 merge the source catalogs. 210 @param[in] schema the schema of the detection catalogs used as input to this one 211 @param[in] butler a butler used to read the input schema from disk, if schema is None 212 @param[in] initInputs This a PipelineTask-only argument that holds all inputs passed in 213 through the PipelineTask middleware 214 @param[in] **kwargs keyword arguments to be passed to CmdLineTask.__init__ 216 The task will set its own self.schema attribute to the schema of the output merged catalog. 219 if initInputs
is not None:
220 schema = initInputs[
'schema'].schema
222 self.makeSubtask(
"skyObjects")
226 filterNames += [self.config.skyFilterName]
230 return {
"outputSchema": afwTable.SourceCatalog(self.
schema)}
233 catalogs = dict(
readCatalog(self, patchRef)
for patchRef
in patchRefList)
234 skyInfo =
getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRefList[0])
236 skySeed = patchRefList[0].get(self.config.coaddName +
"MergedCoaddId")
237 mergeCatalogStruct = self.
run(catalogs, skyInfo, idFactory, skySeed)
238 self.
write(patchRefList[0], mergeCatalogStruct.outputCatalog)
242 inputData[
"skySeed"] = 0
243 inputData[
"idFactory"] = afwTable.IdFactory.makeSimple()
244 catalogDict = {dataId[
'abstract_filter']: cat
for dataId, cat
in zip(inputDataIds[
'catalogs'],
245 inputData[
'catalogs'])}
246 inputData[
'catalogs'] = catalogDict
247 skyMap = inputData.pop(
'skyMap')
249 tractNumber = inputDataIds[
'catalogs'][0][
'tract']
250 tractInfo = skyMap[tractNumber]
251 patchInfo = tractInfo.getPatchInfo(inputDataIds[
'catalogs'][0][
'patch'])
256 wcs=tractInfo.getWcs(),
257 bbox=patchInfo.getOuterBBox()
259 inputData[
'skyInfo'] = skyInfo
260 return self.
run(**inputData)
262 def run(self, catalogs, skyInfo, idFactory, skySeed):
264 @brief Merge multiple catalogs. 266 After ordering the catalogs and filters in priority order, 267 @ref getMergedSourceCatalog of the @ref FootprintMergeList_ "FootprintMergeList" created by 268 @ref \_\_init\_\_ is used to perform the actual merging. Finally, @ref cullPeaks is used to remove 269 garbage peaks detected around bright objects. 273 @param[out] mergedList 277 tractWcs = skyInfo.wcs
278 peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
279 samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
282 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
284 if band
in catalogs.keys()]
286 mergedList = self.
merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
294 if skySourceFootprints:
295 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
296 for foot
in skySourceFootprints:
297 s = mergedList.addNew()
302 for record
in mergedList:
303 record.getFootprint().sortPeaks()
304 self.log.info(
"Merged to %d sources" % len(mergedList))
307 return Struct(outputCatalog=mergedList)
311 @brief Attempt to remove garbage peaks (mostly on the outskirts of large blends). 313 @param[in] catalog Source catalog 315 keys = [item.key
for item
in self.
merged.getPeakSchema().extract(
"merge_peak_*").values()]
316 assert len(keys) > 0,
"Error finding flags that associate peaks with their detection bands." 319 for parentSource
in catalog:
322 keptPeaks = parentSource.getFootprint().getPeaks()
323 oldPeaks = list(keptPeaks)
325 familySize = len(oldPeaks)
326 totalPeaks += familySize
327 for rank, peak
in enumerate(oldPeaks):
328 if ((rank < self.config.cullPeaks.rankSufficient)
or 329 (sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
or 330 (rank < self.config.cullPeaks.rankConsidered
and 331 rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
332 keptPeaks.append(peak)
335 self.log.info(
"Culled %d of %d peaks" % (culledPeaks, totalPeaks))
339 Return a dict of empty catalogs for each catalog dataset produced by this task. 341 @param[out] dictionary of empty catalogs 343 mergeDet = afwTable.SourceCatalog(self.
schema)
344 peak = afwDetect.PeakCatalog(self.
merged.getPeakSchema())
345 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
346 self.config.coaddName +
"Coadd_peak": peak}
350 @brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList 352 @param mergedList The merged Footprints from all the input bands 353 @param skyInfo A description of the patch 354 @param seed Seed for the random number generator 356 mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
357 detected = mask.getPlaneBitMask(
"DETECTED")
359 s.getFootprint().spans.setMask(mask, detected)
361 footprints = self.skyObjects.
run(mask, seed)
366 schema = self.
merged.getPeakSchema()
367 mergeKey = schema.find(
"merge_peak_%s" % self.config.skyFilterName).key
369 for oldFoot
in footprints:
370 assert len(oldFoot.getPeaks()) == 1,
"Should be a single peak only" 371 peak = oldFoot.getPeaks()[0]
372 newFoot = afwDetect.Footprint(oldFoot.spans, schema)
373 newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
374 newFoot.getPeaks()[0].set(mergeKey,
True)
375 converted.append(newFoot)
381 @brief Write the output. 383 @param[in] patchRef data reference for patch 384 @param[in] catalog catalog 386 We write as the dataset provided by the 'outputDataset' 389 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
392 mergeDataId = patchRef.dataId.copy()
393 del mergeDataId[
"filter"]
394 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
398 @brief No metadata to write, and not sure how to write it for a list of dataRefs. def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
def makeMergeArgumentParser(name, dataset)
Create a suitable ArgumentParser.
Merge coadd detections from multiple bands.
def readCatalog(task, patchRef)
Read input catalog.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def __init__(self, butler=None, schema=None, initInputs=None, kwargs)
Initialize the merge detections task.
def getInitOutputDatasets(self)
def run(self, catalogs, skyInfo, idFactory, skySeed)
Merge multiple catalogs.
Configuration parameters for the MergeDetectionsTask.
def getShortFilterName(name)
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
def write(self, patchRef, catalog)
Write the output.
def getInputSchema(self, butler=None, schema=None)
def runDataRef(self, patchRefList)
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don't overlap with anything in mergedList.