22__all__ = [
"MergeDetectionsConfig",
"MergeDetectionsTask"]
25from numpy.lib.recfunctions
import rec_join
28from .multiBandUtils
import CullPeaksConfig
36from lsst.pex.config import Config, Field, ListField, ConfigurableField, ConfigField
37from lsst.pipe.base import (PipelineTask, PipelineTaskConfig, Struct,
38 PipelineTaskConnections)
39import lsst.pipe.base.connectionTypes
as cT
44 """Match two catalogs derived from the same mergeDet catalog.
46 When testing downstream features, like deblending methods/parameters
47 and measurement algorithms/parameters, it
is useful to to compare
48 the same sources
in two catalogs. In most cases this must be done
49 by matching on either RA/DEC
or XY positions, which occassionally
50 will mismatch one source
with another.
52 For a more robust solution,
as long
as the downstream catalog
is
53 derived
from the same mergeDet catalog, exact source matching
54 can be done via the unique ``(parent, deblend_peakID)``
55 combination. So this function performs this exact matching
for
56 all sources both catalogs.
61 The two catalogs to merge
62 patch1, patch2 : `array` of `int`
63 Patch
for each row, converted into an integer.
68 List of matches
for each source (using an inner join).
72 sidx1 = catalog1[
"parent"] != 0
73 sidx2 = catalog2[
"parent"] != 0
76 parents1 = np.array(catalog1[
"parent"][sidx1])
77 peaks1 = np.array(catalog1[
"deblend_peakId"][sidx1])
78 index1 = np.arange(len(catalog1))[sidx1]
79 parents2 = np.array(catalog2[
"parent"][sidx2])
80 peaks2 = np.array(catalog2[
"deblend_peakId"][sidx2])
81 index2 = np.arange(len(catalog2))[sidx2]
83 if patch1
is not None:
85 msg = (
"If the catalogs are from different patches then patch1 and patch2 must be specified"
86 ", got {} and {}").format(patch1, patch2)
88 patch1 = patch1[sidx1]
89 patch2 = patch2[sidx2]
91 key1 = np.rec.array((parents1, peaks1, patch1, index1),
92 dtype=[(
'parent', np.int64), (
'peakId', np.int32),
93 (
"patch", patch1.dtype), (
"index", np.int32)])
94 key2 = np.rec.array((parents2, peaks2, patch2, index2),
95 dtype=[(
'parent', np.int64), (
'peakId', np.int32),
96 (
"patch", patch2.dtype), (
"index", np.int32)])
97 matchColumns = (
"parent",
"peakId",
"patch")
99 key1 = np.rec.array((parents1, peaks1, index1),
100 dtype=[(
'parent', np.int64), (
'peakId', np.int32), (
"index", np.int32)])
101 key2 = np.rec.array((parents2, peaks2, index2),
102 dtype=[(
'parent', np.int64), (
'peakId', np.int32), (
"index", np.int32)])
103 matchColumns = (
"parent",
"peakId")
108 matched = rec_join(matchColumns, key1, key2, jointype=
"inner")
111 indices1 = matched[
"index1"]
112 indices2 = matched[
"index2"]
116 afwTable.SourceMatch(catalog1[int(i1)], catalog2[int(i2)], 0.0)
117 for i1, i2
in zip(indices1, indices2)
124 dimensions=(
"tract",
"patch",
"skymap"),
125 defaultTemplates={
"inputCoaddName":
'deep',
"outputCoaddName":
"deep"}):
126 schema = cT.InitInput(
127 doc=
"Schema of the input detection catalog",
128 name=
"{inputCoaddName}Coadd_det_schema",
129 storageClass=
"SourceCatalog"
132 outputSchema = cT.InitOutput(
133 doc=
"Schema of the merged detection catalog",
134 name=
"{outputCoaddName}Coadd_mergeDet_schema",
135 storageClass=
"SourceCatalog"
138 outputPeakSchema = cT.InitOutput(
139 doc=
"Output schema of the Footprint peak catalog",
140 name=
"{outputCoaddName}Coadd_peak_schema",
141 storageClass=
"PeakCatalog"
145 doc=
"Detection Catalogs to be merged",
146 name=
"{inputCoaddName}Coadd_det",
147 storageClass=
"SourceCatalog",
148 dimensions=(
"tract",
"patch",
"skymap",
"band"),
153 doc=
"SkyMap to be used in merging",
154 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
155 storageClass=
"SkyMap",
156 dimensions=(
"skymap",),
159 outputCatalog = cT.Output(
160 doc=
"Merged Detection catalog",
161 name=
"{outputCoaddName}Coadd_mergeDet",
162 storageClass=
"SourceCatalog",
163 dimensions=(
"tract",
"patch",
"skymap"),
167class MergeDetectionsConfig(PipelineTaskConfig, pipelineConnections=MergeDetectionsConnections):
168 """Configuration parameters for the MergeDetectionsTask.
170 minNewPeak = Field(dtype=float, default=1,
171 doc="Minimum distance from closest peak to create a new one (in arcsec).")
173 maxSamePeak = Field(dtype=float, default=0.3,
174 doc=
"When adding new catalogs to the merge, all peaks less than this distance "
175 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
176 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
178 skyFilterName = Field(dtype=str, default=
"sky",
179 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
180 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
181 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects")
182 priorityList = ListField(dtype=str, default=[],
183 doc=
"Priority-ordered list of filter bands for the merge.")
184 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
185 idGenerator = SkyMapIdGeneratorConfig.make_field()
187 def setDefaults(self):
188 Config.setDefaults(self)
189 self.skyObjects.avoidMask = [
"DETECTED"]
193 if len(self.priorityList) == 0:
194 raise RuntimeError(
"No priority list provided")
197class MergeDetectionsTask(PipelineTask):
198 """Merge sources detected in coadds of exposures obtained with different filters.
200 Merge sources detected in coadds of exposures obtained
with different
201 filters. To perform photometry consistently across coadds
in multiple
202 filter bands, we create a master catalog of sources
from all bands by
203 merging the sources (peaks & footprints) detected
in each coadd,
while
204 keeping track of which band each source originates
in. The catalog merge
206 `~lsst.afw.detection.FootprintMergeList.getMergedSourceCatalog`. Spurious
207 peaks detected around bright objects are culled
as described
in
210 MergeDetectionsTask
is meant to be run after detecting sources
in coadds
211 generated
for the chosen subset of the available bands. The purpose of the
212 task
is to merge sources (peaks & footprints) detected
in the coadds
213 generated
from the chosen subset of filters. Subsequent tasks
in the
214 multi-band processing procedure will deblend the generated master list of
215 sources
and, eventually, perform forced photometry.
219 butler : `
None`, optional
220 Compatibility parameter. Should always be `
None`.
222 The schema of the detection catalogs used
as input to this task.
223 initInputs : `dict`, optional
224 Dictionary that can contain a key ``schema`` containing the
225 input schema. If present will override the value of ``schema``.
227 Additional keyword arguments.
229 ConfigClass = MergeDetectionsConfig
230 _DefaultName = "mergeCoaddDetections"
232 def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
233 super().__init__(**kwargs)
235 if butler
is not None:
236 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
237 category=FutureWarning, stacklevel=2)
240 if initInputs
is not None:
241 schema = initInputs[
'schema'].schema
244 raise ValueError(
"No input schema or initInputs['schema'] provided.")
248 self.makeSubtask(
"skyObjects")
250 filterNames = list(self.config.priorityList)
251 filterNames.append(self.config.skyFilterName)
252 self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
253 self.outputSchema = afwTable.SourceCatalog(self.schema)
254 self.outputPeakSchema = afwDetect.PeakCatalog(self.merged.getPeakSchema())
256 def runQuantum(self, butlerQC, inputRefs, outputRefs):
257 inputs = butlerQC.get(inputRefs)
258 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId)
259 inputs[
"skySeed"] = idGenerator.catalog_id
260 inputs[
"idFactory"] = idGenerator.make_table_id_factory()
261 catalogDict = {ref.dataId[
'band']: cat
for ref, cat
in zip(inputRefs.catalogs,
263 inputs[
'catalogs'] = catalogDict
264 skyMap = inputs.pop(
'skyMap')
266 tractNumber = inputRefs.catalogs[0].dataId[
'tract']
267 tractInfo = skyMap[tractNumber]
268 patchInfo = tractInfo.getPatchInfo(inputRefs.catalogs[0].dataId[
'patch'])
273 wcs=tractInfo.getWcs(),
274 bbox=patchInfo.getOuterBBox()
276 inputs[
'skyInfo'] = skyInfo
278 outputs = self.run(**inputs)
279 butlerQC.put(outputs, outputRefs)
281 def run(self, catalogs, skyInfo, idFactory, skySeed):
282 """Merge multiple catalogs.
284 After ordering the catalogs and filters
in priority order,
285 ``getMergedSourceCatalog`` of the
287 used to perform the actual merging. Finally, `cullPeaks`
is used to
288 remove garbage peaks detected around bright objects.
293 Catalogs to be merged.
299 result : `lsst.pipe.base.Struct`
300 Results
as a struct
with attributes:
306 tractWcs = skyInfo.wcs
307 peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
308 samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
311 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
312 orderedBands = [band
for band
in self.config.priorityList
if band
in catalogs.keys()]
314 mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
315 self.schema, idFactory,
321 skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
322 if skySourceFootprints:
323 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
324 for foot
in skySourceFootprints:
325 s = mergedList.addNew()
330 for record
in mergedList:
331 record.getFootprint().sortPeaks()
332 self.log.info(
"Merged to %d sources", len(mergedList))
334 self.cullPeaks(mergedList)
335 return Struct(outputCatalog=mergedList)
337 def cullPeaks(self, catalog):
338 """Attempt to remove garbage peaks (mostly on the outskirts of large blends).
345 keys = [item.key for item
in self.merged.getPeakSchema().extract(
"merge_peak_*").values()]
346 assert len(keys) > 0,
"Error finding flags that associate peaks with their detection bands."
349 for parentSource
in catalog:
352 keptPeaks = parentSource.getFootprint().getPeaks()
353 oldPeaks = list(keptPeaks)
355 familySize = len(oldPeaks)
356 totalPeaks += familySize
357 for rank, peak
in enumerate(oldPeaks):
358 if ((rank < self.config.cullPeaks.rankSufficient)
359 or (sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
360 or (rank < self.config.cullPeaks.rankConsidered
361 and rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
362 keptPeaks.append(peak)
365 self.log.info(
"Culled %d of %d peaks", culledPeaks, totalPeaks)
367 def getSkySourceFootprints(self, mergedList, skyInfo, seed):
368 """Return a list of Footprints of sky objects which don't overlap with anything in mergedList.
373 The merged Footprints from all the input bands.
374 skyInfo : `lsst.pipe.base.Struct`
375 A description of the patch.
377 Seed
for the random number generator.
379 mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
380 detected = mask.getPlaneBitMask("DETECTED")
382 s.getFootprint().spans.setMask(mask, detected)
384 footprints = self.skyObjects.run(mask, seed)
389 schema = self.merged.getPeakSchema()
390 mergeKey = schema.find(
"merge_peak_%s" % self.config.skyFilterName).key
392 for oldFoot
in footprints:
393 assert len(oldFoot.getPeaks()) == 1,
"Should be a single peak only"
394 peak = oldFoot.getPeaks()[0]
395 newFoot = afwDetect.Footprint(oldFoot.spans, schema)
396 newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
397 newFoot.getPeaks()[0].set(mergeKey,
True)
398 converted.append(newFoot)
def matchCatalogsExact(catalog1, catalog2, patch1=None, patch2=None)