25from numpy.lib.recfunctions
import rec_join
28from .multiBandUtils
import CullPeaksConfig
36from lsst.pex.config import Config, Field, ListField, ConfigurableField, ConfigField
37from lsst.pipe.base import (PipelineTask, PipelineTaskConfig, Struct,
38 PipelineTaskConnections)
39import lsst.pipe.base.connectionTypes
as cT
40from lsst.obs.base
import ExposureIdInfo
44 """Match two catalogs derived from the same mergeDet catalog
46 When testing downstream features, like deblending methods/parameters
47 and measurement algorithms/parameters, it
is useful to to compare
48 the same sources
in two catalogs. In most cases this must be done
49 by matching on either RA/DEC
or XY positions, which occassionally
50 will mismatch one source
with another.
52 For a more robust solution,
as long
as the downstream catalog
is
53 derived
from the same mergeDet catalog, exact source matching
54 can be done via the unique ``(parent, deblend_peakID)``
55 combination. So this function performs this exact matching
for
56 all sources both catalogs.
61 The two catalogs to merge
63 patch1, patch2 : array of int
64 Patch
for each row, converted into an integer.
69 List of matches
for each source (using an inner join).
73 sidx1 = catalog1[
"parent"] != 0
74 sidx2 = catalog2[
"parent"] != 0
77 parents1 = np.array(catalog1[
"parent"][sidx1])
78 peaks1 = np.array(catalog1[
"deblend_peakId"][sidx1])
79 index1 = np.arange(len(catalog1))[sidx1]
80 parents2 = np.array(catalog2[
"parent"][sidx2])
81 peaks2 = np.array(catalog2[
"deblend_peakId"][sidx2])
82 index2 = np.arange(len(catalog2))[sidx2]
84 if patch1
is not None:
86 msg = (
"If the catalogs are from different patches then patch1 and patch2 must be specified"
87 ", got {} and {}").format(patch1, patch2)
89 patch1 = patch1[sidx1]
90 patch2 = patch2[sidx2]
92 key1 = np.rec.array((parents1, peaks1, patch1, index1),
93 dtype=[(
'parent', np.int64), (
'peakId', np.int32),
94 (
"patch", patch1.dtype), (
"index", np.int32)])
95 key2 = np.rec.array((parents2, peaks2, patch2, index2),
96 dtype=[(
'parent', np.int64), (
'peakId', np.int32),
97 (
"patch", patch2.dtype), (
"index", np.int32)])
98 matchColumns = (
"parent",
"peakId",
"patch")
100 key1 = np.rec.array((parents1, peaks1, index1),
101 dtype=[(
'parent', np.int64), (
'peakId', np.int32), (
"index", np.int32)])
102 key2 = np.rec.array((parents2, peaks2, index2),
103 dtype=[(
'parent', np.int64), (
'peakId', np.int32), (
"index", np.int32)])
104 matchColumns = (
"parent",
"peakId")
109 matched = rec_join(matchColumns, key1, key2, jointype=
"inner")
112 indices1 = matched[
"index1"]
113 indices2 = matched[
"index2"]
117 afwTable.SourceMatch(catalog1[int(i1)], catalog2[int(i2)], 0.0)
118 for i1, i2
in zip(indices1, indices2)
125 dimensions=(
"tract",
"patch",
"skymap"),
126 defaultTemplates={
"inputCoaddName":
'deep',
"outputCoaddName":
"deep"}):
127 schema = cT.InitInput(
128 doc=
"Schema of the input detection catalog",
129 name=
"{inputCoaddName}Coadd_det_schema",
130 storageClass=
"SourceCatalog"
133 outputSchema = cT.InitOutput(
134 doc=
"Schema of the merged detection catalog",
135 name=
"{outputCoaddName}Coadd_mergeDet_schema",
136 storageClass=
"SourceCatalog"
139 outputPeakSchema = cT.InitOutput(
140 doc=
"Output schema of the Footprint peak catalog",
141 name=
"{outputCoaddName}Coadd_peak_schema",
142 storageClass=
"PeakCatalog"
146 doc=
"Detection Catalogs to be merged",
147 name=
"{inputCoaddName}Coadd_det",
148 storageClass=
"SourceCatalog",
149 dimensions=(
"tract",
"patch",
"skymap",
"band"),
154 doc=
"SkyMap to be used in merging",
155 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
156 storageClass=
"SkyMap",
157 dimensions=(
"skymap",),
160 outputCatalog = cT.Output(
161 doc=
"Merged Detection catalog",
162 name=
"{outputCoaddName}Coadd_mergeDet",
163 storageClass=
"SourceCatalog",
164 dimensions=(
"tract",
"patch",
"skymap"),
168class MergeDetectionsConfig(PipelineTaskConfig, pipelineConnections=MergeDetectionsConnections):
170 @anchor MergeDetectionsConfig_
172 @brief Configuration parameters
for the MergeDetectionsTask.
174 minNewPeak = Field(dtype=float, default=1,
175 doc="Minimum distance from closest peak to create a new one (in arcsec).")
177 maxSamePeak = Field(dtype=float, default=0.3,
178 doc=
"When adding new catalogs to the merge, all peaks less than this distance "
179 " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
180 cullPeaks = ConfigField(dtype=CullPeaksConfig, doc=
"Configuration for how to cull peaks.")
182 skyFilterName = Field(dtype=str, default=
"sky",
183 doc=
"Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
184 "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
185 skyObjects = ConfigurableField(target=SkyObjectsTask, doc=
"Generate sky objects")
186 priorityList = ListField(dtype=str, default=[],
187 doc=
"Priority-ordered list of filter bands for the merge.")
188 coaddName = Field(dtype=str, default=
"deep", doc=
"Name of coadd")
190 def setDefaults(self):
191 Config.setDefaults(self)
192 self.skyObjects.avoidMask = [
"DETECTED"]
196 if len(self.priorityList) == 0:
197 raise RuntimeError(
"No priority list provided")
200class MergeDetectionsTask(PipelineTask):
201 """Task to merge coadd tetections from multiple bands.
206 Compatibility parameter. Should always be `
None`.
208 The schema of the detection catalogs used
as input to this task.
209 initInputs : `dict`, optional
210 Dictionary that can contain a key ``schema`` containing the
211 input schema. If present will override the value of ``schema``.
213 ConfigClass = MergeDetectionsConfig
214 _DefaultName = "mergeCoaddDetections"
216 def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
217 super().__init__(**kwargs)
219 if butler
is not None:
220 warnings.warn(
"The 'butler' parameter is no longer used and can be safely removed.",
221 category=FutureWarning, stacklevel=2)
224 if initInputs
is not None:
225 schema = initInputs[
'schema'].schema
228 raise ValueError(
"No input schema or initInputs['schema'] provided.")
232 self.makeSubtask(
"skyObjects")
234 filterNames = list(self.config.priorityList)
235 filterNames.append(self.config.skyFilterName)
236 self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
237 self.outputSchema = afwTable.SourceCatalog(self.schema)
238 self.outputPeakSchema = afwDetect.PeakCatalog(self.merged.getPeakSchema())
240 def runQuantum(self, butlerQC, inputRefs, outputRefs):
241 inputs = butlerQC.get(inputRefs)
242 exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
243 inputs[
"skySeed"] = exposureIdInfo.expId
244 inputs[
"idFactory"] = exposureIdInfo.makeSourceIdFactory()
245 catalogDict = {ref.dataId[
'band']: cat
for ref, cat
in zip(inputRefs.catalogs,
247 inputs[
'catalogs'] = catalogDict
248 skyMap = inputs.pop(
'skyMap')
250 tractNumber = inputRefs.catalogs[0].dataId[
'tract']
251 tractInfo = skyMap[tractNumber]
252 patchInfo = tractInfo.getPatchInfo(inputRefs.catalogs[0].dataId[
'patch'])
257 wcs=tractInfo.getWcs(),
258 bbox=patchInfo.getOuterBBox()
260 inputs[
'skyInfo'] = skyInfo
262 outputs = self.run(**inputs)
263 butlerQC.put(outputs, outputRefs)
265 def run(self, catalogs, skyInfo, idFactory, skySeed):
267 @brief Merge multiple catalogs.
269 After ordering the catalogs
and filters
in priority order,
270 @ref getMergedSourceCatalog of the
@ref FootprintMergeList_
"FootprintMergeList" created by
271 @ref \_\_init\_\_
is used to perform the actual merging. Finally,
@ref cullPeaks
is used to remove
272 garbage peaks detected around bright objects.
276 @param[out] mergedList
280 tractWcs = skyInfo.wcs
281 peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
282 samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
285 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
286 orderedBands = [band
for band
in self.config.priorityList
if band
in catalogs.keys()]
288 mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
289 self.schema, idFactory,
295 skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
296 if skySourceFootprints:
297 key = mergedList.schema.find(
"merge_footprint_%s" % self.config.skyFilterName).key
298 for foot
in skySourceFootprints:
299 s = mergedList.addNew()
304 for record
in mergedList:
305 record.getFootprint().sortPeaks()
306 self.log.info(
"Merged to %d sources", len(mergedList))
308 self.cullPeaks(mergedList)
309 return Struct(outputCatalog=mergedList)
311 def cullPeaks(self, catalog):
313 @brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
315 @param[
in] catalog Source catalog
317 keys = [item.key for item
in self.merged.getPeakSchema().extract(
"merge_peak_*").values()]
318 assert len(keys) > 0,
"Error finding flags that associate peaks with their detection bands."
321 for parentSource
in catalog:
324 keptPeaks = parentSource.getFootprint().getPeaks()
325 oldPeaks = list(keptPeaks)
327 familySize = len(oldPeaks)
328 totalPeaks += familySize
329 for rank, peak
in enumerate(oldPeaks):
330 if ((rank < self.config.cullPeaks.rankSufficient)
331 or (sum([peak.get(k)
for k
in keys]) >= self.config.cullPeaks.nBandsSufficient)
332 or (rank < self.config.cullPeaks.rankConsidered
333 and rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
334 keptPeaks.append(peak)
337 self.log.info(
"Culled %d of %d peaks", culledPeaks, totalPeaks)
339 def getSchemaCatalogs(self):
341 Return a dict of empty catalogs for each catalog dataset produced by this task.
343 @param[out] dictionary of empty catalogs
345 mergeDet = afwTable.SourceCatalog(self.schema)
346 peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
347 return {self.config.coaddName +
"Coadd_mergeDet": mergeDet,
348 self.config.coaddName +
"Coadd_peak": peak}
350 def getSkySourceFootprints(self, mergedList, skyInfo, seed):
352 @brief Return a list of Footprints of sky objects which don
't overlap with anything in mergedList
354 @param mergedList The merged Footprints
from all the input bands
355 @param skyInfo A description of the patch
356 @param seed Seed
for the random number generator
358 mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
359 detected = mask.getPlaneBitMask("DETECTED")
361 s.getFootprint().spans.setMask(mask, detected)
363 footprints = self.skyObjects.run(mask, seed)
368 schema = self.merged.getPeakSchema()
369 mergeKey = schema.find(
"merge_peak_%s" % self.config.skyFilterName).key
371 for oldFoot
in footprints:
372 assert len(oldFoot.getPeaks()) == 1,
"Should be a single peak only"
373 peak = oldFoot.getPeaks()[0]
374 newFoot = afwDetect.Footprint(oldFoot.spans, schema)
375 newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
376 newFoot.getPeaks()[0].set(mergeKey,
True)
377 converted.append(newFoot)
def matchCatalogsExact(catalog1, catalog2, patch1=None, patch2=None)