25 from .multiBandUtils
import (MergeSourcesRunner, _makeGetSchemaCatalogs, makeMergeArgumentParser,
26 getInputSchema, getShortFilterName, readCatalog)
35 @anchor MergeMeasurementsConfig_ 37 @brief Configuration parameters for the MergeMeasurementsTask 40 inputSchema = pipeBase.InitInputDatasetField(
41 doc=
"Schema for the input measurement catalogs.",
43 nameTemplate=
"{inputCoaddName}Coadd_meas_schema",
44 storageClass=
"SourceCatalog",
46 outputSchema = pipeBase.InitOutputDatasetField(
47 doc=
"Schema for the output merged measurement catalog.",
49 nameTemplate=
"{outputCoaddName}Coadd_ref_schema",
50 storageClass=
"SourceCatalog",
52 catalogs = pipeBase.InputDatasetField(
53 doc=
"Input catalogs to merge.",
55 nameTemplate=
"{inputCoaddName}Coadd_meas",
57 storageClass=
"SourceCatalog",
58 dimensions=[
"AbstractFilter",
"SkyMap",
"Tract",
"Patch"],
60 mergedCatalog = pipeBase.OutputDatasetField(
61 doc=
"Output merged catalog.",
63 nameTemplate=
"{outputCoaddName}Coadd_ref",
65 storageClass=
"SourceCatalog",
66 dimensions=[
"SkyMap",
"Tract",
"Patch"],
69 pseudoFilterList = pexConfig.ListField(
72 doc=
"Names of filters which may have no associated detection\n" 73 "(N.b. should include MergeDetectionsConfig.skyFilterName)" 75 snName = pexConfig.Field(
77 default=
"base_PsfFlux",
78 doc=
"Name of flux measurement for calculating the S/N when choosing the reference band." 80 minSN = pexConfig.Field(
83 doc=
"If the S/N from the priority band is below this value (and the S/N " 84 "is larger than minSNDiff compared to the priority band), use the band with " 85 "the largest S/N as the reference band." 87 minSNDiff = pexConfig.Field(
90 doc=
"If the difference in S/N between another band and the priority band is larger " 91 "than this value (and the S/N in the priority band is less than minSN) " 92 "use the band with the largest S/N as the reference band" 94 flags = pexConfig.ListField(
96 doc=
"Require that these flags, if available, are not set",
97 default=[
"base_PixelFlags_flag_interpolatedCenter",
"base_PsfFlux_flag",
98 "ext_photometryKron_KronFlux_flag",
"modelfit_CModel_flag", ]
100 priorityList = pexConfig.ListField(
103 doc=
"Priority-ordered list of bands for the merge." 105 coaddName = pexConfig.Field(
114 raise RuntimeError(
"No priority list provided")
118 self.formatTemplateNames({
"inputCoaddName":
"deep",
119 "outputCoaddName":
"deep"})
120 self.quantum.dimensions = (
"SkyMap",
"Tract",
"Patch")
132 @anchor MergeMeasurementsTask_ 134 @brief Merge measurements from multiple bands 136 @section pipe_tasks_multiBand_Contents Contents 138 - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose 139 - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize 140 - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Run 141 - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Config 142 - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug 143 - @ref pipe_tasks_multiband_MergeMeasurementsTask_Example 145 @section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description 147 Command-line task that merges measurements from multiple bands. 149 Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter 150 bands to construct a unified catalog that is suitable for driving forced photometry. Every source is 151 required to have centroid, shape and flux measurements in each band. 154 deepCoadd_meas{tract,patch,filter}: SourceCatalog 156 deepCoadd_ref{tract,patch}: SourceCatalog 160 MergeMeasurementsTask subclasses @ref CmdLineTask_ "CmdLineTask". 162 @section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization 164 @copydoc \_\_init\_\_ 166 @section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task 170 @section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters 172 See @ref MergeMeasurementsConfig_ 174 @section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables 176 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 177 flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py 180 MergeMeasurementsTask has no debug variables. 182 @section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example 183 of using MergeMeasurementsTask 185 MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band. 186 The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in 187 coadds and individual exposures. 188 Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list 189 of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help` 190 command line argument: 192 mergeCoaddMeasurements.py --help 195 To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we 196 will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished 197 step 7 at @ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring 200 mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R 202 This will merge the HSC-I & HSC-R band catalogs. The results are written in 203 `$CI_HSC_DIR/DATA/deepCoadd-results/`. 205 _DefaultName =
"mergeCoaddMeasurements" 206 ConfigClass = MergeMeasurementsConfig
207 RunnerClass = MergeSourcesRunner
208 inputDataset =
"meas" 209 outputDataset =
"ref" 210 getSchemaCatalogs = _makeGetSchemaCatalogs(
"ref")
213 def _makeArgumentParser(cls):
220 return {
"outputSchema": afwTable.SourceCatalog(self.
schema), }
223 catalogDict = {dataId[
'abstract_filter']: cat
for dataId, cat
in zip(inputDataIds[
'catalogs'],
224 inputData[
'catalogs'])}
225 inputData[
'catalogs'] = catalogDict
227 return super().
adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
229 def __init__(self, butler=None, schema=None, initInputs=None, **kwargs):
233 @param[in] schema: the schema of the detection catalogs used as input to this one 234 @param[in] butler: a butler used to read the input schema from disk, if schema is None 236 The task will set its own self.schema attribute to the schema of the output merged catalog. 240 if initInputs
is not None:
241 inputSchema = initInputs[
'inputSchema'].schema
246 self.
instFluxKey = inputSchema.find(self.config.snName +
"_instFlux").getKey()
247 self.
instFluxErrKey = inputSchema.find(self.config.snName +
"_instFluxErr").getKey()
248 self.
fluxFlagKey = inputSchema.find(self.config.snName +
"_flag").getKey()
251 for band
in self.config.priorityList:
253 outputKey = self.
schemaMapper.editOutputSchema().addField(
254 "merge_measurement_%s" % short,
256 doc=
"Flag field set if the measurements here are from the %s filter" % band
258 peakKey = inputSchema.find(
"merge_peak_%s" % short).key
259 footprintKey = inputSchema.find(
"merge_footprint_%s" % short).key
260 self.
flagKeys[band] = pipeBase.Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
264 for filt
in self.config.pseudoFilterList:
267 except Exception
as e:
268 self.log.warn(
"merge_peak is not set for pseudo-filter %s: %s" % (filt, e))
271 for flag
in self.config.flags:
274 except KeyError
as exc:
275 self.log.warn(
"Can't find flag %s in schema: %s" % (flag, exc,))
279 @brief Merge coadd sources from multiple bands. Calls @ref `run`. 280 @param[in] patchRefList list of data references for each filter 282 catalogs = dict(
readCatalog(self, patchRef)
for patchRef
in patchRefList)
283 mergedCatalog = self.
run(catalogs).mergedCatalog
284 self.
write(patchRefList[0], mergedCatalog)
288 Merge measurement catalogs to create a single reference catalog for forced photometry 290 @param[in] catalogs: the catalogs to be merged 292 For parent sources, we choose the first band in config.priorityList for which the 293 merge_footprint flag for that band is is True. 295 For child sources, the logic is the same, except that we use the merge_peak flags. 298 orderedCatalogs = [catalogs[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
299 orderedKeys = [self.
flagKeys[band]
for band
in self.config.priorityList
if band
in catalogs.keys()]
301 mergedCatalog = afwTable.SourceCatalog(self.
schema)
302 mergedCatalog.reserve(len(orderedCatalogs[0]))
304 idKey = orderedCatalogs[0].table.getIdKey()
305 for catalog
in orderedCatalogs[1:]:
306 if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
307 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: source IDs do not match")
311 for orderedRecords
in zip(*orderedCatalogs):
316 priorityRecord =
None 317 priorityFlagKeys =
None 319 hasPseudoFilter =
False 323 for inputRecord, flagKeys
in zip(orderedRecords, orderedKeys):
324 parent = (inputRecord.getParent() == 0
and inputRecord.get(flagKeys.footprint))
325 child = (inputRecord.getParent() != 0
and inputRecord.get(flagKeys.peak))
327 if not (parent
or child):
329 if inputRecord.get(pseudoFilterKey):
330 hasPseudoFilter =
True 331 priorityRecord = inputRecord
332 priorityFlagKeys = flagKeys
337 isBad = any(inputRecord.get(flag)
for flag
in self.
badFlags)
342 if numpy.isnan(sn)
or sn < 0.:
344 if (parent
or child)
and priorityRecord
is None:
345 priorityRecord = inputRecord
346 priorityFlagKeys = flagKeys
349 maxSNRecord = inputRecord
350 maxSNFlagKeys = flagKeys
363 bestRecord = priorityRecord
364 bestFlagKeys = priorityFlagKeys
365 elif (prioritySN < self.config.minSN
and (maxSN - prioritySN) > self.config.minSNDiff
and 366 maxSNRecord
is not None):
367 bestRecord = maxSNRecord
368 bestFlagKeys = maxSNFlagKeys
369 elif priorityRecord
is not None:
370 bestRecord = priorityRecord
371 bestFlagKeys = priorityFlagKeys
373 if bestRecord
is not None and bestFlagKeys
is not None:
374 outputRecord = mergedCatalog.addNew()
376 outputRecord.set(bestFlagKeys.output,
True)
378 raise ValueError(
"Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
382 for inputCatalog
in orderedCatalogs:
383 if len(mergedCatalog) != len(inputCatalog):
384 raise ValueError(
"Mismatch between catalog sizes: %s != %s" %
385 (len(mergedCatalog), len(orderedCatalogs)))
387 return pipeBase.Struct(
388 mergedCatalog=mergedCatalog
393 @brief Write the output. 395 @param[in] patchRef data reference for patch 396 @param[in] catalog catalog 398 We write as the dataset provided by the 'outputDataset' 401 patchRef.put(catalog, self.config.coaddName +
"Coadd_" + self.
outputDataset)
404 mergeDataId = patchRef.dataId.copy()
405 del mergeDataId[
"filter"]
406 self.log.info(
"Wrote merged catalog: %s" % (mergeDataId,))
410 @brief No metadata to write, and not sure how to write it for a list of dataRefs. def runDataRef(self, patchRefList)
Merge coadd sources from multiple bands.
def makeMergeArgumentParser(name, dataset)
Create a suitable ArgumentParser.
def readCatalog(task, patchRef)
Read input catalog.
Merge measurements from multiple bands.
def run(self, catalogs)
Merge measurement catalogs to create a single reference catalog for forced photometry.
def getInitOutputDatasets(self)
Configuration parameters for the MergeMeasurementsTask.
def getInputSchema(self, butler=None, schema=None)
def write(self, patchRef, catalog)
Write the output.
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def __init__(self, butler=None, schema=None, initInputs=None, kwargs)
Initialize the task.
def getShortFilterName(name)
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.