lsst.pipe.tasks  14.0-54-gb4e84373+1
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo
35 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
36 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
37 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
38 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
39 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
40 import lsst.afw.image as afwImage
41 import lsst.afw.table as afwTable
42 import lsst.afw.math as afwMath
43 import lsst.afw.geom as afwGeom
44 import lsst.afw.detection as afwDetect
45 from lsst.daf.base import PropertyList
46 
47 """
48 New dataset types:
49 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
50 * deepCoadd_mergeDet: merged detections (tract, patch)
51 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
52 * deepCoadd_ref: reference sources (tract, patch)
53 All of these have associated *_schema catalogs that require no data ID and hold no records.
54 
55 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
56 the mergeDet, meas, and ref dataset Footprints:
57 * deepCoadd_peak_schema
58 """
59 
60 
61 def _makeGetSchemaCatalogs(datasetSuffix):
62  """Construct a getSchemaCatalogs instance method
63 
64  These are identical for most of the classes here, so we'll consolidate
65  the code.
66 
67  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
68  """
69 
70  def getSchemaCatalogs(self):
71  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
72  src = afwTable.SourceCatalog(self.schema)
73  if hasattr(self, "algMetadata"):
74  src.getTable().setMetadata(self.algMetadata)
75  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
76  return getSchemaCatalogs
77 
78 
79 def _makeMakeIdFactory(datasetName):
80  """Construct a makeIdFactory instance method
81 
82  These are identical for all the classes here, so this consolidates
83  the code.
84 
85  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
86  """
87 
88  def makeIdFactory(self, dataRef):
89  """Return an IdFactory for setting the detection identifiers
90 
91  The actual parameters used in the IdFactory are provided by
92  the butler (through the provided data reference.
93  """
94  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
95  expId = int(dataRef.get(self.config.coaddName + datasetName))
96  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
97  return makeIdFactory
98 
99 
101  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
102  """
103  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
104  # and its the only way I could get it to work.
105  return afwImage.Filter(name).getFilterProperty().getName()
106 
107 
108 
109 
111  """!
112  \anchor DetectCoaddSourcesConfig_
113 
114  \brief Configuration parameters for the DetectCoaddSourcesTask
115  """
116  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
117  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
118  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
119  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131  self.detection.reEstimateBackground = False
132  self.detection.background.useApprox = False
133  self.detection.background.binSize = 4096
134  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
136 
137 
143 
144 
145 class DetectCoaddSourcesTask(CmdLineTask):
146  """!
147  \anchor DetectCoaddSourcesTask_
148 
149  \brief Detect sources on a coadd
150 
151  \section pipe_tasks_multiBand_Contents Contents
152 
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
156  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
157  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
158  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
159 
160  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
161 
162  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
163 
164  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
165  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
166  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
167  propagate the full covariance matrix -- but it is simple and works well in practice.
168 
169  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
170  SourceDetectionTask_ "detection" subtask.
171 
172  \par Inputs:
173  deepCoadd{tract,patch,filter}: ExposureF
174  \par Outputs:
175  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
176  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
177  exposure (ExposureF)
178  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
179  \par Data Unit:
180  tract, patch, filter
181 
182  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
183  You can retarget this subtask if you wish.
184 
185  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
186 
187  \copydoc \_\_init\_\_
188 
189  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
190 
191  \copydoc run
192 
193  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
194 
195  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
196 
197  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
198 
199  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
200  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
201  files.
202 
203  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
204  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
205  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
206 
207  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
208  of using DetectCoaddSourcesTask
209 
210  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
211  the task is to update the background, detect all sources in a single band and generate a set of parent
212  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
213  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
214  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
215  calling detectCoaddSources.py with the `--help` command line argument:
216  \code
217  detectCoaddSources.py --help
218  \endcode
219 
220  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
221  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
222  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
223  \code
224  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
225  \endcode
226  that will process the HSC-I band data. The results are written to
227  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
228 
229  It is also necessary to run:
230  \code
231  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
232  \endcode
233  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
234  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
235  """
236  _DefaultName = "detectCoaddSources"
237  ConfigClass = DetectCoaddSourcesConfig
238  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
239  makeIdFactory = _makeMakeIdFactory("CoaddId")
240 
241  @classmethod
242  def _makeArgumentParser(cls):
243  parser = ArgumentParser(name=cls._DefaultName)
244  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
245  ContainerClass=ExistingCoaddDataIdContainer)
246  return parser
247 
248  def __init__(self, schema=None, **kwargs):
249  """!
250  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
251 
252  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
253 
254  \param[in] schema: initial schema for the output catalog, modified-in place to include all
255  fields set by this task. If None, the source minimal schema will be used.
256  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
257  """
258  CmdLineTask.__init__(self, **kwargs)
259  if schema is None:
260  schema = afwTable.SourceTable.makeMinimalSchema()
261  if self.config.doInsertFakes:
262  self.makeSubtask("insertFakes")
263  self.schema = schema
264  self.makeSubtask("detection", schema=self.schema)
265  if self.config.doScaleVariance:
266  self.makeSubtask("scaleVariance")
267 
268  def run(self, patchRef):
269  """!
270  \brief Run detection on a coadd.
271 
272  Invokes \ref runDetection and then uses \ref write to output the
273  results.
274 
275  \param[in] patchRef: data reference for patch
276  """
277  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
278  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
279  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
280  self.write(exposure, results, patchRef)
281  return results
282 
283  def runDetection(self, exposure, idFactory, expId):
284  """!
285  \brief Run detection on an exposure.
286 
287  First scale the variance plane to match the observed variance
288  using \ref ScaleVarianceTask. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
289  detect sources.
290 
291  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
292  depending on configuration).
293  \param[in] idFactory: IdFactory to set source identifiers
294  \param[in] expId: Exposure identifier (integer) for RNG seed
295 
296  \return a pipe.base.Struct with fields
297  - sources: catalog of detections
298  - backgrounds: list of backgrounds
299  """
300  if self.config.doScaleVariance:
301  varScale = self.scaleVariance.run(exposure.maskedImage)
302  exposure.getMetadata().add("variance_scale", varScale)
303  backgrounds = afwMath.BackgroundList()
304  if self.config.doInsertFakes:
305  self.insertFakes.run(exposure, background=backgrounds)
306  table = afwTable.SourceTable.make(self.schema, idFactory)
307  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
308  sources = detections.sources
309  fpSets = detections.fpSets
310  if hasattr(fpSets, "background") and fpSets.background:
311  for bg in fpSets.background:
312  backgrounds.append(bg)
313  return Struct(sources=sources, backgrounds=backgrounds)
314 
315  def write(self, exposure, results, patchRef):
316  """!
317  \brief Write out results from runDetection.
318 
319  \param[in] exposure: Exposure to write out
320  \param[in] results: Struct returned from runDetection
321  \param[in] patchRef: data reference for patch
322  """
323  coaddName = self.config.coaddName + "Coadd"
324  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
325  patchRef.put(results.sources, coaddName + "_det")
326  patchRef.put(exposure, coaddName + "_calexp")
327 
328 
329 
330 
331 class MergeSourcesRunner(TaskRunner):
332  """!
333  \anchor MergeSourcesRunner_
334 
335  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
336  requires a list of dataRefs rather than a single dataRef.
337  """
338 
339  def makeTask(self, parsedCmd=None, args=None):
340  """!
341  \brief Provide a butler to the Task constructor.
342 
343  \param[in] parsedCmd the parsed command
344  \param[in] args tuple of a list of data references and kwargs (un-used)
345  \throws RuntimeError if both parsedCmd & args are None
346  """
347  if parsedCmd is not None:
348  butler = parsedCmd.butler
349  elif args is not None:
350  dataRefList, kwargs = args
351  butler = dataRefList[0].getButler()
352  else:
353  raise RuntimeError("Neither parsedCmd or args specified")
354  return self.TaskClass(config=self.config, log=self.log, butler=butler)
355 
356  @staticmethod
357  def getTargetList(parsedCmd, **kwargs):
358  """!
359  \brief Provide a list of patch references for each patch.
360 
361  The patch references within the list will have different filters.
362 
363  \param[in] parsedCmd the parsed command
364  \param **kwargs key word arguments (unused)
365  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
366  filter
367  """
368  refList = {} # Will index this as refList[tract][patch][filter] = ref
369  for ref in parsedCmd.id.refList:
370  tract = ref.dataId["tract"]
371  patch = ref.dataId["patch"]
372  filter = ref.dataId["filter"]
373  if tract not in refList:
374  refList[tract] = {}
375  if patch not in refList[tract]:
376  refList[tract][patch] = {}
377  if filter in refList[tract][patch]:
378  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
379  refList[tract][patch][filter] = ref
380  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
381 
382 
383 class MergeSourcesConfig(Config):
384  """!
385  \anchor MergeSourcesConfig_
386 
387  \brief Configuration for merging sources.
388  """
389  priorityList = ListField(dtype=str, default=[],
390  doc="Priority-ordered list of bands for the merge.")
391  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
392 
393  def validate(self):
394  Config.validate(self)
395  if len(self.priorityList) == 0:
396  raise RuntimeError("No priority list provided")
397 
398 
399 class MergeSourcesTask(CmdLineTask):
400  """!
401  \anchor MergeSourcesTask_
402 
403  \brief A base class for merging source catalogs.
404 
405  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
406  so similar that it makes sense to re-use the code, in the form of this abstract base class.
407 
408  NB: Do not use this class directly. Instead use one of the child classes that inherit from
409  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
410  "MergeMeasurementsTask"
411 
412  Sub-classes should set the following class variables:
413  * `_DefaultName`: name of Task
414  * `inputDataset`: name of dataset to read
415  * `outputDataset`: name of dataset to write
416  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
417 
418  In addition, sub-classes must implement the mergeCatalogs method.
419  """
420  _DefaultName = None
421  ConfigClass = MergeSourcesConfig
422  RunnerClass = MergeSourcesRunner
423  inputDataset = None
424  outputDataset = None
425  getSchemaCatalogs = None
426 
427  @classmethod
428  def _makeArgumentParser(cls):
429  """!
430  \brief Create a suitable ArgumentParser.
431 
432  We will use the ArgumentParser to get a provide a list of data
433  references for patches; the RunnerClass will sort them into lists
434  of data references for the same patch
435  """
436  parser = ArgumentParser(name=cls._DefaultName)
437  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
438  ContainerClass=ExistingCoaddDataIdContainer,
439  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
440  return parser
441 
442  def getInputSchema(self, butler=None, schema=None):
443  """!
444  \brief Obtain the input schema either directly or froma butler reference.
445 
446  \param[in] butler butler reference to obtain the input schema from
447  \param[in] schema the input schema
448  """
449  if schema is None:
450  assert butler is not None, "Neither butler nor schema specified"
451  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
452  immediate=True).schema
453  return schema
454 
455  def __init__(self, butler=None, schema=None, **kwargs):
456  """!
457  \brief Initialize the task.
458 
459  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
460  \param[in] schema the schema of the detection catalogs used as input to this one
461  \param[in] butler a butler used to read the input schema from disk, if schema is None
462 
463  Derived classes should use the getInputSchema() method to handle the additional
464  arguments and retreive the actual input schema.
465  """
466  CmdLineTask.__init__(self, **kwargs)
467 
468  def run(self, patchRefList):
469  """!
470  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
471  subclasses that inherit from MergeSourcesTask.
472 
473  \param[in] patchRefList list of data references for each filter
474  """
475  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
476  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
477  self.write(patchRefList[0], mergedCatalog)
478 
479  def readCatalog(self, patchRef):
480  """!
481  \brief Read input catalog.
482 
483  We read the input dataset provided by the 'inputDataset'
484  class variable.
485 
486  \param[in] patchRef data reference for patch
487  \return tuple consisting of the filter name and the catalog
488  """
489  filterName = patchRef.dataId["filter"]
490  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
491  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
492  return filterName, catalog
493 
494  def mergeCatalogs(self, catalogs, patchRef):
495  """!
496  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
497  MergeSourcesTask.
498 
499  \param[in] catalogs dict mapping filter name to source catalog
500 
501  \return merged catalog
502  """
503  raise NotImplementedError()
504 
505  def write(self, patchRef, catalog):
506  """!
507  \brief Write the output.
508 
509  \param[in] patchRef data reference for patch
510  \param[in] catalog catalog
511 
512  We write as the dataset provided by the 'outputDataset'
513  class variable.
514  """
515  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
516  # since the filter isn't actually part of the data ID for the dataset we're saving,
517  # it's confusing to see it in the log message, even if the butler simply ignores it.
518  mergeDataId = patchRef.dataId.copy()
519  del mergeDataId["filter"]
520  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
521 
522  def writeMetadata(self, dataRefList):
523  """!
524  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
525  """
526  pass
527 
528 
529 class CullPeaksConfig(Config):
530  """!
531  \anchor CullPeaksConfig_
532 
533  \brief Configuration for culling garbage peaks after merging footprints.
534 
535  Peaks may also be culled after detection or during deblending; this configuration object
536  only deals with culling after merging Footprints.
537 
538  These cuts are based on three quantities:
539  - nBands: the number of bands in which the peak was detected
540  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
541  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
542 
543  The formula that identifie peaks to cull is:
544 
545  nBands < nBandsSufficient
546  AND (rank >= rankSufficient)
547  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
548 
549  To disable peak culling, simply set nBandsSufficient=1.
550  """
551 
552  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
553  doc="Always keep peaks detected in this many bands")
554  rankSufficient = RangeField(dtype=int, default=20, min=1,
555  doc="Always keep this many peaks in each family")
556  rankConsidered = RangeField(dtype=int, default=30, min=1,
557  doc=("Keep peaks with less than this rank that also match the "
558  "rankNormalizedConsidered condition."))
559  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
560  doc=("Keep peaks with less than this normalized rank that"
561  " also match the rankConsidered condition."))
562 
563 
565  """!
566  \anchor MergeDetectionsConfig_
567 
568  \brief Configuration parameters for the MergeDetectionsTask.
569  """
570  minNewPeak = Field(dtype=float, default=1,
571  doc="Minimum distance from closest peak to create a new one (in arcsec).")
572 
573  maxSamePeak = Field(dtype=float, default=0.3,
574  doc="When adding new catalogs to the merge, all peaks less than this distance "
575  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
576  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
577 
578  skyFilterName = Field(dtype=str, default="sky",
579  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
580  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
581  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
582 
583  def setDefaults(self):
584  MergeSourcesConfig.setDefaults(self)
585  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
586 
587 
588 
594 
595 
597  """!
598  \anchor MergeDetectionsTask_
599 
600  \brief Merge coadd detections from multiple bands.
601 
602  \section pipe_tasks_multiBand_Contents Contents
603 
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
608  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
609  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
610 
611  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
612 
613  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
614 
615  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
616  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
617  track of which band each source originates in.
618 
619  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
620  objects are culled as described in \ref CullPeaksConfig_.
621 
622  \par Inputs:
623  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
624  \par Outputs:
625  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
626  \par Data Unit:
627  tract, patch
628 
629  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
630 
631  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
632 
633  \copydoc \_\_init\_\_
634 
635  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
636 
637  \copydoc run
638 
639  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
640 
641  See \ref MergeDetectionsConfig_
642 
643  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
644 
645  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
646  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
647 
648  MergeDetectionsTask has no debug variables.
649 
650  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
651 
652  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
653  of the available bands.
654  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
655  chosen subset of filters.
656  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
657  and, eventually, perform forced photometry.
658  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
659  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
660  `--help` command line argument:
661  \code
662  mergeCoaddDetections.py --help
663  \endcode
664 
665  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
666  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
667  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
668  \code
669  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
670  \endcode
671  This will merge the HSC-I & -R band parent source catalogs and write the results to
672  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
673 
674  The next step in the multi-band processing procedure is
675  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
676  """
677  ConfigClass = MergeDetectionsConfig
678  _DefaultName = "mergeCoaddDetections"
679  inputDataset = "det"
680  outputDataset = "mergeDet"
681  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
682 
683  def __init__(self, butler=None, schema=None, **kwargs):
684  """!
685  \brief Initialize the merge detections task.
686 
687  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
688  merge the source catalogs.
689 
690  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
691  \param[in] schema the schema of the detection catalogs used as input to this one
692  \param[in] butler a butler used to read the input schema from disk, if schema is None
693  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
694 
695  The task will set its own self.schema attribute to the schema of the output merged catalog.
696  """
697  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
698  self.makeSubtask("skyObjects")
699  self.schema = self.getInputSchema(butler=butler, schema=schema)
700 
701  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
702  filterNames += [self.config.skyFilterName]
703  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
704 
705  def mergeCatalogs(self, catalogs, patchRef):
706  """!
707  \brief Merge multiple catalogs.
708 
709  After ordering the catalogs and filters in priority order,
710  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
711  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
712  garbage peaks detected around bright objects.
713 
714  \param[in] catalogs
715  \param[in] patchRef
716  \param[out] mergedList
717  """
718 
719  # Convert distance to tract coordinate
720  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
721  tractWcs = skyInfo.wcs
722  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
723  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
724 
725  # Put catalogs, filters in priority order
726  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
727  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
728  if band in catalogs.keys()]
729 
730  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
731  self.schema, self.makeIdFactory(patchRef),
732  samePeakDistance)
733 
734  #
735  # Add extra sources that correspond to blank sky
736  #
737  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
738  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
739  if skySourceFootprints:
740  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
741  for foot in skySourceFootprints:
742  s = mergedList.addNew()
743  s.setFootprint(foot)
744  s.set(key, True)
745 
746  # Sort Peaks from brightest to faintest
747  for record in mergedList:
748  record.getFootprint().sortPeaks()
749  self.log.info("Merged to %d sources" % len(mergedList))
750  # Attempt to remove garbage peaks
751  self.cullPeaks(mergedList)
752  return mergedList
753 
754  def cullPeaks(self, catalog):
755  """!
756  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
757 
758  \param[in] catalog Source catalog
759  """
760  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
761  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
762  totalPeaks = 0
763  culledPeaks = 0
764  for parentSource in catalog:
765  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
766  # to it (which is easier than deleting as we iterate).
767  keptPeaks = parentSource.getFootprint().getPeaks()
768  oldPeaks = list(keptPeaks)
769  keptPeaks.clear()
770  familySize = len(oldPeaks)
771  totalPeaks += familySize
772  for rank, peak in enumerate(oldPeaks):
773  if ((rank < self.config.cullPeaks.rankSufficient) or
774  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
775  (rank < self.config.cullPeaks.rankConsidered and
776  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
777  keptPeaks.append(peak)
778  else:
779  culledPeaks += 1
780  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
781 
782  def getSchemaCatalogs(self):
783  """!
784  Return a dict of empty catalogs for each catalog dataset produced by this task.
785 
786  \param[out] dictionary of empty catalogs
787  """
788  mergeDet = afwTable.SourceCatalog(self.schema)
789  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
790  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
791  self.config.coaddName + "Coadd_peak": peak}
792 
793  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
794  """!
795  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
796 
797  \param mergedList The merged Footprints from all the input bands
798  \param skyInfo A description of the patch
799  \param seed Seed for the random number generator
800  """
801  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
802  detected = mask.getPlaneBitMask("DETECTED")
803  for s in mergedList:
804  s.getFootprint().spans.setMask(mask, detected)
805 
806  footprints = self.skyObjects.run(mask, seed)
807  if not footprints:
808  return footprints
809 
810  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
811  schema = self.merged.getPeakSchema()
812  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
813  converted = []
814  for oldFoot in footprints:
815  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
816  peak = oldFoot.getPeaks()[0]
817  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
818  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
819  newFoot.getPeaks()[0].set(mergeKey, True)
820  converted.append(newFoot)
821 
822  return converted
823 
824 
826  """!
827  \anchor MeasureMergedCoaddSourcesConfig_
828 
829  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
830  """
831  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
832  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
833  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
834  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
835  doPropagateFlags = Field(
836  dtype=bool, default=True,
837  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
838  )
839  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
840  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
841  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
842  doWriteMatchesDenormalized = Field(
843  dtype=bool,
844  default=False,
845  doc=("Write reference matches in denormalized format? "
846  "This format uses more disk space, but is more convenient to read."),
847  )
848  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
849  checkUnitsParseStrict = Field(
850  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
851  dtype=str,
852  default="raise",
853  )
854  doApCorr = Field(
855  dtype=bool,
856  default=True,
857  doc="Apply aperture corrections"
858  )
859  applyApCorr = ConfigurableField(
860  target=ApplyApCorrTask,
861  doc="Subtask to apply aperture corrections"
862  )
863  doRunCatalogCalculation = Field(
864  dtype=bool,
865  default=True,
866  doc='Run catalogCalculation task'
867  )
868  catalogCalculation = ConfigurableField(
869  target=CatalogCalculationTask,
870  doc="Subtask to run catalogCalculation plugins on catalog"
871  )
872 
873  def setDefaults(self):
874  Config.setDefaults(self)
875  self.deblend.propagateAllPeaks = True
876  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
877  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
878  'INEXACT_PSF']
879  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
880  'INEXACT_PSF']
881 
882 
888 
889 
890 class MeasureMergedCoaddSourcesTask(CmdLineTask):
891  """!
892  \anchor MeasureMergedCoaddSourcesTask_
893 
894  \brief Deblend sources from master catalog in each coadd seperately and measure.
895 
896  \section pipe_tasks_multiBand_Contents Contents
897 
898  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
899  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
900  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
901  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
902  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
903  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
904 
905  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
906 
907  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
908  measurement in each coadd.
909 
910  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
911  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
912  consistent set of child sources.
913 
914  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
915  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
916  flags are propagated to the coadd sources.
917 
918  Optionally, we can match the coadd sources to an external reference catalog.
919 
920  \par Inputs:
921  deepCoadd_mergeDet{tract,patch}: SourceCatalog
922  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
923  \par Outputs:
924  deepCoadd_meas{tract,patch,filter}: SourceCatalog
925  \par Data Unit:
926  tract, patch, filter
927 
928  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
929 
930  <DL>
931  <DT> \ref SourceDeblendTask_ "deblend"
932  <DD> Deblend all the sources from the master catalog.</DD>
933  <DT> \ref SingleFrameMeasurementTask_ "measurement"
934  <DD> Measure source properties of deblended sources.</DD>
935  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
936  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
937  not at the edge of the field and that have either not been deblended or are the children of deblended
938  sources</DD>
939  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
940  <DD> Propagate flags set in individual visits to the coadd.</DD>
941  <DT> \ref DirectMatchTask_ "match"
942  <DD> Match input sources to a reference catalog (optional).
943  </DD>
944  </DL>
945  These subtasks may be retargeted as required.
946 
947  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
948 
949  \copydoc \_\_init\_\_
950 
951  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
952 
953  \copydoc run
954 
955  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
956 
957  See \ref MeasureMergedCoaddSourcesConfig_
958 
959  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
960 
961  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
962  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
963  files.
964 
965  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
966  the various sub-tasks. See the documetation for individual sub-tasks for more information.
967 
968  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
969  MeasureMergedCoaddSourcesTask
970 
971  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
972  The next stage in the multi-band processing procedure will merge these measurements into a suitable
973  catalog for driving forced photometry.
974 
975  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
976  to be processed.
977  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
978  `--help` command line argument:
979  \code
980  measureCoaddSources.py --help
981  \endcode
982 
983  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
984  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
985  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
986  coadd as follows:
987  \code
988  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
989  \endcode
990  This will process the HSC-I band data. The results are written in
991  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
992 
993  It is also necessary to run
994  \code
995  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
996  \endcode
997  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
998  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
999  """
1000  _DefaultName = "measureCoaddSources"
1001  ConfigClass = MeasureMergedCoaddSourcesConfig
1002  RunnerClass = ButlerInitializedTaskRunner
1003  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1004  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1005 
1006  @classmethod
1007  def _makeArgumentParser(cls):
1008  parser = ArgumentParser(name=cls._DefaultName)
1009  parser.add_id_argument("--id", "deepCoadd_calexp",
1010  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1011  ContainerClass=ExistingCoaddDataIdContainer)
1012  return parser
1013 
1014  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1015  """!
1016  \brief Initialize the task.
1017 
1018  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1019  \param[in] schema: the schema of the merged detection catalog used as input to this one
1020  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1021  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1022  catalog. May be None if the loader can be constructed from the butler argument or all steps
1023  requiring a reference catalog are disabled.
1024  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1025  catalog loader, if schema or peakSchema or refObjLoader is None
1026 
1027  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1028  This will include all fields from the input schema, as well as additional fields for all the
1029  measurements.
1030  """
1031  CmdLineTask.__init__(self, **kwargs)
1032  if schema is None:
1033  assert butler is not None, "Neither butler nor schema is defined"
1034  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1035  self.schemaMapper = afwTable.SchemaMapper(schema)
1036  self.schemaMapper.addMinimalSchema(schema)
1037  self.schema = self.schemaMapper.getOutputSchema()
1039  if self.config.doDeblend:
1040  if peakSchema is None:
1041  assert butler is not None, "Neither butler nor peakSchema is defined"
1042  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1043  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1044  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1045  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1046  if self.config.doMatchSources:
1047  if refObjLoader is None:
1048  assert butler is not None, "Neither butler nor refObjLoader is defined"
1049  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1050  if self.config.doPropagateFlags:
1051  self.makeSubtask("propagateFlags", schema=self.schema)
1052  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1053  if self.config.doApCorr:
1054  self.makeSubtask("applyApCorr", schema=self.schema)
1055  if self.config.doRunCatalogCalculation:
1056  self.makeSubtask("catalogCalculation", schema=self.schema)
1057 
1058  def run(self, patchRef):
1059  """!
1060  \brief Deblend and measure.
1061 
1062  \param[in] patchRef: Patch reference.
1063 
1064  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1065  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1066  Finally, write the deblended sources and measurements out.
1067  """
1068  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1069  sources = self.readSources(patchRef)
1070  if self.config.doDeblend:
1071  self.deblend.run(exposure, sources)
1072 
1073  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1074  # catalog is non-contiguous so can't extract column
1075  numBig = sum((s.get(bigKey) for s in sources))
1076  if numBig > 0:
1077  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1078  (patchRef.dataId, numBig))
1079 
1080  table = sources.getTable()
1081  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1082 
1083  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1084 
1085  if self.config.doApCorr:
1086  self.applyApCorr.run(
1087  catalog=sources,
1088  apCorrMap=exposure.getInfo().getApCorrMap()
1089  )
1090 
1091  if self.config.doRunCatalogCalculation:
1092  self.catalogCalculation.run(sources)
1093 
1094  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1095  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1096  includeDeblend=self.config.doDeblend)
1097  if self.config.doPropagateFlags:
1098  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1099  exposure.getWcs())
1100  if self.config.doMatchSources:
1101  self.writeMatches(patchRef, exposure, sources)
1102  self.write(patchRef, sources)
1103 
1104  def readSources(self, dataRef):
1105  """!
1106  \brief Read input sources.
1107 
1108  \param[in] dataRef: Data reference for catalog of merged detections
1109  \return List of sources in merged catalog
1110 
1111  We also need to add columns to hold the measurements we're about to make
1112  so we can measure in-place.
1113  """
1114  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1115  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1116  idFactory = self.makeIdFactory(dataRef)
1117  for s in merged:
1118  idFactory.notify(s.getId())
1119  table = afwTable.SourceTable.make(self.schema, idFactory)
1120  sources = afwTable.SourceCatalog(table)
1121  sources.extend(merged, self.schemaMapper)
1122  return sources
1123 
1124  def writeMatches(self, dataRef, exposure, sources):
1125  """!
1126  \brief Write matches of the sources to the astrometric reference catalog.
1127 
1128  We use the Wcs in the exposure to match sources.
1129 
1130  \param[in] dataRef: data reference
1131  \param[in] exposure: exposure with Wcs
1132  \param[in] sources: source catalog
1133  """
1134  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1135  if result.matches:
1136  matches = afwTable.packMatches(result.matches)
1137  matches.table.setMetadata(result.matchMeta)
1138  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1139  if self.config.doWriteMatchesDenormalized:
1140  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1141  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1142 
1143  def write(self, dataRef, sources):
1144  """!
1145  \brief Write the source catalog.
1146 
1147  \param[in] dataRef: data reference
1148  \param[in] sources: source catalog
1149  """
1150  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1151  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1152 
1153  def getExposureId(self, dataRef):
1154  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1155 
1156 
1158  """!
1159  \anchor MergeMeasurementsConfig_
1160 
1161  \brief Configuration parameters for the MergeMeasurementsTask
1162  """
1163  pseudoFilterList = ListField(dtype=str, default=["sky"],
1164  doc="Names of filters which may have no associated detection\n"
1165  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1166  snName = Field(dtype=str, default="base_PsfFlux",
1167  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1168  minSN = Field(dtype=float, default=10.,
1169  doc="If the S/N from the priority band is below this value (and the S/N "
1170  "is larger than minSNDiff compared to the priority band), use the band with "
1171  "the largest S/N as the reference band.")
1172  minSNDiff = Field(dtype=float, default=3.,
1173  doc="If the difference in S/N between another band and the priority band is larger "
1174  "than this value (and the S/N in the priority band is less than minSN) "
1175  "use the band with the largest S/N as the reference band")
1176  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1177  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1178  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1179 
1180 
1186 
1187 
1189  """!
1190  \anchor MergeMeasurementsTask_
1191 
1192  \brief Merge measurements from multiple bands
1193 
1194  \section pipe_tasks_multiBand_Contents Contents
1195 
1196  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1197  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1198  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1199  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1200  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1201  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1202 
1203  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1204 
1205  Command-line task that merges measurements from multiple bands.
1206 
1207  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1208  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1209  required to have centroid, shape and flux measurements in each band.
1210 
1211  \par Inputs:
1212  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1213  \par Outputs:
1214  deepCoadd_ref{tract,patch}: SourceCatalog
1215  \par Data Unit:
1216  tract, patch
1217 
1218  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1219 
1220  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1221 
1222  \copydoc \_\_init\_\_
1223 
1224  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1225 
1226  \copydoc run
1227 
1228  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1229 
1230  See \ref MergeMeasurementsConfig_
1231 
1232  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1233 
1234  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1235  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1236  files.
1237 
1238  MergeMeasurementsTask has no debug variables.
1239 
1240  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1241  of using MergeMeasurementsTask
1242 
1243  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1244  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1245  coadds and individual exposures.
1246  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1247  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1248  command line argument:
1249  \code
1250  mergeCoaddMeasurements.py --help
1251  \endcode
1252 
1253  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1254  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1255  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1256  as follows:
1257  \code
1258  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1259  \endcode
1260  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1261  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1262  """
1263  _DefaultName = "mergeCoaddMeasurements"
1264  ConfigClass = MergeMeasurementsConfig
1265  inputDataset = "meas"
1266  outputDataset = "ref"
1267  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1268 
1269  def __init__(self, butler=None, schema=None, **kwargs):
1270  """!
1271  Initialize the task.
1272 
1273  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1274  \param[in] schema: the schema of the detection catalogs used as input to this one
1275  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1276 
1277  The task will set its own self.schema attribute to the schema of the output merged catalog.
1278  """
1279  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1280  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1281  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1282  self.schemaMapper.addMinimalSchema(inputSchema, True)
1283  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1284  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1285  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1286 
1287  self.flagKeys = {}
1288  for band in self.config.priorityList:
1289  short = getShortFilterName(band)
1290  outputKey = self.schemaMapper.editOutputSchema().addField(
1291  "merge_measurement_%s" % short,
1292  type="Flag",
1293  doc="Flag field set if the measurements here are from the %s filter" % band
1294  )
1295  peakKey = inputSchema.find("merge_peak_%s" % short).key
1296  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1297  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1298  self.schema = self.schemaMapper.getOutputSchema()
1299 
1301  for filt in self.config.pseudoFilterList:
1302  try:
1303  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1304  except:
1305  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1306 
1307  self.badFlags = {}
1308  for flag in self.config.flags:
1309  try:
1310  self.badFlags[flag] = self.schema.find(flag).getKey()
1311  except KeyError as exc:
1312  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1313 
1314  def mergeCatalogs(self, catalogs, patchRef):
1315  """!
1316  Merge measurement catalogs to create a single reference catalog for forced photometry
1317 
1318  \param[in] catalogs: the catalogs to be merged
1319  \param[in] patchRef: patch reference for data
1320 
1321  For parent sources, we choose the first band in config.priorityList for which the
1322  merge_footprint flag for that band is is True.
1323 
1324  For child sources, the logic is the same, except that we use the merge_peak flags.
1325  """
1326  # Put catalogs, filters in priority order
1327  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1328  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1329 
1330  mergedCatalog = afwTable.SourceCatalog(self.schema)
1331  mergedCatalog.reserve(len(orderedCatalogs[0]))
1332 
1333  idKey = orderedCatalogs[0].table.getIdKey()
1334  for catalog in orderedCatalogs[1:]:
1335  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1336  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1337 
1338  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1339  # record for each band, in priority order.
1340  for orderedRecords in zip(*orderedCatalogs):
1341 
1342  maxSNRecord = None
1343  maxSNFlagKeys = None
1344  maxSN = 0.
1345  priorityRecord = None
1346  priorityFlagKeys = None
1347  prioritySN = 0.
1348  hasPseudoFilter = False
1349 
1350  # Now we iterate over those record-band pairs, keeping track of the priority and the
1351  # largest S/N band.
1352  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1353  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1354  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1355 
1356  if not (parent or child):
1357  for pseudoFilterKey in self.pseudoFilterKeys:
1358  if inputRecord.get(pseudoFilterKey):
1359  hasPseudoFilter = True
1360  priorityRecord = inputRecord
1361  priorityFlagKeys = flagKeys
1362  break
1363  if hasPseudoFilter:
1364  break
1365 
1366  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1367  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1368  sn = 0.
1369  else:
1370  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1371  if numpy.isnan(sn) or sn < 0.:
1372  sn = 0.
1373  if (parent or child) and priorityRecord is None:
1374  priorityRecord = inputRecord
1375  priorityFlagKeys = flagKeys
1376  prioritySN = sn
1377  if sn > maxSN:
1378  maxSNRecord = inputRecord
1379  maxSNFlagKeys = flagKeys
1380  maxSN = sn
1381 
1382  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1383  # the reference band instead. However, we only want to choose the highest S/N band if it is
1384  # significantly better than the priority band. Therefore, to choose a band other than the
1385  # priority, we require that the priority S/N is below the minimum threshold and that the
1386  # difference between the priority and highest S/N is larger than the difference threshold.
1387  #
1388  # For pseudo code objects we always choose the first band in the priority list.
1389  bestRecord = None
1390  bestFlagKeys = None
1391  if hasPseudoFilter:
1392  bestRecord = priorityRecord
1393  bestFlagKeys = priorityFlagKeys
1394  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1395  maxSNRecord is not None):
1396  bestRecord = maxSNRecord
1397  bestFlagKeys = maxSNFlagKeys
1398  elif priorityRecord is not None:
1399  bestRecord = priorityRecord
1400  bestFlagKeys = priorityFlagKeys
1401 
1402  if bestRecord is not None and bestFlagKeys is not None:
1403  outputRecord = mergedCatalog.addNew()
1404  outputRecord.assign(bestRecord, self.schemaMapper)
1405  outputRecord.set(bestFlagKeys.output, True)
1406  else: # if we didn't find any records
1407  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1408  inputRecord.getId())
1409 
1410  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1411  for inputCatalog in orderedCatalogs:
1412  if len(mergedCatalog) != len(inputCatalog):
1413  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1414  (len(mergedCatalog), len(orderedCatalogs)))
1415 
1416  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:793
Merge coadd detections from multiple bands.
Definition: multiBand.py:596
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:339
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:442
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:782
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:283
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:754
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1058
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:331
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:315
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:455
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1269
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:110
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:248
Merge measurements from multiple bands.
Definition: multiBand.py:1188
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:468
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:890
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1124
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:268
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1314
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:494
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1157
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1104
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:564
Configuration for merging sources.
Definition: multiBand.py:383
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:505
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:705
A base class for merging source catalogs.
Definition: multiBand.py:399
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1143
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:479
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:825
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:256
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:522
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1014
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:683
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:357