lsst.pipe.tasks  13.0-29-g7046ce1+1
 All Classes Namespaces Files Functions Variables Groups Pages
multiBand.py
Go to the documentation of this file.
1 
2 #!/usr/bin/env python
3 #
4 # LSST Data Management System
5 # Copyright 2008-2015 AURA/LSST.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <https://www.lsstcorp.org/LegalNotices/>.
23 #
24 from __future__ import absolute_import, division, print_function
25 from builtins import zip
26 from builtins import range
27 import numpy
28 
29 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
30 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
31 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
32 from lsst.meas.algorithms import SourceDetectionTask
33 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
34 from lsst.meas.deblender import SourceDeblendTask
35 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
36 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 ##############################################################################################################
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=SourceDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120 
121  def setDefaults(self):
122  Config.setDefaults(self)
123  self.detection.thresholdType = "pixel_stdev"
124  self.detection.isotropicGrow = True
125  # Coadds are made from background-subtracted CCDs, so background subtraction should be very basic
126  self.detection.background.useApprox = False
127  self.detection.background.binSize = 4096
128  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
129 
130 ## \addtogroup LSST_task_documentation
131 ## \{
132 ## \page DetectCoaddSourcesTask
133 ## \ref DetectCoaddSourcesTask_ "DetectCoaddSourcesTask"
134 ## \copybrief DetectCoaddSourcesTask
135 ## \}
136 
137 
138 class DetectCoaddSourcesTask(CmdLineTask):
139  """!
140  \anchor DetectCoaddSourcesTask_
141 
142  \brief Detect sources on a coadd
143 
144  \section pipe_tasks_multiBand_Contents Contents
145 
146  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
147  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
148  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
149  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
150  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
151  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
152 
153  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
154 
155  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
156 
157  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
158  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
159  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
160  propagate the full covariance matrix -- but it is simple and works well in practice.
161 
162  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
163  SourceDetectionTask_ "detection" subtask.
164 
165  \par Inputs:
166  deepCoadd{tract,patch,filter}: ExposureF
167  \par Outputs:
168  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
169  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
170  exposure (ExposureF)
171  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
172  \par Data Unit:
173  tract, patch, filter
174 
175  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
176  You can retarget this subtask if you wish.
177 
178  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
179 
180  \copydoc \_\_init\_\_
181 
182  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
183 
184  \copydoc run
185 
186  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
187 
188  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
189 
190  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
191 
192  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
193  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
194  files.
195 
196  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
197  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
198  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
199 
200  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
201 
202  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
203  the task is to update the background, detect all sources in a single band and generate a set of parent
204  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
205  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
206  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
207  calling detectCoaddSources.py with the `--help` command line argument:
208  \code
209  detectCoaddSources.py --help
210  \endcode
211 
212  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
213  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
214  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
215  \code
216  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
217  \endcode
218  that will process the HSC-I band data. The results are written to
219  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
220 
221  It is also necessary to run:
222  \code
223  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
224  \endcode
225  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
226  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
227  """
228  _DefaultName = "detectCoaddSources"
229  ConfigClass = DetectCoaddSourcesConfig
230  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
231  makeIdFactory = _makeMakeIdFactory("CoaddId")
232 
233  @classmethod
234  def _makeArgumentParser(cls):
235  parser = ArgumentParser(name=cls._DefaultName)
236  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
237  ContainerClass=ExistingCoaddDataIdContainer)
238  return parser
239 
240  def __init__(self, schema=None, **kwargs):
241  """!
242  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
243 
244  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
245 
246  \param[in] schema: initial schema for the output catalog, modified-in place to include all
247  fields set by this task. If None, the source minimal schema will be used.
248  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
249  """
250  CmdLineTask.__init__(self, **kwargs)
251  if schema is None:
252  schema = afwTable.SourceTable.makeMinimalSchema()
253  self.schema = schema
254  self.makeSubtask("detection", schema=self.schema)
255 
256  def run(self, patchRef):
257  """!
258  \brief Run detection on a coadd.
259 
260  Invokes \ref runDetection and then uses \ref write to output the
261  results.
262 
263  \param[in] patchRef: data reference for patch
264  """
265  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
266  results = self.runDetection(exposure, self.makeIdFactory(patchRef))
267  self.write(exposure, results, patchRef)
268  return results
269 
270  def runDetection(self, exposure, idFactory):
271  """!
272  \brief Run detection on an exposure.
273 
274  First scale the variance plane to match the observed variance
275  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
276  detect sources.
277 
278  \param[in] exposure: Exposure on which to detect
279  \param[in] idFactory: IdFactory to set source identifiers
280 
281  \return a pipe.base.Struct with fields
282  - sources: catalog of detections
283  - backgrounds: list of backgrounds
284  """
285  if self.config.doScaleVariance:
286  scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
287  backgrounds = afwMath.BackgroundList()
288  table = afwTable.SourceTable.make(self.schema, idFactory)
289  detections = self.detection.makeSourceCatalog(table, exposure)
290  sources = detections.sources
291  fpSets = detections.fpSets
292  if fpSets.background:
293  backgrounds.append(fpSets.background)
294  return Struct(sources=sources, backgrounds=backgrounds)
295 
296  def write(self, exposure, results, patchRef):
297  """!
298  \brief Write out results from runDetection.
299 
300  \param[in] exposure: Exposure to write out
301  \param[in] results: Struct returned from runDetection
302  \param[in] patchRef: data reference for patch
303  """
304  coaddName = self.config.coaddName + "Coadd"
305  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
306  patchRef.put(results.sources, coaddName + "_det")
307  patchRef.put(exposure, coaddName + "_calexp")
308 
309 ##############################################################################################################
310 
311 
312 class MergeSourcesRunner(TaskRunner):
313  """!
314  \anchor MergeSourcesRunner_
315 
316  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
317  requires a list of dataRefs rather than a single dataRef.
318  """
319 
320  def makeTask(self, parsedCmd=None, args=None):
321  """!
322  \brief Provide a butler to the Task constructor.
323 
324  \param[in] parsedCmd the parsed command
325  \param[in] args tuple of a list of data references and kwargs (un-used)
326  \throws RuntimeError if both parsedCmd & args are None
327  """
328  if parsedCmd is not None:
329  butler = parsedCmd.butler
330  elif args is not None:
331  dataRefList, kwargs = args
332  butler = dataRefList[0].getButler()
333  else:
334  raise RuntimeError("Neither parsedCmd or args specified")
335  return self.TaskClass(config=self.config, log=self.log, butler=butler)
336 
337  @staticmethod
338  def getTargetList(parsedCmd, **kwargs):
339  """!
340  \brief Provide a list of patch references for each patch.
341 
342  The patch references within the list will have different filters.
343 
344  \param[in] parsedCmd the parsed command
345  \param **kwargs key word arguments (unused)
346  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
347  filter
348  """
349  refList = {} # Will index this as refList[tract][patch][filter] = ref
350  for ref in parsedCmd.id.refList:
351  tract = ref.dataId["tract"]
352  patch = ref.dataId["patch"]
353  filter = ref.dataId["filter"]
354  if not tract in refList:
355  refList[tract] = {}
356  if not patch in refList[tract]:
357  refList[tract][patch] = {}
358  if filter in refList[tract][patch]:
359  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
360  refList[tract][patch][filter] = ref
361  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
362 
363 
364 class MergeSourcesConfig(Config):
365  """!
366  \anchor MergeSourcesConfig_
367 
368  \brief Configuration for merging sources.
369  """
370  priorityList = ListField(dtype=str, default=[],
371  doc="Priority-ordered list of bands for the merge.")
372  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
373 
374  def validate(self):
375  Config.validate(self)
376  if len(self.priorityList) == 0:
377  raise RuntimeError("No priority list provided")
378 
379 
380 class MergeSourcesTask(CmdLineTask):
381  """!
382  \anchor MergeSourcesTask_
383 
384  \brief A base class for merging source catalogs.
385 
386  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
387  so similar that it makes sense to re-use the code, in the form of this abstract base class.
388 
389  NB: Do not use this class directly. Instead use one of the child classes that inherit from
390  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
391  "MergeMeasurementsTask"
392 
393  Sub-classes should set the following class variables:
394  * `_DefaultName`: name of Task
395  * `inputDataset`: name of dataset to read
396  * `outputDataset`: name of dataset to write
397  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
398 
399  In addition, sub-classes must implement the mergeCatalogs method.
400  """
401  _DefaultName = None
402  ConfigClass = MergeSourcesConfig
403  RunnerClass = MergeSourcesRunner
404  inputDataset = None
405  outputDataset = None
406  getSchemaCatalogs = None
407 
408  @classmethod
409  def _makeArgumentParser(cls):
410  """!
411  \brief Create a suitable ArgumentParser.
412 
413  We will use the ArgumentParser to get a provide a list of data
414  references for patches; the RunnerClass will sort them into lists
415  of data references for the same patch
416  """
417  parser = ArgumentParser(name=cls._DefaultName)
418  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
419  ContainerClass=ExistingCoaddDataIdContainer,
420  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
421  return parser
422 
423  def getInputSchema(self, butler=None, schema=None):
424  """!
425  \brief Obtain the input schema either directly or froma butler reference.
426 
427  \param[in] butler butler reference to obtain the input schema from
428  \param[in] schema the input schema
429  """
430  if schema is None:
431  assert butler is not None, "Neither butler nor schema specified"
432  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
433  immediate=True).schema
434  return schema
435 
436  def __init__(self, butler=None, schema=None, **kwargs):
437  """!
438  \brief Initialize the task.
439 
440  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
441  \param[in] schema the schema of the detection catalogs used as input to this one
442  \param[in] butler a butler used to read the input schema from disk, if schema is None
443 
444  Derived classes should use the getInputSchema() method to handle the additional
445  arguments and retreive the actual input schema.
446  """
447  CmdLineTask.__init__(self, **kwargs)
448 
449  def run(self, patchRefList):
450  """!
451  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
452  subclasses that inherit from MergeSourcesTask.
453 
454  \param[in] patchRefList list of data references for each filter
455  """
456  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
457  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
458  self.write(patchRefList[0], mergedCatalog)
459 
460  def readCatalog(self, patchRef):
461  """!
462  \brief Read input catalog.
463 
464  We read the input dataset provided by the 'inputDataset'
465  class variable.
466 
467  \param[in] patchRef data reference for patch
468  \return tuple consisting of the filter name and the catalog
469  """
470  filterName = patchRef.dataId["filter"]
471  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
472  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
473  return filterName, catalog
474 
475  def mergeCatalogs(self, catalogs, patchRef):
476  """!
477  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
478  MergeSourcesTask.
479 
480  \param[in] catalogs dict mapping filter name to source catalog
481 
482  \return merged catalog
483  """
484  raise NotImplementedError()
485 
486  def write(self, patchRef, catalog):
487  """!
488  \brief Write the output.
489 
490  \param[in] patchRef data reference for patch
491  \param[in] catalog catalog
492 
493  We write as the dataset provided by the 'outputDataset'
494  class variable.
495  """
496  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
497  # since the filter isn't actually part of the data ID for the dataset we're saving,
498  # it's confusing to see it in the log message, even if the butler simply ignores it.
499  mergeDataId = patchRef.dataId.copy()
500  del mergeDataId["filter"]
501  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
502 
503  def writeMetadata(self, dataRefList):
504  """!
505  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
506  """
507  pass
508 
509 
510 class CullPeaksConfig(Config):
511  """!
512  \anchor CullPeaksConfig_
513 
514  \brief Configuration for culling garbage peaks after merging footprints.
515 
516  Peaks may also be culled after detection or during deblending; this configuration object
517  only deals with culling after merging Footprints.
518 
519  These cuts are based on three quantities:
520  - nBands: the number of bands in which the peak was detected
521  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
522  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
523 
524  The formula that identifie peaks to cull is:
525 
526  nBands < nBandsSufficient
527  AND (rank >= rankSufficient)
528  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
529 
530  To disable peak culling, simply set nBandsSafe=1.
531  """
532 
533  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
534  doc="Always keep peaks detected in this many bands")
535  rankSufficient = RangeField(dtype=int, default=20, min=1,
536  doc="Always keep this many peaks in each family")
537  rankConsidered = RangeField(dtype=int, default=30, min=1,
538  doc=("Keep peaks with less than this rank that also match the "
539  "rankNormalizedConsidered condition."))
540  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
541  doc=("Keep peaks with less than this normalized rank that"
542  " also match the rankConsidered condition."))
543 
544 
546  """!
547  \anchor MergeDetectionsConfig_
548 
549  \brief Configuration parameters for the MergeDetectionsTask.
550  """
551  minNewPeak = Field(dtype=float, default=1,
552  doc="Minimum distance from closest peak to create a new one (in arcsec).")
553 
554  maxSamePeak = Field(dtype=float, default=0.3,
555  doc="When adding new catalogs to the merge, all peaks less than this distance "
556  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
557  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
558 
559  skyFilterName = Field(dtype=str, default="sky",
560  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
561  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
562  skySourceRadius = Field(dtype=float, default=8,
563  doc="Radius, in pixels, of sky objects")
564  skyGrowDetectedFootprints = Field(dtype=int, default=0,
565  doc="Number of pixels to grow the detected footprint mask "
566  "when adding sky objects")
567  nSkySourcesPerPatch = Field(dtype=int, default=100,
568  doc="Try to add this many sky objects to the mergeDet list, which will\n"
569  "then be measured along with the detected objects in sourceMeasurementTask")
570  nTrialSkySourcesPerPatch = Field(dtype=int, default=None, optional=True,
571  doc="Maximum number of trial sky object positions\n"
572  "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
573  nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
574  doc="Set nTrialSkySourcesPerPatch to\n"
575  " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
576  "if nTrialSkySourcesPerPatch is None")
577 
578 ## \addtogroup LSST_task_documentation
579 ## \{
580 ## \page MergeDetectionsTask
581 ## \ref MergeDetectionsTask_ "MergeDetectionsTask"
582 ## \copybrief MergeDetectionsTask
583 ## \}
584 
585 
587  """!
588  \anchor MergeDetectionsTask_
589 
590  \brief Merge coadd detections from multiple bands.
591 
592  \section pipe_tasks_multiBand_Contents Contents
593 
594  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
595  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
596  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
597  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
598  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
599  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
600 
601  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
602 
603  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
604 
605  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
606  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
607  track of which band each source originates in.
608 
609  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
610  objects are culled as described in \ref CullPeaksConfig_.
611 
612  \par Inputs:
613  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
614  \par Outputs:
615  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
616  \par Data Unit:
617  tract, patch
618 
619  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
620 
621  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
622 
623  \copydoc \_\_init\_\_
624 
625  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
626 
627  \copydoc run
628 
629  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
630 
631  See \ref MergeDetectionsConfig_
632 
633  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
634 
635  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
636  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
637 
638  MergeDetectionsTask has no debug variables.
639 
640  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
641 
642  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
643  of the available bands.
644  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
645  chosen subset of filters.
646  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
647  and, eventually, perform forced photometry.
648  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
649  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
650  `--help` command line argument:
651  \code
652  mergeCoaddDetections.py --help
653  \endcode
654 
655  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
656  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
657  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
658  \code
659  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
660  \endcode
661  This will merge the HSC-I & -R band parent source catalogs and write the results to
662  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
663 
664  The next step in the multi-band processing procedure is
665  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
666  """
667  ConfigClass = MergeDetectionsConfig
668  _DefaultName = "mergeCoaddDetections"
669  inputDataset = "det"
670  outputDataset = "mergeDet"
671  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
672 
673  def __init__(self, butler=None, schema=None, **kwargs):
674  """!
675  \brief Initialize the merge detections task.
676 
677  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
678  merge the source catalogs.
679 
680  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
681  \param[in] schema the schema of the detection catalogs used as input to this one
682  \param[in] butler a butler used to read the input schema from disk, if schema is None
683  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
684 
685  The task will set its own self.schema attribute to the schema of the output merged catalog.
686  """
687  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
688  self.schema = self.getInputSchema(butler=butler, schema=schema)
689 
690  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
691  if self.config.nSkySourcesPerPatch > 0:
692  filterNames += [self.config.skyFilterName]
693  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
694 
695  def mergeCatalogs(self, catalogs, patchRef):
696  """!
697  \brief Merge multiple catalogs.
698 
699  After ordering the catalogs and filters in priority order,
700  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
701  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
702  garbage peaks detected around bright objects.
703 
704  \param[in] catalogs
705  \param[in] patchRef
706  \param[out] mergedList
707  """
708 
709  # Convert distance to tract coordinate
710  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
711  tractWcs = skyInfo.wcs
712  peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
713  samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
714 
715  # Put catalogs, filters in priority order
716  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
717  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
718  if band in catalogs.keys()]
719 
720  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
721  self.schema, self.makeIdFactory(patchRef),
722  samePeakDistance)
723 
724  #
725  # Add extra sources that correspond to blank sky
726  #
727  skySourceFootprints = self.getSkySourceFootprints(
728  mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
729  if skySourceFootprints:
730  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
731 
732  for foot in skySourceFootprints:
733  s = mergedList.addNew()
734  s.setFootprint(foot)
735  s.set(key, True)
736 
737  self.log.info("Added %d sky sources (%.0f%% of requested)",
738  len(skySourceFootprints),
739  100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
740 
741  # Sort Peaks from brightest to faintest
742  for record in mergedList:
743  record.getFootprint().sortPeaks()
744  self.log.info("Merged to %d sources" % len(mergedList))
745  # Attempt to remove garbage peaks
746  self.cullPeaks(mergedList)
747  return mergedList
748 
749  def cullPeaks(self, catalog):
750  """!
751  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
752 
753  \param[in] catalog Source catalog
754  """
755  keys = [item.key for item in self.merged.getPeakSchema().extract("merge.peak.*").values()]
756  totalPeaks = 0
757  culledPeaks = 0
758  for parentSource in catalog:
759  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
760  # to it (which is easier than deleting as we iterate).
761  keptPeaks = parentSource.getFootprint().getPeaks()
762  oldPeaks = list(keptPeaks)
763  keptPeaks.clear()
764  familySize = len(oldPeaks)
765  totalPeaks += familySize
766  for rank, peak in enumerate(oldPeaks):
767  if ((rank < self.config.cullPeaks.rankSufficient) or
768  (self.config.cullPeaks.nBandsSufficient > 1 and
769  sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
770  (rank < self.config.cullPeaks.rankConsidered and
771  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
772  keptPeaks.append(peak)
773  else:
774  culledPeaks += 1
775  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
776 
777  def getSchemaCatalogs(self):
778  """!
779  Return a dict of empty catalogs for each catalog dataset produced by this task.
780 
781  \param[out] dictionary of empty catalogs
782  """
783  mergeDet = afwTable.SourceCatalog(self.schema)
784  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
785  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
786  self.config.coaddName + "Coadd_peak": peak}
787 
788  def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0):
789  """!
790  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
791 
792  \param mergedList The merged Footprints from all the input bands
793  \param skyInfo A description of the patch
794  \param growDetectedFootprints The number of pixels to grow the detected footprints
795  """
796 
797  if self.config.nSkySourcesPerPatch <= 0:
798  return []
799 
800  skySourceRadius = self.config.skySourceRadius
801  nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
802  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
803  if nTrialSkySourcesPerPatch is None:
804  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
805  #
806  # We are going to find circular Footprints that don't intersect any pre-existing Footprints,
807  # and the easiest way to do this is to generate a Mask containing all the detected pixels (as
808  # merged by this task).
809  #
810  patchBBox = skyInfo.patchInfo.getOuterBBox()
811  mask = afwImage.MaskU(patchBBox)
812  detectedMask = mask.getPlaneBitMask("DETECTED")
813  for s in mergedList:
814  foot = s.getFootprint()
815  if growDetectedFootprints > 0:
816  foot.dilate(growDetectedFootprints)
817  foot.spans.setMask(mask, detectedMask)
818 
819  xmin, ymin = patchBBox.getMin()
820  xmax, ymax = patchBBox.getMax()
821  # Avoid objects partially off the image
822  xmin += skySourceRadius + 1
823  ymin += skySourceRadius + 1
824  xmax -= skySourceRadius + 1
825  ymax -= skySourceRadius + 1
826 
827  skySourceFootprints = []
828  maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
829  for i in range(nTrialSkySourcesPerPatch):
830  if len(skySourceFootprints) == nSkySourcesPerPatch:
831  break
832 
833  x = int(numpy.random.uniform(xmin, xmax))
834  y = int(numpy.random.uniform(ymin, ymax))
835  spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
836  offset=(x, y))
837  foot = afwDetect.Footprint(spans, patchBBox)
838  foot.setPeakSchema(self.merged.getPeakSchema())
839 
840  if not foot.spans.overlaps(maskToSpanSet):
841  foot.addPeak(x, y, 0)
842  foot.getPeaks()[0].set("merge_peak_%s" % self.config.skyFilterName, True)
843  skySourceFootprints.append(foot)
844 
845  return skySourceFootprints
846 
847 
849  """!
850  \anchor MeasureMergedCoaddSourcesConfig_
851 
852  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
853  """
854  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
855  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
856  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
857  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
858  doPropagateFlags = Field(
859  dtype=bool, default=True,
860  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
861  )
862  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
863  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
864  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
865  doWriteMatchesDenormalized = Field(
866  dtype=bool,
867  default=False,
868  doc=("Write reference matches in denormalized format? "
869  "This format uses more disk space, but is more convenient to read."),
870  )
871  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
872  checkUnitsParseStrict = Field(
873  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
874  dtype=str,
875  default="raise",
876  )
877  doApCorr = Field(
878  dtype=bool,
879  default=True,
880  doc="Apply aperture corrections"
881  )
882  applyApCorr = ConfigurableField(
883  target=ApplyApCorrTask,
884  doc="Subtask to apply aperture corrections"
885  )
886  doRunCatalogCalculation = Field(
887  dtype=bool,
888  default=True,
889  doc='Run catalogCalculation task'
890  )
891  catalogCalculation = ConfigurableField(
892  target=CatalogCalculationTask,
893  doc="Subtask to run catalogCalculation plugins on catalog"
894  )
895 
896  def setDefaults(self):
897  Config.setDefaults(self)
898  self.deblend.propagateAllPeaks = True
899  self.measurement.plugins.names |= ['base_InputCount']
900  # The following line must be set if clipped pixel flags are to be added to the output table
901  # The clipped mask plane is added by running SafeClipAssembleCoaddTask
902  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED']
903 
904 ## \addtogroup LSST_task_documentation
905 ## \{
906 ## \page MeasureMergedCoaddSourcesTask
907 ## \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
908 ## \copybrief MeasureMergedCoaddSourcesTask
909 ## \}
910 
911 
912 class MeasureMergedCoaddSourcesTask(CmdLineTask):
913  """!
914  \anchor MeasureMergedCoaddSourcesTask_
915 
916  \brief Deblend sources from master catalog in each coadd seperately and measure.
917 
918  \section pipe_tasks_multiBand_Contents Contents
919 
920  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
921  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
922  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
923  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
924  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
925  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
926 
927  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
928 
929  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
930  measurement in each coadd.
931 
932  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
933  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
934  consistent set of child sources.
935 
936  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
937  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
938  flags are propagated to the coadd sources.
939 
940  Optionally, we can match the coadd sources to an external reference catalog.
941 
942  \par Inputs:
943  deepCoadd_mergeDet{tract,patch}: SourceCatalog
944  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
945  \par Outputs:
946  deepCoadd_meas{tract,patch,filter}: SourceCatalog
947  \par Data Unit:
948  tract, patch, filter
949 
950  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
951 
952  <DL>
953  <DT> \ref SourceDeblendTask_ "deblend"
954  <DD> Deblend all the sources from the master catalog.</DD>
955  <DT> \ref SingleFrameMeasurementTask_ "measurement"
956  <DD> Measure source properties of deblended sources.</DD>
957  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
958  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
959  not at the edge of the field and that have either not been deblended or are the children of deblended
960  sources</DD>
961  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
962  <DD> Propagate flags set in individual visits to the coadd.</DD>
963  <DT> \ref DirectMatchTask_ "match"
964  <DD> Match input sources to a reference catalog (optional).
965  </DD>
966  </DL>
967  These subtasks may be retargeted as required.
968 
969  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
970 
971  \copydoc \_\_init\_\_
972 
973  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
974 
975  \copydoc run
976 
977  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
978 
979  See \ref MeasureMergedCoaddSourcesConfig_
980 
981  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
982 
983  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
984  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
985  files.
986 
987  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
988  the various sub-tasks. See the documetation for individual sub-tasks for more information.
989 
990  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
991 
992  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
993  The next stage in the multi-band processing procedure will merge these measurements into a suitable
994  catalog for driving forced photometry.
995 
996  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
997  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
998  `--help` command line argument:
999  \code
1000  measureCoaddSources.py --help
1001  \endcode
1002 
1003  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1004  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1005  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1006  coadd as follows:
1007  \code
1008  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1009  \endcode
1010  This will process the HSC-I band data. The results are written in
1011  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1012 
1013  It is also necessary to run
1014  \code
1015  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1016  \endcode
1017  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1018  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1019  """
1020  _DefaultName = "measureCoaddSources"
1021  ConfigClass = MeasureMergedCoaddSourcesConfig
1022  RunnerClass = ButlerInitializedTaskRunner
1023  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1024  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1025 
1026  @classmethod
1027  def _makeArgumentParser(cls):
1028  parser = ArgumentParser(name=cls._DefaultName)
1029  parser.add_id_argument("--id", "deepCoadd_calexp",
1030  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1031  ContainerClass=ExistingCoaddDataIdContainer)
1032  return parser
1033 
1034  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1035  """!
1036  \brief Initialize the task.
1037 
1038  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1039  \param[in] schema: the schema of the merged detection catalog used as input to this one
1040  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1041  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1042  catalog. May be None if the loader can be constructed from the butler argument or all steps
1043  requiring a reference catalog are disabled.
1044  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1045  catalog loader, if schema or peakSchema or refObjLoader is None
1046 
1047  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1048  This will include all fields from the input schema, as well as additional fields for all the
1049  measurements.
1050  """
1051  CmdLineTask.__init__(self, **kwargs)
1052  if schema is None:
1053  assert butler is not None, "Neither butler nor schema is defined"
1054  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1055  self.schemaMapper = afwTable.SchemaMapper(schema)
1056  self.schemaMapper.addMinimalSchema(schema)
1057  self.schema = self.schemaMapper.getOutputSchema()
1058  self.algMetadata = PropertyList()
1059  if self.config.doDeblend:
1060  if peakSchema is None:
1061  assert butler is not None, "Neither butler nor peakSchema is defined"
1062  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1063  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1064  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1065  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1066  if self.config.doMatchSources:
1067  if refObjLoader is None:
1068  assert butler is not None, "Neither butler nor refObjLoader is defined"
1069  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1070  if self.config.doPropagateFlags:
1071  self.makeSubtask("propagateFlags", schema=self.schema)
1072  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1073  if self.config.doApCorr:
1074  self.makeSubtask("applyApCorr", schema=self.schema)
1075  if self.config.doRunCatalogCalculation:
1076  self.makeSubtask("catalogCalculation", schema=self.schema)
1077 
1078  def run(self, patchRef):
1079  """!
1080  \brief Deblend and measure.
1081 
1082  \param[in] patchRef: Patch reference.
1083 
1084  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1085  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1086  Finally, write the deblended sources and measurements out.
1087  """
1088  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1089  sources = self.readSources(patchRef)
1090  if self.config.doDeblend:
1091  self.deblend.run(exposure, sources)
1092 
1093  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1094  # catalog is non-contiguous so can't extract column
1095  numBig = sum((s.get(bigKey) for s in sources))
1096  if numBig > 0:
1097  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1098  (patchRef.dataId, numBig))
1099 
1100  table = sources.getTable()
1101  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1102 
1103  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1104 
1105  if self.config.doApCorr:
1106  self.applyApCorr.run(
1107  catalog=sources,
1108  apCorrMap=exposure.getInfo().getApCorrMap()
1109  )
1110 
1111  if self.config.doRunCatalogCalculation:
1112  self.catalogCalculation.run(sources)
1113 
1114  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1115  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1116  includeDeblend=self.config.doDeblend)
1117  if self.config.doPropagateFlags:
1118  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1119  exposure.getWcs())
1120  if self.config.doMatchSources:
1121  self.writeMatches(patchRef, exposure, sources)
1122  self.write(patchRef, sources)
1123 
1124  def readSources(self, dataRef):
1125  """!
1126  \brief Read input sources.
1127 
1128  \param[in] dataRef: Data reference for catalog of merged detections
1129  \return List of sources in merged catalog
1130 
1131  We also need to add columns to hold the measurements we're about to make
1132  so we can measure in-place.
1133  """
1134  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1135  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1136  idFactory = self.makeIdFactory(dataRef)
1137  for s in merged:
1138  idFactory.notify(s.getId())
1139  table = afwTable.SourceTable.make(self.schema, idFactory)
1140  sources = afwTable.SourceCatalog(table)
1141  sources.extend(merged, self.schemaMapper)
1142  return sources
1143 
1144  def writeMatches(self, dataRef, exposure, sources):
1145  """!
1146  \brief Write matches of the sources to the astrometric reference catalog.
1147 
1148  We use the Wcs in the exposure to match sources.
1149 
1150  \param[in] dataRef: data reference
1151  \param[in] exposure: exposure with Wcs
1152  \param[in] sources: source catalog
1153  """
1154  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1155  if result.matches:
1156  matches = afwTable.packMatches(result.matches)
1157  matches.table.setMetadata(result.matchMeta)
1158  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1159  if self.config.doWriteMatchesDenormalized:
1160  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1161  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1162 
1163 
1164  def write(self, dataRef, sources):
1165  """!
1166  \brief Write the source catalog.
1167 
1168  \param[in] dataRef: data reference
1169  \param[in] sources: source catalog
1170  """
1171  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1172  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1173 
1174  def getExposureId(self, dataRef):
1175  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1176 
1177 ##############################################################################################################
1178 
1179 
1181  """!
1182  \anchor MergeMeasurementsConfig_
1183 
1184  \brief Configuration parameters for the MergeMeasurementsTask
1185  """
1186  pseudoFilterList = ListField(dtype=str, default=["sky"],
1187  doc="Names of filters which may have no associated detection\n"
1188  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1189  snName = Field(dtype=str, default="base_PsfFlux",
1190  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1191  minSN = Field(dtype=float, default=10.,
1192  doc="If the S/N from the priority band is below this value (and the S/N "
1193  "is larger than minSNDiff compared to the priority band), use the band with "
1194  "the largest S/N as the reference band.")
1195  minSNDiff = Field(dtype=float, default=3.,
1196  doc="If the difference in S/N between another band and the priority band is larger "
1197  "than this value (and the S/N in the priority band is less than minSN) "
1198  "use the band with the largest S/N as the reference band")
1199  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1200  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1201  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1202 
1203 ## \addtogroup LSST_task_documentation
1204 ## \{
1205 ## \page MergeMeasurementsTask
1206 ## \ref MergeMeasurementsTask_ "MergeMeasurementsTask"
1207 ## \copybrief MergeMeasurementsTask
1208 ## \}
1209 
1210 
1212  """!
1213  \anchor MergeMeasurementsTask_
1214 
1215  \brief Merge measurements from multiple bands
1216 
1217  \section pipe_tasks_multiBand_Contents Contents
1218 
1219  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1220  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1221  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1222  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1223  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1224  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1225 
1226  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1227 
1228  Command-line task that merges measurements from multiple bands.
1229 
1230  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1231  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1232  required to have centroid, shape and flux measurements in each band.
1233 
1234  \par Inputs:
1235  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1236  \par Outputs:
1237  deepCoadd_ref{tract,patch}: SourceCatalog
1238  \par Data Unit:
1239  tract, patch
1240 
1241  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1242 
1243  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1244 
1245  \copydoc \_\_init\_\_
1246 
1247  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1248 
1249  \copydoc run
1250 
1251  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1252 
1253  See \ref MergeMeasurementsConfig_
1254 
1255  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1256 
1257  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1258  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1259  files.
1260 
1261  MergeMeasurementsTask has no debug variables.
1262 
1263  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1264 
1265  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1266  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1267  coadds and individual exposures.
1268  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1269  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1270  command line argument:
1271  \code
1272  mergeCoaddMeasurements.py --help
1273  \endcode
1274 
1275  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1276  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1277  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1278  as follows:
1279  \code
1280  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1281  \endcode
1282  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1283  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1284  """
1285  _DefaultName = "mergeCoaddMeasurements"
1286  ConfigClass = MergeMeasurementsConfig
1287  inputDataset = "meas"
1288  outputDataset = "ref"
1289  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1290 
1291  def __init__(self, butler=None, schema=None, **kwargs):
1292  """!
1293  Initialize the task.
1294 
1295  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1296  \param[in] schema: the schema of the detection catalogs used as input to this one
1297  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1298 
1299  The task will set its own self.schema attribute to the schema of the output merged catalog.
1300  """
1301  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1302  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1303  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1304  self.schemaMapper.addMinimalSchema(inputSchema, True)
1305  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1306  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1307  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1308 
1309  self.flagKeys = {}
1310  for band in self.config.priorityList:
1311  short = getShortFilterName(band)
1312  outputKey = self.schemaMapper.editOutputSchema().addField(
1313  "merge_measurement_%s" % short,
1314  type="Flag",
1315  doc="Flag field set if the measurements here are from the %s filter" % band
1316  )
1317  peakKey = inputSchema.find("merge_peak_%s" % short).key
1318  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1319  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1320  self.schema = self.schemaMapper.getOutputSchema()
1321 
1323  for filt in self.config.pseudoFilterList:
1324  try:
1325  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1326  except:
1327  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1328 
1329  self.badFlags = {}
1330  for flag in self.config.flags:
1331  try:
1332  self.badFlags[flag] = self.schema.find(flag).getKey()
1333  except KeyError as exc:
1334  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1335 
1336  def mergeCatalogs(self, catalogs, patchRef):
1337  """!
1338  Merge measurement catalogs to create a single reference catalog for forced photometry
1339 
1340  \param[in] catalogs: the catalogs to be merged
1341  \param[in] patchRef: patch reference for data
1342 
1343  For parent sources, we choose the first band in config.priorityList for which the
1344  merge_footprint flag for that band is is True.
1345 
1346  For child sources, the logic is the same, except that we use the merge_peak flags.
1347  """
1348  # Put catalogs, filters in priority order
1349  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1350  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1351 
1352  mergedCatalog = afwTable.SourceCatalog(self.schema)
1353  mergedCatalog.reserve(len(orderedCatalogs[0]))
1354 
1355  idKey = orderedCatalogs[0].table.getIdKey()
1356  for catalog in orderedCatalogs[1:]:
1357  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1358  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1359 
1360  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1361  # record for each band, in priority order.
1362  for orderedRecords in zip(*orderedCatalogs):
1363 
1364  maxSNRecord = None
1365  maxSNFlagKeys = None
1366  maxSN = 0.
1367  priorityRecord = None
1368  priorityFlagKeys = None
1369  prioritySN = 0.
1370  hasPseudoFilter = False
1371 
1372  # Now we iterate over those record-band pairs, keeping track of the priority and the
1373  # largest S/N band.
1374  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1375  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1376  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1377 
1378  if not (parent or child):
1379  for pseudoFilterKey in self.pseudoFilterKeys:
1380  if inputRecord.get(pseudoFilterKey):
1381  hasPseudoFilter = True
1382  priorityRecord = inputRecord
1383  priorityFlagKeys = flagKeys
1384  break
1385  if hasPseudoFilter:
1386  break
1387 
1388  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1389  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1390  sn = 0.
1391  else:
1392  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1393  if numpy.isnan(sn) or sn < 0.:
1394  sn = 0.
1395  if (parent or child) and priorityRecord is None:
1396  priorityRecord = inputRecord
1397  priorityFlagKeys = flagKeys
1398  prioritySN = sn
1399  if sn > maxSN:
1400  maxSNRecord = inputRecord
1401  maxSNFlagKeys = flagKeys
1402  maxSN = sn
1403 
1404  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1405  # the reference band instead. However, we only want to choose the highest S/N band if it is
1406  # significantly better than the priority band. Therefore, to choose a band other than the
1407  # priority, we require that the priority S/N is below the minimum threshold and that the
1408  # difference between the priority and highest S/N is larger than the difference threshold.
1409  #
1410  # For pseudo code objects we always choose the first band in the priority list.
1411  bestRecord = None
1412  bestFlagKeys = None
1413  if hasPseudoFilter:
1414  bestRecord = priorityRecord
1415  bestFlagKeys = priorityFlagKeys
1416  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1417  maxSNRecord is not None):
1418  bestRecord = maxSNRecord
1419  bestFlagKeys = maxSNFlagKeys
1420  elif priorityRecord is not None:
1421  bestRecord = priorityRecord
1422  bestFlagKeys = priorityFlagKeys
1423 
1424  if bestRecord is not None and bestFlagKeys is not None:
1425  outputRecord = mergedCatalog.addNew()
1426  outputRecord.assign(bestRecord, self.schemaMapper)
1427  outputRecord.set(bestFlagKeys.output, True)
1428  else: # if we didn't find any records
1429  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1430  inputRecord.getId())
1431 
1432  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1433  for inputCatalog in orderedCatalogs:
1434  if len(mergedCatalog) != len(inputCatalog):
1435  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1436  (len(mergedCatalog), len(orderedCatalogs)))
1437 
1438  return mergedCatalog
def writeMetadata
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:503
def cullPeaks
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:749
Merge coadd detections from multiple bands.
Definition: multiBand.py:586
def mergeCatalogs
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1336
def runDetection
Run detection on an exposure.
Definition: multiBand.py:270
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:312
def run
Merge coadd sources from multiple bands.
Definition: multiBand.py:449
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
Merge measurements from multiple bands.
Definition: multiBand.py:1211
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:912
def getTargetList
Provide a list of patch references for each patch.
Definition: multiBand.py:338
def scaleVariance
Scale the variance in a maskedImage.
Definition: coaddBase.py:263
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1180
def mergeCatalogs
Merge multiple catalogs.
Definition: multiBand.py:695
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:545
Configuration for merging sources.
Definition: multiBand.py:364
def mergeCatalogs
Merge multiple catalogs.
Definition: multiBand.py:475
def makeTask
Provide a butler to the Task constructor.
Definition: multiBand.py:320
A base class for merging source catalogs.
Definition: multiBand.py:380
def getSkyInfo
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:232
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:848
def getSkySourceFootprints
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:788
def __init__
Initialize the merge detections task.
Definition: multiBand.py:673
def getInputSchema
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:423
def getSchemaCatalogs
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:777
def write
Write out results from runDetection.
Definition: multiBand.py:296
def writeMatches
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1144