lsst.pipe.tasks  14.0-34-g85a33b94
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import SourceDetectionTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
35 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
36 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=SourceDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so background subtraction should be very basic
131  self.detection.background.useApprox = False
132  self.detection.background.binSize = 4096
133  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
134 
135 
141 
142 
143 class DetectCoaddSourcesTask(CmdLineTask):
144  """!
145  \anchor DetectCoaddSourcesTask_
146 
147  \brief Detect sources on a coadd
148 
149  \section pipe_tasks_multiBand_Contents Contents
150 
151  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
156  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
157 
158  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
159 
160  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
161 
162  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
163  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
164  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
165  propagate the full covariance matrix -- but it is simple and works well in practice.
166 
167  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
168  SourceDetectionTask_ "detection" subtask.
169 
170  \par Inputs:
171  deepCoadd{tract,patch,filter}: ExposureF
172  \par Outputs:
173  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
174  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
175  exposure (ExposureF)
176  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
177  \par Data Unit:
178  tract, patch, filter
179 
180  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
181  You can retarget this subtask if you wish.
182 
183  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
184 
185  \copydoc \_\_init\_\_
186 
187  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
188 
189  \copydoc run
190 
191  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
192 
193  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
194 
195  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
196 
197  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
198  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
199  files.
200 
201  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
202  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
203  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
204 
205  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
206 
207  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
208  the task is to update the background, detect all sources in a single band and generate a set of parent
209  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
210  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
211  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
212  calling detectCoaddSources.py with the `--help` command line argument:
213  \code
214  detectCoaddSources.py --help
215  \endcode
216 
217  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
218  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
219  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
220  \code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
222  \endcode
223  that will process the HSC-I band data. The results are written to
224  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
225 
226  It is also necessary to run:
227  \code
228  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
229  \endcode
230  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
231  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
232  """
233  _DefaultName = "detectCoaddSources"
234  ConfigClass = DetectCoaddSourcesConfig
235  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
236  makeIdFactory = _makeMakeIdFactory("CoaddId")
237 
238  @classmethod
239  def _makeArgumentParser(cls):
240  parser = ArgumentParser(name=cls._DefaultName)
241  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242  ContainerClass=ExistingCoaddDataIdContainer)
243  return parser
244 
245  def __init__(self, schema=None, **kwargs):
246  """!
247  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
248 
249  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
250 
251  \param[in] schema: initial schema for the output catalog, modified-in place to include all
252  fields set by this task. If None, the source minimal schema will be used.
253  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
254  """
255  CmdLineTask.__init__(self, **kwargs)
256  if schema is None:
257  schema = afwTable.SourceTable.makeMinimalSchema()
258  if self.config.doInsertFakes:
259  self.makeSubtask("insertFakes")
260  self.schema = schema
261  self.makeSubtask("detection", schema=self.schema)
262 
263  def run(self, patchRef):
264  """!
265  \brief Run detection on a coadd.
266 
267  Invokes \ref runDetection and then uses \ref write to output the
268  results.
269 
270  \param[in] patchRef: data reference for patch
271  """
272  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
273  results = self.runDetection(exposure, self.makeIdFactory(patchRef))
274  self.write(exposure, results, patchRef)
275  return results
276 
277  def runDetection(self, exposure, idFactory):
278  """!
279  \brief Run detection on an exposure.
280 
281  First scale the variance plane to match the observed variance
282  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
283  detect sources.
284 
285  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
286  depending on configuration).
287  \param[in] idFactory: IdFactory to set source identifiers
288 
289  \return a pipe.base.Struct with fields
290  - sources: catalog of detections
291  - backgrounds: list of backgrounds
292  """
293  if self.config.doScaleVariance:
294  varScale = scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
295  exposure.getMetadata().add("variance_scale", varScale)
296  backgrounds = afwMath.BackgroundList()
297  if self.config.doInsertFakes:
298  self.insertFakes.run(exposure, background=backgrounds)
299  table = afwTable.SourceTable.make(self.schema, idFactory)
300  detections = self.detection.makeSourceCatalog(table, exposure)
301  sources = detections.sources
302  fpSets = detections.fpSets
303  if fpSets.background:
304  backgrounds.append(fpSets.background)
305  return Struct(sources=sources, backgrounds=backgrounds)
306 
307  def write(self, exposure, results, patchRef):
308  """!
309  \brief Write out results from runDetection.
310 
311  \param[in] exposure: Exposure to write out
312  \param[in] results: Struct returned from runDetection
313  \param[in] patchRef: data reference for patch
314  """
315  coaddName = self.config.coaddName + "Coadd"
316  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
317  patchRef.put(results.sources, coaddName + "_det")
318  patchRef.put(exposure, coaddName + "_calexp")
319 
320 
321 
322 
323 class MergeSourcesRunner(TaskRunner):
324  """!
325  \anchor MergeSourcesRunner_
326 
327  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
328  requires a list of dataRefs rather than a single dataRef.
329  """
330 
331  def makeTask(self, parsedCmd=None, args=None):
332  """!
333  \brief Provide a butler to the Task constructor.
334 
335  \param[in] parsedCmd the parsed command
336  \param[in] args tuple of a list of data references and kwargs (un-used)
337  \throws RuntimeError if both parsedCmd & args are None
338  """
339  if parsedCmd is not None:
340  butler = parsedCmd.butler
341  elif args is not None:
342  dataRefList, kwargs = args
343  butler = dataRefList[0].getButler()
344  else:
345  raise RuntimeError("Neither parsedCmd or args specified")
346  return self.TaskClass(config=self.config, log=self.log, butler=butler)
347 
348  @staticmethod
349  def getTargetList(parsedCmd, **kwargs):
350  """!
351  \brief Provide a list of patch references for each patch.
352 
353  The patch references within the list will have different filters.
354 
355  \param[in] parsedCmd the parsed command
356  \param **kwargs key word arguments (unused)
357  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
358  filter
359  """
360  refList = {} # Will index this as refList[tract][patch][filter] = ref
361  for ref in parsedCmd.id.refList:
362  tract = ref.dataId["tract"]
363  patch = ref.dataId["patch"]
364  filter = ref.dataId["filter"]
365  if not tract in refList:
366  refList[tract] = {}
367  if not patch in refList[tract]:
368  refList[tract][patch] = {}
369  if filter in refList[tract][patch]:
370  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
371  refList[tract][patch][filter] = ref
372  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
373 
374 
375 class MergeSourcesConfig(Config):
376  """!
377  \anchor MergeSourcesConfig_
378 
379  \brief Configuration for merging sources.
380  """
381  priorityList = ListField(dtype=str, default=[],
382  doc="Priority-ordered list of bands for the merge.")
383  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
384 
385  def validate(self):
386  Config.validate(self)
387  if len(self.priorityList) == 0:
388  raise RuntimeError("No priority list provided")
389 
390 
391 class MergeSourcesTask(CmdLineTask):
392  """!
393  \anchor MergeSourcesTask_
394 
395  \brief A base class for merging source catalogs.
396 
397  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
398  so similar that it makes sense to re-use the code, in the form of this abstract base class.
399 
400  NB: Do not use this class directly. Instead use one of the child classes that inherit from
401  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
402  "MergeMeasurementsTask"
403 
404  Sub-classes should set the following class variables:
405  * `_DefaultName`: name of Task
406  * `inputDataset`: name of dataset to read
407  * `outputDataset`: name of dataset to write
408  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
409 
410  In addition, sub-classes must implement the mergeCatalogs method.
411  """
412  _DefaultName = None
413  ConfigClass = MergeSourcesConfig
414  RunnerClass = MergeSourcesRunner
415  inputDataset = None
416  outputDataset = None
417  getSchemaCatalogs = None
418 
419  @classmethod
420  def _makeArgumentParser(cls):
421  """!
422  \brief Create a suitable ArgumentParser.
423 
424  We will use the ArgumentParser to get a provide a list of data
425  references for patches; the RunnerClass will sort them into lists
426  of data references for the same patch
427  """
428  parser = ArgumentParser(name=cls._DefaultName)
429  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
430  ContainerClass=ExistingCoaddDataIdContainer,
431  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
432  return parser
433 
434  def getInputSchema(self, butler=None, schema=None):
435  """!
436  \brief Obtain the input schema either directly or froma butler reference.
437 
438  \param[in] butler butler reference to obtain the input schema from
439  \param[in] schema the input schema
440  """
441  if schema is None:
442  assert butler is not None, "Neither butler nor schema specified"
443  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
444  immediate=True).schema
445  return schema
446 
447  def __init__(self, butler=None, schema=None, **kwargs):
448  """!
449  \brief Initialize the task.
450 
451  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
452  \param[in] schema the schema of the detection catalogs used as input to this one
453  \param[in] butler a butler used to read the input schema from disk, if schema is None
454 
455  Derived classes should use the getInputSchema() method to handle the additional
456  arguments and retreive the actual input schema.
457  """
458  CmdLineTask.__init__(self, **kwargs)
459 
460  def run(self, patchRefList):
461  """!
462  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
463  subclasses that inherit from MergeSourcesTask.
464 
465  \param[in] patchRefList list of data references for each filter
466  """
467  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
468  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
469  self.write(patchRefList[0], mergedCatalog)
470 
471  def readCatalog(self, patchRef):
472  """!
473  \brief Read input catalog.
474 
475  We read the input dataset provided by the 'inputDataset'
476  class variable.
477 
478  \param[in] patchRef data reference for patch
479  \return tuple consisting of the filter name and the catalog
480  """
481  filterName = patchRef.dataId["filter"]
482  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
483  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
484  return filterName, catalog
485 
486  def mergeCatalogs(self, catalogs, patchRef):
487  """!
488  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
489  MergeSourcesTask.
490 
491  \param[in] catalogs dict mapping filter name to source catalog
492 
493  \return merged catalog
494  """
495  raise NotImplementedError()
496 
497  def write(self, patchRef, catalog):
498  """!
499  \brief Write the output.
500 
501  \param[in] patchRef data reference for patch
502  \param[in] catalog catalog
503 
504  We write as the dataset provided by the 'outputDataset'
505  class variable.
506  """
507  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
508  # since the filter isn't actually part of the data ID for the dataset we're saving,
509  # it's confusing to see it in the log message, even if the butler simply ignores it.
510  mergeDataId = patchRef.dataId.copy()
511  del mergeDataId["filter"]
512  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
513 
514  def writeMetadata(self, dataRefList):
515  """!
516  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
517  """
518  pass
519 
520 
521 class CullPeaksConfig(Config):
522  """!
523  \anchor CullPeaksConfig_
524 
525  \brief Configuration for culling garbage peaks after merging footprints.
526 
527  Peaks may also be culled after detection or during deblending; this configuration object
528  only deals with culling after merging Footprints.
529 
530  These cuts are based on three quantities:
531  - nBands: the number of bands in which the peak was detected
532  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
533  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
534 
535  The formula that identifie peaks to cull is:
536 
537  nBands < nBandsSufficient
538  AND (rank >= rankSufficient)
539  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
540 
541  To disable peak culling, simply set nBandsSufficient=1.
542  """
543 
544  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
545  doc="Always keep peaks detected in this many bands")
546  rankSufficient = RangeField(dtype=int, default=20, min=1,
547  doc="Always keep this many peaks in each family")
548  rankConsidered = RangeField(dtype=int, default=30, min=1,
549  doc=("Keep peaks with less than this rank that also match the "
550  "rankNormalizedConsidered condition."))
551  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
552  doc=("Keep peaks with less than this normalized rank that"
553  " also match the rankConsidered condition."))
554 
555 
557  """!
558  \anchor MergeDetectionsConfig_
559 
560  \brief Configuration parameters for the MergeDetectionsTask.
561  """
562  minNewPeak = Field(dtype=float, default=1,
563  doc="Minimum distance from closest peak to create a new one (in arcsec).")
564 
565  maxSamePeak = Field(dtype=float, default=0.3,
566  doc="When adding new catalogs to the merge, all peaks less than this distance "
567  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
568  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
569 
570  skyFilterName = Field(dtype=str, default="sky",
571  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
572  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
573  skySourceRadius = Field(dtype=float, default=8,
574  doc="Radius, in pixels, of sky objects")
575  skyGrowDetectedFootprints = Field(dtype=int, default=0,
576  doc="Number of pixels to grow the detected footprint mask "
577  "when adding sky objects")
578  nSkySourcesPerPatch = Field(dtype=int, default=100,
579  doc="Try to add this many sky objects to the mergeDet list, which will\n"
580  "then be measured along with the detected objects in sourceMeasurementTask")
581  nTrialSkySourcesPerPatch = Field(dtype=int, default=None, optional=True,
582  doc="Maximum number of trial sky object positions\n"
583  "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
584  nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
585  doc="Set nTrialSkySourcesPerPatch to\n"
586  " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
587  "if nTrialSkySourcesPerPatch is None")
588 
589 
595 
596 
598  """!
599  \anchor MergeDetectionsTask_
600 
601  \brief Merge coadd detections from multiple bands.
602 
603  \section pipe_tasks_multiBand_Contents Contents
604 
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
608  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
609  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
610  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
611 
612  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
613 
614  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
615 
616  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
617  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
618  track of which band each source originates in.
619 
620  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
621  objects are culled as described in \ref CullPeaksConfig_.
622 
623  \par Inputs:
624  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
625  \par Outputs:
626  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
627  \par Data Unit:
628  tract, patch
629 
630  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
631 
632  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
633 
634  \copydoc \_\_init\_\_
635 
636  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
637 
638  \copydoc run
639 
640  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
641 
642  See \ref MergeDetectionsConfig_
643 
644  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
645 
646  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
647  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
648 
649  MergeDetectionsTask has no debug variables.
650 
651  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
652 
653  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
654  of the available bands.
655  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
656  chosen subset of filters.
657  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
658  and, eventually, perform forced photometry.
659  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
660  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
661  `--help` command line argument:
662  \code
663  mergeCoaddDetections.py --help
664  \endcode
665 
666  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
667  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
668  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
669  \code
670  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
671  \endcode
672  This will merge the HSC-I & -R band parent source catalogs and write the results to
673  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
674 
675  The next step in the multi-band processing procedure is
676  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
677  """
678  ConfigClass = MergeDetectionsConfig
679  _DefaultName = "mergeCoaddDetections"
680  inputDataset = "det"
681  outputDataset = "mergeDet"
682  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
683 
684  def __init__(self, butler=None, schema=None, **kwargs):
685  """!
686  \brief Initialize the merge detections task.
687 
688  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
689  merge the source catalogs.
690 
691  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
692  \param[in] schema the schema of the detection catalogs used as input to this one
693  \param[in] butler a butler used to read the input schema from disk, if schema is None
694  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
695 
696  The task will set its own self.schema attribute to the schema of the output merged catalog.
697  """
698  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
699  self.schema = self.getInputSchema(butler=butler, schema=schema)
700 
701  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
702  if self.config.nSkySourcesPerPatch > 0:
703  filterNames += [self.config.skyFilterName]
704  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
705 
706  def mergeCatalogs(self, catalogs, patchRef):
707  """!
708  \brief Merge multiple catalogs.
709 
710  After ordering the catalogs and filters in priority order,
711  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
712  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
713  garbage peaks detected around bright objects.
714 
715  \param[in] catalogs
716  \param[in] patchRef
717  \param[out] mergedList
718  """
719 
720  # Convert distance to tract coordinate
721  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
722  tractWcs = skyInfo.wcs
723  peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
724  samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
725 
726  # Put catalogs, filters in priority order
727  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
728  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
729  if band in catalogs.keys()]
730 
731  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
732  self.schema, self.makeIdFactory(patchRef),
733  samePeakDistance)
734 
735  #
736  # Add extra sources that correspond to blank sky
737  #
738  skySourceFootprints = self.getSkySourceFootprints(
739  mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
740  if skySourceFootprints:
741  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
742 
743  for foot in skySourceFootprints:
744  s = mergedList.addNew()
745  s.setFootprint(foot)
746  s.set(key, True)
747 
748  self.log.info("Added %d sky sources (%.0f%% of requested)",
749  len(skySourceFootprints),
750  100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
751 
752  # Sort Peaks from brightest to faintest
753  for record in mergedList:
754  record.getFootprint().sortPeaks()
755  self.log.info("Merged to %d sources" % len(mergedList))
756  # Attempt to remove garbage peaks
757  self.cullPeaks(mergedList)
758  return mergedList
759 
760  def cullPeaks(self, catalog):
761  """!
762  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
763 
764  \param[in] catalog Source catalog
765  """
766  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
767  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
768  totalPeaks = 0
769  culledPeaks = 0
770  for parentSource in catalog:
771  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
772  # to it (which is easier than deleting as we iterate).
773  keptPeaks = parentSource.getFootprint().getPeaks()
774  oldPeaks = list(keptPeaks)
775  keptPeaks.clear()
776  familySize = len(oldPeaks)
777  totalPeaks += familySize
778  for rank, peak in enumerate(oldPeaks):
779  if ((rank < self.config.cullPeaks.rankSufficient) or
780  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
781  (rank < self.config.cullPeaks.rankConsidered and
782  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
783  keptPeaks.append(peak)
784  else:
785  culledPeaks += 1
786  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
787 
788  def getSchemaCatalogs(self):
789  """!
790  Return a dict of empty catalogs for each catalog dataset produced by this task.
791 
792  \param[out] dictionary of empty catalogs
793  """
794  mergeDet = afwTable.SourceCatalog(self.schema)
795  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
796  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
797  self.config.coaddName + "Coadd_peak": peak}
798 
799  def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0):
800  """!
801  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
802 
803  \param mergedList The merged Footprints from all the input bands
804  \param skyInfo A description of the patch
805  \param growDetectedFootprints The number of pixels to grow the detected footprints
806  """
807 
808  if self.config.nSkySourcesPerPatch <= 0:
809  return []
810 
811  skySourceRadius = self.config.skySourceRadius
812  nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
813  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
814  if nTrialSkySourcesPerPatch is None:
815  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
816  #
817  # We are going to find circular Footprints that don't intersect any pre-existing Footprints,
818  # and the easiest way to do this is to generate a Mask containing all the detected pixels (as
819  # merged by this task).
820  #
821  patchBBox = skyInfo.patchInfo.getOuterBBox()
822  mask = afwImage.Mask(patchBBox)
823  detectedMask = mask.getPlaneBitMask("DETECTED")
824  for s in mergedList:
825  foot = s.getFootprint()
826  if growDetectedFootprints > 0:
827  foot.dilate(growDetectedFootprints)
828  foot.spans.setMask(mask, detectedMask)
829 
830  xmin, ymin = patchBBox.getMin()
831  xmax, ymax = patchBBox.getMax()
832  # Avoid objects partially off the image
833  xmin += skySourceRadius + 1
834  ymin += skySourceRadius + 1
835  xmax -= skySourceRadius + 1
836  ymax -= skySourceRadius + 1
837 
838  skySourceFootprints = []
839  maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
840  for i in range(nTrialSkySourcesPerPatch):
841  if len(skySourceFootprints) == nSkySourcesPerPatch:
842  break
843 
844  x = int(numpy.random.uniform(xmin, xmax))
845  y = int(numpy.random.uniform(ymin, ymax))
846  spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
847  offset=(x, y))
848  foot = afwDetect.Footprint(spans, patchBBox)
849  foot.setPeakSchema(self.merged.getPeakSchema())
850 
851  if not foot.spans.overlaps(maskToSpanSet):
852  foot.addPeak(x, y, 0)
853  foot.getPeaks()[0].set("merge_peak_%s" % self.config.skyFilterName, True)
854  skySourceFootprints.append(foot)
855 
856  return skySourceFootprints
857 
858 
860  """!
861  \anchor MeasureMergedCoaddSourcesConfig_
862 
863  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
864  """
865  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
866  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
867  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
868  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
869  doPropagateFlags = Field(
870  dtype=bool, default=True,
871  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
872  )
873  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
874  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
875  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
876  doWriteMatchesDenormalized = Field(
877  dtype=bool,
878  default=False,
879  doc=("Write reference matches in denormalized format? "
880  "This format uses more disk space, but is more convenient to read."),
881  )
882  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
883  checkUnitsParseStrict = Field(
884  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
885  dtype=str,
886  default="raise",
887  )
888  doApCorr = Field(
889  dtype=bool,
890  default=True,
891  doc="Apply aperture corrections"
892  )
893  applyApCorr = ConfigurableField(
894  target=ApplyApCorrTask,
895  doc="Subtask to apply aperture corrections"
896  )
897  doRunCatalogCalculation = Field(
898  dtype=bool,
899  default=True,
900  doc='Run catalogCalculation task'
901  )
902  catalogCalculation = ConfigurableField(
903  target=CatalogCalculationTask,
904  doc="Subtask to run catalogCalculation plugins on catalog"
905  )
906 
907  def setDefaults(self):
908  Config.setDefaults(self)
909  self.deblend.propagateAllPeaks = True
910  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
911  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
912  'INEXACT_PSF']
913 
914 
920 
921 
922 class MeasureMergedCoaddSourcesTask(CmdLineTask):
923  """!
924  \anchor MeasureMergedCoaddSourcesTask_
925 
926  \brief Deblend sources from master catalog in each coadd seperately and measure.
927 
928  \section pipe_tasks_multiBand_Contents Contents
929 
930  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
931  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
932  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
933  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
934  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
935  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
936 
937  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
938 
939  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
940  measurement in each coadd.
941 
942  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
943  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
944  consistent set of child sources.
945 
946  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
947  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
948  flags are propagated to the coadd sources.
949 
950  Optionally, we can match the coadd sources to an external reference catalog.
951 
952  \par Inputs:
953  deepCoadd_mergeDet{tract,patch}: SourceCatalog
954  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
955  \par Outputs:
956  deepCoadd_meas{tract,patch,filter}: SourceCatalog
957  \par Data Unit:
958  tract, patch, filter
959 
960  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
961 
962  <DL>
963  <DT> \ref SourceDeblendTask_ "deblend"
964  <DD> Deblend all the sources from the master catalog.</DD>
965  <DT> \ref SingleFrameMeasurementTask_ "measurement"
966  <DD> Measure source properties of deblended sources.</DD>
967  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
968  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
969  not at the edge of the field and that have either not been deblended or are the children of deblended
970  sources</DD>
971  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
972  <DD> Propagate flags set in individual visits to the coadd.</DD>
973  <DT> \ref DirectMatchTask_ "match"
974  <DD> Match input sources to a reference catalog (optional).
975  </DD>
976  </DL>
977  These subtasks may be retargeted as required.
978 
979  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
980 
981  \copydoc \_\_init\_\_
982 
983  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
984 
985  \copydoc run
986 
987  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
988 
989  See \ref MeasureMergedCoaddSourcesConfig_
990 
991  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
992 
993  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
994  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
995  files.
996 
997  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
998  the various sub-tasks. See the documetation for individual sub-tasks for more information.
999 
1000  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
1001 
1002  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
1003  The next stage in the multi-band processing procedure will merge these measurements into a suitable
1004  catalog for driving forced photometry.
1005 
1006  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
1007  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
1008  `--help` command line argument:
1009  \code
1010  measureCoaddSources.py --help
1011  \endcode
1012 
1013  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1014  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1015  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1016  coadd as follows:
1017  \code
1018  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1019  \endcode
1020  This will process the HSC-I band data. The results are written in
1021  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1022 
1023  It is also necessary to run
1024  \code
1025  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1026  \endcode
1027  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1028  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1029  """
1030  _DefaultName = "measureCoaddSources"
1031  ConfigClass = MeasureMergedCoaddSourcesConfig
1032  RunnerClass = ButlerInitializedTaskRunner
1033  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1034  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1035 
1036  @classmethod
1037  def _makeArgumentParser(cls):
1038  parser = ArgumentParser(name=cls._DefaultName)
1039  parser.add_id_argument("--id", "deepCoadd_calexp",
1040  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1041  ContainerClass=ExistingCoaddDataIdContainer)
1042  return parser
1043 
1044  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1045  """!
1046  \brief Initialize the task.
1047 
1048  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1049  \param[in] schema: the schema of the merged detection catalog used as input to this one
1050  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1051  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1052  catalog. May be None if the loader can be constructed from the butler argument or all steps
1053  requiring a reference catalog are disabled.
1054  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1055  catalog loader, if schema or peakSchema or refObjLoader is None
1056 
1057  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1058  This will include all fields from the input schema, as well as additional fields for all the
1059  measurements.
1060  """
1061  CmdLineTask.__init__(self, **kwargs)
1062  if schema is None:
1063  assert butler is not None, "Neither butler nor schema is defined"
1064  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1065  self.schemaMapper = afwTable.SchemaMapper(schema)
1066  self.schemaMapper.addMinimalSchema(schema)
1067  self.schema = self.schemaMapper.getOutputSchema()
1069  if self.config.doDeblend:
1070  if peakSchema is None:
1071  assert butler is not None, "Neither butler nor peakSchema is defined"
1072  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1073  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1074  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1075  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1076  if self.config.doMatchSources:
1077  if refObjLoader is None:
1078  assert butler is not None, "Neither butler nor refObjLoader is defined"
1079  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1080  if self.config.doPropagateFlags:
1081  self.makeSubtask("propagateFlags", schema=self.schema)
1082  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1083  if self.config.doApCorr:
1084  self.makeSubtask("applyApCorr", schema=self.schema)
1085  if self.config.doRunCatalogCalculation:
1086  self.makeSubtask("catalogCalculation", schema=self.schema)
1087 
1088  def run(self, patchRef):
1089  """!
1090  \brief Deblend and measure.
1091 
1092  \param[in] patchRef: Patch reference.
1093 
1094  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1095  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1096  Finally, write the deblended sources and measurements out.
1097  """
1098  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1099  sources = self.readSources(patchRef)
1100  if self.config.doDeblend:
1101  self.deblend.run(exposure, sources)
1102 
1103  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1104  # catalog is non-contiguous so can't extract column
1105  numBig = sum((s.get(bigKey) for s in sources))
1106  if numBig > 0:
1107  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1108  (patchRef.dataId, numBig))
1109 
1110  table = sources.getTable()
1111  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1112 
1113  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1114 
1115  if self.config.doApCorr:
1116  self.applyApCorr.run(
1117  catalog=sources,
1118  apCorrMap=exposure.getInfo().getApCorrMap()
1119  )
1120 
1121  if self.config.doRunCatalogCalculation:
1122  self.catalogCalculation.run(sources)
1123 
1124  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1125  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1126  includeDeblend=self.config.doDeblend)
1127  if self.config.doPropagateFlags:
1128  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1129  exposure.getWcs())
1130  if self.config.doMatchSources:
1131  self.writeMatches(patchRef, exposure, sources)
1132  self.write(patchRef, sources)
1133 
1134  def readSources(self, dataRef):
1135  """!
1136  \brief Read input sources.
1137 
1138  \param[in] dataRef: Data reference for catalog of merged detections
1139  \return List of sources in merged catalog
1140 
1141  We also need to add columns to hold the measurements we're about to make
1142  so we can measure in-place.
1143  """
1144  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1145  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1146  idFactory = self.makeIdFactory(dataRef)
1147  for s in merged:
1148  idFactory.notify(s.getId())
1149  table = afwTable.SourceTable.make(self.schema, idFactory)
1150  sources = afwTable.SourceCatalog(table)
1151  sources.extend(merged, self.schemaMapper)
1152  return sources
1153 
1154  def writeMatches(self, dataRef, exposure, sources):
1155  """!
1156  \brief Write matches of the sources to the astrometric reference catalog.
1157 
1158  We use the Wcs in the exposure to match sources.
1159 
1160  \param[in] dataRef: data reference
1161  \param[in] exposure: exposure with Wcs
1162  \param[in] sources: source catalog
1163  """
1164  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1165  if result.matches:
1166  matches = afwTable.packMatches(result.matches)
1167  matches.table.setMetadata(result.matchMeta)
1168  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1169  if self.config.doWriteMatchesDenormalized:
1170  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1171  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1172 
1173 
1174  def write(self, dataRef, sources):
1175  """!
1176  \brief Write the source catalog.
1177 
1178  \param[in] dataRef: data reference
1179  \param[in] sources: source catalog
1180  """
1181  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1182  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1183 
1184  def getExposureId(self, dataRef):
1185  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1186 
1187 
1188 
1189 
1191  """!
1192  \anchor MergeMeasurementsConfig_
1193 
1194  \brief Configuration parameters for the MergeMeasurementsTask
1195  """
1196  pseudoFilterList = ListField(dtype=str, default=["sky"],
1197  doc="Names of filters which may have no associated detection\n"
1198  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1199  snName = Field(dtype=str, default="base_PsfFlux",
1200  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1201  minSN = Field(dtype=float, default=10.,
1202  doc="If the S/N from the priority band is below this value (and the S/N "
1203  "is larger than minSNDiff compared to the priority band), use the band with "
1204  "the largest S/N as the reference band.")
1205  minSNDiff = Field(dtype=float, default=3.,
1206  doc="If the difference in S/N between another band and the priority band is larger "
1207  "than this value (and the S/N in the priority band is less than minSN) "
1208  "use the band with the largest S/N as the reference band")
1209  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1210  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1211  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1212 
1213 
1219 
1220 
1222  """!
1223  \anchor MergeMeasurementsTask_
1224 
1225  \brief Merge measurements from multiple bands
1226 
1227  \section pipe_tasks_multiBand_Contents Contents
1228 
1229  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1230  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1231  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1232  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1233  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1234  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1235 
1236  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1237 
1238  Command-line task that merges measurements from multiple bands.
1239 
1240  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1241  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1242  required to have centroid, shape and flux measurements in each band.
1243 
1244  \par Inputs:
1245  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1246  \par Outputs:
1247  deepCoadd_ref{tract,patch}: SourceCatalog
1248  \par Data Unit:
1249  tract, patch
1250 
1251  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1252 
1253  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1254 
1255  \copydoc \_\_init\_\_
1256 
1257  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1258 
1259  \copydoc run
1260 
1261  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1262 
1263  See \ref MergeMeasurementsConfig_
1264 
1265  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1266 
1267  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1268  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1269  files.
1270 
1271  MergeMeasurementsTask has no debug variables.
1272 
1273  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1274 
1275  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1276  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1277  coadds and individual exposures.
1278  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1279  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1280  command line argument:
1281  \code
1282  mergeCoaddMeasurements.py --help
1283  \endcode
1284 
1285  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1286  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1287  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1288  as follows:
1289  \code
1290  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1291  \endcode
1292  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1293  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1294  """
1295  _DefaultName = "mergeCoaddMeasurements"
1296  ConfigClass = MergeMeasurementsConfig
1297  inputDataset = "meas"
1298  outputDataset = "ref"
1299  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1300 
1301  def __init__(self, butler=None, schema=None, **kwargs):
1302  """!
1303  Initialize the task.
1304 
1305  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1306  \param[in] schema: the schema of the detection catalogs used as input to this one
1307  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1308 
1309  The task will set its own self.schema attribute to the schema of the output merged catalog.
1310  """
1311  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1312  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1313  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1314  self.schemaMapper.addMinimalSchema(inputSchema, True)
1315  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1316  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1317  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1318 
1319  self.flagKeys = {}
1320  for band in self.config.priorityList:
1321  short = getShortFilterName(band)
1322  outputKey = self.schemaMapper.editOutputSchema().addField(
1323  "merge_measurement_%s" % short,
1324  type="Flag",
1325  doc="Flag field set if the measurements here are from the %s filter" % band
1326  )
1327  peakKey = inputSchema.find("merge_peak_%s" % short).key
1328  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1329  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1330  self.schema = self.schemaMapper.getOutputSchema()
1331 
1333  for filt in self.config.pseudoFilterList:
1334  try:
1335  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1336  except:
1337  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1338 
1339  self.badFlags = {}
1340  for flag in self.config.flags:
1341  try:
1342  self.badFlags[flag] = self.schema.find(flag).getKey()
1343  except KeyError as exc:
1344  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1345 
1346  def mergeCatalogs(self, catalogs, patchRef):
1347  """!
1348  Merge measurement catalogs to create a single reference catalog for forced photometry
1349 
1350  \param[in] catalogs: the catalogs to be merged
1351  \param[in] patchRef: patch reference for data
1352 
1353  For parent sources, we choose the first band in config.priorityList for which the
1354  merge_footprint flag for that band is is True.
1355 
1356  For child sources, the logic is the same, except that we use the merge_peak flags.
1357  """
1358  # Put catalogs, filters in priority order
1359  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1360  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1361 
1362  mergedCatalog = afwTable.SourceCatalog(self.schema)
1363  mergedCatalog.reserve(len(orderedCatalogs[0]))
1364 
1365  idKey = orderedCatalogs[0].table.getIdKey()
1366  for catalog in orderedCatalogs[1:]:
1367  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1368  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1369 
1370  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1371  # record for each band, in priority order.
1372  for orderedRecords in zip(*orderedCatalogs):
1373 
1374  maxSNRecord = None
1375  maxSNFlagKeys = None
1376  maxSN = 0.
1377  priorityRecord = None
1378  priorityFlagKeys = None
1379  prioritySN = 0.
1380  hasPseudoFilter = False
1381 
1382  # Now we iterate over those record-band pairs, keeping track of the priority and the
1383  # largest S/N band.
1384  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1385  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1386  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1387 
1388  if not (parent or child):
1389  for pseudoFilterKey in self.pseudoFilterKeys:
1390  if inputRecord.get(pseudoFilterKey):
1391  hasPseudoFilter = True
1392  priorityRecord = inputRecord
1393  priorityFlagKeys = flagKeys
1394  break
1395  if hasPseudoFilter:
1396  break
1397 
1398  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1399  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1400  sn = 0.
1401  else:
1402  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1403  if numpy.isnan(sn) or sn < 0.:
1404  sn = 0.
1405  if (parent or child) and priorityRecord is None:
1406  priorityRecord = inputRecord
1407  priorityFlagKeys = flagKeys
1408  prioritySN = sn
1409  if sn > maxSN:
1410  maxSNRecord = inputRecord
1411  maxSNFlagKeys = flagKeys
1412  maxSN = sn
1413 
1414  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1415  # the reference band instead. However, we only want to choose the highest S/N band if it is
1416  # significantly better than the priority band. Therefore, to choose a band other than the
1417  # priority, we require that the priority S/N is below the minimum threshold and that the
1418  # difference between the priority and highest S/N is larger than the difference threshold.
1419  #
1420  # For pseudo code objects we always choose the first band in the priority list.
1421  bestRecord = None
1422  bestFlagKeys = None
1423  if hasPseudoFilter:
1424  bestRecord = priorityRecord
1425  bestFlagKeys = priorityFlagKeys
1426  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1427  maxSNRecord is not None):
1428  bestRecord = maxSNRecord
1429  bestFlagKeys = maxSNFlagKeys
1430  elif priorityRecord is not None:
1431  bestRecord = priorityRecord
1432  bestFlagKeys = priorityFlagKeys
1433 
1434  if bestRecord is not None and bestFlagKeys is not None:
1435  outputRecord = mergedCatalog.addNew()
1436  outputRecord.assign(bestRecord, self.schemaMapper)
1437  outputRecord.set(bestFlagKeys.output, True)
1438  else: # if we didn't find any records
1439  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1440  inputRecord.getId())
1441 
1442  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1443  for inputCatalog in orderedCatalogs:
1444  if len(mergedCatalog) != len(inputCatalog):
1445  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1446  (len(mergedCatalog), len(orderedCatalogs)))
1447 
1448  return mergedCatalog
Merge coadd detections from multiple bands.
Definition: multiBand.py:597
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:331
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:434
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:788
def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:799
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:760
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1088
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:323
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:307
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:447
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1301
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:245
Merge measurements from multiple bands.
Definition: multiBand.py:1221
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:460
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:922
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1154
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:263
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1346
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:486
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1190
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1134
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:556
Configuration for merging sources.
Definition: multiBand.py:375
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:497
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:706
A base class for merging source catalogs.
Definition: multiBand.py:391
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1174
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:471
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:859
def runDetection(self, exposure, idFactory)
Run detection on an exposure.
Definition: multiBand.py:277
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:249
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:514
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1044
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:684
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:349