lsst.pipe.tasks  13.0-66-gfbf2f2ce+5
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import SourceDetectionTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
35 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
36 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=SourceDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so background subtraction should be very basic
131  self.detection.background.useApprox = False
132  self.detection.background.binSize = 4096
133  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
134 
135 
141 
142 
143 class DetectCoaddSourcesTask(CmdLineTask):
144  """!
145  \anchor DetectCoaddSourcesTask_
146 
147  \brief Detect sources on a coadd
148 
149  \section pipe_tasks_multiBand_Contents Contents
150 
151  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
156  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
157 
158  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
159 
160  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
161 
162  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
163  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
164  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
165  propagate the full covariance matrix -- but it is simple and works well in practice.
166 
167  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
168  SourceDetectionTask_ "detection" subtask.
169 
170  \par Inputs:
171  deepCoadd{tract,patch,filter}: ExposureF
172  \par Outputs:
173  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
174  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
175  exposure (ExposureF)
176  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
177  \par Data Unit:
178  tract, patch, filter
179 
180  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
181  You can retarget this subtask if you wish.
182 
183  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
184 
185  \copydoc \_\_init\_\_
186 
187  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
188 
189  \copydoc run
190 
191  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
192 
193  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
194 
195  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
196 
197  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
198  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
199  files.
200 
201  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
202  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
203  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
204 
205  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
206 
207  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
208  the task is to update the background, detect all sources in a single band and generate a set of parent
209  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
210  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
211  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
212  calling detectCoaddSources.py with the `--help` command line argument:
213  \code
214  detectCoaddSources.py --help
215  \endcode
216 
217  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
218  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
219  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
220  \code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
222  \endcode
223  that will process the HSC-I band data. The results are written to
224  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
225 
226  It is also necessary to run:
227  \code
228  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
229  \endcode
230  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
231  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
232  """
233  _DefaultName = "detectCoaddSources"
234  ConfigClass = DetectCoaddSourcesConfig
235  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
236  makeIdFactory = _makeMakeIdFactory("CoaddId")
237 
238  @classmethod
239  def _makeArgumentParser(cls):
240  parser = ArgumentParser(name=cls._DefaultName)
241  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242  ContainerClass=ExistingCoaddDataIdContainer)
243  return parser
244 
245  def __init__(self, schema=None, **kwargs):
246  """!
247  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
248 
249  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
250 
251  \param[in] schema: initial schema for the output catalog, modified-in place to include all
252  fields set by this task. If None, the source minimal schema will be used.
253  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
254  """
255  CmdLineTask.__init__(self, **kwargs)
256  if schema is None:
257  schema = afwTable.SourceTable.makeMinimalSchema()
258  if self.config.doInsertFakes:
259  self.makeSubtask("insertFakes")
260  self.schema = schema
261  self.makeSubtask("detection", schema=self.schema)
262 
263  def run(self, patchRef):
264  """!
265  \brief Run detection on a coadd.
266 
267  Invokes \ref runDetection and then uses \ref write to output the
268  results.
269 
270  \param[in] patchRef: data reference for patch
271  """
272  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
273  results = self.runDetection(exposure, self.makeIdFactory(patchRef))
274  self.write(exposure, results, patchRef)
275  return results
276 
277  def runDetection(self, exposure, idFactory):
278  """!
279  \brief Run detection on an exposure.
280 
281  First scale the variance plane to match the observed variance
282  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
283  detect sources.
284 
285  \param[in] exposure: Exposure on which to detect
286  \param[in] idFactory: IdFactory to set source identifiers
287 
288  \return a pipe.base.Struct with fields
289  - sources: catalog of detections
290  - backgrounds: list of backgrounds
291  """
292  if self.config.doScaleVariance:
293  scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
294  backgrounds = afwMath.BackgroundList()
295  if self.config.doInsertFakes:
296  self.insertFakes.run(exposure, background=backgrounds)
297  table = afwTable.SourceTable.make(self.schema, idFactory)
298  detections = self.detection.makeSourceCatalog(table, exposure)
299  sources = detections.sources
300  fpSets = detections.fpSets
301  if fpSets.background:
302  backgrounds.append(fpSets.background)
303  return Struct(sources=sources, backgrounds=backgrounds)
304 
305  def write(self, exposure, results, patchRef):
306  """!
307  \brief Write out results from runDetection.
308 
309  \param[in] exposure: Exposure to write out
310  \param[in] results: Struct returned from runDetection
311  \param[in] patchRef: data reference for patch
312  """
313  coaddName = self.config.coaddName + "Coadd"
314  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
315  patchRef.put(results.sources, coaddName + "_det")
316  patchRef.put(exposure, coaddName + "_calexp")
317 
318 
319 
320 
321 class MergeSourcesRunner(TaskRunner):
322  """!
323  \anchor MergeSourcesRunner_
324 
325  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
326  requires a list of dataRefs rather than a single dataRef.
327  """
328 
329  def makeTask(self, parsedCmd=None, args=None):
330  """!
331  \brief Provide a butler to the Task constructor.
332 
333  \param[in] parsedCmd the parsed command
334  \param[in] args tuple of a list of data references and kwargs (un-used)
335  \throws RuntimeError if both parsedCmd & args are None
336  """
337  if parsedCmd is not None:
338  butler = parsedCmd.butler
339  elif args is not None:
340  dataRefList, kwargs = args
341  butler = dataRefList[0].getButler()
342  else:
343  raise RuntimeError("Neither parsedCmd or args specified")
344  return self.TaskClass(config=self.config, log=self.log, butler=butler)
345 
346  @staticmethod
347  def getTargetList(parsedCmd, **kwargs):
348  """!
349  \brief Provide a list of patch references for each patch.
350 
351  The patch references within the list will have different filters.
352 
353  \param[in] parsedCmd the parsed command
354  \param **kwargs key word arguments (unused)
355  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
356  filter
357  """
358  refList = {} # Will index this as refList[tract][patch][filter] = ref
359  for ref in parsedCmd.id.refList:
360  tract = ref.dataId["tract"]
361  patch = ref.dataId["patch"]
362  filter = ref.dataId["filter"]
363  if not tract in refList:
364  refList[tract] = {}
365  if not patch in refList[tract]:
366  refList[tract][patch] = {}
367  if filter in refList[tract][patch]:
368  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
369  refList[tract][patch][filter] = ref
370  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
371 
372 
373 class MergeSourcesConfig(Config):
374  """!
375  \anchor MergeSourcesConfig_
376 
377  \brief Configuration for merging sources.
378  """
379  priorityList = ListField(dtype=str, default=[],
380  doc="Priority-ordered list of bands for the merge.")
381  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
382 
383  def validate(self):
384  Config.validate(self)
385  if len(self.priorityList) == 0:
386  raise RuntimeError("No priority list provided")
387 
388 
389 class MergeSourcesTask(CmdLineTask):
390  """!
391  \anchor MergeSourcesTask_
392 
393  \brief A base class for merging source catalogs.
394 
395  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
396  so similar that it makes sense to re-use the code, in the form of this abstract base class.
397 
398  NB: Do not use this class directly. Instead use one of the child classes that inherit from
399  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
400  "MergeMeasurementsTask"
401 
402  Sub-classes should set the following class variables:
403  * `_DefaultName`: name of Task
404  * `inputDataset`: name of dataset to read
405  * `outputDataset`: name of dataset to write
406  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
407 
408  In addition, sub-classes must implement the mergeCatalogs method.
409  """
410  _DefaultName = None
411  ConfigClass = MergeSourcesConfig
412  RunnerClass = MergeSourcesRunner
413  inputDataset = None
414  outputDataset = None
415  getSchemaCatalogs = None
416 
417  @classmethod
418  def _makeArgumentParser(cls):
419  """!
420  \brief Create a suitable ArgumentParser.
421 
422  We will use the ArgumentParser to get a provide a list of data
423  references for patches; the RunnerClass will sort them into lists
424  of data references for the same patch
425  """
426  parser = ArgumentParser(name=cls._DefaultName)
427  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
428  ContainerClass=ExistingCoaddDataIdContainer,
429  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
430  return parser
431 
432  def getInputSchema(self, butler=None, schema=None):
433  """!
434  \brief Obtain the input schema either directly or froma butler reference.
435 
436  \param[in] butler butler reference to obtain the input schema from
437  \param[in] schema the input schema
438  """
439  if schema is None:
440  assert butler is not None, "Neither butler nor schema specified"
441  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
442  immediate=True).schema
443  return schema
444 
445  def __init__(self, butler=None, schema=None, **kwargs):
446  """!
447  \brief Initialize the task.
448 
449  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
450  \param[in] schema the schema of the detection catalogs used as input to this one
451  \param[in] butler a butler used to read the input schema from disk, if schema is None
452 
453  Derived classes should use the getInputSchema() method to handle the additional
454  arguments and retreive the actual input schema.
455  """
456  CmdLineTask.__init__(self, **kwargs)
457 
458  def run(self, patchRefList):
459  """!
460  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
461  subclasses that inherit from MergeSourcesTask.
462 
463  \param[in] patchRefList list of data references for each filter
464  """
465  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
466  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
467  self.write(patchRefList[0], mergedCatalog)
468 
469  def readCatalog(self, patchRef):
470  """!
471  \brief Read input catalog.
472 
473  We read the input dataset provided by the 'inputDataset'
474  class variable.
475 
476  \param[in] patchRef data reference for patch
477  \return tuple consisting of the filter name and the catalog
478  """
479  filterName = patchRef.dataId["filter"]
480  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
481  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
482  return filterName, catalog
483 
484  def mergeCatalogs(self, catalogs, patchRef):
485  """!
486  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
487  MergeSourcesTask.
488 
489  \param[in] catalogs dict mapping filter name to source catalog
490 
491  \return merged catalog
492  """
493  raise NotImplementedError()
494 
495  def write(self, patchRef, catalog):
496  """!
497  \brief Write the output.
498 
499  \param[in] patchRef data reference for patch
500  \param[in] catalog catalog
501 
502  We write as the dataset provided by the 'outputDataset'
503  class variable.
504  """
505  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
506  # since the filter isn't actually part of the data ID for the dataset we're saving,
507  # it's confusing to see it in the log message, even if the butler simply ignores it.
508  mergeDataId = patchRef.dataId.copy()
509  del mergeDataId["filter"]
510  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
511 
512  def writeMetadata(self, dataRefList):
513  """!
514  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
515  """
516  pass
517 
518 
519 class CullPeaksConfig(Config):
520  """!
521  \anchor CullPeaksConfig_
522 
523  \brief Configuration for culling garbage peaks after merging footprints.
524 
525  Peaks may also be culled after detection or during deblending; this configuration object
526  only deals with culling after merging Footprints.
527 
528  These cuts are based on three quantities:
529  - nBands: the number of bands in which the peak was detected
530  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
531  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
532 
533  The formula that identifie peaks to cull is:
534 
535  nBands < nBandsSufficient
536  AND (rank >= rankSufficient)
537  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
538 
539  To disable peak culling, simply set nBandsSufficient=1.
540  """
541 
542  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
543  doc="Always keep peaks detected in this many bands")
544  rankSufficient = RangeField(dtype=int, default=20, min=1,
545  doc="Always keep this many peaks in each family")
546  rankConsidered = RangeField(dtype=int, default=30, min=1,
547  doc=("Keep peaks with less than this rank that also match the "
548  "rankNormalizedConsidered condition."))
549  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
550  doc=("Keep peaks with less than this normalized rank that"
551  " also match the rankConsidered condition."))
552 
553 
555  """!
556  \anchor MergeDetectionsConfig_
557 
558  \brief Configuration parameters for the MergeDetectionsTask.
559  """
560  minNewPeak = Field(dtype=float, default=1,
561  doc="Minimum distance from closest peak to create a new one (in arcsec).")
562 
563  maxSamePeak = Field(dtype=float, default=0.3,
564  doc="When adding new catalogs to the merge, all peaks less than this distance "
565  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
566  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
567 
568  skyFilterName = Field(dtype=str, default="sky",
569  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
570  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
571  skySourceRadius = Field(dtype=float, default=8,
572  doc="Radius, in pixels, of sky objects")
573  skyGrowDetectedFootprints = Field(dtype=int, default=0,
574  doc="Number of pixels to grow the detected footprint mask "
575  "when adding sky objects")
576  nSkySourcesPerPatch = Field(dtype=int, default=100,
577  doc="Try to add this many sky objects to the mergeDet list, which will\n"
578  "then be measured along with the detected objects in sourceMeasurementTask")
579  nTrialSkySourcesPerPatch = Field(dtype=int, default=None, optional=True,
580  doc="Maximum number of trial sky object positions\n"
581  "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
582  nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
583  doc="Set nTrialSkySourcesPerPatch to\n"
584  " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
585  "if nTrialSkySourcesPerPatch is None")
586 
587 
593 
594 
596  """!
597  \anchor MergeDetectionsTask_
598 
599  \brief Merge coadd detections from multiple bands.
600 
601  \section pipe_tasks_multiBand_Contents Contents
602 
603  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
608  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
609 
610  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
611 
612  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
613 
614  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
615  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
616  track of which band each source originates in.
617 
618  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
619  objects are culled as described in \ref CullPeaksConfig_.
620 
621  \par Inputs:
622  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
623  \par Outputs:
624  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
625  \par Data Unit:
626  tract, patch
627 
628  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
629 
630  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
631 
632  \copydoc \_\_init\_\_
633 
634  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
635 
636  \copydoc run
637 
638  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
639 
640  See \ref MergeDetectionsConfig_
641 
642  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
643 
644  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
645  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
646 
647  MergeDetectionsTask has no debug variables.
648 
649  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
650 
651  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
652  of the available bands.
653  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
654  chosen subset of filters.
655  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
656  and, eventually, perform forced photometry.
657  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
658  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
659  `--help` command line argument:
660  \code
661  mergeCoaddDetections.py --help
662  \endcode
663 
664  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
665  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
666  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
667  \code
668  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
669  \endcode
670  This will merge the HSC-I & -R band parent source catalogs and write the results to
671  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
672 
673  The next step in the multi-band processing procedure is
674  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
675  """
676  ConfigClass = MergeDetectionsConfig
677  _DefaultName = "mergeCoaddDetections"
678  inputDataset = "det"
679  outputDataset = "mergeDet"
680  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
681 
682  def __init__(self, butler=None, schema=None, **kwargs):
683  """!
684  \brief Initialize the merge detections task.
685 
686  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
687  merge the source catalogs.
688 
689  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
690  \param[in] schema the schema of the detection catalogs used as input to this one
691  \param[in] butler a butler used to read the input schema from disk, if schema is None
692  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
693 
694  The task will set its own self.schema attribute to the schema of the output merged catalog.
695  """
696  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
697  self.schema = self.getInputSchema(butler=butler, schema=schema)
698 
699  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
700  if self.config.nSkySourcesPerPatch > 0:
701  filterNames += [self.config.skyFilterName]
702  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
703 
704  def mergeCatalogs(self, catalogs, patchRef):
705  """!
706  \brief Merge multiple catalogs.
707 
708  After ordering the catalogs and filters in priority order,
709  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
710  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
711  garbage peaks detected around bright objects.
712 
713  \param[in] catalogs
714  \param[in] patchRef
715  \param[out] mergedList
716  """
717 
718  # Convert distance to tract coordinate
719  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
720  tractWcs = skyInfo.wcs
721  peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
722  samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
723 
724  # Put catalogs, filters in priority order
725  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
726  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
727  if band in catalogs.keys()]
728 
729  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
730  self.schema, self.makeIdFactory(patchRef),
731  samePeakDistance)
732 
733  #
734  # Add extra sources that correspond to blank sky
735  #
736  skySourceFootprints = self.getSkySourceFootprints(
737  mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
738  if skySourceFootprints:
739  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
740 
741  for foot in skySourceFootprints:
742  s = mergedList.addNew()
743  s.setFootprint(foot)
744  s.set(key, True)
745 
746  self.log.info("Added %d sky sources (%.0f%% of requested)",
747  len(skySourceFootprints),
748  100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
749 
750  # Sort Peaks from brightest to faintest
751  for record in mergedList:
752  record.getFootprint().sortPeaks()
753  self.log.info("Merged to %d sources" % len(mergedList))
754  # Attempt to remove garbage peaks
755  self.cullPeaks(mergedList)
756  return mergedList
757 
758  def cullPeaks(self, catalog):
759  """!
760  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
761 
762  \param[in] catalog Source catalog
763  """
764  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
765  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
766  totalPeaks = 0
767  culledPeaks = 0
768  for parentSource in catalog:
769  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
770  # to it (which is easier than deleting as we iterate).
771  keptPeaks = parentSource.getFootprint().getPeaks()
772  oldPeaks = list(keptPeaks)
773  keptPeaks.clear()
774  familySize = len(oldPeaks)
775  totalPeaks += familySize
776  for rank, peak in enumerate(oldPeaks):
777  if ((rank < self.config.cullPeaks.rankSufficient) or
778  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
779  (rank < self.config.cullPeaks.rankConsidered and
780  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
781  keptPeaks.append(peak)
782  else:
783  culledPeaks += 1
784  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
785 
786  def getSchemaCatalogs(self):
787  """!
788  Return a dict of empty catalogs for each catalog dataset produced by this task.
789 
790  \param[out] dictionary of empty catalogs
791  """
792  mergeDet = afwTable.SourceCatalog(self.schema)
793  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
794  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
795  self.config.coaddName + "Coadd_peak": peak}
796 
797  def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0):
798  """!
799  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
800 
801  \param mergedList The merged Footprints from all the input bands
802  \param skyInfo A description of the patch
803  \param growDetectedFootprints The number of pixels to grow the detected footprints
804  """
805 
806  if self.config.nSkySourcesPerPatch <= 0:
807  return []
808 
809  skySourceRadius = self.config.skySourceRadius
810  nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
811  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
812  if nTrialSkySourcesPerPatch is None:
813  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
814  #
815  # We are going to find circular Footprints that don't intersect any pre-existing Footprints,
816  # and the easiest way to do this is to generate a Mask containing all the detected pixels (as
817  # merged by this task).
818  #
819  patchBBox = skyInfo.patchInfo.getOuterBBox()
820  mask = afwImage.Mask(patchBBox)
821  detectedMask = mask.getPlaneBitMask("DETECTED")
822  for s in mergedList:
823  foot = s.getFootprint()
824  if growDetectedFootprints > 0:
825  foot.dilate(growDetectedFootprints)
826  foot.spans.setMask(mask, detectedMask)
827 
828  xmin, ymin = patchBBox.getMin()
829  xmax, ymax = patchBBox.getMax()
830  # Avoid objects partially off the image
831  xmin += skySourceRadius + 1
832  ymin += skySourceRadius + 1
833  xmax -= skySourceRadius + 1
834  ymax -= skySourceRadius + 1
835 
836  skySourceFootprints = []
837  maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
838  for i in range(nTrialSkySourcesPerPatch):
839  if len(skySourceFootprints) == nSkySourcesPerPatch:
840  break
841 
842  x = int(numpy.random.uniform(xmin, xmax))
843  y = int(numpy.random.uniform(ymin, ymax))
844  spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
845  offset=(x, y))
846  foot = afwDetect.Footprint(spans, patchBBox)
847  foot.setPeakSchema(self.merged.getPeakSchema())
848 
849  if not foot.spans.overlaps(maskToSpanSet):
850  foot.addPeak(x, y, 0)
851  foot.getPeaks()[0].set("merge_peak_%s" % self.config.skyFilterName, True)
852  skySourceFootprints.append(foot)
853 
854  return skySourceFootprints
855 
856 
858  """!
859  \anchor MeasureMergedCoaddSourcesConfig_
860 
861  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
862  """
863  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
864  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
865  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
866  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
867  doPropagateFlags = Field(
868  dtype=bool, default=True,
869  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
870  )
871  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
872  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
873  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
874  doWriteMatchesDenormalized = Field(
875  dtype=bool,
876  default=False,
877  doc=("Write reference matches in denormalized format? "
878  "This format uses more disk space, but is more convenient to read."),
879  )
880  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
881  checkUnitsParseStrict = Field(
882  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
883  dtype=str,
884  default="raise",
885  )
886  doApCorr = Field(
887  dtype=bool,
888  default=True,
889  doc="Apply aperture corrections"
890  )
891  applyApCorr = ConfigurableField(
892  target=ApplyApCorrTask,
893  doc="Subtask to apply aperture corrections"
894  )
895  doRunCatalogCalculation = Field(
896  dtype=bool,
897  default=True,
898  doc='Run catalogCalculation task'
899  )
900  catalogCalculation = ConfigurableField(
901  target=CatalogCalculationTask,
902  doc="Subtask to run catalogCalculation plugins on catalog"
903  )
904 
905  def setDefaults(self):
906  Config.setDefaults(self)
907  self.deblend.propagateAllPeaks = True
908  self.measurement.plugins.names |= ['base_InputCount']
909  # The following line must be set if clipped pixel flags are to be added to the output table
910  # The clipped mask plane is added by running SafeClipAssembleCoaddTask
911  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED']
912 
913 
919 
920 
921 class MeasureMergedCoaddSourcesTask(CmdLineTask):
922  """!
923  \anchor MeasureMergedCoaddSourcesTask_
924 
925  \brief Deblend sources from master catalog in each coadd seperately and measure.
926 
927  \section pipe_tasks_multiBand_Contents Contents
928 
929  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
930  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
931  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
932  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
933  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
934  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
935 
936  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
937 
938  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
939  measurement in each coadd.
940 
941  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
942  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
943  consistent set of child sources.
944 
945  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
946  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
947  flags are propagated to the coadd sources.
948 
949  Optionally, we can match the coadd sources to an external reference catalog.
950 
951  \par Inputs:
952  deepCoadd_mergeDet{tract,patch}: SourceCatalog
953  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
954  \par Outputs:
955  deepCoadd_meas{tract,patch,filter}: SourceCatalog
956  \par Data Unit:
957  tract, patch, filter
958 
959  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
960 
961  <DL>
962  <DT> \ref SourceDeblendTask_ "deblend"
963  <DD> Deblend all the sources from the master catalog.</DD>
964  <DT> \ref SingleFrameMeasurementTask_ "measurement"
965  <DD> Measure source properties of deblended sources.</DD>
966  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
967  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
968  not at the edge of the field and that have either not been deblended or are the children of deblended
969  sources</DD>
970  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
971  <DD> Propagate flags set in individual visits to the coadd.</DD>
972  <DT> \ref DirectMatchTask_ "match"
973  <DD> Match input sources to a reference catalog (optional).
974  </DD>
975  </DL>
976  These subtasks may be retargeted as required.
977 
978  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
979 
980  \copydoc \_\_init\_\_
981 
982  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
983 
984  \copydoc run
985 
986  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
987 
988  See \ref MeasureMergedCoaddSourcesConfig_
989 
990  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
991 
992  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
993  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
994  files.
995 
996  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
997  the various sub-tasks. See the documetation for individual sub-tasks for more information.
998 
999  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
1000 
1001  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
1002  The next stage in the multi-band processing procedure will merge these measurements into a suitable
1003  catalog for driving forced photometry.
1004 
1005  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
1006  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
1007  `--help` command line argument:
1008  \code
1009  measureCoaddSources.py --help
1010  \endcode
1011 
1012  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1013  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1014  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1015  coadd as follows:
1016  \code
1017  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1018  \endcode
1019  This will process the HSC-I band data. The results are written in
1020  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1021 
1022  It is also necessary to run
1023  \code
1024  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1025  \endcode
1026  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1027  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1028  """
1029  _DefaultName = "measureCoaddSources"
1030  ConfigClass = MeasureMergedCoaddSourcesConfig
1031  RunnerClass = ButlerInitializedTaskRunner
1032  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1033  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1034 
1035  @classmethod
1036  def _makeArgumentParser(cls):
1037  parser = ArgumentParser(name=cls._DefaultName)
1038  parser.add_id_argument("--id", "deepCoadd_calexp",
1039  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1040  ContainerClass=ExistingCoaddDataIdContainer)
1041  return parser
1042 
1043  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1044  """!
1045  \brief Initialize the task.
1046 
1047  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1048  \param[in] schema: the schema of the merged detection catalog used as input to this one
1049  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1050  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1051  catalog. May be None if the loader can be constructed from the butler argument or all steps
1052  requiring a reference catalog are disabled.
1053  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1054  catalog loader, if schema or peakSchema or refObjLoader is None
1055 
1056  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1057  This will include all fields from the input schema, as well as additional fields for all the
1058  measurements.
1059  """
1060  CmdLineTask.__init__(self, **kwargs)
1061  if schema is None:
1062  assert butler is not None, "Neither butler nor schema is defined"
1063  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1064  self.schemaMapper = afwTable.SchemaMapper(schema)
1065  self.schemaMapper.addMinimalSchema(schema)
1066  self.schema = self.schemaMapper.getOutputSchema()
1067  self.algMetadata = PropertyList()
1068  if self.config.doDeblend:
1069  if peakSchema is None:
1070  assert butler is not None, "Neither butler nor peakSchema is defined"
1071  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1072  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1073  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1074  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1075  if self.config.doMatchSources:
1076  if refObjLoader is None:
1077  assert butler is not None, "Neither butler nor refObjLoader is defined"
1078  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1079  if self.config.doPropagateFlags:
1080  self.makeSubtask("propagateFlags", schema=self.schema)
1081  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1082  if self.config.doApCorr:
1083  self.makeSubtask("applyApCorr", schema=self.schema)
1084  if self.config.doRunCatalogCalculation:
1085  self.makeSubtask("catalogCalculation", schema=self.schema)
1086 
1087  def run(self, patchRef):
1088  """!
1089  \brief Deblend and measure.
1090 
1091  \param[in] patchRef: Patch reference.
1092 
1093  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1094  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1095  Finally, write the deblended sources and measurements out.
1096  """
1097  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1098  sources = self.readSources(patchRef)
1099  if self.config.doDeblend:
1100  self.deblend.run(exposure, sources)
1101 
1102  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1103  # catalog is non-contiguous so can't extract column
1104  numBig = sum((s.get(bigKey) for s in sources))
1105  if numBig > 0:
1106  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1107  (patchRef.dataId, numBig))
1108 
1109  table = sources.getTable()
1110  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1111 
1112  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1113 
1114  if self.config.doApCorr:
1115  self.applyApCorr.run(
1116  catalog=sources,
1117  apCorrMap=exposure.getInfo().getApCorrMap()
1118  )
1119 
1120  if self.config.doRunCatalogCalculation:
1121  self.catalogCalculation.run(sources)
1122 
1123  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1124  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1125  includeDeblend=self.config.doDeblend)
1126  if self.config.doPropagateFlags:
1127  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1128  exposure.getWcs())
1129  if self.config.doMatchSources:
1130  self.writeMatches(patchRef, exposure, sources)
1131  self.write(patchRef, sources)
1132 
1133  def readSources(self, dataRef):
1134  """!
1135  \brief Read input sources.
1136 
1137  \param[in] dataRef: Data reference for catalog of merged detections
1138  \return List of sources in merged catalog
1139 
1140  We also need to add columns to hold the measurements we're about to make
1141  so we can measure in-place.
1142  """
1143  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1144  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1145  idFactory = self.makeIdFactory(dataRef)
1146  for s in merged:
1147  idFactory.notify(s.getId())
1148  table = afwTable.SourceTable.make(self.schema, idFactory)
1149  sources = afwTable.SourceCatalog(table)
1150  sources.extend(merged, self.schemaMapper)
1151  return sources
1152 
1153  def writeMatches(self, dataRef, exposure, sources):
1154  """!
1155  \brief Write matches of the sources to the astrometric reference catalog.
1156 
1157  We use the Wcs in the exposure to match sources.
1158 
1159  \param[in] dataRef: data reference
1160  \param[in] exposure: exposure with Wcs
1161  \param[in] sources: source catalog
1162  """
1163  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1164  if result.matches:
1165  matches = afwTable.packMatches(result.matches)
1166  matches.table.setMetadata(result.matchMeta)
1167  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1168  if self.config.doWriteMatchesDenormalized:
1169  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1170  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1171 
1172 
1173  def write(self, dataRef, sources):
1174  """!
1175  \brief Write the source catalog.
1176 
1177  \param[in] dataRef: data reference
1178  \param[in] sources: source catalog
1179  """
1180  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1181  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1182 
1183  def getExposureId(self, dataRef):
1184  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1185 
1186 
1187 
1188 
1190  """!
1191  \anchor MergeMeasurementsConfig_
1192 
1193  \brief Configuration parameters for the MergeMeasurementsTask
1194  """
1195  pseudoFilterList = ListField(dtype=str, default=["sky"],
1196  doc="Names of filters which may have no associated detection\n"
1197  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1198  snName = Field(dtype=str, default="base_PsfFlux",
1199  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1200  minSN = Field(dtype=float, default=10.,
1201  doc="If the S/N from the priority band is below this value (and the S/N "
1202  "is larger than minSNDiff compared to the priority band), use the band with "
1203  "the largest S/N as the reference band.")
1204  minSNDiff = Field(dtype=float, default=3.,
1205  doc="If the difference in S/N between another band and the priority band is larger "
1206  "than this value (and the S/N in the priority band is less than minSN) "
1207  "use the band with the largest S/N as the reference band")
1208  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1209  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1210  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1211 
1212 
1218 
1219 
1221  """!
1222  \anchor MergeMeasurementsTask_
1223 
1224  \brief Merge measurements from multiple bands
1225 
1226  \section pipe_tasks_multiBand_Contents Contents
1227 
1228  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1229  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1230  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1231  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1232  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1233  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1234 
1235  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1236 
1237  Command-line task that merges measurements from multiple bands.
1238 
1239  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1240  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1241  required to have centroid, shape and flux measurements in each band.
1242 
1243  \par Inputs:
1244  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1245  \par Outputs:
1246  deepCoadd_ref{tract,patch}: SourceCatalog
1247  \par Data Unit:
1248  tract, patch
1249 
1250  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1251 
1252  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1253 
1254  \copydoc \_\_init\_\_
1255 
1256  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1257 
1258  \copydoc run
1259 
1260  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1261 
1262  See \ref MergeMeasurementsConfig_
1263 
1264  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1265 
1266  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1267  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1268  files.
1269 
1270  MergeMeasurementsTask has no debug variables.
1271 
1272  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1273 
1274  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1275  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1276  coadds and individual exposures.
1277  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1278  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1279  command line argument:
1280  \code
1281  mergeCoaddMeasurements.py --help
1282  \endcode
1283 
1284  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1285  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1286  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1287  as follows:
1288  \code
1289  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1290  \endcode
1291  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1292  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1293  """
1294  _DefaultName = "mergeCoaddMeasurements"
1295  ConfigClass = MergeMeasurementsConfig
1296  inputDataset = "meas"
1297  outputDataset = "ref"
1298  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1299 
1300  def __init__(self, butler=None, schema=None, **kwargs):
1301  """!
1302  Initialize the task.
1303 
1304  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1305  \param[in] schema: the schema of the detection catalogs used as input to this one
1306  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1307 
1308  The task will set its own self.schema attribute to the schema of the output merged catalog.
1309  """
1310  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1311  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1312  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1313  self.schemaMapper.addMinimalSchema(inputSchema, True)
1314  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1315  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1316  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1317 
1318  self.flagKeys = {}
1319  for band in self.config.priorityList:
1320  short = getShortFilterName(band)
1321  outputKey = self.schemaMapper.editOutputSchema().addField(
1322  "merge_measurement_%s" % short,
1323  type="Flag",
1324  doc="Flag field set if the measurements here are from the %s filter" % band
1325  )
1326  peakKey = inputSchema.find("merge_peak_%s" % short).key
1327  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1328  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1329  self.schema = self.schemaMapper.getOutputSchema()
1330 
1332  for filt in self.config.pseudoFilterList:
1333  try:
1334  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1335  except:
1336  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1337 
1338  self.badFlags = {}
1339  for flag in self.config.flags:
1340  try:
1341  self.badFlags[flag] = self.schema.find(flag).getKey()
1342  except KeyError as exc:
1343  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1344 
1345  def mergeCatalogs(self, catalogs, patchRef):
1346  """!
1347  Merge measurement catalogs to create a single reference catalog for forced photometry
1348 
1349  \param[in] catalogs: the catalogs to be merged
1350  \param[in] patchRef: patch reference for data
1351 
1352  For parent sources, we choose the first band in config.priorityList for which the
1353  merge_footprint flag for that band is is True.
1354 
1355  For child sources, the logic is the same, except that we use the merge_peak flags.
1356  """
1357  # Put catalogs, filters in priority order
1358  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1359  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1360 
1361  mergedCatalog = afwTable.SourceCatalog(self.schema)
1362  mergedCatalog.reserve(len(orderedCatalogs[0]))
1363 
1364  idKey = orderedCatalogs[0].table.getIdKey()
1365  for catalog in orderedCatalogs[1:]:
1366  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1367  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1368 
1369  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1370  # record for each band, in priority order.
1371  for orderedRecords in zip(*orderedCatalogs):
1372 
1373  maxSNRecord = None
1374  maxSNFlagKeys = None
1375  maxSN = 0.
1376  priorityRecord = None
1377  priorityFlagKeys = None
1378  prioritySN = 0.
1379  hasPseudoFilter = False
1380 
1381  # Now we iterate over those record-band pairs, keeping track of the priority and the
1382  # largest S/N band.
1383  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1384  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1385  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1386 
1387  if not (parent or child):
1388  for pseudoFilterKey in self.pseudoFilterKeys:
1389  if inputRecord.get(pseudoFilterKey):
1390  hasPseudoFilter = True
1391  priorityRecord = inputRecord
1392  priorityFlagKeys = flagKeys
1393  break
1394  if hasPseudoFilter:
1395  break
1396 
1397  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1398  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1399  sn = 0.
1400  else:
1401  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1402  if numpy.isnan(sn) or sn < 0.:
1403  sn = 0.
1404  if (parent or child) and priorityRecord is None:
1405  priorityRecord = inputRecord
1406  priorityFlagKeys = flagKeys
1407  prioritySN = sn
1408  if sn > maxSN:
1409  maxSNRecord = inputRecord
1410  maxSNFlagKeys = flagKeys
1411  maxSN = sn
1412 
1413  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1414  # the reference band instead. However, we only want to choose the highest S/N band if it is
1415  # significantly better than the priority band. Therefore, to choose a band other than the
1416  # priority, we require that the priority S/N is below the minimum threshold and that the
1417  # difference between the priority and highest S/N is larger than the difference threshold.
1418  #
1419  # For pseudo code objects we always choose the first band in the priority list.
1420  bestRecord = None
1421  bestFlagKeys = None
1422  if hasPseudoFilter:
1423  bestRecord = priorityRecord
1424  bestFlagKeys = priorityFlagKeys
1425  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1426  maxSNRecord is not None):
1427  bestRecord = maxSNRecord
1428  bestFlagKeys = maxSNFlagKeys
1429  elif priorityRecord is not None:
1430  bestRecord = priorityRecord
1431  bestFlagKeys = priorityFlagKeys
1432 
1433  if bestRecord is not None and bestFlagKeys is not None:
1434  outputRecord = mergedCatalog.addNew()
1435  outputRecord.assign(bestRecord, self.schemaMapper)
1436  outputRecord.set(bestFlagKeys.output, True)
1437  else: # if we didn't find any records
1438  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1439  inputRecord.getId())
1440 
1441  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1442  for inputCatalog in orderedCatalogs:
1443  if len(mergedCatalog) != len(inputCatalog):
1444  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1445  (len(mergedCatalog), len(orderedCatalogs)))
1446 
1447  return mergedCatalog
Merge coadd detections from multiple bands.
Definition: multiBand.py:595
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:329
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:432
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:786
def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:797
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:758
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1087
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:321
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:305
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:445
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1300
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:245
Merge measurements from multiple bands.
Definition: multiBand.py:1220
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:458
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:921
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1153
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:263
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1345
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:484
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1189
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1133
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:554
Configuration for merging sources.
Definition: multiBand.py:373
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:495
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:704
A base class for merging source catalogs.
Definition: multiBand.py:389
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1173
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:469
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:857
def runDetection(self, exposure, idFactory)
Run detection on an exposure.
Definition: multiBand.py:277
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:249
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:512
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1043
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:682
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:347