lsst.pipe.tasks  14.0-19-g29094cd5+1
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import SourceDetectionTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
35 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
36 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=SourceDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so background subtraction should be very basic
131  self.detection.background.useApprox = False
132  self.detection.background.binSize = 4096
133  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
134 
135 
141 
142 
143 class DetectCoaddSourcesTask(CmdLineTask):
144  """!
145  \anchor DetectCoaddSourcesTask_
146 
147  \brief Detect sources on a coadd
148 
149  \section pipe_tasks_multiBand_Contents Contents
150 
151  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
156  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
157 
158  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
159 
160  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
161 
162  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
163  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
164  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
165  propagate the full covariance matrix -- but it is simple and works well in practice.
166 
167  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
168  SourceDetectionTask_ "detection" subtask.
169 
170  \par Inputs:
171  deepCoadd{tract,patch,filter}: ExposureF
172  \par Outputs:
173  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
174  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
175  exposure (ExposureF)
176  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
177  \par Data Unit:
178  tract, patch, filter
179 
180  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
181  You can retarget this subtask if you wish.
182 
183  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
184 
185  \copydoc \_\_init\_\_
186 
187  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
188 
189  \copydoc run
190 
191  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
192 
193  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
194 
195  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
196 
197  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
198  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
199  files.
200 
201  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
202  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
203  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
204 
205  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
206 
207  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
208  the task is to update the background, detect all sources in a single band and generate a set of parent
209  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
210  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
211  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
212  calling detectCoaddSources.py with the `--help` command line argument:
213  \code
214  detectCoaddSources.py --help
215  \endcode
216 
217  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
218  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
219  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
220  \code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
222  \endcode
223  that will process the HSC-I band data. The results are written to
224  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
225 
226  It is also necessary to run:
227  \code
228  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
229  \endcode
230  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
231  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
232  """
233  _DefaultName = "detectCoaddSources"
234  ConfigClass = DetectCoaddSourcesConfig
235  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
236  makeIdFactory = _makeMakeIdFactory("CoaddId")
237 
238  @classmethod
239  def _makeArgumentParser(cls):
240  parser = ArgumentParser(name=cls._DefaultName)
241  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242  ContainerClass=ExistingCoaddDataIdContainer)
243  return parser
244 
245  def __init__(self, schema=None, **kwargs):
246  """!
247  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
248 
249  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
250 
251  \param[in] schema: initial schema for the output catalog, modified-in place to include all
252  fields set by this task. If None, the source minimal schema will be used.
253  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
254  """
255  CmdLineTask.__init__(self, **kwargs)
256  if schema is None:
257  schema = afwTable.SourceTable.makeMinimalSchema()
258  if self.config.doInsertFakes:
259  self.makeSubtask("insertFakes")
260  self.schema = schema
261  self.makeSubtask("detection", schema=self.schema)
262 
263  def run(self, patchRef):
264  """!
265  \brief Run detection on a coadd.
266 
267  Invokes \ref runDetection and then uses \ref write to output the
268  results.
269 
270  \param[in] patchRef: data reference for patch
271  """
272  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
273  results = self.runDetection(exposure, self.makeIdFactory(patchRef))
274  self.write(exposure, results, patchRef)
275  return results
276 
277  def runDetection(self, exposure, idFactory):
278  """!
279  \brief Run detection on an exposure.
280 
281  First scale the variance plane to match the observed variance
282  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
283  detect sources.
284 
285  \param[in] exposure: Exposure on which to detect
286  \param[in] idFactory: IdFactory to set source identifiers
287 
288  \return a pipe.base.Struct with fields
289  - sources: catalog of detections
290  - backgrounds: list of backgrounds
291  """
292  if self.config.doScaleVariance:
293  varScale = scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
294  self.metadata.add("variance_scale", varScale)
295  backgrounds = afwMath.BackgroundList()
296  if self.config.doInsertFakes:
297  self.insertFakes.run(exposure, background=backgrounds)
298  table = afwTable.SourceTable.make(self.schema, idFactory)
299  detections = self.detection.makeSourceCatalog(table, exposure)
300  sources = detections.sources
301  fpSets = detections.fpSets
302  if fpSets.background:
303  backgrounds.append(fpSets.background)
304  return Struct(sources=sources, backgrounds=backgrounds)
305 
306  def write(self, exposure, results, patchRef):
307  """!
308  \brief Write out results from runDetection.
309 
310  \param[in] exposure: Exposure to write out
311  \param[in] results: Struct returned from runDetection
312  \param[in] patchRef: data reference for patch
313  """
314  coaddName = self.config.coaddName + "Coadd"
315  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
316  patchRef.put(results.sources, coaddName + "_det")
317  patchRef.put(exposure, coaddName + "_calexp")
318 
319 
320 
321 
322 class MergeSourcesRunner(TaskRunner):
323  """!
324  \anchor MergeSourcesRunner_
325 
326  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
327  requires a list of dataRefs rather than a single dataRef.
328  """
329 
330  def makeTask(self, parsedCmd=None, args=None):
331  """!
332  \brief Provide a butler to the Task constructor.
333 
334  \param[in] parsedCmd the parsed command
335  \param[in] args tuple of a list of data references and kwargs (un-used)
336  \throws RuntimeError if both parsedCmd & args are None
337  """
338  if parsedCmd is not None:
339  butler = parsedCmd.butler
340  elif args is not None:
341  dataRefList, kwargs = args
342  butler = dataRefList[0].getButler()
343  else:
344  raise RuntimeError("Neither parsedCmd or args specified")
345  return self.TaskClass(config=self.config, log=self.log, butler=butler)
346 
347  @staticmethod
348  def getTargetList(parsedCmd, **kwargs):
349  """!
350  \brief Provide a list of patch references for each patch.
351 
352  The patch references within the list will have different filters.
353 
354  \param[in] parsedCmd the parsed command
355  \param **kwargs key word arguments (unused)
356  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
357  filter
358  """
359  refList = {} # Will index this as refList[tract][patch][filter] = ref
360  for ref in parsedCmd.id.refList:
361  tract = ref.dataId["tract"]
362  patch = ref.dataId["patch"]
363  filter = ref.dataId["filter"]
364  if not tract in refList:
365  refList[tract] = {}
366  if not patch in refList[tract]:
367  refList[tract][patch] = {}
368  if filter in refList[tract][patch]:
369  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
370  refList[tract][patch][filter] = ref
371  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
372 
373 
374 class MergeSourcesConfig(Config):
375  """!
376  \anchor MergeSourcesConfig_
377 
378  \brief Configuration for merging sources.
379  """
380  priorityList = ListField(dtype=str, default=[],
381  doc="Priority-ordered list of bands for the merge.")
382  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
383 
384  def validate(self):
385  Config.validate(self)
386  if len(self.priorityList) == 0:
387  raise RuntimeError("No priority list provided")
388 
389 
390 class MergeSourcesTask(CmdLineTask):
391  """!
392  \anchor MergeSourcesTask_
393 
394  \brief A base class for merging source catalogs.
395 
396  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
397  so similar that it makes sense to re-use the code, in the form of this abstract base class.
398 
399  NB: Do not use this class directly. Instead use one of the child classes that inherit from
400  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
401  "MergeMeasurementsTask"
402 
403  Sub-classes should set the following class variables:
404  * `_DefaultName`: name of Task
405  * `inputDataset`: name of dataset to read
406  * `outputDataset`: name of dataset to write
407  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
408 
409  In addition, sub-classes must implement the mergeCatalogs method.
410  """
411  _DefaultName = None
412  ConfigClass = MergeSourcesConfig
413  RunnerClass = MergeSourcesRunner
414  inputDataset = None
415  outputDataset = None
416  getSchemaCatalogs = None
417 
418  @classmethod
419  def _makeArgumentParser(cls):
420  """!
421  \brief Create a suitable ArgumentParser.
422 
423  We will use the ArgumentParser to get a provide a list of data
424  references for patches; the RunnerClass will sort them into lists
425  of data references for the same patch
426  """
427  parser = ArgumentParser(name=cls._DefaultName)
428  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
429  ContainerClass=ExistingCoaddDataIdContainer,
430  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
431  return parser
432 
433  def getInputSchema(self, butler=None, schema=None):
434  """!
435  \brief Obtain the input schema either directly or froma butler reference.
436 
437  \param[in] butler butler reference to obtain the input schema from
438  \param[in] schema the input schema
439  """
440  if schema is None:
441  assert butler is not None, "Neither butler nor schema specified"
442  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
443  immediate=True).schema
444  return schema
445 
446  def __init__(self, butler=None, schema=None, **kwargs):
447  """!
448  \brief Initialize the task.
449 
450  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
451  \param[in] schema the schema of the detection catalogs used as input to this one
452  \param[in] butler a butler used to read the input schema from disk, if schema is None
453 
454  Derived classes should use the getInputSchema() method to handle the additional
455  arguments and retreive the actual input schema.
456  """
457  CmdLineTask.__init__(self, **kwargs)
458 
459  def run(self, patchRefList):
460  """!
461  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
462  subclasses that inherit from MergeSourcesTask.
463 
464  \param[in] patchRefList list of data references for each filter
465  """
466  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
467  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
468  self.write(patchRefList[0], mergedCatalog)
469 
470  def readCatalog(self, patchRef):
471  """!
472  \brief Read input catalog.
473 
474  We read the input dataset provided by the 'inputDataset'
475  class variable.
476 
477  \param[in] patchRef data reference for patch
478  \return tuple consisting of the filter name and the catalog
479  """
480  filterName = patchRef.dataId["filter"]
481  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
482  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
483  return filterName, catalog
484 
485  def mergeCatalogs(self, catalogs, patchRef):
486  """!
487  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
488  MergeSourcesTask.
489 
490  \param[in] catalogs dict mapping filter name to source catalog
491 
492  \return merged catalog
493  """
494  raise NotImplementedError()
495 
496  def write(self, patchRef, catalog):
497  """!
498  \brief Write the output.
499 
500  \param[in] patchRef data reference for patch
501  \param[in] catalog catalog
502 
503  We write as the dataset provided by the 'outputDataset'
504  class variable.
505  """
506  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
507  # since the filter isn't actually part of the data ID for the dataset we're saving,
508  # it's confusing to see it in the log message, even if the butler simply ignores it.
509  mergeDataId = patchRef.dataId.copy()
510  del mergeDataId["filter"]
511  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
512 
513  def writeMetadata(self, dataRefList):
514  """!
515  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
516  """
517  pass
518 
519 
520 class CullPeaksConfig(Config):
521  """!
522  \anchor CullPeaksConfig_
523 
524  \brief Configuration for culling garbage peaks after merging footprints.
525 
526  Peaks may also be culled after detection or during deblending; this configuration object
527  only deals with culling after merging Footprints.
528 
529  These cuts are based on three quantities:
530  - nBands: the number of bands in which the peak was detected
531  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
532  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
533 
534  The formula that identifie peaks to cull is:
535 
536  nBands < nBandsSufficient
537  AND (rank >= rankSufficient)
538  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
539 
540  To disable peak culling, simply set nBandsSufficient=1.
541  """
542 
543  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
544  doc="Always keep peaks detected in this many bands")
545  rankSufficient = RangeField(dtype=int, default=20, min=1,
546  doc="Always keep this many peaks in each family")
547  rankConsidered = RangeField(dtype=int, default=30, min=1,
548  doc=("Keep peaks with less than this rank that also match the "
549  "rankNormalizedConsidered condition."))
550  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
551  doc=("Keep peaks with less than this normalized rank that"
552  " also match the rankConsidered condition."))
553 
554 
556  """!
557  \anchor MergeDetectionsConfig_
558 
559  \brief Configuration parameters for the MergeDetectionsTask.
560  """
561  minNewPeak = Field(dtype=float, default=1,
562  doc="Minimum distance from closest peak to create a new one (in arcsec).")
563 
564  maxSamePeak = Field(dtype=float, default=0.3,
565  doc="When adding new catalogs to the merge, all peaks less than this distance "
566  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
567  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
568 
569  skyFilterName = Field(dtype=str, default="sky",
570  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
571  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
572  skySourceRadius = Field(dtype=float, default=8,
573  doc="Radius, in pixels, of sky objects")
574  skyGrowDetectedFootprints = Field(dtype=int, default=0,
575  doc="Number of pixels to grow the detected footprint mask "
576  "when adding sky objects")
577  nSkySourcesPerPatch = Field(dtype=int, default=100,
578  doc="Try to add this many sky objects to the mergeDet list, which will\n"
579  "then be measured along with the detected objects in sourceMeasurementTask")
580  nTrialSkySourcesPerPatch = Field(dtype=int, default=None, optional=True,
581  doc="Maximum number of trial sky object positions\n"
582  "(default: nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier)")
583  nTrialSkySourcesPerPatchMultiplier = Field(dtype=int, default=5,
584  doc="Set nTrialSkySourcesPerPatch to\n"
585  " nSkySourcesPerPatch*nTrialSkySourcesPerPatchMultiplier\n"
586  "if nTrialSkySourcesPerPatch is None")
587 
588 
594 
595 
597  """!
598  \anchor MergeDetectionsTask_
599 
600  \brief Merge coadd detections from multiple bands.
601 
602  \section pipe_tasks_multiBand_Contents Contents
603 
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
608  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
609  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
610 
611  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
612 
613  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
614 
615  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
616  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
617  track of which band each source originates in.
618 
619  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
620  objects are culled as described in \ref CullPeaksConfig_.
621 
622  \par Inputs:
623  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
624  \par Outputs:
625  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
626  \par Data Unit:
627  tract, patch
628 
629  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
630 
631  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
632 
633  \copydoc \_\_init\_\_
634 
635  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
636 
637  \copydoc run
638 
639  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
640 
641  See \ref MergeDetectionsConfig_
642 
643  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
644 
645  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
646  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
647 
648  MergeDetectionsTask has no debug variables.
649 
650  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
651 
652  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
653  of the available bands.
654  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
655  chosen subset of filters.
656  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
657  and, eventually, perform forced photometry.
658  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
659  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
660  `--help` command line argument:
661  \code
662  mergeCoaddDetections.py --help
663  \endcode
664 
665  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
666  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
667  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
668  \code
669  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
670  \endcode
671  This will merge the HSC-I & -R band parent source catalogs and write the results to
672  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
673 
674  The next step in the multi-band processing procedure is
675  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
676  """
677  ConfigClass = MergeDetectionsConfig
678  _DefaultName = "mergeCoaddDetections"
679  inputDataset = "det"
680  outputDataset = "mergeDet"
681  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
682 
683  def __init__(self, butler=None, schema=None, **kwargs):
684  """!
685  \brief Initialize the merge detections task.
686 
687  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
688  merge the source catalogs.
689 
690  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
691  \param[in] schema the schema of the detection catalogs used as input to this one
692  \param[in] butler a butler used to read the input schema from disk, if schema is None
693  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
694 
695  The task will set its own self.schema attribute to the schema of the output merged catalog.
696  """
697  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
698  self.schema = self.getInputSchema(butler=butler, schema=schema)
699 
700  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
701  if self.config.nSkySourcesPerPatch > 0:
702  filterNames += [self.config.skyFilterName]
703  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
704 
705  def mergeCatalogs(self, catalogs, patchRef):
706  """!
707  \brief Merge multiple catalogs.
708 
709  After ordering the catalogs and filters in priority order,
710  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
711  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
712  garbage peaks detected around bright objects.
713 
714  \param[in] catalogs
715  \param[in] patchRef
716  \param[out] mergedList
717  """
718 
719  # Convert distance to tract coordinate
720  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
721  tractWcs = skyInfo.wcs
722  peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
723  samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
724 
725  # Put catalogs, filters in priority order
726  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
727  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
728  if band in catalogs.keys()]
729 
730  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
731  self.schema, self.makeIdFactory(patchRef),
732  samePeakDistance)
733 
734  #
735  # Add extra sources that correspond to blank sky
736  #
737  skySourceFootprints = self.getSkySourceFootprints(
738  mergedList, skyInfo, self.config.skyGrowDetectedFootprints)
739  if skySourceFootprints:
740  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
741 
742  for foot in skySourceFootprints:
743  s = mergedList.addNew()
744  s.setFootprint(foot)
745  s.set(key, True)
746 
747  self.log.info("Added %d sky sources (%.0f%% of requested)",
748  len(skySourceFootprints),
749  100*len(skySourceFootprints)/float(self.config.nSkySourcesPerPatch))
750 
751  # Sort Peaks from brightest to faintest
752  for record in mergedList:
753  record.getFootprint().sortPeaks()
754  self.log.info("Merged to %d sources" % len(mergedList))
755  # Attempt to remove garbage peaks
756  self.cullPeaks(mergedList)
757  return mergedList
758 
759  def cullPeaks(self, catalog):
760  """!
761  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
762 
763  \param[in] catalog Source catalog
764  """
765  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
766  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
767  totalPeaks = 0
768  culledPeaks = 0
769  for parentSource in catalog:
770  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
771  # to it (which is easier than deleting as we iterate).
772  keptPeaks = parentSource.getFootprint().getPeaks()
773  oldPeaks = list(keptPeaks)
774  keptPeaks.clear()
775  familySize = len(oldPeaks)
776  totalPeaks += familySize
777  for rank, peak in enumerate(oldPeaks):
778  if ((rank < self.config.cullPeaks.rankSufficient) or
779  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
780  (rank < self.config.cullPeaks.rankConsidered and
781  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
782  keptPeaks.append(peak)
783  else:
784  culledPeaks += 1
785  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
786 
787  def getSchemaCatalogs(self):
788  """!
789  Return a dict of empty catalogs for each catalog dataset produced by this task.
790 
791  \param[out] dictionary of empty catalogs
792  """
793  mergeDet = afwTable.SourceCatalog(self.schema)
794  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
795  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
796  self.config.coaddName + "Coadd_peak": peak}
797 
798  def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0):
799  """!
800  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
801 
802  \param mergedList The merged Footprints from all the input bands
803  \param skyInfo A description of the patch
804  \param growDetectedFootprints The number of pixels to grow the detected footprints
805  """
806 
807  if self.config.nSkySourcesPerPatch <= 0:
808  return []
809 
810  skySourceRadius = self.config.skySourceRadius
811  nSkySourcesPerPatch = self.config.nSkySourcesPerPatch
812  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatch
813  if nTrialSkySourcesPerPatch is None:
814  nTrialSkySourcesPerPatch = self.config.nTrialSkySourcesPerPatchMultiplier*nSkySourcesPerPatch
815  #
816  # We are going to find circular Footprints that don't intersect any pre-existing Footprints,
817  # and the easiest way to do this is to generate a Mask containing all the detected pixels (as
818  # merged by this task).
819  #
820  patchBBox = skyInfo.patchInfo.getOuterBBox()
821  mask = afwImage.Mask(patchBBox)
822  detectedMask = mask.getPlaneBitMask("DETECTED")
823  for s in mergedList:
824  foot = s.getFootprint()
825  if growDetectedFootprints > 0:
826  foot.dilate(growDetectedFootprints)
827  foot.spans.setMask(mask, detectedMask)
828 
829  xmin, ymin = patchBBox.getMin()
830  xmax, ymax = patchBBox.getMax()
831  # Avoid objects partially off the image
832  xmin += skySourceRadius + 1
833  ymin += skySourceRadius + 1
834  xmax -= skySourceRadius + 1
835  ymax -= skySourceRadius + 1
836 
837  skySourceFootprints = []
838  maskToSpanSet = afwGeom.SpanSet.fromMask(mask)
839  for i in range(nTrialSkySourcesPerPatch):
840  if len(skySourceFootprints) == nSkySourcesPerPatch:
841  break
842 
843  x = int(numpy.random.uniform(xmin, xmax))
844  y = int(numpy.random.uniform(ymin, ymax))
845  spans = afwGeom.SpanSet.fromShape(int(skySourceRadius),
846  offset=(x, y))
847  foot = afwDetect.Footprint(spans, patchBBox)
848  foot.setPeakSchema(self.merged.getPeakSchema())
849 
850  if not foot.spans.overlaps(maskToSpanSet):
851  foot.addPeak(x, y, 0)
852  foot.getPeaks()[0].set("merge_peak_%s" % self.config.skyFilterName, True)
853  skySourceFootprints.append(foot)
854 
855  return skySourceFootprints
856 
857 
859  """!
860  \anchor MeasureMergedCoaddSourcesConfig_
861 
862  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
863  """
864  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
865  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
866  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
867  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
868  doPropagateFlags = Field(
869  dtype=bool, default=True,
870  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
871  )
872  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
873  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
874  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
875  doWriteMatchesDenormalized = Field(
876  dtype=bool,
877  default=False,
878  doc=("Write reference matches in denormalized format? "
879  "This format uses more disk space, but is more convenient to read."),
880  )
881  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
882  checkUnitsParseStrict = Field(
883  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
884  dtype=str,
885  default="raise",
886  )
887  doApCorr = Field(
888  dtype=bool,
889  default=True,
890  doc="Apply aperture corrections"
891  )
892  applyApCorr = ConfigurableField(
893  target=ApplyApCorrTask,
894  doc="Subtask to apply aperture corrections"
895  )
896  doRunCatalogCalculation = Field(
897  dtype=bool,
898  default=True,
899  doc='Run catalogCalculation task'
900  )
901  catalogCalculation = ConfigurableField(
902  target=CatalogCalculationTask,
903  doc="Subtask to run catalogCalculation plugins on catalog"
904  )
905 
906  def setDefaults(self):
907  Config.setDefaults(self)
908  self.deblend.propagateAllPeaks = True
909  self.measurement.plugins.names |= ['base_InputCount']
910  # The following line must be set if clipped pixel flags are to be added to the output table
911  # The clipped mask plane is added by running SafeClipAssembleCoaddTask
912  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED']
913 
914 
920 
921 
922 class MeasureMergedCoaddSourcesTask(CmdLineTask):
923  """!
924  \anchor MeasureMergedCoaddSourcesTask_
925 
926  \brief Deblend sources from master catalog in each coadd seperately and measure.
927 
928  \section pipe_tasks_multiBand_Contents Contents
929 
930  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
931  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
932  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
933  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
934  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
935  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
936 
937  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
938 
939  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
940  measurement in each coadd.
941 
942  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
943  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
944  consistent set of child sources.
945 
946  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
947  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
948  flags are propagated to the coadd sources.
949 
950  Optionally, we can match the coadd sources to an external reference catalog.
951 
952  \par Inputs:
953  deepCoadd_mergeDet{tract,patch}: SourceCatalog
954  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
955  \par Outputs:
956  deepCoadd_meas{tract,patch,filter}: SourceCatalog
957  \par Data Unit:
958  tract, patch, filter
959 
960  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
961 
962  <DL>
963  <DT> \ref SourceDeblendTask_ "deblend"
964  <DD> Deblend all the sources from the master catalog.</DD>
965  <DT> \ref SingleFrameMeasurementTask_ "measurement"
966  <DD> Measure source properties of deblended sources.</DD>
967  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
968  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
969  not at the edge of the field and that have either not been deblended or are the children of deblended
970  sources</DD>
971  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
972  <DD> Propagate flags set in individual visits to the coadd.</DD>
973  <DT> \ref DirectMatchTask_ "match"
974  <DD> Match input sources to a reference catalog (optional).
975  </DD>
976  </DL>
977  These subtasks may be retargeted as required.
978 
979  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
980 
981  \copydoc \_\_init\_\_
982 
983  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
984 
985  \copydoc run
986 
987  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
988 
989  See \ref MeasureMergedCoaddSourcesConfig_
990 
991  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
992 
993  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
994  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
995  files.
996 
997  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
998  the various sub-tasks. See the documetation for individual sub-tasks for more information.
999 
1000  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
1001 
1002  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
1003  The next stage in the multi-band processing procedure will merge these measurements into a suitable
1004  catalog for driving forced photometry.
1005 
1006  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
1007  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
1008  `--help` command line argument:
1009  \code
1010  measureCoaddSources.py --help
1011  \endcode
1012 
1013  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1014  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1015  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
1016  coadd as follows:
1017  \code
1018  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
1019  \endcode
1020  This will process the HSC-I band data. The results are written in
1021  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
1022 
1023  It is also necessary to run
1024  \code
1025  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1026  \endcode
1027  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1028  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1029  """
1030  _DefaultName = "measureCoaddSources"
1031  ConfigClass = MeasureMergedCoaddSourcesConfig
1032  RunnerClass = ButlerInitializedTaskRunner
1033  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1034  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1035 
1036  @classmethod
1037  def _makeArgumentParser(cls):
1038  parser = ArgumentParser(name=cls._DefaultName)
1039  parser.add_id_argument("--id", "deepCoadd_calexp",
1040  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1041  ContainerClass=ExistingCoaddDataIdContainer)
1042  return parser
1043 
1044  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1045  """!
1046  \brief Initialize the task.
1047 
1048  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1049  \param[in] schema: the schema of the merged detection catalog used as input to this one
1050  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1051  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1052  catalog. May be None if the loader can be constructed from the butler argument or all steps
1053  requiring a reference catalog are disabled.
1054  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1055  catalog loader, if schema or peakSchema or refObjLoader is None
1056 
1057  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1058  This will include all fields from the input schema, as well as additional fields for all the
1059  measurements.
1060  """
1061  CmdLineTask.__init__(self, **kwargs)
1062  if schema is None:
1063  assert butler is not None, "Neither butler nor schema is defined"
1064  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1065  self.schemaMapper = afwTable.SchemaMapper(schema)
1066  self.schemaMapper.addMinimalSchema(schema)
1067  self.schema = self.schemaMapper.getOutputSchema()
1069  if self.config.doDeblend:
1070  if peakSchema is None:
1071  assert butler is not None, "Neither butler nor peakSchema is defined"
1072  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1073  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1074  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1075  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1076  if self.config.doMatchSources:
1077  if refObjLoader is None:
1078  assert butler is not None, "Neither butler nor refObjLoader is defined"
1079  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1080  if self.config.doPropagateFlags:
1081  self.makeSubtask("propagateFlags", schema=self.schema)
1082  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1083  if self.config.doApCorr:
1084  self.makeSubtask("applyApCorr", schema=self.schema)
1085  if self.config.doRunCatalogCalculation:
1086  self.makeSubtask("catalogCalculation", schema=self.schema)
1087 
1088  def run(self, patchRef):
1089  """!
1090  \brief Deblend and measure.
1091 
1092  \param[in] patchRef: Patch reference.
1093 
1094  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1095  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1096  Finally, write the deblended sources and measurements out.
1097  """
1098  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1099  sources = self.readSources(patchRef)
1100  if self.config.doDeblend:
1101  self.deblend.run(exposure, sources)
1102 
1103  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1104  # catalog is non-contiguous so can't extract column
1105  numBig = sum((s.get(bigKey) for s in sources))
1106  if numBig > 0:
1107  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1108  (patchRef.dataId, numBig))
1109 
1110  table = sources.getTable()
1111  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1112 
1113  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1114 
1115  if self.config.doApCorr:
1116  self.applyApCorr.run(
1117  catalog=sources,
1118  apCorrMap=exposure.getInfo().getApCorrMap()
1119  )
1120 
1121  if self.config.doRunCatalogCalculation:
1122  self.catalogCalculation.run(sources)
1123 
1124  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1125  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1126  includeDeblend=self.config.doDeblend)
1127  if self.config.doPropagateFlags:
1128  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1129  exposure.getWcs())
1130  if self.config.doMatchSources:
1131  self.writeMatches(patchRef, exposure, sources)
1132  self.write(patchRef, sources)
1133 
1134  def readSources(self, dataRef):
1135  """!
1136  \brief Read input sources.
1137 
1138  \param[in] dataRef: Data reference for catalog of merged detections
1139  \return List of sources in merged catalog
1140 
1141  We also need to add columns to hold the measurements we're about to make
1142  so we can measure in-place.
1143  """
1144  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1145  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1146  idFactory = self.makeIdFactory(dataRef)
1147  for s in merged:
1148  idFactory.notify(s.getId())
1149  table = afwTable.SourceTable.make(self.schema, idFactory)
1150  sources = afwTable.SourceCatalog(table)
1151  sources.extend(merged, self.schemaMapper)
1152  return sources
1153 
1154  def writeMatches(self, dataRef, exposure, sources):
1155  """!
1156  \brief Write matches of the sources to the astrometric reference catalog.
1157 
1158  We use the Wcs in the exposure to match sources.
1159 
1160  \param[in] dataRef: data reference
1161  \param[in] exposure: exposure with Wcs
1162  \param[in] sources: source catalog
1163  """
1164  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1165  if result.matches:
1166  matches = afwTable.packMatches(result.matches)
1167  matches.table.setMetadata(result.matchMeta)
1168  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1169  if self.config.doWriteMatchesDenormalized:
1170  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1171  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1172 
1173 
1174  def write(self, dataRef, sources):
1175  """!
1176  \brief Write the source catalog.
1177 
1178  \param[in] dataRef: data reference
1179  \param[in] sources: source catalog
1180  """
1181  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1182  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1183 
1184  def getExposureId(self, dataRef):
1185  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1186 
1187 
1188 
1189 
1191  """!
1192  \anchor MergeMeasurementsConfig_
1193 
1194  \brief Configuration parameters for the MergeMeasurementsTask
1195  """
1196  pseudoFilterList = ListField(dtype=str, default=["sky"],
1197  doc="Names of filters which may have no associated detection\n"
1198  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1199  snName = Field(dtype=str, default="base_PsfFlux",
1200  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1201  minSN = Field(dtype=float, default=10.,
1202  doc="If the S/N from the priority band is below this value (and the S/N "
1203  "is larger than minSNDiff compared to the priority band), use the band with "
1204  "the largest S/N as the reference band.")
1205  minSNDiff = Field(dtype=float, default=3.,
1206  doc="If the difference in S/N between another band and the priority band is larger "
1207  "than this value (and the S/N in the priority band is less than minSN) "
1208  "use the band with the largest S/N as the reference band")
1209  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1210  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1211  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1212 
1213 
1219 
1220 
1222  """!
1223  \anchor MergeMeasurementsTask_
1224 
1225  \brief Merge measurements from multiple bands
1226 
1227  \section pipe_tasks_multiBand_Contents Contents
1228 
1229  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1230  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1231  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1232  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1233  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1234  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1235 
1236  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1237 
1238  Command-line task that merges measurements from multiple bands.
1239 
1240  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1241  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1242  required to have centroid, shape and flux measurements in each band.
1243 
1244  \par Inputs:
1245  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1246  \par Outputs:
1247  deepCoadd_ref{tract,patch}: SourceCatalog
1248  \par Data Unit:
1249  tract, patch
1250 
1251  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1252 
1253  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1254 
1255  \copydoc \_\_init\_\_
1256 
1257  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1258 
1259  \copydoc run
1260 
1261  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1262 
1263  See \ref MergeMeasurementsConfig_
1264 
1265  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1266 
1267  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1268  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1269  files.
1270 
1271  MergeMeasurementsTask has no debug variables.
1272 
1273  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1274 
1275  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1276  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1277  coadds and individual exposures.
1278  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1279  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1280  command line argument:
1281  \code
1282  mergeCoaddMeasurements.py --help
1283  \endcode
1284 
1285  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1286  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1287  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1288  as follows:
1289  \code
1290  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1291  \endcode
1292  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1293  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1294  """
1295  _DefaultName = "mergeCoaddMeasurements"
1296  ConfigClass = MergeMeasurementsConfig
1297  inputDataset = "meas"
1298  outputDataset = "ref"
1299  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1300 
1301  def __init__(self, butler=None, schema=None, **kwargs):
1302  """!
1303  Initialize the task.
1304 
1305  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1306  \param[in] schema: the schema of the detection catalogs used as input to this one
1307  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1308 
1309  The task will set its own self.schema attribute to the schema of the output merged catalog.
1310  """
1311  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1312  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1313  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1314  self.schemaMapper.addMinimalSchema(inputSchema, True)
1315  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1316  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1317  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1318 
1319  self.flagKeys = {}
1320  for band in self.config.priorityList:
1321  short = getShortFilterName(band)
1322  outputKey = self.schemaMapper.editOutputSchema().addField(
1323  "merge_measurement_%s" % short,
1324  type="Flag",
1325  doc="Flag field set if the measurements here are from the %s filter" % band
1326  )
1327  peakKey = inputSchema.find("merge_peak_%s" % short).key
1328  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1329  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1330  self.schema = self.schemaMapper.getOutputSchema()
1331 
1333  for filt in self.config.pseudoFilterList:
1334  try:
1335  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1336  except:
1337  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1338 
1339  self.badFlags = {}
1340  for flag in self.config.flags:
1341  try:
1342  self.badFlags[flag] = self.schema.find(flag).getKey()
1343  except KeyError as exc:
1344  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1345 
1346  def mergeCatalogs(self, catalogs, patchRef):
1347  """!
1348  Merge measurement catalogs to create a single reference catalog for forced photometry
1349 
1350  \param[in] catalogs: the catalogs to be merged
1351  \param[in] patchRef: patch reference for data
1352 
1353  For parent sources, we choose the first band in config.priorityList for which the
1354  merge_footprint flag for that band is is True.
1355 
1356  For child sources, the logic is the same, except that we use the merge_peak flags.
1357  """
1358  # Put catalogs, filters in priority order
1359  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1360  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1361 
1362  mergedCatalog = afwTable.SourceCatalog(self.schema)
1363  mergedCatalog.reserve(len(orderedCatalogs[0]))
1364 
1365  idKey = orderedCatalogs[0].table.getIdKey()
1366  for catalog in orderedCatalogs[1:]:
1367  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1368  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1369 
1370  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1371  # record for each band, in priority order.
1372  for orderedRecords in zip(*orderedCatalogs):
1373 
1374  maxSNRecord = None
1375  maxSNFlagKeys = None
1376  maxSN = 0.
1377  priorityRecord = None
1378  priorityFlagKeys = None
1379  prioritySN = 0.
1380  hasPseudoFilter = False
1381 
1382  # Now we iterate over those record-band pairs, keeping track of the priority and the
1383  # largest S/N band.
1384  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1385  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1386  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1387 
1388  if not (parent or child):
1389  for pseudoFilterKey in self.pseudoFilterKeys:
1390  if inputRecord.get(pseudoFilterKey):
1391  hasPseudoFilter = True
1392  priorityRecord = inputRecord
1393  priorityFlagKeys = flagKeys
1394  break
1395  if hasPseudoFilter:
1396  break
1397 
1398  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1399  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1400  sn = 0.
1401  else:
1402  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1403  if numpy.isnan(sn) or sn < 0.:
1404  sn = 0.
1405  if (parent or child) and priorityRecord is None:
1406  priorityRecord = inputRecord
1407  priorityFlagKeys = flagKeys
1408  prioritySN = sn
1409  if sn > maxSN:
1410  maxSNRecord = inputRecord
1411  maxSNFlagKeys = flagKeys
1412  maxSN = sn
1413 
1414  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1415  # the reference band instead. However, we only want to choose the highest S/N band if it is
1416  # significantly better than the priority band. Therefore, to choose a band other than the
1417  # priority, we require that the priority S/N is below the minimum threshold and that the
1418  # difference between the priority and highest S/N is larger than the difference threshold.
1419  #
1420  # For pseudo code objects we always choose the first band in the priority list.
1421  bestRecord = None
1422  bestFlagKeys = None
1423  if hasPseudoFilter:
1424  bestRecord = priorityRecord
1425  bestFlagKeys = priorityFlagKeys
1426  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1427  maxSNRecord is not None):
1428  bestRecord = maxSNRecord
1429  bestFlagKeys = maxSNFlagKeys
1430  elif priorityRecord is not None:
1431  bestRecord = priorityRecord
1432  bestFlagKeys = priorityFlagKeys
1433 
1434  if bestRecord is not None and bestFlagKeys is not None:
1435  outputRecord = mergedCatalog.addNew()
1436  outputRecord.assign(bestRecord, self.schemaMapper)
1437  outputRecord.set(bestFlagKeys.output, True)
1438  else: # if we didn't find any records
1439  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1440  inputRecord.getId())
1441 
1442  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1443  for inputCatalog in orderedCatalogs:
1444  if len(mergedCatalog) != len(inputCatalog):
1445  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1446  (len(mergedCatalog), len(orderedCatalogs)))
1447 
1448  return mergedCatalog
Merge coadd detections from multiple bands.
Definition: multiBand.py:596
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:330
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:433
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:787
def getSkySourceFootprints(self, mergedList, skyInfo, growDetectedFootprints=0)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:798
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:759
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1088
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:322
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:306
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:446
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1301
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:245
Merge measurements from multiple bands.
Definition: multiBand.py:1221
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:459
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:922
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1154
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:263
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1346
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:485
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1190
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1134
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:555
Configuration for merging sources.
Definition: multiBand.py:374
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:496
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:705
A base class for merging source catalogs.
Definition: multiBand.py:390
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1174
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:470
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:858
def runDetection(self, exposure, idFactory)
Run detection on an exposure.
Definition: multiBand.py:277
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:249
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:513
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1044
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:683
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:348