lsst.pipe.tasks  14.0-57-ga659d1f3+2
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo
35 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
36 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
37 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
38 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
39 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
40 import lsst.afw.image as afwImage
41 import lsst.afw.table as afwTable
42 import lsst.afw.math as afwMath
43 import lsst.afw.geom as afwGeom
44 import lsst.afw.detection as afwDetect
45 from lsst.daf.base import PropertyList
46 
47 """
48 New dataset types:
49 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
50 * deepCoadd_mergeDet: merged detections (tract, patch)
51 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
52 * deepCoadd_ref: reference sources (tract, patch)
53 All of these have associated *_schema catalogs that require no data ID and hold no records.
54 
55 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
56 the mergeDet, meas, and ref dataset Footprints:
57 * deepCoadd_peak_schema
58 """
59 
60 
61 def _makeGetSchemaCatalogs(datasetSuffix):
62  """Construct a getSchemaCatalogs instance method
63 
64  These are identical for most of the classes here, so we'll consolidate
65  the code.
66 
67  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
68  """
69 
70  def getSchemaCatalogs(self):
71  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
72  src = afwTable.SourceCatalog(self.schema)
73  if hasattr(self, "algMetadata"):
74  src.getTable().setMetadata(self.algMetadata)
75  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
76  return getSchemaCatalogs
77 
78 
79 def _makeMakeIdFactory(datasetName):
80  """Construct a makeIdFactory instance method
81 
82  These are identical for all the classes here, so this consolidates
83  the code.
84 
85  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
86  """
87 
88  def makeIdFactory(self, dataRef):
89  """Return an IdFactory for setting the detection identifiers
90 
91  The actual parameters used in the IdFactory are provided by
92  the butler (through the provided data reference.
93  """
94  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
95  expId = int(dataRef.get(self.config.coaddName + datasetName))
96  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
97  return makeIdFactory
98 
99 
101  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
102  """
103  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
104  # and its the only way I could get it to work.
105  return afwImage.Filter(name).getFilterProperty().getName()
106 
107 
108 
109 
111  """!
112  \anchor DetectCoaddSourcesConfig_
113 
114  \brief Configuration parameters for the DetectCoaddSourcesTask
115  """
116  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
117  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
118  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
119  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131  self.detection.reEstimateBackground = False
132  self.detection.background.useApprox = False
133  self.detection.background.binSize = 4096
134  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
136 
137 
143 
144 
145 class DetectCoaddSourcesTask(CmdLineTask):
146  """!
147  \anchor DetectCoaddSourcesTask_
148 
149  \brief Detect sources on a coadd
150 
151  \section pipe_tasks_multiBand_Contents Contents
152 
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
156  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
157  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
158  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
159 
160  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
161 
162  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
163 
164  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
165  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
166  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
167  propagate the full covariance matrix -- but it is simple and works well in practice.
168 
169  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
170  SourceDetectionTask_ "detection" subtask.
171 
172  \par Inputs:
173  deepCoadd{tract,patch,filter}: ExposureF
174  \par Outputs:
175  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
176  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
177  exposure (ExposureF)
178  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
179  \par Data Unit:
180  tract, patch, filter
181 
182  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
183  You can retarget this subtask if you wish.
184 
185  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
186 
187  \copydoc \_\_init\_\_
188 
189  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
190 
191  \copydoc run
192 
193  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
194 
195  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
196 
197  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
198 
199  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
200  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
201  files.
202 
203  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
204  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
205  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
206 
207  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
208  of using DetectCoaddSourcesTask
209 
210  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
211  the task is to update the background, detect all sources in a single band and generate a set of parent
212  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
213  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
214  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
215  calling detectCoaddSources.py with the `--help` command line argument:
216  \code
217  detectCoaddSources.py --help
218  \endcode
219 
220  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
221  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
222  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
223  \code
224  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
225  \endcode
226  that will process the HSC-I band data. The results are written to
227  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
228 
229  It is also necessary to run:
230  \code
231  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
232  \endcode
233  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
234  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
235  """
236  _DefaultName = "detectCoaddSources"
237  ConfigClass = DetectCoaddSourcesConfig
238  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
239  makeIdFactory = _makeMakeIdFactory("CoaddId")
240 
241  @classmethod
242  def _makeArgumentParser(cls):
243  parser = ArgumentParser(name=cls._DefaultName)
244  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
245  ContainerClass=ExistingCoaddDataIdContainer)
246  return parser
247 
248  def __init__(self, schema=None, **kwargs):
249  """!
250  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
251 
252  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
253 
254  \param[in] schema: initial schema for the output catalog, modified-in place to include all
255  fields set by this task. If None, the source minimal schema will be used.
256  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
257  """
258  CmdLineTask.__init__(self, **kwargs)
259  if schema is None:
260  schema = afwTable.SourceTable.makeMinimalSchema()
261  if self.config.doInsertFakes:
262  self.makeSubtask("insertFakes")
263  self.schema = schema
264  self.makeSubtask("detection", schema=self.schema)
265  if self.config.doScaleVariance:
266  self.makeSubtask("scaleVariance")
267 
268  def run(self, patchRef):
269  """!
270  \brief Run detection on a coadd.
271 
272  Invokes \ref runDetection and then uses \ref write to output the
273  results.
274 
275  \param[in] patchRef: data reference for patch
276  """
277  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
278  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
279  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
280  self.write(exposure, results, patchRef)
281  return results
282 
283  def runDetection(self, exposure, idFactory, expId):
284  """!
285  \brief Run detection on an exposure.
286 
287  First scale the variance plane to match the observed variance
288  using \ref ScaleVarianceTask. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
289  detect sources.
290 
291  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
292  depending on configuration).
293  \param[in] idFactory: IdFactory to set source identifiers
294  \param[in] expId: Exposure identifier (integer) for RNG seed
295 
296  \return a pipe.base.Struct with fields
297  - sources: catalog of detections
298  - backgrounds: list of backgrounds
299  """
300  if self.config.doScaleVariance:
301  varScale = self.scaleVariance.run(exposure.maskedImage)
302  exposure.getMetadata().add("variance_scale", varScale)
303  backgrounds = afwMath.BackgroundList()
304  if self.config.doInsertFakes:
305  self.insertFakes.run(exposure, background=backgrounds)
306  table = afwTable.SourceTable.make(self.schema, idFactory)
307  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
308  sources = detections.sources
309  fpSets = detections.fpSets
310  if hasattr(fpSets, "background") and fpSets.background:
311  for bg in fpSets.background:
312  backgrounds.append(bg)
313  return Struct(sources=sources, backgrounds=backgrounds)
314 
315  def write(self, exposure, results, patchRef):
316  """!
317  \brief Write out results from runDetection.
318 
319  \param[in] exposure: Exposure to write out
320  \param[in] results: Struct returned from runDetection
321  \param[in] patchRef: data reference for patch
322  """
323  coaddName = self.config.coaddName + "Coadd"
324  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
325  patchRef.put(results.sources, coaddName + "_det")
326  patchRef.put(exposure, coaddName + "_calexp")
327 
328 
329 
330 
331 class MergeSourcesRunner(TaskRunner):
332  """!
333  \anchor MergeSourcesRunner_
334 
335  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
336  requires a list of dataRefs rather than a single dataRef.
337  """
338 
339  def makeTask(self, parsedCmd=None, args=None):
340  """!
341  \brief Provide a butler to the Task constructor.
342 
343  \param[in] parsedCmd the parsed command
344  \param[in] args tuple of a list of data references and kwargs (un-used)
345  \throws RuntimeError if both parsedCmd & args are None
346  """
347  if parsedCmd is not None:
348  butler = parsedCmd.butler
349  elif args is not None:
350  dataRefList, kwargs = args
351  butler = dataRefList[0].getButler()
352  else:
353  raise RuntimeError("Neither parsedCmd or args specified")
354  return self.TaskClass(config=self.config, log=self.log, butler=butler)
355 
356  @staticmethod
357  def getTargetList(parsedCmd, **kwargs):
358  """!
359  \brief Provide a list of patch references for each patch.
360 
361  The patch references within the list will have different filters.
362 
363  \param[in] parsedCmd the parsed command
364  \param **kwargs key word arguments (unused)
365  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
366  filter
367  """
368  refList = {} # Will index this as refList[tract][patch][filter] = ref
369  for ref in parsedCmd.id.refList:
370  tract = ref.dataId["tract"]
371  patch = ref.dataId["patch"]
372  filter = ref.dataId["filter"]
373  if tract not in refList:
374  refList[tract] = {}
375  if patch not in refList[tract]:
376  refList[tract][patch] = {}
377  if filter in refList[tract][patch]:
378  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
379  refList[tract][patch][filter] = ref
380  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
381 
382 
383 class MergeSourcesConfig(Config):
384  """!
385  \anchor MergeSourcesConfig_
386 
387  \brief Configuration for merging sources.
388  """
389  priorityList = ListField(dtype=str, default=[],
390  doc="Priority-ordered list of bands for the merge.")
391  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
392 
393  def validate(self):
394  Config.validate(self)
395  if len(self.priorityList) == 0:
396  raise RuntimeError("No priority list provided")
397 
398 
399 class MergeSourcesTask(CmdLineTask):
400  """!
401  \anchor MergeSourcesTask_
402 
403  \brief A base class for merging source catalogs.
404 
405  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
406  so similar that it makes sense to re-use the code, in the form of this abstract base class.
407 
408  NB: Do not use this class directly. Instead use one of the child classes that inherit from
409  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
410  "MergeMeasurementsTask"
411 
412  Sub-classes should set the following class variables:
413  * `_DefaultName`: name of Task
414  * `inputDataset`: name of dataset to read
415  * `outputDataset`: name of dataset to write
416  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
417 
418  In addition, sub-classes must implement the mergeCatalogs method.
419  """
420  _DefaultName = None
421  ConfigClass = MergeSourcesConfig
422  RunnerClass = MergeSourcesRunner
423  inputDataset = None
424  outputDataset = None
425  getSchemaCatalogs = None
426 
427  @classmethod
428  def _makeArgumentParser(cls):
429  """!
430  \brief Create a suitable ArgumentParser.
431 
432  We will use the ArgumentParser to get a provide a list of data
433  references for patches; the RunnerClass will sort them into lists
434  of data references for the same patch
435  """
436  parser = ArgumentParser(name=cls._DefaultName)
437  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
438  ContainerClass=ExistingCoaddDataIdContainer,
439  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
440  return parser
441 
442  def getInputSchema(self, butler=None, schema=None):
443  """!
444  \brief Obtain the input schema either directly or froma butler reference.
445 
446  \param[in] butler butler reference to obtain the input schema from
447  \param[in] schema the input schema
448  """
449  if schema is None:
450  assert butler is not None, "Neither butler nor schema specified"
451  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
452  immediate=True).schema
453  return schema
454 
455  def __init__(self, butler=None, schema=None, **kwargs):
456  """!
457  \brief Initialize the task.
458 
459  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
460  \param[in] schema the schema of the detection catalogs used as input to this one
461  \param[in] butler a butler used to read the input schema from disk, if schema is None
462 
463  Derived classes should use the getInputSchema() method to handle the additional
464  arguments and retreive the actual input schema.
465  """
466  CmdLineTask.__init__(self, **kwargs)
467 
468  def run(self, patchRefList):
469  """!
470  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
471  subclasses that inherit from MergeSourcesTask.
472 
473  \param[in] patchRefList list of data references for each filter
474  """
475  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
476  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
477  self.write(patchRefList[0], mergedCatalog)
478 
479  def readCatalog(self, patchRef):
480  """!
481  \brief Read input catalog.
482 
483  We read the input dataset provided by the 'inputDataset'
484  class variable.
485 
486  \param[in] patchRef data reference for patch
487  \return tuple consisting of the filter name and the catalog
488  """
489  filterName = patchRef.dataId["filter"]
490  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
491  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
492  return filterName, catalog
493 
494  def mergeCatalogs(self, catalogs, patchRef):
495  """!
496  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
497  MergeSourcesTask.
498 
499  \param[in] catalogs dict mapping filter name to source catalog
500 
501  \return merged catalog
502  """
503  raise NotImplementedError()
504 
505  def write(self, patchRef, catalog):
506  """!
507  \brief Write the output.
508 
509  \param[in] patchRef data reference for patch
510  \param[in] catalog catalog
511 
512  We write as the dataset provided by the 'outputDataset'
513  class variable.
514  """
515  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
516  # since the filter isn't actually part of the data ID for the dataset we're saving,
517  # it's confusing to see it in the log message, even if the butler simply ignores it.
518  mergeDataId = patchRef.dataId.copy()
519  del mergeDataId["filter"]
520  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
521 
522  def writeMetadata(self, dataRefList):
523  """!
524  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
525  """
526  pass
527 
528 
529 class CullPeaksConfig(Config):
530  """!
531  \anchor CullPeaksConfig_
532 
533  \brief Configuration for culling garbage peaks after merging footprints.
534 
535  Peaks may also be culled after detection or during deblending; this configuration object
536  only deals with culling after merging Footprints.
537 
538  These cuts are based on three quantities:
539  - nBands: the number of bands in which the peak was detected
540  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
541  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
542 
543  The formula that identifie peaks to cull is:
544 
545  nBands < nBandsSufficient
546  AND (rank >= rankSufficient)
547  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
548 
549  To disable peak culling, simply set nBandsSufficient=1.
550  """
551 
552  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
553  doc="Always keep peaks detected in this many bands")
554  rankSufficient = RangeField(dtype=int, default=20, min=1,
555  doc="Always keep this many peaks in each family")
556  rankConsidered = RangeField(dtype=int, default=30, min=1,
557  doc=("Keep peaks with less than this rank that also match the "
558  "rankNormalizedConsidered condition."))
559  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
560  doc=("Keep peaks with less than this normalized rank that"
561  " also match the rankConsidered condition."))
562 
563 
565  """!
566  \anchor MergeDetectionsConfig_
567 
568  \brief Configuration parameters for the MergeDetectionsTask.
569  """
570  minNewPeak = Field(dtype=float, default=1,
571  doc="Minimum distance from closest peak to create a new one (in arcsec).")
572 
573  maxSamePeak = Field(dtype=float, default=0.3,
574  doc="When adding new catalogs to the merge, all peaks less than this distance "
575  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
576  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
577 
578  skyFilterName = Field(dtype=str, default="sky",
579  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
580  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
581  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
582 
583  def setDefaults(self):
584  MergeSourcesConfig.setDefaults(self)
585  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
586 
587 
588 
594 
595 
597  """!
598  \anchor MergeDetectionsTask_
599 
600  \brief Merge coadd detections from multiple bands.
601 
602  \section pipe_tasks_multiBand_Contents Contents
603 
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
608  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
609  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
610 
611  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
612 
613  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
614 
615  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
616  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
617  track of which band each source originates in.
618 
619  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
620  objects are culled as described in \ref CullPeaksConfig_.
621 
622  \par Inputs:
623  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
624  \par Outputs:
625  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
626  \par Data Unit:
627  tract, patch
628 
629  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
630 
631  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
632 
633  \copydoc \_\_init\_\_
634 
635  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
636 
637  \copydoc run
638 
639  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
640 
641  See \ref MergeDetectionsConfig_
642 
643  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
644 
645  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
646  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
647 
648  MergeDetectionsTask has no debug variables.
649 
650  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
651 
652  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
653  of the available bands.
654  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
655  chosen subset of filters.
656  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
657  and, eventually, perform forced photometry.
658  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
659  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
660  `--help` command line argument:
661  \code
662  mergeCoaddDetections.py --help
663  \endcode
664 
665  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
666  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
667  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
668  \code
669  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
670  \endcode
671  This will merge the HSC-I & -R band parent source catalogs and write the results to
672  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
673 
674  The next step in the multi-band processing procedure is
675  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
676  """
677  ConfigClass = MergeDetectionsConfig
678  _DefaultName = "mergeCoaddDetections"
679  inputDataset = "det"
680  outputDataset = "mergeDet"
681  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
682 
683  def __init__(self, butler=None, schema=None, **kwargs):
684  """!
685  \brief Initialize the merge detections task.
686 
687  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
688  merge the source catalogs.
689 
690  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
691  \param[in] schema the schema of the detection catalogs used as input to this one
692  \param[in] butler a butler used to read the input schema from disk, if schema is None
693  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
694 
695  The task will set its own self.schema attribute to the schema of the output merged catalog.
696  """
697  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
698  self.makeSubtask("skyObjects")
699  self.schema = self.getInputSchema(butler=butler, schema=schema)
700 
701  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
702  filterNames += [self.config.skyFilterName]
703  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
704 
705  def mergeCatalogs(self, catalogs, patchRef):
706  """!
707  \brief Merge multiple catalogs.
708 
709  After ordering the catalogs and filters in priority order,
710  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
711  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
712  garbage peaks detected around bright objects.
713 
714  \param[in] catalogs
715  \param[in] patchRef
716  \param[out] mergedList
717  """
718 
719  # Convert distance to tract coordinate
720  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
721  tractWcs = skyInfo.wcs
722  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
723  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
724 
725  # Put catalogs, filters in priority order
726  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
727  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
728  if band in catalogs.keys()]
729 
730  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
731  self.schema, self.makeIdFactory(patchRef),
732  samePeakDistance)
733 
734  #
735  # Add extra sources that correspond to blank sky
736  #
737  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
738  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
739  if skySourceFootprints:
740  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
741  for foot in skySourceFootprints:
742  s = mergedList.addNew()
743  s.setFootprint(foot)
744  s.set(key, True)
745 
746  # Sort Peaks from brightest to faintest
747  for record in mergedList:
748  record.getFootprint().sortPeaks()
749  self.log.info("Merged to %d sources" % len(mergedList))
750  # Attempt to remove garbage peaks
751  self.cullPeaks(mergedList)
752  return mergedList
753 
754  def cullPeaks(self, catalog):
755  """!
756  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
757 
758  \param[in] catalog Source catalog
759  """
760  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
761  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
762  totalPeaks = 0
763  culledPeaks = 0
764  for parentSource in catalog:
765  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
766  # to it (which is easier than deleting as we iterate).
767  keptPeaks = parentSource.getFootprint().getPeaks()
768  oldPeaks = list(keptPeaks)
769  keptPeaks.clear()
770  familySize = len(oldPeaks)
771  totalPeaks += familySize
772  for rank, peak in enumerate(oldPeaks):
773  if ((rank < self.config.cullPeaks.rankSufficient) or
774  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
775  (rank < self.config.cullPeaks.rankConsidered and
776  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
777  keptPeaks.append(peak)
778  else:
779  culledPeaks += 1
780  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
781 
782  def getSchemaCatalogs(self):
783  """!
784  Return a dict of empty catalogs for each catalog dataset produced by this task.
785 
786  \param[out] dictionary of empty catalogs
787  """
788  mergeDet = afwTable.SourceCatalog(self.schema)
789  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
790  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
791  self.config.coaddName + "Coadd_peak": peak}
792 
793  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
794  """!
795  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
796 
797  \param mergedList The merged Footprints from all the input bands
798  \param skyInfo A description of the patch
799  \param seed Seed for the random number generator
800  """
801  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
802  detected = mask.getPlaneBitMask("DETECTED")
803  for s in mergedList:
804  s.getFootprint().spans.setMask(mask, detected)
805 
806  footprints = self.skyObjects.run(mask, seed)
807  if not footprints:
808  return footprints
809 
810  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
811  schema = self.merged.getPeakSchema()
812  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
813  converted = []
814  for oldFoot in footprints:
815  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
816  peak = oldFoot.getPeaks()[0]
817  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
818  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
819  newFoot.getPeaks()[0].set(mergeKey, True)
820  converted.append(newFoot)
821 
822  return converted
823 
824 
826  """!
827  \anchor MeasureMergedCoaddSourcesConfig_
828 
829  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
830  """
831  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
832  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
833  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
834  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
835  doPropagateFlags = Field(
836  dtype=bool, default=True,
837  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
838  )
839  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
840  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
841  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
842  doWriteMatchesDenormalized = Field(
843  dtype=bool,
844  default=False,
845  doc=("Write reference matches in denormalized format? "
846  "This format uses more disk space, but is more convenient to read."),
847  )
848  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
849  checkUnitsParseStrict = Field(
850  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
851  dtype=str,
852  default="raise",
853  )
854  doApCorr = Field(
855  dtype=bool,
856  default=True,
857  doc="Apply aperture corrections"
858  )
859  applyApCorr = ConfigurableField(
860  target=ApplyApCorrTask,
861  doc="Subtask to apply aperture corrections"
862  )
863  doRunCatalogCalculation = Field(
864  dtype=bool,
865  default=True,
866  doc='Run catalogCalculation task'
867  )
868  catalogCalculation = ConfigurableField(
869  target=CatalogCalculationTask,
870  doc="Subtask to run catalogCalculation plugins on catalog"
871  )
872 
873  def setDefaults(self):
874  Config.setDefaults(self)
875  self.deblend.propagateAllPeaks = True
876  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
877  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
878  'INEXACT_PSF']
879  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
880  'INEXACT_PSF']
881 
882 
888 
889 
890 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
891  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
892  @staticmethod
893  def getTargetList(parsedCmd, **kwargs):
894  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
895 
896 
897 class MeasureMergedCoaddSourcesTask(CmdLineTask):
898  """!
899  \anchor MeasureMergedCoaddSourcesTask_
900 
901  \brief Deblend sources from master catalog in each coadd seperately and measure.
902 
903  \section pipe_tasks_multiBand_Contents Contents
904 
905  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
906  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
907  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
908  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
909  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
910  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
911 
912  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
913 
914  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
915  measurement in each coadd.
916 
917  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
918  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
919  consistent set of child sources.
920 
921  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
922  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
923  flags are propagated to the coadd sources.
924 
925  Optionally, we can match the coadd sources to an external reference catalog.
926 
927  \par Inputs:
928  deepCoadd_mergeDet{tract,patch}: SourceCatalog
929  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
930  \par Outputs:
931  deepCoadd_meas{tract,patch,filter}: SourceCatalog
932  \par Data Unit:
933  tract, patch, filter
934 
935  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
936 
937  <DL>
938  <DT> \ref SourceDeblendTask_ "deblend"
939  <DD> Deblend all the sources from the master catalog.</DD>
940  <DT> \ref SingleFrameMeasurementTask_ "measurement"
941  <DD> Measure source properties of deblended sources.</DD>
942  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
943  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
944  not at the edge of the field and that have either not been deblended or are the children of deblended
945  sources</DD>
946  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
947  <DD> Propagate flags set in individual visits to the coadd.</DD>
948  <DT> \ref DirectMatchTask_ "match"
949  <DD> Match input sources to a reference catalog (optional).
950  </DD>
951  </DL>
952  These subtasks may be retargeted as required.
953 
954  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
955 
956  \copydoc \_\_init\_\_
957 
958  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
959 
960  \copydoc run
961 
962  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
963 
964  See \ref MeasureMergedCoaddSourcesConfig_
965 
966  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
967 
968  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
969  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
970  files.
971 
972  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
973  the various sub-tasks. See the documetation for individual sub-tasks for more information.
974 
975  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
976  MeasureMergedCoaddSourcesTask
977 
978  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
979  The next stage in the multi-band processing procedure will merge these measurements into a suitable
980  catalog for driving forced photometry.
981 
982  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
983  to be processed.
984  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
985  `--help` command line argument:
986  \code
987  measureCoaddSources.py --help
988  \endcode
989 
990  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
991  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
992  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
993  coadd as follows:
994  \code
995  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
996  \endcode
997  This will process the HSC-I band data. The results are written in
998  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
999 
1000  It is also necessary to run
1001  \code
1002  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1003  \endcode
1004  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1005  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1006  """
1007  _DefaultName = "measureCoaddSources"
1008  ConfigClass = MeasureMergedCoaddSourcesConfig
1009  RunnerClass = MeasureMergedCoaddSourcesRunner
1010  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1011  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1012 
1013  @classmethod
1014  def _makeArgumentParser(cls):
1015  parser = ArgumentParser(name=cls._DefaultName)
1016  parser.add_id_argument("--id", "deepCoadd_calexp",
1017  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1018  ContainerClass=ExistingCoaddDataIdContainer)
1019  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
1020  return parser
1021 
1022  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1023  """!
1024  \brief Initialize the task.
1025 
1026  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1027  \param[in] schema: the schema of the merged detection catalog used as input to this one
1028  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1029  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1030  catalog. May be None if the loader can be constructed from the butler argument or all steps
1031  requiring a reference catalog are disabled.
1032  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1033  catalog loader, if schema or peakSchema or refObjLoader is None
1034 
1035  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1036  This will include all fields from the input schema, as well as additional fields for all the
1037  measurements.
1038  """
1039  CmdLineTask.__init__(self, **kwargs)
1040  if schema is None:
1041  assert butler is not None, "Neither butler nor schema is defined"
1042  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1043  self.schemaMapper = afwTable.SchemaMapper(schema)
1044  self.schemaMapper.addMinimalSchema(schema)
1045  self.schema = self.schemaMapper.getOutputSchema()
1047  if self.config.doDeblend:
1048  if peakSchema is None:
1049  assert butler is not None, "Neither butler nor peakSchema is defined"
1050  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1051  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1052  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1053  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1054  if self.config.doMatchSources:
1055  if refObjLoader is None:
1056  assert butler is not None, "Neither butler nor refObjLoader is defined"
1057  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1058  if self.config.doPropagateFlags:
1059  self.makeSubtask("propagateFlags", schema=self.schema)
1060  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1061  if self.config.doApCorr:
1062  self.makeSubtask("applyApCorr", schema=self.schema)
1063  if self.config.doRunCatalogCalculation:
1064  self.makeSubtask("catalogCalculation", schema=self.schema)
1065 
1066  def run(self, patchRef, psfCache=100):
1067  """!
1068  \brief Deblend and measure.
1069 
1070  \param[in] patchRef: Patch reference.
1071 
1072  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1073  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1074  Finally, write the deblended sources and measurements out.
1075  """
1076  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1077  exposure.getPsf().setCacheCapacity(psfCache)
1078  sources = self.readSources(patchRef)
1079  if self.config.doDeblend:
1080  self.deblend.run(exposure, sources)
1081 
1082  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1083  # catalog is non-contiguous so can't extract column
1084  numBig = sum((s.get(bigKey) for s in sources))
1085  if numBig > 0:
1086  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1087  (patchRef.dataId, numBig))
1088 
1089  table = sources.getTable()
1090  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1091 
1092  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1093 
1094  if self.config.doApCorr:
1095  self.applyApCorr.run(
1096  catalog=sources,
1097  apCorrMap=exposure.getInfo().getApCorrMap()
1098  )
1099 
1100  if self.config.doRunCatalogCalculation:
1101  self.catalogCalculation.run(sources)
1102 
1103  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1104  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1105  includeDeblend=self.config.doDeblend)
1106  if self.config.doPropagateFlags:
1107  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1108  exposure.getWcs())
1109  if self.config.doMatchSources:
1110  self.writeMatches(patchRef, exposure, sources)
1111  self.write(patchRef, sources)
1112 
1113  def readSources(self, dataRef):
1114  """!
1115  \brief Read input sources.
1116 
1117  \param[in] dataRef: Data reference for catalog of merged detections
1118  \return List of sources in merged catalog
1119 
1120  We also need to add columns to hold the measurements we're about to make
1121  so we can measure in-place.
1122  """
1123  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1124  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1125  idFactory = self.makeIdFactory(dataRef)
1126  for s in merged:
1127  idFactory.notify(s.getId())
1128  table = afwTable.SourceTable.make(self.schema, idFactory)
1129  sources = afwTable.SourceCatalog(table)
1130  sources.extend(merged, self.schemaMapper)
1131  return sources
1132 
1133  def writeMatches(self, dataRef, exposure, sources):
1134  """!
1135  \brief Write matches of the sources to the astrometric reference catalog.
1136 
1137  We use the Wcs in the exposure to match sources.
1138 
1139  \param[in] dataRef: data reference
1140  \param[in] exposure: exposure with Wcs
1141  \param[in] sources: source catalog
1142  """
1143  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1144  if result.matches:
1145  matches = afwTable.packMatches(result.matches)
1146  matches.table.setMetadata(result.matchMeta)
1147  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1148  if self.config.doWriteMatchesDenormalized:
1149  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1150  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1151 
1152  def write(self, dataRef, sources):
1153  """!
1154  \brief Write the source catalog.
1155 
1156  \param[in] dataRef: data reference
1157  \param[in] sources: source catalog
1158  """
1159  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1160  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1161 
1162  def getExposureId(self, dataRef):
1163  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1164 
1165 
1167  """!
1168  \anchor MergeMeasurementsConfig_
1169 
1170  \brief Configuration parameters for the MergeMeasurementsTask
1171  """
1172  pseudoFilterList = ListField(dtype=str, default=["sky"],
1173  doc="Names of filters which may have no associated detection\n"
1174  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1175  snName = Field(dtype=str, default="base_PsfFlux",
1176  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1177  minSN = Field(dtype=float, default=10.,
1178  doc="If the S/N from the priority band is below this value (and the S/N "
1179  "is larger than minSNDiff compared to the priority band), use the band with "
1180  "the largest S/N as the reference band.")
1181  minSNDiff = Field(dtype=float, default=3.,
1182  doc="If the difference in S/N between another band and the priority band is larger "
1183  "than this value (and the S/N in the priority band is less than minSN) "
1184  "use the band with the largest S/N as the reference band")
1185  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1186  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1187  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1188 
1189 
1195 
1196 
1198  """!
1199  \anchor MergeMeasurementsTask_
1200 
1201  \brief Merge measurements from multiple bands
1202 
1203  \section pipe_tasks_multiBand_Contents Contents
1204 
1205  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1206  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1207  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1208  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1209  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1210  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1211 
1212  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1213 
1214  Command-line task that merges measurements from multiple bands.
1215 
1216  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1217  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1218  required to have centroid, shape and flux measurements in each band.
1219 
1220  \par Inputs:
1221  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1222  \par Outputs:
1223  deepCoadd_ref{tract,patch}: SourceCatalog
1224  \par Data Unit:
1225  tract, patch
1226 
1227  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1228 
1229  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1230 
1231  \copydoc \_\_init\_\_
1232 
1233  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1234 
1235  \copydoc run
1236 
1237  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1238 
1239  See \ref MergeMeasurementsConfig_
1240 
1241  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1242 
1243  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1244  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1245  files.
1246 
1247  MergeMeasurementsTask has no debug variables.
1248 
1249  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1250  of using MergeMeasurementsTask
1251 
1252  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1253  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1254  coadds and individual exposures.
1255  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1256  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1257  command line argument:
1258  \code
1259  mergeCoaddMeasurements.py --help
1260  \endcode
1261 
1262  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1263  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1264  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1265  as follows:
1266  \code
1267  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1268  \endcode
1269  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1270  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1271  """
1272  _DefaultName = "mergeCoaddMeasurements"
1273  ConfigClass = MergeMeasurementsConfig
1274  inputDataset = "meas"
1275  outputDataset = "ref"
1276  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1277 
1278  def __init__(self, butler=None, schema=None, **kwargs):
1279  """!
1280  Initialize the task.
1281 
1282  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1283  \param[in] schema: the schema of the detection catalogs used as input to this one
1284  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1285 
1286  The task will set its own self.schema attribute to the schema of the output merged catalog.
1287  """
1288  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1289  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1290  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1291  self.schemaMapper.addMinimalSchema(inputSchema, True)
1292  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1293  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1294  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1295 
1296  self.flagKeys = {}
1297  for band in self.config.priorityList:
1298  short = getShortFilterName(band)
1299  outputKey = self.schemaMapper.editOutputSchema().addField(
1300  "merge_measurement_%s" % short,
1301  type="Flag",
1302  doc="Flag field set if the measurements here are from the %s filter" % band
1303  )
1304  peakKey = inputSchema.find("merge_peak_%s" % short).key
1305  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1306  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1307  self.schema = self.schemaMapper.getOutputSchema()
1308 
1310  for filt in self.config.pseudoFilterList:
1311  try:
1312  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1313  except:
1314  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1315 
1316  self.badFlags = {}
1317  for flag in self.config.flags:
1318  try:
1319  self.badFlags[flag] = self.schema.find(flag).getKey()
1320  except KeyError as exc:
1321  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1322 
1323  def mergeCatalogs(self, catalogs, patchRef):
1324  """!
1325  Merge measurement catalogs to create a single reference catalog for forced photometry
1326 
1327  \param[in] catalogs: the catalogs to be merged
1328  \param[in] patchRef: patch reference for data
1329 
1330  For parent sources, we choose the first band in config.priorityList for which the
1331  merge_footprint flag for that band is is True.
1332 
1333  For child sources, the logic is the same, except that we use the merge_peak flags.
1334  """
1335  # Put catalogs, filters in priority order
1336  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1337  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1338 
1339  mergedCatalog = afwTable.SourceCatalog(self.schema)
1340  mergedCatalog.reserve(len(orderedCatalogs[0]))
1341 
1342  idKey = orderedCatalogs[0].table.getIdKey()
1343  for catalog in orderedCatalogs[1:]:
1344  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1345  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1346 
1347  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1348  # record for each band, in priority order.
1349  for orderedRecords in zip(*orderedCatalogs):
1350 
1351  maxSNRecord = None
1352  maxSNFlagKeys = None
1353  maxSN = 0.
1354  priorityRecord = None
1355  priorityFlagKeys = None
1356  prioritySN = 0.
1357  hasPseudoFilter = False
1358 
1359  # Now we iterate over those record-band pairs, keeping track of the priority and the
1360  # largest S/N band.
1361  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1362  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1363  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1364 
1365  if not (parent or child):
1366  for pseudoFilterKey in self.pseudoFilterKeys:
1367  if inputRecord.get(pseudoFilterKey):
1368  hasPseudoFilter = True
1369  priorityRecord = inputRecord
1370  priorityFlagKeys = flagKeys
1371  break
1372  if hasPseudoFilter:
1373  break
1374 
1375  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1376  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1377  sn = 0.
1378  else:
1379  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1380  if numpy.isnan(sn) or sn < 0.:
1381  sn = 0.
1382  if (parent or child) and priorityRecord is None:
1383  priorityRecord = inputRecord
1384  priorityFlagKeys = flagKeys
1385  prioritySN = sn
1386  if sn > maxSN:
1387  maxSNRecord = inputRecord
1388  maxSNFlagKeys = flagKeys
1389  maxSN = sn
1390 
1391  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1392  # the reference band instead. However, we only want to choose the highest S/N band if it is
1393  # significantly better than the priority band. Therefore, to choose a band other than the
1394  # priority, we require that the priority S/N is below the minimum threshold and that the
1395  # difference between the priority and highest S/N is larger than the difference threshold.
1396  #
1397  # For pseudo code objects we always choose the first band in the priority list.
1398  bestRecord = None
1399  bestFlagKeys = None
1400  if hasPseudoFilter:
1401  bestRecord = priorityRecord
1402  bestFlagKeys = priorityFlagKeys
1403  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1404  maxSNRecord is not None):
1405  bestRecord = maxSNRecord
1406  bestFlagKeys = maxSNFlagKeys
1407  elif priorityRecord is not None:
1408  bestRecord = priorityRecord
1409  bestFlagKeys = priorityFlagKeys
1410 
1411  if bestRecord is not None and bestFlagKeys is not None:
1412  outputRecord = mergedCatalog.addNew()
1413  outputRecord.assign(bestRecord, self.schemaMapper)
1414  outputRecord.set(bestFlagKeys.output, True)
1415  else: # if we didn't find any records
1416  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1417  inputRecord.getId())
1418 
1419  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1420  for inputCatalog in orderedCatalogs:
1421  if len(mergedCatalog) != len(inputCatalog):
1422  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1423  (len(mergedCatalog), len(orderedCatalogs)))
1424 
1425  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:793
Merge coadd detections from multiple bands.
Definition: multiBand.py:596
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:339
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:442
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:782
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:283
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:754
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:331
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:315
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:455
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1278
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:110
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:248
Merge measurements from multiple bands.
Definition: multiBand.py:1197
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:468
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:897
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1133
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:268
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1323
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:494
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1166
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1113
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:564
Configuration for merging sources.
Definition: multiBand.py:383
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:505
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:705
def run(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:1066
A base class for merging source catalogs.
Definition: multiBand.py:399
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1152
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:479
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:825
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:256
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:522
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1022
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:683
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:357