lsst.pipe.tasks  14.0-50-g5874b18d+1
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo
35 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
36 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
37 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
38 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
39 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
40 import lsst.afw.image as afwImage
41 import lsst.afw.table as afwTable
42 import lsst.afw.math as afwMath
43 import lsst.afw.geom as afwGeom
44 import lsst.afw.detection as afwDetect
45 from lsst.daf.base import PropertyList
46 
47 """
48 New dataset types:
49 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
50 * deepCoadd_mergeDet: merged detections (tract, patch)
51 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
52 * deepCoadd_ref: reference sources (tract, patch)
53 All of these have associated *_schema catalogs that require no data ID and hold no records.
54 
55 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
56 the mergeDet, meas, and ref dataset Footprints:
57 * deepCoadd_peak_schema
58 """
59 
60 
61 def _makeGetSchemaCatalogs(datasetSuffix):
62  """Construct a getSchemaCatalogs instance method
63 
64  These are identical for most of the classes here, so we'll consolidate
65  the code.
66 
67  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
68  """
69 
70  def getSchemaCatalogs(self):
71  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
72  src = afwTable.SourceCatalog(self.schema)
73  if hasattr(self, "algMetadata"):
74  src.getTable().setMetadata(self.algMetadata)
75  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
76  return getSchemaCatalogs
77 
78 
79 def _makeMakeIdFactory(datasetName):
80  """Construct a makeIdFactory instance method
81 
82  These are identical for all the classes here, so this consolidates
83  the code.
84 
85  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
86  """
87 
88  def makeIdFactory(self, dataRef):
89  """Return an IdFactory for setting the detection identifiers
90 
91  The actual parameters used in the IdFactory are provided by
92  the butler (through the provided data reference.
93  """
94  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
95  expId = int(dataRef.get(self.config.coaddName + datasetName))
96  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
97  return makeIdFactory
98 
99 
101  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
102  """
103  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
104  # and its the only way I could get it to work.
105  return afwImage.Filter(name).getFilterProperty().getName()
106 
107 
108 
109 
111  """!
112  \anchor DetectCoaddSourcesConfig_
113 
114  \brief Configuration parameters for the DetectCoaddSourcesTask
115  """
116  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
117  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
118  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
119  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131  self.detection.reEstimateBackground = False
132  self.detection.background.useApprox = False
133  self.detection.background.binSize = 4096
134  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
136 
137 
143 
144 
145 class DetectCoaddSourcesTask(CmdLineTask):
146  """!
147  \anchor DetectCoaddSourcesTask_
148 
149  \brief Detect sources on a coadd
150 
151  \section pipe_tasks_multiBand_Contents Contents
152 
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
156  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
157  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
158  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
159 
160  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
161 
162  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
163 
164  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
165  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
166  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
167  propagate the full covariance matrix -- but it is simple and works well in practice.
168 
169  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
170  SourceDetectionTask_ "detection" subtask.
171 
172  \par Inputs:
173  deepCoadd{tract,patch,filter}: ExposureF
174  \par Outputs:
175  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
176  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
177  exposure (ExposureF)
178  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
179  \par Data Unit:
180  tract, patch, filter
181 
182  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
183  You can retarget this subtask if you wish.
184 
185  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
186 
187  \copydoc \_\_init\_\_
188 
189  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
190 
191  \copydoc run
192 
193  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
194 
195  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
196 
197  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
198 
199  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
200  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
201  files.
202 
203  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
204  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
205  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
206 
207  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
208  of using DetectCoaddSourcesTask
209 
210  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
211  the task is to update the background, detect all sources in a single band and generate a set of parent
212  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
213  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
214  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
215  calling detectCoaddSources.py with the `--help` command line argument:
216  \code
217  detectCoaddSources.py --help
218  \endcode
219 
220  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
221  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
222  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
223  \code
224  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
225  \endcode
226  that will process the HSC-I band data. The results are written to
227  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
228 
229  It is also necessary to run:
230  \code
231  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
232  \endcode
233  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
234  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
235  """
236  _DefaultName = "detectCoaddSources"
237  ConfigClass = DetectCoaddSourcesConfig
238  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
239  makeIdFactory = _makeMakeIdFactory("CoaddId")
240 
241  @classmethod
242  def _makeArgumentParser(cls):
243  parser = ArgumentParser(name=cls._DefaultName)
244  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
245  ContainerClass=ExistingCoaddDataIdContainer)
246  return parser
247 
248  def __init__(self, schema=None, **kwargs):
249  """!
250  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
251 
252  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
253 
254  \param[in] schema: initial schema for the output catalog, modified-in place to include all
255  fields set by this task. If None, the source minimal schema will be used.
256  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
257  """
258  CmdLineTask.__init__(self, **kwargs)
259  if schema is None:
260  schema = afwTable.SourceTable.makeMinimalSchema()
261  if self.config.doInsertFakes:
262  self.makeSubtask("insertFakes")
263  self.schema = schema
264  self.makeSubtask("detection", schema=self.schema)
265  if self.config.doScaleVariance:
266  self.makeSubtask("scaleVariance")
267 
268  def run(self, patchRef):
269  """!
270  \brief Run detection on a coadd.
271 
272  Invokes \ref runDetection and then uses \ref write to output the
273  results.
274 
275  \param[in] patchRef: data reference for patch
276  """
277  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
278  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
279  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
280  self.write(exposure, results, patchRef)
281  return results
282 
283  def runDetection(self, exposure, idFactory, expId):
284  """!
285  \brief Run detection on an exposure.
286 
287  First scale the variance plane to match the observed variance
288  using \ref ScaleVarianceTask. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
289  detect sources.
290 
291  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
292  depending on configuration).
293  \param[in] idFactory: IdFactory to set source identifiers
294  \param[in] expId: Exposure identifier (integer) for RNG seed
295 
296  \return a pipe.base.Struct with fields
297  - sources: catalog of detections
298  - backgrounds: list of backgrounds
299  """
300  if self.config.doScaleVariance:
301  varScale = self.scaleVariance.run(exposure.maskedImage)
302  exposure.getMetadata().add("variance_scale", varScale)
303  backgrounds = afwMath.BackgroundList()
304  if self.config.doInsertFakes:
305  self.insertFakes.run(exposure, background=backgrounds)
306  table = afwTable.SourceTable.make(self.schema, idFactory)
307  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
308  sources = detections.sources
309  fpSets = detections.fpSets
310  if hasattr(fpSets, "background") and fpSets.background:
311  backgrounds.append(fpSets.background)
312  return Struct(sources=sources, backgrounds=backgrounds)
313 
314  def write(self, exposure, results, patchRef):
315  """!
316  \brief Write out results from runDetection.
317 
318  \param[in] exposure: Exposure to write out
319  \param[in] results: Struct returned from runDetection
320  \param[in] patchRef: data reference for patch
321  """
322  coaddName = self.config.coaddName + "Coadd"
323  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
324  patchRef.put(results.sources, coaddName + "_det")
325  patchRef.put(exposure, coaddName + "_calexp")
326 
327 
328 
329 
330 class MergeSourcesRunner(TaskRunner):
331  """!
332  \anchor MergeSourcesRunner_
333 
334  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
335  requires a list of dataRefs rather than a single dataRef.
336  """
337 
338  def makeTask(self, parsedCmd=None, args=None):
339  """!
340  \brief Provide a butler to the Task constructor.
341 
342  \param[in] parsedCmd the parsed command
343  \param[in] args tuple of a list of data references and kwargs (un-used)
344  \throws RuntimeError if both parsedCmd & args are None
345  """
346  if parsedCmd is not None:
347  butler = parsedCmd.butler
348  elif args is not None:
349  dataRefList, kwargs = args
350  butler = dataRefList[0].getButler()
351  else:
352  raise RuntimeError("Neither parsedCmd or args specified")
353  return self.TaskClass(config=self.config, log=self.log, butler=butler)
354 
355  @staticmethod
356  def getTargetList(parsedCmd, **kwargs):
357  """!
358  \brief Provide a list of patch references for each patch.
359 
360  The patch references within the list will have different filters.
361 
362  \param[in] parsedCmd the parsed command
363  \param **kwargs key word arguments (unused)
364  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
365  filter
366  """
367  refList = {} # Will index this as refList[tract][patch][filter] = ref
368  for ref in parsedCmd.id.refList:
369  tract = ref.dataId["tract"]
370  patch = ref.dataId["patch"]
371  filter = ref.dataId["filter"]
372  if tract not in refList:
373  refList[tract] = {}
374  if patch not in refList[tract]:
375  refList[tract][patch] = {}
376  if filter in refList[tract][patch]:
377  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
378  refList[tract][patch][filter] = ref
379  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
380 
381 
382 class MergeSourcesConfig(Config):
383  """!
384  \anchor MergeSourcesConfig_
385 
386  \brief Configuration for merging sources.
387  """
388  priorityList = ListField(dtype=str, default=[],
389  doc="Priority-ordered list of bands for the merge.")
390  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
391 
392  def validate(self):
393  Config.validate(self)
394  if len(self.priorityList) == 0:
395  raise RuntimeError("No priority list provided")
396 
397 
398 class MergeSourcesTask(CmdLineTask):
399  """!
400  \anchor MergeSourcesTask_
401 
402  \brief A base class for merging source catalogs.
403 
404  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
405  so similar that it makes sense to re-use the code, in the form of this abstract base class.
406 
407  NB: Do not use this class directly. Instead use one of the child classes that inherit from
408  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
409  "MergeMeasurementsTask"
410 
411  Sub-classes should set the following class variables:
412  * `_DefaultName`: name of Task
413  * `inputDataset`: name of dataset to read
414  * `outputDataset`: name of dataset to write
415  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
416 
417  In addition, sub-classes must implement the mergeCatalogs method.
418  """
419  _DefaultName = None
420  ConfigClass = MergeSourcesConfig
421  RunnerClass = MergeSourcesRunner
422  inputDataset = None
423  outputDataset = None
424  getSchemaCatalogs = None
425 
426  @classmethod
427  def _makeArgumentParser(cls):
428  """!
429  \brief Create a suitable ArgumentParser.
430 
431  We will use the ArgumentParser to get a provide a list of data
432  references for patches; the RunnerClass will sort them into lists
433  of data references for the same patch
434  """
435  parser = ArgumentParser(name=cls._DefaultName)
436  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
437  ContainerClass=ExistingCoaddDataIdContainer,
438  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
439  return parser
440 
441  def getInputSchema(self, butler=None, schema=None):
442  """!
443  \brief Obtain the input schema either directly or froma butler reference.
444 
445  \param[in] butler butler reference to obtain the input schema from
446  \param[in] schema the input schema
447  """
448  if schema is None:
449  assert butler is not None, "Neither butler nor schema specified"
450  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
451  immediate=True).schema
452  return schema
453 
454  def __init__(self, butler=None, schema=None, **kwargs):
455  """!
456  \brief Initialize the task.
457 
458  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
459  \param[in] schema the schema of the detection catalogs used as input to this one
460  \param[in] butler a butler used to read the input schema from disk, if schema is None
461 
462  Derived classes should use the getInputSchema() method to handle the additional
463  arguments and retreive the actual input schema.
464  """
465  CmdLineTask.__init__(self, **kwargs)
466 
467  def run(self, patchRefList):
468  """!
469  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
470  subclasses that inherit from MergeSourcesTask.
471 
472  \param[in] patchRefList list of data references for each filter
473  """
474  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
475  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
476  self.write(patchRefList[0], mergedCatalog)
477 
478  def readCatalog(self, patchRef):
479  """!
480  \brief Read input catalog.
481 
482  We read the input dataset provided by the 'inputDataset'
483  class variable.
484 
485  \param[in] patchRef data reference for patch
486  \return tuple consisting of the filter name and the catalog
487  """
488  filterName = patchRef.dataId["filter"]
489  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
490  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
491  return filterName, catalog
492 
493  def mergeCatalogs(self, catalogs, patchRef):
494  """!
495  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
496  MergeSourcesTask.
497 
498  \param[in] catalogs dict mapping filter name to source catalog
499 
500  \return merged catalog
501  """
502  raise NotImplementedError()
503 
504  def write(self, patchRef, catalog):
505  """!
506  \brief Write the output.
507 
508  \param[in] patchRef data reference for patch
509  \param[in] catalog catalog
510 
511  We write as the dataset provided by the 'outputDataset'
512  class variable.
513  """
514  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
515  # since the filter isn't actually part of the data ID for the dataset we're saving,
516  # it's confusing to see it in the log message, even if the butler simply ignores it.
517  mergeDataId = patchRef.dataId.copy()
518  del mergeDataId["filter"]
519  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
520 
521  def writeMetadata(self, dataRefList):
522  """!
523  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
524  """
525  pass
526 
527 
528 class CullPeaksConfig(Config):
529  """!
530  \anchor CullPeaksConfig_
531 
532  \brief Configuration for culling garbage peaks after merging footprints.
533 
534  Peaks may also be culled after detection or during deblending; this configuration object
535  only deals with culling after merging Footprints.
536 
537  These cuts are based on three quantities:
538  - nBands: the number of bands in which the peak was detected
539  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
540  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
541 
542  The formula that identifie peaks to cull is:
543 
544  nBands < nBandsSufficient
545  AND (rank >= rankSufficient)
546  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
547 
548  To disable peak culling, simply set nBandsSufficient=1.
549  """
550 
551  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
552  doc="Always keep peaks detected in this many bands")
553  rankSufficient = RangeField(dtype=int, default=20, min=1,
554  doc="Always keep this many peaks in each family")
555  rankConsidered = RangeField(dtype=int, default=30, min=1,
556  doc=("Keep peaks with less than this rank that also match the "
557  "rankNormalizedConsidered condition."))
558  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
559  doc=("Keep peaks with less than this normalized rank that"
560  " also match the rankConsidered condition."))
561 
562 
564  """!
565  \anchor MergeDetectionsConfig_
566 
567  \brief Configuration parameters for the MergeDetectionsTask.
568  """
569  minNewPeak = Field(dtype=float, default=1,
570  doc="Minimum distance from closest peak to create a new one (in arcsec).")
571 
572  maxSamePeak = Field(dtype=float, default=0.3,
573  doc="When adding new catalogs to the merge, all peaks less than this distance "
574  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
575  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
576 
577  skyFilterName = Field(dtype=str, default="sky",
578  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
579  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
580  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
581 
582  def setDefaults(self):
583  MergeSourcesConfig.setDefaults(self)
584  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
585 
586 
587 
593 
594 
596  """!
597  \anchor MergeDetectionsTask_
598 
599  \brief Merge coadd detections from multiple bands.
600 
601  \section pipe_tasks_multiBand_Contents Contents
602 
603  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
606  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
607  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
608  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
609 
610  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
611 
612  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
613 
614  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
615  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
616  track of which band each source originates in.
617 
618  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
619  objects are culled as described in \ref CullPeaksConfig_.
620 
621  \par Inputs:
622  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
623  \par Outputs:
624  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
625  \par Data Unit:
626  tract, patch
627 
628  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
629 
630  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
631 
632  \copydoc \_\_init\_\_
633 
634  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
635 
636  \copydoc run
637 
638  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
639 
640  See \ref MergeDetectionsConfig_
641 
642  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
643 
644  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
645  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
646 
647  MergeDetectionsTask has no debug variables.
648 
649  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
650 
651  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
652  of the available bands.
653  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
654  chosen subset of filters.
655  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
656  and, eventually, perform forced photometry.
657  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
658  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
659  `--help` command line argument:
660  \code
661  mergeCoaddDetections.py --help
662  \endcode
663 
664  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
665  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
666  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
667  \code
668  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
669  \endcode
670  This will merge the HSC-I & -R band parent source catalogs and write the results to
671  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
672 
673  The next step in the multi-band processing procedure is
674  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
675  """
676  ConfigClass = MergeDetectionsConfig
677  _DefaultName = "mergeCoaddDetections"
678  inputDataset = "det"
679  outputDataset = "mergeDet"
680  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
681 
682  def __init__(self, butler=None, schema=None, **kwargs):
683  """!
684  \brief Initialize the merge detections task.
685 
686  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
687  merge the source catalogs.
688 
689  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
690  \param[in] schema the schema of the detection catalogs used as input to this one
691  \param[in] butler a butler used to read the input schema from disk, if schema is None
692  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
693 
694  The task will set its own self.schema attribute to the schema of the output merged catalog.
695  """
696  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
697  self.makeSubtask("skyObjects")
698  self.schema = self.getInputSchema(butler=butler, schema=schema)
699 
700  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
701  filterNames += [self.config.skyFilterName]
702  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
703 
704  def mergeCatalogs(self, catalogs, patchRef):
705  """!
706  \brief Merge multiple catalogs.
707 
708  After ordering the catalogs and filters in priority order,
709  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
710  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
711  garbage peaks detected around bright objects.
712 
713  \param[in] catalogs
714  \param[in] patchRef
715  \param[out] mergedList
716  """
717 
718  # Convert distance to tract coordinate
719  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
720  tractWcs = skyInfo.wcs
721  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
722  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
723 
724  # Put catalogs, filters in priority order
725  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
726  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
727  if band in catalogs.keys()]
728 
729  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
730  self.schema, self.makeIdFactory(patchRef),
731  samePeakDistance)
732 
733  #
734  # Add extra sources that correspond to blank sky
735  #
736  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
737  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
738  if skySourceFootprints:
739  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
740  for foot in skySourceFootprints:
741  s = mergedList.addNew()
742  s.setFootprint(foot)
743  s.set(key, True)
744 
745  # Sort Peaks from brightest to faintest
746  for record in mergedList:
747  record.getFootprint().sortPeaks()
748  self.log.info("Merged to %d sources" % len(mergedList))
749  # Attempt to remove garbage peaks
750  self.cullPeaks(mergedList)
751  return mergedList
752 
753  def cullPeaks(self, catalog):
754  """!
755  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
756 
757  \param[in] catalog Source catalog
758  """
759  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
760  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
761  totalPeaks = 0
762  culledPeaks = 0
763  for parentSource in catalog:
764  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
765  # to it (which is easier than deleting as we iterate).
766  keptPeaks = parentSource.getFootprint().getPeaks()
767  oldPeaks = list(keptPeaks)
768  keptPeaks.clear()
769  familySize = len(oldPeaks)
770  totalPeaks += familySize
771  for rank, peak in enumerate(oldPeaks):
772  if ((rank < self.config.cullPeaks.rankSufficient) or
773  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
774  (rank < self.config.cullPeaks.rankConsidered and
775  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
776  keptPeaks.append(peak)
777  else:
778  culledPeaks += 1
779  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
780 
781  def getSchemaCatalogs(self):
782  """!
783  Return a dict of empty catalogs for each catalog dataset produced by this task.
784 
785  \param[out] dictionary of empty catalogs
786  """
787  mergeDet = afwTable.SourceCatalog(self.schema)
788  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
789  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
790  self.config.coaddName + "Coadd_peak": peak}
791 
792  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
793  """!
794  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
795 
796  \param mergedList The merged Footprints from all the input bands
797  \param skyInfo A description of the patch
798  \param seed Seed for the random number generator
799  """
800  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
801  detected = mask.getPlaneBitMask("DETECTED")
802  for s in mergedList:
803  s.getFootprint().spans.setMask(mask, detected)
804 
805  footprints = self.skyObjects.run(mask, seed)
806  if not footprints:
807  return footprints
808 
809  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
810  schema = self.merged.getPeakSchema()
811  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
812  converted = []
813  for oldFoot in footprints:
814  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
815  peak = oldFoot.getPeaks()[0]
816  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
817  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
818  newFoot.getPeaks()[0].set(mergeKey, True)
819  converted.append(newFoot)
820 
821  return converted
822 
823 
825  """!
826  \anchor MeasureMergedCoaddSourcesConfig_
827 
828  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
829  """
830  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
831  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
832  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
833  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
834  doPropagateFlags = Field(
835  dtype=bool, default=True,
836  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
837  )
838  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
839  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
840  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
841  doWriteMatchesDenormalized = Field(
842  dtype=bool,
843  default=False,
844  doc=("Write reference matches in denormalized format? "
845  "This format uses more disk space, but is more convenient to read."),
846  )
847  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
848  checkUnitsParseStrict = Field(
849  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
850  dtype=str,
851  default="raise",
852  )
853  doApCorr = Field(
854  dtype=bool,
855  default=True,
856  doc="Apply aperture corrections"
857  )
858  applyApCorr = ConfigurableField(
859  target=ApplyApCorrTask,
860  doc="Subtask to apply aperture corrections"
861  )
862  doRunCatalogCalculation = Field(
863  dtype=bool,
864  default=True,
865  doc='Run catalogCalculation task'
866  )
867  catalogCalculation = ConfigurableField(
868  target=CatalogCalculationTask,
869  doc="Subtask to run catalogCalculation plugins on catalog"
870  )
871 
872  def setDefaults(self):
873  Config.setDefaults(self)
874  self.deblend.propagateAllPeaks = True
875  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
876  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
877  'INEXACT_PSF']
878  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
879  'INEXACT_PSF']
880 
881 
887 
888 
889 class MeasureMergedCoaddSourcesTask(CmdLineTask):
890  """!
891  \anchor MeasureMergedCoaddSourcesTask_
892 
893  \brief Deblend sources from master catalog in each coadd seperately and measure.
894 
895  \section pipe_tasks_multiBand_Contents Contents
896 
897  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
898  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
899  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
900  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
901  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
902  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
903 
904  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
905 
906  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
907  measurement in each coadd.
908 
909  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
910  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
911  consistent set of child sources.
912 
913  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
914  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
915  flags are propagated to the coadd sources.
916 
917  Optionally, we can match the coadd sources to an external reference catalog.
918 
919  \par Inputs:
920  deepCoadd_mergeDet{tract,patch}: SourceCatalog
921  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
922  \par Outputs:
923  deepCoadd_meas{tract,patch,filter}: SourceCatalog
924  \par Data Unit:
925  tract, patch, filter
926 
927  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
928 
929  <DL>
930  <DT> \ref SourceDeblendTask_ "deblend"
931  <DD> Deblend all the sources from the master catalog.</DD>
932  <DT> \ref SingleFrameMeasurementTask_ "measurement"
933  <DD> Measure source properties of deblended sources.</DD>
934  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
935  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
936  not at the edge of the field and that have either not been deblended or are the children of deblended
937  sources</DD>
938  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
939  <DD> Propagate flags set in individual visits to the coadd.</DD>
940  <DT> \ref DirectMatchTask_ "match"
941  <DD> Match input sources to a reference catalog (optional).
942  </DD>
943  </DL>
944  These subtasks may be retargeted as required.
945 
946  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
947 
948  \copydoc \_\_init\_\_
949 
950  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
951 
952  \copydoc run
953 
954  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
955 
956  See \ref MeasureMergedCoaddSourcesConfig_
957 
958  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
959 
960  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
961  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
962  files.
963 
964  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
965  the various sub-tasks. See the documetation for individual sub-tasks for more information.
966 
967  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
968  MeasureMergedCoaddSourcesTask
969 
970  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
971  The next stage in the multi-band processing procedure will merge these measurements into a suitable
972  catalog for driving forced photometry.
973 
974  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
975  to be processed.
976  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
977  `--help` command line argument:
978  \code
979  measureCoaddSources.py --help
980  \endcode
981 
982  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
983  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
984  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
985  coadd as follows:
986  \code
987  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
988  \endcode
989  This will process the HSC-I band data. The results are written in
990  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
991 
992  It is also necessary to run
993  \code
994  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
995  \endcode
996  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
997  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
998  """
999  _DefaultName = "measureCoaddSources"
1000  ConfigClass = MeasureMergedCoaddSourcesConfig
1001  RunnerClass = ButlerInitializedTaskRunner
1002  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1003  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1004 
1005  @classmethod
1006  def _makeArgumentParser(cls):
1007  parser = ArgumentParser(name=cls._DefaultName)
1008  parser.add_id_argument("--id", "deepCoadd_calexp",
1009  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1010  ContainerClass=ExistingCoaddDataIdContainer)
1011  return parser
1012 
1013  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1014  """!
1015  \brief Initialize the task.
1016 
1017  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1018  \param[in] schema: the schema of the merged detection catalog used as input to this one
1019  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1020  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1021  catalog. May be None if the loader can be constructed from the butler argument or all steps
1022  requiring a reference catalog are disabled.
1023  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1024  catalog loader, if schema or peakSchema or refObjLoader is None
1025 
1026  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1027  This will include all fields from the input schema, as well as additional fields for all the
1028  measurements.
1029  """
1030  CmdLineTask.__init__(self, **kwargs)
1031  if schema is None:
1032  assert butler is not None, "Neither butler nor schema is defined"
1033  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1034  self.schemaMapper = afwTable.SchemaMapper(schema)
1035  self.schemaMapper.addMinimalSchema(schema)
1036  self.schema = self.schemaMapper.getOutputSchema()
1038  if self.config.doDeblend:
1039  if peakSchema is None:
1040  assert butler is not None, "Neither butler nor peakSchema is defined"
1041  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1042  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1043  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1044  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1045  if self.config.doMatchSources:
1046  if refObjLoader is None:
1047  assert butler is not None, "Neither butler nor refObjLoader is defined"
1048  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1049  if self.config.doPropagateFlags:
1050  self.makeSubtask("propagateFlags", schema=self.schema)
1051  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1052  if self.config.doApCorr:
1053  self.makeSubtask("applyApCorr", schema=self.schema)
1054  if self.config.doRunCatalogCalculation:
1055  self.makeSubtask("catalogCalculation", schema=self.schema)
1056 
1057  def run(self, patchRef):
1058  """!
1059  \brief Deblend and measure.
1060 
1061  \param[in] patchRef: Patch reference.
1062 
1063  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1064  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1065  Finally, write the deblended sources and measurements out.
1066  """
1067  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1068  sources = self.readSources(patchRef)
1069  if self.config.doDeblend:
1070  self.deblend.run(exposure, sources)
1071 
1072  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1073  # catalog is non-contiguous so can't extract column
1074  numBig = sum((s.get(bigKey) for s in sources))
1075  if numBig > 0:
1076  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1077  (patchRef.dataId, numBig))
1078 
1079  table = sources.getTable()
1080  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1081 
1082  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1083 
1084  if self.config.doApCorr:
1085  self.applyApCorr.run(
1086  catalog=sources,
1087  apCorrMap=exposure.getInfo().getApCorrMap()
1088  )
1089 
1090  if self.config.doRunCatalogCalculation:
1091  self.catalogCalculation.run(sources)
1092 
1093  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1094  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1095  includeDeblend=self.config.doDeblend)
1096  if self.config.doPropagateFlags:
1097  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1098  exposure.getWcs())
1099  if self.config.doMatchSources:
1100  self.writeMatches(patchRef, exposure, sources)
1101  self.write(patchRef, sources)
1102 
1103  def readSources(self, dataRef):
1104  """!
1105  \brief Read input sources.
1106 
1107  \param[in] dataRef: Data reference for catalog of merged detections
1108  \return List of sources in merged catalog
1109 
1110  We also need to add columns to hold the measurements we're about to make
1111  so we can measure in-place.
1112  """
1113  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1114  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1115  idFactory = self.makeIdFactory(dataRef)
1116  for s in merged:
1117  idFactory.notify(s.getId())
1118  table = afwTable.SourceTable.make(self.schema, idFactory)
1119  sources = afwTable.SourceCatalog(table)
1120  sources.extend(merged, self.schemaMapper)
1121  return sources
1122 
1123  def writeMatches(self, dataRef, exposure, sources):
1124  """!
1125  \brief Write matches of the sources to the astrometric reference catalog.
1126 
1127  We use the Wcs in the exposure to match sources.
1128 
1129  \param[in] dataRef: data reference
1130  \param[in] exposure: exposure with Wcs
1131  \param[in] sources: source catalog
1132  """
1133  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1134  if result.matches:
1135  matches = afwTable.packMatches(result.matches)
1136  matches.table.setMetadata(result.matchMeta)
1137  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1138  if self.config.doWriteMatchesDenormalized:
1139  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1140  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1141 
1142  def write(self, dataRef, sources):
1143  """!
1144  \brief Write the source catalog.
1145 
1146  \param[in] dataRef: data reference
1147  \param[in] sources: source catalog
1148  """
1149  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1150  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1151 
1152  def getExposureId(self, dataRef):
1153  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1154 
1155 
1157  """!
1158  \anchor MergeMeasurementsConfig_
1159 
1160  \brief Configuration parameters for the MergeMeasurementsTask
1161  """
1162  pseudoFilterList = ListField(dtype=str, default=["sky"],
1163  doc="Names of filters which may have no associated detection\n"
1164  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1165  snName = Field(dtype=str, default="base_PsfFlux",
1166  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1167  minSN = Field(dtype=float, default=10.,
1168  doc="If the S/N from the priority band is below this value (and the S/N "
1169  "is larger than minSNDiff compared to the priority band), use the band with "
1170  "the largest S/N as the reference band.")
1171  minSNDiff = Field(dtype=float, default=3.,
1172  doc="If the difference in S/N between another band and the priority band is larger "
1173  "than this value (and the S/N in the priority band is less than minSN) "
1174  "use the band with the largest S/N as the reference band")
1175  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1176  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1177  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1178 
1179 
1185 
1186 
1188  """!
1189  \anchor MergeMeasurementsTask_
1190 
1191  \brief Merge measurements from multiple bands
1192 
1193  \section pipe_tasks_multiBand_Contents Contents
1194 
1195  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1196  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1197  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1198  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1199  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1200  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1201 
1202  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1203 
1204  Command-line task that merges measurements from multiple bands.
1205 
1206  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1207  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1208  required to have centroid, shape and flux measurements in each band.
1209 
1210  \par Inputs:
1211  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1212  \par Outputs:
1213  deepCoadd_ref{tract,patch}: SourceCatalog
1214  \par Data Unit:
1215  tract, patch
1216 
1217  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1218 
1219  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1220 
1221  \copydoc \_\_init\_\_
1222 
1223  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1224 
1225  \copydoc run
1226 
1227  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1228 
1229  See \ref MergeMeasurementsConfig_
1230 
1231  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1232 
1233  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1234  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1235  files.
1236 
1237  MergeMeasurementsTask has no debug variables.
1238 
1239  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1240  of using MergeMeasurementsTask
1241 
1242  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1243  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1244  coadds and individual exposures.
1245  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1246  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1247  command line argument:
1248  \code
1249  mergeCoaddMeasurements.py --help
1250  \endcode
1251 
1252  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1253  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1254  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1255  as follows:
1256  \code
1257  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1258  \endcode
1259  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1260  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1261  """
1262  _DefaultName = "mergeCoaddMeasurements"
1263  ConfigClass = MergeMeasurementsConfig
1264  inputDataset = "meas"
1265  outputDataset = "ref"
1266  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1267 
1268  def __init__(self, butler=None, schema=None, **kwargs):
1269  """!
1270  Initialize the task.
1271 
1272  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1273  \param[in] schema: the schema of the detection catalogs used as input to this one
1274  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1275 
1276  The task will set its own self.schema attribute to the schema of the output merged catalog.
1277  """
1278  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1279  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1280  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1281  self.schemaMapper.addMinimalSchema(inputSchema, True)
1282  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1283  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1284  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1285 
1286  self.flagKeys = {}
1287  for band in self.config.priorityList:
1288  short = getShortFilterName(band)
1289  outputKey = self.schemaMapper.editOutputSchema().addField(
1290  "merge_measurement_%s" % short,
1291  type="Flag",
1292  doc="Flag field set if the measurements here are from the %s filter" % band
1293  )
1294  peakKey = inputSchema.find("merge_peak_%s" % short).key
1295  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1296  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1297  self.schema = self.schemaMapper.getOutputSchema()
1298 
1300  for filt in self.config.pseudoFilterList:
1301  try:
1302  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1303  except:
1304  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1305 
1306  self.badFlags = {}
1307  for flag in self.config.flags:
1308  try:
1309  self.badFlags[flag] = self.schema.find(flag).getKey()
1310  except KeyError as exc:
1311  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1312 
1313  def mergeCatalogs(self, catalogs, patchRef):
1314  """!
1315  Merge measurement catalogs to create a single reference catalog for forced photometry
1316 
1317  \param[in] catalogs: the catalogs to be merged
1318  \param[in] patchRef: patch reference for data
1319 
1320  For parent sources, we choose the first band in config.priorityList for which the
1321  merge_footprint flag for that band is is True.
1322 
1323  For child sources, the logic is the same, except that we use the merge_peak flags.
1324  """
1325  # Put catalogs, filters in priority order
1326  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1327  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1328 
1329  mergedCatalog = afwTable.SourceCatalog(self.schema)
1330  mergedCatalog.reserve(len(orderedCatalogs[0]))
1331 
1332  idKey = orderedCatalogs[0].table.getIdKey()
1333  for catalog in orderedCatalogs[1:]:
1334  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1335  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1336 
1337  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1338  # record for each band, in priority order.
1339  for orderedRecords in zip(*orderedCatalogs):
1340 
1341  maxSNRecord = None
1342  maxSNFlagKeys = None
1343  maxSN = 0.
1344  priorityRecord = None
1345  priorityFlagKeys = None
1346  prioritySN = 0.
1347  hasPseudoFilter = False
1348 
1349  # Now we iterate over those record-band pairs, keeping track of the priority and the
1350  # largest S/N band.
1351  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1352  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1353  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1354 
1355  if not (parent or child):
1356  for pseudoFilterKey in self.pseudoFilterKeys:
1357  if inputRecord.get(pseudoFilterKey):
1358  hasPseudoFilter = True
1359  priorityRecord = inputRecord
1360  priorityFlagKeys = flagKeys
1361  break
1362  if hasPseudoFilter:
1363  break
1364 
1365  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1366  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1367  sn = 0.
1368  else:
1369  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1370  if numpy.isnan(sn) or sn < 0.:
1371  sn = 0.
1372  if (parent or child) and priorityRecord is None:
1373  priorityRecord = inputRecord
1374  priorityFlagKeys = flagKeys
1375  prioritySN = sn
1376  if sn > maxSN:
1377  maxSNRecord = inputRecord
1378  maxSNFlagKeys = flagKeys
1379  maxSN = sn
1380 
1381  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1382  # the reference band instead. However, we only want to choose the highest S/N band if it is
1383  # significantly better than the priority band. Therefore, to choose a band other than the
1384  # priority, we require that the priority S/N is below the minimum threshold and that the
1385  # difference between the priority and highest S/N is larger than the difference threshold.
1386  #
1387  # For pseudo code objects we always choose the first band in the priority list.
1388  bestRecord = None
1389  bestFlagKeys = None
1390  if hasPseudoFilter:
1391  bestRecord = priorityRecord
1392  bestFlagKeys = priorityFlagKeys
1393  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1394  maxSNRecord is not None):
1395  bestRecord = maxSNRecord
1396  bestFlagKeys = maxSNFlagKeys
1397  elif priorityRecord is not None:
1398  bestRecord = priorityRecord
1399  bestFlagKeys = priorityFlagKeys
1400 
1401  if bestRecord is not None and bestFlagKeys is not None:
1402  outputRecord = mergedCatalog.addNew()
1403  outputRecord.assign(bestRecord, self.schemaMapper)
1404  outputRecord.set(bestFlagKeys.output, True)
1405  else: # if we didn't find any records
1406  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1407  inputRecord.getId())
1408 
1409  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1410  for inputCatalog in orderedCatalogs:
1411  if len(mergedCatalog) != len(inputCatalog):
1412  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1413  (len(mergedCatalog), len(orderedCatalogs)))
1414 
1415  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:792
Merge coadd detections from multiple bands.
Definition: multiBand.py:595
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:338
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:441
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:781
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:283
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:753
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1057
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:330
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:314
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:454
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1268
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:110
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:248
Merge measurements from multiple bands.
Definition: multiBand.py:1187
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:467
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:889
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1123
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:268
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1313
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:493
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1156
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1103
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:563
Configuration for merging sources.
Definition: multiBand.py:382
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:504
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:704
A base class for merging source catalogs.
Definition: multiBand.py:398
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1142
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:478
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:824
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:256
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:521
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1013
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:682
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:356