lsst.pipe.tasks  14.0-47-gd71a8703
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
35 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
36 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131  self.detection.reEstimateBackground = False
132  self.detection.background.useApprox = False
133  self.detection.background.binSize = 4096
134  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135 
136 
142 
143 
144 class DetectCoaddSourcesTask(CmdLineTask):
145  """!
146  \anchor DetectCoaddSourcesTask_
147 
148  \brief Detect sources on a coadd
149 
150  \section pipe_tasks_multiBand_Contents Contents
151 
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
156  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
157  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
158 
159  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
160 
161  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
162 
163  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
164  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
165  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
166  propagate the full covariance matrix -- but it is simple and works well in practice.
167 
168  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
169  SourceDetectionTask_ "detection" subtask.
170 
171  \par Inputs:
172  deepCoadd{tract,patch,filter}: ExposureF
173  \par Outputs:
174  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
175  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
176  exposure (ExposureF)
177  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
178  \par Data Unit:
179  tract, patch, filter
180 
181  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
182  You can retarget this subtask if you wish.
183 
184  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
185 
186  \copydoc \_\_init\_\_
187 
188  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
189 
190  \copydoc run
191 
192  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
193 
194  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
195 
196  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
197 
198  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
199  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
200  files.
201 
202  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
203  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
204  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
205 
206  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
207  of using DetectCoaddSourcesTask
208 
209  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
210  the task is to update the background, detect all sources in a single band and generate a set of parent
211  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
212  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
213  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
214  calling detectCoaddSources.py with the `--help` command line argument:
215  \code
216  detectCoaddSources.py --help
217  \endcode
218 
219  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
220  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
221  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
222  \code
223  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
224  \endcode
225  that will process the HSC-I band data. The results are written to
226  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
227 
228  It is also necessary to run:
229  \code
230  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
231  \endcode
232  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
233  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
234  """
235  _DefaultName = "detectCoaddSources"
236  ConfigClass = DetectCoaddSourcesConfig
237  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
238  makeIdFactory = _makeMakeIdFactory("CoaddId")
239 
240  @classmethod
241  def _makeArgumentParser(cls):
242  parser = ArgumentParser(name=cls._DefaultName)
243  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
244  ContainerClass=ExistingCoaddDataIdContainer)
245  return parser
246 
247  def __init__(self, schema=None, **kwargs):
248  """!
249  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
250 
251  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
252 
253  \param[in] schema: initial schema for the output catalog, modified-in place to include all
254  fields set by this task. If None, the source minimal schema will be used.
255  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
256  """
257  CmdLineTask.__init__(self, **kwargs)
258  if schema is None:
259  schema = afwTable.SourceTable.makeMinimalSchema()
260  if self.config.doInsertFakes:
261  self.makeSubtask("insertFakes")
262  self.schema = schema
263  self.makeSubtask("detection", schema=self.schema)
264 
265  def run(self, patchRef):
266  """!
267  \brief Run detection on a coadd.
268 
269  Invokes \ref runDetection and then uses \ref write to output the
270  results.
271 
272  \param[in] patchRef: data reference for patch
273  """
274  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
275  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
276  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
277  self.write(exposure, results, patchRef)
278  return results
279 
280  def runDetection(self, exposure, idFactory, expId):
281  """!
282  \brief Run detection on an exposure.
283 
284  First scale the variance plane to match the observed variance
285  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
286  detect sources.
287 
288  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
289  depending on configuration).
290  \param[in] idFactory: IdFactory to set source identifiers
291  \param[in] expId: Exposure identifier (integer) for RNG seed
292 
293  \return a pipe.base.Struct with fields
294  - sources: catalog of detections
295  - backgrounds: list of backgrounds
296  """
297  if self.config.doScaleVariance:
298  varScale = scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
299  exposure.getMetadata().add("variance_scale", varScale)
300  backgrounds = afwMath.BackgroundList()
301  if self.config.doInsertFakes:
302  self.insertFakes.run(exposure, background=backgrounds)
303  table = afwTable.SourceTable.make(self.schema, idFactory)
304  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
305  sources = detections.sources
306  fpSets = detections.fpSets
307  if hasattr(fpSets, "background") and fpSets.background:
308  backgrounds.append(fpSets.background)
309  return Struct(sources=sources, backgrounds=backgrounds)
310 
311  def write(self, exposure, results, patchRef):
312  """!
313  \brief Write out results from runDetection.
314 
315  \param[in] exposure: Exposure to write out
316  \param[in] results: Struct returned from runDetection
317  \param[in] patchRef: data reference for patch
318  """
319  coaddName = self.config.coaddName + "Coadd"
320  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
321  patchRef.put(results.sources, coaddName + "_det")
322  patchRef.put(exposure, coaddName + "_calexp")
323 
324 
325 
326 
327 class MergeSourcesRunner(TaskRunner):
328  """!
329  \anchor MergeSourcesRunner_
330 
331  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
332  requires a list of dataRefs rather than a single dataRef.
333  """
334 
335  def makeTask(self, parsedCmd=None, args=None):
336  """!
337  \brief Provide a butler to the Task constructor.
338 
339  \param[in] parsedCmd the parsed command
340  \param[in] args tuple of a list of data references and kwargs (un-used)
341  \throws RuntimeError if both parsedCmd & args are None
342  """
343  if parsedCmd is not None:
344  butler = parsedCmd.butler
345  elif args is not None:
346  dataRefList, kwargs = args
347  butler = dataRefList[0].getButler()
348  else:
349  raise RuntimeError("Neither parsedCmd or args specified")
350  return self.TaskClass(config=self.config, log=self.log, butler=butler)
351 
352  @staticmethod
353  def getTargetList(parsedCmd, **kwargs):
354  """!
355  \brief Provide a list of patch references for each patch.
356 
357  The patch references within the list will have different filters.
358 
359  \param[in] parsedCmd the parsed command
360  \param **kwargs key word arguments (unused)
361  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
362  filter
363  """
364  refList = {} # Will index this as refList[tract][patch][filter] = ref
365  for ref in parsedCmd.id.refList:
366  tract = ref.dataId["tract"]
367  patch = ref.dataId["patch"]
368  filter = ref.dataId["filter"]
369  if tract not in refList:
370  refList[tract] = {}
371  if patch not in refList[tract]:
372  refList[tract][patch] = {}
373  if filter in refList[tract][patch]:
374  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
375  refList[tract][patch][filter] = ref
376  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
377 
378 
379 class MergeSourcesConfig(Config):
380  """!
381  \anchor MergeSourcesConfig_
382 
383  \brief Configuration for merging sources.
384  """
385  priorityList = ListField(dtype=str, default=[],
386  doc="Priority-ordered list of bands for the merge.")
387  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
388 
389  def validate(self):
390  Config.validate(self)
391  if len(self.priorityList) == 0:
392  raise RuntimeError("No priority list provided")
393 
394 
395 class MergeSourcesTask(CmdLineTask):
396  """!
397  \anchor MergeSourcesTask_
398 
399  \brief A base class for merging source catalogs.
400 
401  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
402  so similar that it makes sense to re-use the code, in the form of this abstract base class.
403 
404  NB: Do not use this class directly. Instead use one of the child classes that inherit from
405  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
406  "MergeMeasurementsTask"
407 
408  Sub-classes should set the following class variables:
409  * `_DefaultName`: name of Task
410  * `inputDataset`: name of dataset to read
411  * `outputDataset`: name of dataset to write
412  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
413 
414  In addition, sub-classes must implement the mergeCatalogs method.
415  """
416  _DefaultName = None
417  ConfigClass = MergeSourcesConfig
418  RunnerClass = MergeSourcesRunner
419  inputDataset = None
420  outputDataset = None
421  getSchemaCatalogs = None
422 
423  @classmethod
424  def _makeArgumentParser(cls):
425  """!
426  \brief Create a suitable ArgumentParser.
427 
428  We will use the ArgumentParser to get a provide a list of data
429  references for patches; the RunnerClass will sort them into lists
430  of data references for the same patch
431  """
432  parser = ArgumentParser(name=cls._DefaultName)
433  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
434  ContainerClass=ExistingCoaddDataIdContainer,
435  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
436  return parser
437 
438  def getInputSchema(self, butler=None, schema=None):
439  """!
440  \brief Obtain the input schema either directly or froma butler reference.
441 
442  \param[in] butler butler reference to obtain the input schema from
443  \param[in] schema the input schema
444  """
445  if schema is None:
446  assert butler is not None, "Neither butler nor schema specified"
447  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
448  immediate=True).schema
449  return schema
450 
451  def __init__(self, butler=None, schema=None, **kwargs):
452  """!
453  \brief Initialize the task.
454 
455  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
456  \param[in] schema the schema of the detection catalogs used as input to this one
457  \param[in] butler a butler used to read the input schema from disk, if schema is None
458 
459  Derived classes should use the getInputSchema() method to handle the additional
460  arguments and retreive the actual input schema.
461  """
462  CmdLineTask.__init__(self, **kwargs)
463 
464  def run(self, patchRefList):
465  """!
466  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
467  subclasses that inherit from MergeSourcesTask.
468 
469  \param[in] patchRefList list of data references for each filter
470  """
471  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
472  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
473  self.write(patchRefList[0], mergedCatalog)
474 
475  def readCatalog(self, patchRef):
476  """!
477  \brief Read input catalog.
478 
479  We read the input dataset provided by the 'inputDataset'
480  class variable.
481 
482  \param[in] patchRef data reference for patch
483  \return tuple consisting of the filter name and the catalog
484  """
485  filterName = patchRef.dataId["filter"]
486  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
487  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
488  return filterName, catalog
489 
490  def mergeCatalogs(self, catalogs, patchRef):
491  """!
492  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
493  MergeSourcesTask.
494 
495  \param[in] catalogs dict mapping filter name to source catalog
496 
497  \return merged catalog
498  """
499  raise NotImplementedError()
500 
501  def write(self, patchRef, catalog):
502  """!
503  \brief Write the output.
504 
505  \param[in] patchRef data reference for patch
506  \param[in] catalog catalog
507 
508  We write as the dataset provided by the 'outputDataset'
509  class variable.
510  """
511  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
512  # since the filter isn't actually part of the data ID for the dataset we're saving,
513  # it's confusing to see it in the log message, even if the butler simply ignores it.
514  mergeDataId = patchRef.dataId.copy()
515  del mergeDataId["filter"]
516  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
517 
518  def writeMetadata(self, dataRefList):
519  """!
520  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
521  """
522  pass
523 
524 
525 class CullPeaksConfig(Config):
526  """!
527  \anchor CullPeaksConfig_
528 
529  \brief Configuration for culling garbage peaks after merging footprints.
530 
531  Peaks may also be culled after detection or during deblending; this configuration object
532  only deals with culling after merging Footprints.
533 
534  These cuts are based on three quantities:
535  - nBands: the number of bands in which the peak was detected
536  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
537  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
538 
539  The formula that identifie peaks to cull is:
540 
541  nBands < nBandsSufficient
542  AND (rank >= rankSufficient)
543  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
544 
545  To disable peak culling, simply set nBandsSufficient=1.
546  """
547 
548  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
549  doc="Always keep peaks detected in this many bands")
550  rankSufficient = RangeField(dtype=int, default=20, min=1,
551  doc="Always keep this many peaks in each family")
552  rankConsidered = RangeField(dtype=int, default=30, min=1,
553  doc=("Keep peaks with less than this rank that also match the "
554  "rankNormalizedConsidered condition."))
555  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
556  doc=("Keep peaks with less than this normalized rank that"
557  " also match the rankConsidered condition."))
558 
559 
561  """!
562  \anchor MergeDetectionsConfig_
563 
564  \brief Configuration parameters for the MergeDetectionsTask.
565  """
566  minNewPeak = Field(dtype=float, default=1,
567  doc="Minimum distance from closest peak to create a new one (in arcsec).")
568 
569  maxSamePeak = Field(dtype=float, default=0.3,
570  doc="When adding new catalogs to the merge, all peaks less than this distance "
571  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
572  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
573 
574  skyFilterName = Field(dtype=str, default="sky",
575  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
576  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
577  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
578 
579  def setDefaults(self):
580  MergeSourcesConfig.setDefaults(self)
581  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
582 
583 
584 
590 
591 
593  """!
594  \anchor MergeDetectionsTask_
595 
596  \brief Merge coadd detections from multiple bands.
597 
598  \section pipe_tasks_multiBand_Contents Contents
599 
600  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
601  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
602  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
603  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
605  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
606 
607  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
608 
609  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
610 
611  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
612  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
613  track of which band each source originates in.
614 
615  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
616  objects are culled as described in \ref CullPeaksConfig_.
617 
618  \par Inputs:
619  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
620  \par Outputs:
621  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
622  \par Data Unit:
623  tract, patch
624 
625  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
626 
627  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
628 
629  \copydoc \_\_init\_\_
630 
631  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
632 
633  \copydoc run
634 
635  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
636 
637  See \ref MergeDetectionsConfig_
638 
639  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
640 
641  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
642  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
643 
644  MergeDetectionsTask has no debug variables.
645 
646  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
647 
648  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
649  of the available bands.
650  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
651  chosen subset of filters.
652  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
653  and, eventually, perform forced photometry.
654  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
655  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
656  `--help` command line argument:
657  \code
658  mergeCoaddDetections.py --help
659  \endcode
660 
661  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
662  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
663  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
664  \code
665  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
666  \endcode
667  This will merge the HSC-I & -R band parent source catalogs and write the results to
668  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
669 
670  The next step in the multi-band processing procedure is
671  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
672  """
673  ConfigClass = MergeDetectionsConfig
674  _DefaultName = "mergeCoaddDetections"
675  inputDataset = "det"
676  outputDataset = "mergeDet"
677  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
678 
679  def __init__(self, butler=None, schema=None, **kwargs):
680  """!
681  \brief Initialize the merge detections task.
682 
683  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
684  merge the source catalogs.
685 
686  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
687  \param[in] schema the schema of the detection catalogs used as input to this one
688  \param[in] butler a butler used to read the input schema from disk, if schema is None
689  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
690 
691  The task will set its own self.schema attribute to the schema of the output merged catalog.
692  """
693  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
694  self.makeSubtask("skyObjects")
695  self.schema = self.getInputSchema(butler=butler, schema=schema)
696 
697  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
698  filterNames += [self.config.skyFilterName]
699  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
700 
701  def mergeCatalogs(self, catalogs, patchRef):
702  """!
703  \brief Merge multiple catalogs.
704 
705  After ordering the catalogs and filters in priority order,
706  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
707  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
708  garbage peaks detected around bright objects.
709 
710  \param[in] catalogs
711  \param[in] patchRef
712  \param[out] mergedList
713  """
714 
715  # Convert distance to tract coordinate
716  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
717  tractWcs = skyInfo.wcs
718  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
719  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
720 
721  # Put catalogs, filters in priority order
722  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
723  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
724  if band in catalogs.keys()]
725 
726  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
727  self.schema, self.makeIdFactory(patchRef),
728  samePeakDistance)
729 
730  #
731  # Add extra sources that correspond to blank sky
732  #
733  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
734  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
735  if skySourceFootprints:
736  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
737  for foot in skySourceFootprints:
738  s = mergedList.addNew()
739  s.setFootprint(foot)
740  s.set(key, True)
741 
742  # Sort Peaks from brightest to faintest
743  for record in mergedList:
744  record.getFootprint().sortPeaks()
745  self.log.info("Merged to %d sources" % len(mergedList))
746  # Attempt to remove garbage peaks
747  self.cullPeaks(mergedList)
748  return mergedList
749 
750  def cullPeaks(self, catalog):
751  """!
752  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
753 
754  \param[in] catalog Source catalog
755  """
756  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
757  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
758  totalPeaks = 0
759  culledPeaks = 0
760  for parentSource in catalog:
761  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
762  # to it (which is easier than deleting as we iterate).
763  keptPeaks = parentSource.getFootprint().getPeaks()
764  oldPeaks = list(keptPeaks)
765  keptPeaks.clear()
766  familySize = len(oldPeaks)
767  totalPeaks += familySize
768  for rank, peak in enumerate(oldPeaks):
769  if ((rank < self.config.cullPeaks.rankSufficient) or
770  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
771  (rank < self.config.cullPeaks.rankConsidered and
772  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
773  keptPeaks.append(peak)
774  else:
775  culledPeaks += 1
776  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
777 
778  def getSchemaCatalogs(self):
779  """!
780  Return a dict of empty catalogs for each catalog dataset produced by this task.
781 
782  \param[out] dictionary of empty catalogs
783  """
784  mergeDet = afwTable.SourceCatalog(self.schema)
785  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
786  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
787  self.config.coaddName + "Coadd_peak": peak}
788 
789  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
790  """!
791  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
792 
793  \param mergedList The merged Footprints from all the input bands
794  \param skyInfo A description of the patch
795  \param seed Seed for the random number generator
796  """
797  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
798  detected = mask.getPlaneBitMask("DETECTED")
799  for s in mergedList:
800  s.getFootprint().spans.setMask(mask, detected)
801 
802  footprints = self.skyObjects.run(mask, seed)
803  if not footprints:
804  return footprints
805 
806  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
807  schema = self.merged.getPeakSchema()
808  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
809  converted = []
810  for oldFoot in footprints:
811  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
812  peak = oldFoot.getPeaks()[0]
813  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
814  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
815  newFoot.getPeaks()[0].set(mergeKey, True)
816  converted.append(newFoot)
817 
818  return converted
819 
820 
822  """!
823  \anchor MeasureMergedCoaddSourcesConfig_
824 
825  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
826  """
827  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
828  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
829  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
830  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
831  doPropagateFlags = Field(
832  dtype=bool, default=True,
833  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
834  )
835  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
836  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
837  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
838  doWriteMatchesDenormalized = Field(
839  dtype=bool,
840  default=False,
841  doc=("Write reference matches in denormalized format? "
842  "This format uses more disk space, but is more convenient to read."),
843  )
844  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
845  checkUnitsParseStrict = Field(
846  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
847  dtype=str,
848  default="raise",
849  )
850  doApCorr = Field(
851  dtype=bool,
852  default=True,
853  doc="Apply aperture corrections"
854  )
855  applyApCorr = ConfigurableField(
856  target=ApplyApCorrTask,
857  doc="Subtask to apply aperture corrections"
858  )
859  doRunCatalogCalculation = Field(
860  dtype=bool,
861  default=True,
862  doc='Run catalogCalculation task'
863  )
864  catalogCalculation = ConfigurableField(
865  target=CatalogCalculationTask,
866  doc="Subtask to run catalogCalculation plugins on catalog"
867  )
868 
869  def setDefaults(self):
870  Config.setDefaults(self)
871  self.deblend.propagateAllPeaks = True
872  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
873  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
874  'INEXACT_PSF']
875  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
876  'INEXACT_PSF']
877 
878 
884 
885 
886 class MeasureMergedCoaddSourcesTask(CmdLineTask):
887  """!
888  \anchor MeasureMergedCoaddSourcesTask_
889 
890  \brief Deblend sources from master catalog in each coadd seperately and measure.
891 
892  \section pipe_tasks_multiBand_Contents Contents
893 
894  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
895  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
896  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
897  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
898  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
899  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
900 
901  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
902 
903  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
904  measurement in each coadd.
905 
906  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
907  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
908  consistent set of child sources.
909 
910  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
911  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
912  flags are propagated to the coadd sources.
913 
914  Optionally, we can match the coadd sources to an external reference catalog.
915 
916  \par Inputs:
917  deepCoadd_mergeDet{tract,patch}: SourceCatalog
918  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
919  \par Outputs:
920  deepCoadd_meas{tract,patch,filter}: SourceCatalog
921  \par Data Unit:
922  tract, patch, filter
923 
924  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
925 
926  <DL>
927  <DT> \ref SourceDeblendTask_ "deblend"
928  <DD> Deblend all the sources from the master catalog.</DD>
929  <DT> \ref SingleFrameMeasurementTask_ "measurement"
930  <DD> Measure source properties of deblended sources.</DD>
931  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
932  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
933  not at the edge of the field and that have either not been deblended or are the children of deblended
934  sources</DD>
935  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
936  <DD> Propagate flags set in individual visits to the coadd.</DD>
937  <DT> \ref DirectMatchTask_ "match"
938  <DD> Match input sources to a reference catalog (optional).
939  </DD>
940  </DL>
941  These subtasks may be retargeted as required.
942 
943  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
944 
945  \copydoc \_\_init\_\_
946 
947  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
948 
949  \copydoc run
950 
951  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
952 
953  See \ref MeasureMergedCoaddSourcesConfig_
954 
955  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
956 
957  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
958  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
959  files.
960 
961  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
962  the various sub-tasks. See the documetation for individual sub-tasks for more information.
963 
964  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
965  MeasureMergedCoaddSourcesTask
966 
967  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
968  The next stage in the multi-band processing procedure will merge these measurements into a suitable
969  catalog for driving forced photometry.
970 
971  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
972  to be processed.
973  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
974  `--help` command line argument:
975  \code
976  measureCoaddSources.py --help
977  \endcode
978 
979  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
980  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
981  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
982  coadd as follows:
983  \code
984  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
985  \endcode
986  This will process the HSC-I band data. The results are written in
987  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
988 
989  It is also necessary to run
990  \code
991  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
992  \endcode
993  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
994  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
995  """
996  _DefaultName = "measureCoaddSources"
997  ConfigClass = MeasureMergedCoaddSourcesConfig
998  RunnerClass = ButlerInitializedTaskRunner
999  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1000  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1001 
1002  @classmethod
1003  def _makeArgumentParser(cls):
1004  parser = ArgumentParser(name=cls._DefaultName)
1005  parser.add_id_argument("--id", "deepCoadd_calexp",
1006  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1007  ContainerClass=ExistingCoaddDataIdContainer)
1008  return parser
1009 
1010  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1011  """!
1012  \brief Initialize the task.
1013 
1014  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1015  \param[in] schema: the schema of the merged detection catalog used as input to this one
1016  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1017  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1018  catalog. May be None if the loader can be constructed from the butler argument or all steps
1019  requiring a reference catalog are disabled.
1020  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1021  catalog loader, if schema or peakSchema or refObjLoader is None
1022 
1023  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1024  This will include all fields from the input schema, as well as additional fields for all the
1025  measurements.
1026  """
1027  CmdLineTask.__init__(self, **kwargs)
1028  if schema is None:
1029  assert butler is not None, "Neither butler nor schema is defined"
1030  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1031  self.schemaMapper = afwTable.SchemaMapper(schema)
1032  self.schemaMapper.addMinimalSchema(schema)
1033  self.schema = self.schemaMapper.getOutputSchema()
1035  if self.config.doDeblend:
1036  if peakSchema is None:
1037  assert butler is not None, "Neither butler nor peakSchema is defined"
1038  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1039  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1040  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1041  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1042  if self.config.doMatchSources:
1043  if refObjLoader is None:
1044  assert butler is not None, "Neither butler nor refObjLoader is defined"
1045  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1046  if self.config.doPropagateFlags:
1047  self.makeSubtask("propagateFlags", schema=self.schema)
1048  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1049  if self.config.doApCorr:
1050  self.makeSubtask("applyApCorr", schema=self.schema)
1051  if self.config.doRunCatalogCalculation:
1052  self.makeSubtask("catalogCalculation", schema=self.schema)
1053 
1054  def run(self, patchRef):
1055  """!
1056  \brief Deblend and measure.
1057 
1058  \param[in] patchRef: Patch reference.
1059 
1060  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1061  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1062  Finally, write the deblended sources and measurements out.
1063  """
1064  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1065  sources = self.readSources(patchRef)
1066  if self.config.doDeblend:
1067  self.deblend.run(exposure, sources)
1068 
1069  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1070  # catalog is non-contiguous so can't extract column
1071  numBig = sum((s.get(bigKey) for s in sources))
1072  if numBig > 0:
1073  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1074  (patchRef.dataId, numBig))
1075 
1076  table = sources.getTable()
1077  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1078 
1079  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1080 
1081  if self.config.doApCorr:
1082  self.applyApCorr.run(
1083  catalog=sources,
1084  apCorrMap=exposure.getInfo().getApCorrMap()
1085  )
1086 
1087  if self.config.doRunCatalogCalculation:
1088  self.catalogCalculation.run(sources)
1089 
1090  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1091  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1092  includeDeblend=self.config.doDeblend)
1093  if self.config.doPropagateFlags:
1094  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1095  exposure.getWcs())
1096  if self.config.doMatchSources:
1097  self.writeMatches(patchRef, exposure, sources)
1098  self.write(patchRef, sources)
1099 
1100  def readSources(self, dataRef):
1101  """!
1102  \brief Read input sources.
1103 
1104  \param[in] dataRef: Data reference for catalog of merged detections
1105  \return List of sources in merged catalog
1106 
1107  We also need to add columns to hold the measurements we're about to make
1108  so we can measure in-place.
1109  """
1110  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1111  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1112  idFactory = self.makeIdFactory(dataRef)
1113  for s in merged:
1114  idFactory.notify(s.getId())
1115  table = afwTable.SourceTable.make(self.schema, idFactory)
1116  sources = afwTable.SourceCatalog(table)
1117  sources.extend(merged, self.schemaMapper)
1118  return sources
1119 
1120  def writeMatches(self, dataRef, exposure, sources):
1121  """!
1122  \brief Write matches of the sources to the astrometric reference catalog.
1123 
1124  We use the Wcs in the exposure to match sources.
1125 
1126  \param[in] dataRef: data reference
1127  \param[in] exposure: exposure with Wcs
1128  \param[in] sources: source catalog
1129  """
1130  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1131  if result.matches:
1132  matches = afwTable.packMatches(result.matches)
1133  matches.table.setMetadata(result.matchMeta)
1134  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1135  if self.config.doWriteMatchesDenormalized:
1136  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1137  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1138 
1139  def write(self, dataRef, sources):
1140  """!
1141  \brief Write the source catalog.
1142 
1143  \param[in] dataRef: data reference
1144  \param[in] sources: source catalog
1145  """
1146  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1147  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1148 
1149  def getExposureId(self, dataRef):
1150  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1151 
1152 
1154  """!
1155  \anchor MergeMeasurementsConfig_
1156 
1157  \brief Configuration parameters for the MergeMeasurementsTask
1158  """
1159  pseudoFilterList = ListField(dtype=str, default=["sky"],
1160  doc="Names of filters which may have no associated detection\n"
1161  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1162  snName = Field(dtype=str, default="base_PsfFlux",
1163  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1164  minSN = Field(dtype=float, default=10.,
1165  doc="If the S/N from the priority band is below this value (and the S/N "
1166  "is larger than minSNDiff compared to the priority band), use the band with "
1167  "the largest S/N as the reference band.")
1168  minSNDiff = Field(dtype=float, default=3.,
1169  doc="If the difference in S/N between another band and the priority band is larger "
1170  "than this value (and the S/N in the priority band is less than minSN) "
1171  "use the band with the largest S/N as the reference band")
1172  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1173  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1174  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1175 
1176 
1182 
1183 
1185  """!
1186  \anchor MergeMeasurementsTask_
1187 
1188  \brief Merge measurements from multiple bands
1189 
1190  \section pipe_tasks_multiBand_Contents Contents
1191 
1192  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1193  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1194  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1195  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1196  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1197  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1198 
1199  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1200 
1201  Command-line task that merges measurements from multiple bands.
1202 
1203  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1204  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1205  required to have centroid, shape and flux measurements in each band.
1206 
1207  \par Inputs:
1208  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1209  \par Outputs:
1210  deepCoadd_ref{tract,patch}: SourceCatalog
1211  \par Data Unit:
1212  tract, patch
1213 
1214  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1215 
1216  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1217 
1218  \copydoc \_\_init\_\_
1219 
1220  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1221 
1222  \copydoc run
1223 
1224  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1225 
1226  See \ref MergeMeasurementsConfig_
1227 
1228  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1229 
1230  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1231  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1232  files.
1233 
1234  MergeMeasurementsTask has no debug variables.
1235 
1236  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1237  of using MergeMeasurementsTask
1238 
1239  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1240  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1241  coadds and individual exposures.
1242  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1243  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1244  command line argument:
1245  \code
1246  mergeCoaddMeasurements.py --help
1247  \endcode
1248 
1249  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1250  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1251  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1252  as follows:
1253  \code
1254  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1255  \endcode
1256  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1257  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1258  """
1259  _DefaultName = "mergeCoaddMeasurements"
1260  ConfigClass = MergeMeasurementsConfig
1261  inputDataset = "meas"
1262  outputDataset = "ref"
1263  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1264 
1265  def __init__(self, butler=None, schema=None, **kwargs):
1266  """!
1267  Initialize the task.
1268 
1269  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1270  \param[in] schema: the schema of the detection catalogs used as input to this one
1271  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1272 
1273  The task will set its own self.schema attribute to the schema of the output merged catalog.
1274  """
1275  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1276  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1277  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1278  self.schemaMapper.addMinimalSchema(inputSchema, True)
1279  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1280  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1281  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1282 
1283  self.flagKeys = {}
1284  for band in self.config.priorityList:
1285  short = getShortFilterName(band)
1286  outputKey = self.schemaMapper.editOutputSchema().addField(
1287  "merge_measurement_%s" % short,
1288  type="Flag",
1289  doc="Flag field set if the measurements here are from the %s filter" % band
1290  )
1291  peakKey = inputSchema.find("merge_peak_%s" % short).key
1292  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1293  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1294  self.schema = self.schemaMapper.getOutputSchema()
1295 
1297  for filt in self.config.pseudoFilterList:
1298  try:
1299  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1300  except:
1301  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1302 
1303  self.badFlags = {}
1304  for flag in self.config.flags:
1305  try:
1306  self.badFlags[flag] = self.schema.find(flag).getKey()
1307  except KeyError as exc:
1308  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1309 
1310  def mergeCatalogs(self, catalogs, patchRef):
1311  """!
1312  Merge measurement catalogs to create a single reference catalog for forced photometry
1313 
1314  \param[in] catalogs: the catalogs to be merged
1315  \param[in] patchRef: patch reference for data
1316 
1317  For parent sources, we choose the first band in config.priorityList for which the
1318  merge_footprint flag for that band is is True.
1319 
1320  For child sources, the logic is the same, except that we use the merge_peak flags.
1321  """
1322  # Put catalogs, filters in priority order
1323  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1324  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1325 
1326  mergedCatalog = afwTable.SourceCatalog(self.schema)
1327  mergedCatalog.reserve(len(orderedCatalogs[0]))
1328 
1329  idKey = orderedCatalogs[0].table.getIdKey()
1330  for catalog in orderedCatalogs[1:]:
1331  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1332  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1333 
1334  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1335  # record for each band, in priority order.
1336  for orderedRecords in zip(*orderedCatalogs):
1337 
1338  maxSNRecord = None
1339  maxSNFlagKeys = None
1340  maxSN = 0.
1341  priorityRecord = None
1342  priorityFlagKeys = None
1343  prioritySN = 0.
1344  hasPseudoFilter = False
1345 
1346  # Now we iterate over those record-band pairs, keeping track of the priority and the
1347  # largest S/N band.
1348  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1349  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1350  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1351 
1352  if not (parent or child):
1353  for pseudoFilterKey in self.pseudoFilterKeys:
1354  if inputRecord.get(pseudoFilterKey):
1355  hasPseudoFilter = True
1356  priorityRecord = inputRecord
1357  priorityFlagKeys = flagKeys
1358  break
1359  if hasPseudoFilter:
1360  break
1361 
1362  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1363  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1364  sn = 0.
1365  else:
1366  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1367  if numpy.isnan(sn) or sn < 0.:
1368  sn = 0.
1369  if (parent or child) and priorityRecord is None:
1370  priorityRecord = inputRecord
1371  priorityFlagKeys = flagKeys
1372  prioritySN = sn
1373  if sn > maxSN:
1374  maxSNRecord = inputRecord
1375  maxSNFlagKeys = flagKeys
1376  maxSN = sn
1377 
1378  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1379  # the reference band instead. However, we only want to choose the highest S/N band if it is
1380  # significantly better than the priority band. Therefore, to choose a band other than the
1381  # priority, we require that the priority S/N is below the minimum threshold and that the
1382  # difference between the priority and highest S/N is larger than the difference threshold.
1383  #
1384  # For pseudo code objects we always choose the first band in the priority list.
1385  bestRecord = None
1386  bestFlagKeys = None
1387  if hasPseudoFilter:
1388  bestRecord = priorityRecord
1389  bestFlagKeys = priorityFlagKeys
1390  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1391  maxSNRecord is not None):
1392  bestRecord = maxSNRecord
1393  bestFlagKeys = maxSNFlagKeys
1394  elif priorityRecord is not None:
1395  bestRecord = priorityRecord
1396  bestFlagKeys = priorityFlagKeys
1397 
1398  if bestRecord is not None and bestFlagKeys is not None:
1399  outputRecord = mergedCatalog.addNew()
1400  outputRecord.assign(bestRecord, self.schemaMapper)
1401  outputRecord.set(bestFlagKeys.output, True)
1402  else: # if we didn't find any records
1403  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1404  inputRecord.getId())
1405 
1406  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1407  for inputCatalog in orderedCatalogs:
1408  if len(mergedCatalog) != len(inputCatalog):
1409  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1410  (len(mergedCatalog), len(orderedCatalogs)))
1411 
1412  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:789
Merge coadd detections from multiple bands.
Definition: multiBand.py:592
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:335
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:438
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:778
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:280
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:750
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1054
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:327
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:311
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:451
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1265
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:247
Merge measurements from multiple bands.
Definition: multiBand.py:1184
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:464
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:886
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1120
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:265
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1310
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:490
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1153
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1100
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:560
Configuration for merging sources.
Definition: multiBand.py:379
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:501
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:701
A base class for merging source catalogs.
Definition: multiBand.py:395
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1139
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:475
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:821
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:249
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:518
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1010
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:679
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:353