lsst.pipe.tasks  14.0-41-g36ac3fe5+1
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division, print_function
24 from builtins import zip
25 from builtins import range
26 import numpy
27 
28 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
29 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
30 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
31 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
32 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
33 from lsst.meas.deblender import SourceDeblendTask
34 from lsst.pipe.tasks.coaddBase import getSkyInfo, scaleVariance
35 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
36 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
37 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
38 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
39 import lsst.afw.image as afwImage
40 import lsst.afw.table as afwTable
41 import lsst.afw.math as afwMath
42 import lsst.afw.geom as afwGeom
43 import lsst.afw.detection as afwDetect
44 from lsst.daf.base import PropertyList
45 
46 """
47 New dataset types:
48 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
49 * deepCoadd_mergeDet: merged detections (tract, patch)
50 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
51 * deepCoadd_ref: reference sources (tract, patch)
52 All of these have associated *_schema catalogs that require no data ID and hold no records.
53 
54 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
55 the mergeDet, meas, and ref dataset Footprints:
56 * deepCoadd_peak_schema
57 """
58 
59 
60 def _makeGetSchemaCatalogs(datasetSuffix):
61  """Construct a getSchemaCatalogs instance method
62 
63  These are identical for most of the classes here, so we'll consolidate
64  the code.
65 
66  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
67  """
68 
69  def getSchemaCatalogs(self):
70  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
71  src = afwTable.SourceCatalog(self.schema)
72  if hasattr(self, "algMetadata"):
73  src.getTable().setMetadata(self.algMetadata)
74  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
75  return getSchemaCatalogs
76 
77 
78 def _makeMakeIdFactory(datasetName):
79  """Construct a makeIdFactory instance method
80 
81  These are identical for all the classes here, so this consolidates
82  the code.
83 
84  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
85  """
86 
87  def makeIdFactory(self, dataRef):
88  """Return an IdFactory for setting the detection identifiers
89 
90  The actual parameters used in the IdFactory are provided by
91  the butler (through the provided data reference.
92  """
93  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
94  expId = int(dataRef.get(self.config.coaddName + datasetName))
95  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
96  return makeIdFactory
97 
98 
100  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
101  """
102  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
103  # and its the only way I could get it to work.
104  return afwImage.Filter(name).getFilterProperty().getName()
105 
106 
107 
108 
110  """!
111  \anchor DetectCoaddSourcesConfig_
112 
113  \brief Configuration parameters for the DetectCoaddSourcesTask
114  """
115  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
116  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
117  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
118  mask = ListField(dtype=str, default=["DETECTED", "BAD", "SAT", "NO_DATA", "INTRP"],
119  doc="Mask planes for pixels to ignore when scaling variance")
120  doInsertFakes = Field(dtype=bool, default=False,
121  doc="Run fake sources injection task")
122  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
123  doc="Injection of fake sources for testing "
124  "purposes (must be retargeted)")
125 
126  def setDefaults(self):
127  Config.setDefaults(self)
128  self.detection.thresholdType = "pixel_stdev"
129  self.detection.isotropicGrow = True
130  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
131  self.detection.reEstimateBackground = False
132  self.detection.background.useApprox = False
133  self.detection.background.binSize = 4096
134  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
135 
136 
142 
143 
144 class DetectCoaddSourcesTask(CmdLineTask):
145  """!
146  \anchor DetectCoaddSourcesTask_
147 
148  \brief Detect sources on a coadd
149 
150  \section pipe_tasks_multiBand_Contents Contents
151 
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
155  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
156  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
157  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
158 
159  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
160 
161  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
162 
163  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
164  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
165  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
166  propagate the full covariance matrix -- but it is simple and works well in practice.
167 
168  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
169  SourceDetectionTask_ "detection" subtask.
170 
171  \par Inputs:
172  deepCoadd{tract,patch,filter}: ExposureF
173  \par Outputs:
174  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
175  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
176  exposure (ExposureF)
177  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
178  \par Data Unit:
179  tract, patch, filter
180 
181  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
182  You can retarget this subtask if you wish.
183 
184  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
185 
186  \copydoc \_\_init\_\_
187 
188  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
189 
190  \copydoc run
191 
192  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
193 
194  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
195 
196  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
197 
198  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
199  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
200  files.
201 
202  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
203  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
204  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
205 
206  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example of using DetectCoaddSourcesTask
207 
208  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
209  the task is to update the background, detect all sources in a single band and generate a set of parent
210  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
211  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
212  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
213  calling detectCoaddSources.py with the `--help` command line argument:
214  \code
215  detectCoaddSources.py --help
216  \endcode
217 
218  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
219  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
220  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
221  \code
222  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
223  \endcode
224  that will process the HSC-I band data. The results are written to
225  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
226 
227  It is also necessary to run:
228  \code
229  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
230  \endcode
231  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
232  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
233  """
234  _DefaultName = "detectCoaddSources"
235  ConfigClass = DetectCoaddSourcesConfig
236  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
237  makeIdFactory = _makeMakeIdFactory("CoaddId")
238 
239  @classmethod
240  def _makeArgumentParser(cls):
241  parser = ArgumentParser(name=cls._DefaultName)
242  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
243  ContainerClass=ExistingCoaddDataIdContainer)
244  return parser
245 
246  def __init__(self, schema=None, **kwargs):
247  """!
248  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
249 
250  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
251 
252  \param[in] schema: initial schema for the output catalog, modified-in place to include all
253  fields set by this task. If None, the source minimal schema will be used.
254  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
255  """
256  CmdLineTask.__init__(self, **kwargs)
257  if schema is None:
258  schema = afwTable.SourceTable.makeMinimalSchema()
259  if self.config.doInsertFakes:
260  self.makeSubtask("insertFakes")
261  self.schema = schema
262  self.makeSubtask("detection", schema=self.schema)
263 
264  def run(self, patchRef):
265  """!
266  \brief Run detection on a coadd.
267 
268  Invokes \ref runDetection and then uses \ref write to output the
269  results.
270 
271  \param[in] patchRef: data reference for patch
272  """
273  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
274  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
275  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
276  self.write(exposure, results, patchRef)
277  return results
278 
279  def runDetection(self, exposure, idFactory, expId):
280  """!
281  \brief Run detection on an exposure.
282 
283  First scale the variance plane to match the observed variance
284  using \ref scaleVariance. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
285  detect sources.
286 
287  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
288  depending on configuration).
289  \param[in] idFactory: IdFactory to set source identifiers
290  \param[in] expId: Exposure identifier (integer) for RNG seed
291 
292  \return a pipe.base.Struct with fields
293  - sources: catalog of detections
294  - backgrounds: list of backgrounds
295  """
296  if self.config.doScaleVariance:
297  varScale = scaleVariance(exposure.getMaskedImage(), self.config.mask, log=self.log)
298  exposure.getMetadata().add("variance_scale", varScale)
299  backgrounds = afwMath.BackgroundList()
300  if self.config.doInsertFakes:
301  self.insertFakes.run(exposure, background=backgrounds)
302  table = afwTable.SourceTable.make(self.schema, idFactory)
303  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
304  sources = detections.sources
305  fpSets = detections.fpSets
306  if hasattr(fpSets, "background") and fpSets.background:
307  backgrounds.append(fpSets.background)
308  return Struct(sources=sources, backgrounds=backgrounds)
309 
310  def write(self, exposure, results, patchRef):
311  """!
312  \brief Write out results from runDetection.
313 
314  \param[in] exposure: Exposure to write out
315  \param[in] results: Struct returned from runDetection
316  \param[in] patchRef: data reference for patch
317  """
318  coaddName = self.config.coaddName + "Coadd"
319  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
320  patchRef.put(results.sources, coaddName + "_det")
321  patchRef.put(exposure, coaddName + "_calexp")
322 
323 
324 
325 
326 class MergeSourcesRunner(TaskRunner):
327  """!
328  \anchor MergeSourcesRunner_
329 
330  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
331  requires a list of dataRefs rather than a single dataRef.
332  """
333 
334  def makeTask(self, parsedCmd=None, args=None):
335  """!
336  \brief Provide a butler to the Task constructor.
337 
338  \param[in] parsedCmd the parsed command
339  \param[in] args tuple of a list of data references and kwargs (un-used)
340  \throws RuntimeError if both parsedCmd & args are None
341  """
342  if parsedCmd is not None:
343  butler = parsedCmd.butler
344  elif args is not None:
345  dataRefList, kwargs = args
346  butler = dataRefList[0].getButler()
347  else:
348  raise RuntimeError("Neither parsedCmd or args specified")
349  return self.TaskClass(config=self.config, log=self.log, butler=butler)
350 
351  @staticmethod
352  def getTargetList(parsedCmd, **kwargs):
353  """!
354  \brief Provide a list of patch references for each patch.
355 
356  The patch references within the list will have different filters.
357 
358  \param[in] parsedCmd the parsed command
359  \param **kwargs key word arguments (unused)
360  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
361  filter
362  """
363  refList = {} # Will index this as refList[tract][patch][filter] = ref
364  for ref in parsedCmd.id.refList:
365  tract = ref.dataId["tract"]
366  patch = ref.dataId["patch"]
367  filter = ref.dataId["filter"]
368  if not tract in refList:
369  refList[tract] = {}
370  if not patch in refList[tract]:
371  refList[tract][patch] = {}
372  if filter in refList[tract][patch]:
373  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
374  refList[tract][patch][filter] = ref
375  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
376 
377 
378 class MergeSourcesConfig(Config):
379  """!
380  \anchor MergeSourcesConfig_
381 
382  \brief Configuration for merging sources.
383  """
384  priorityList = ListField(dtype=str, default=[],
385  doc="Priority-ordered list of bands for the merge.")
386  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
387 
388  def validate(self):
389  Config.validate(self)
390  if len(self.priorityList) == 0:
391  raise RuntimeError("No priority list provided")
392 
393 
394 class MergeSourcesTask(CmdLineTask):
395  """!
396  \anchor MergeSourcesTask_
397 
398  \brief A base class for merging source catalogs.
399 
400  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
401  so similar that it makes sense to re-use the code, in the form of this abstract base class.
402 
403  NB: Do not use this class directly. Instead use one of the child classes that inherit from
404  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
405  "MergeMeasurementsTask"
406 
407  Sub-classes should set the following class variables:
408  * `_DefaultName`: name of Task
409  * `inputDataset`: name of dataset to read
410  * `outputDataset`: name of dataset to write
411  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
412 
413  In addition, sub-classes must implement the mergeCatalogs method.
414  """
415  _DefaultName = None
416  ConfigClass = MergeSourcesConfig
417  RunnerClass = MergeSourcesRunner
418  inputDataset = None
419  outputDataset = None
420  getSchemaCatalogs = None
421 
422  @classmethod
423  def _makeArgumentParser(cls):
424  """!
425  \brief Create a suitable ArgumentParser.
426 
427  We will use the ArgumentParser to get a provide a list of data
428  references for patches; the RunnerClass will sort them into lists
429  of data references for the same patch
430  """
431  parser = ArgumentParser(name=cls._DefaultName)
432  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
433  ContainerClass=ExistingCoaddDataIdContainer,
434  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
435  return parser
436 
437  def getInputSchema(self, butler=None, schema=None):
438  """!
439  \brief Obtain the input schema either directly or froma butler reference.
440 
441  \param[in] butler butler reference to obtain the input schema from
442  \param[in] schema the input schema
443  """
444  if schema is None:
445  assert butler is not None, "Neither butler nor schema specified"
446  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
447  immediate=True).schema
448  return schema
449 
450  def __init__(self, butler=None, schema=None, **kwargs):
451  """!
452  \brief Initialize the task.
453 
454  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
455  \param[in] schema the schema of the detection catalogs used as input to this one
456  \param[in] butler a butler used to read the input schema from disk, if schema is None
457 
458  Derived classes should use the getInputSchema() method to handle the additional
459  arguments and retreive the actual input schema.
460  """
461  CmdLineTask.__init__(self, **kwargs)
462 
463  def run(self, patchRefList):
464  """!
465  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
466  subclasses that inherit from MergeSourcesTask.
467 
468  \param[in] patchRefList list of data references for each filter
469  """
470  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
471  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
472  self.write(patchRefList[0], mergedCatalog)
473 
474  def readCatalog(self, patchRef):
475  """!
476  \brief Read input catalog.
477 
478  We read the input dataset provided by the 'inputDataset'
479  class variable.
480 
481  \param[in] patchRef data reference for patch
482  \return tuple consisting of the filter name and the catalog
483  """
484  filterName = patchRef.dataId["filter"]
485  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
486  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
487  return filterName, catalog
488 
489  def mergeCatalogs(self, catalogs, patchRef):
490  """!
491  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
492  MergeSourcesTask.
493 
494  \param[in] catalogs dict mapping filter name to source catalog
495 
496  \return merged catalog
497  """
498  raise NotImplementedError()
499 
500  def write(self, patchRef, catalog):
501  """!
502  \brief Write the output.
503 
504  \param[in] patchRef data reference for patch
505  \param[in] catalog catalog
506 
507  We write as the dataset provided by the 'outputDataset'
508  class variable.
509  """
510  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
511  # since the filter isn't actually part of the data ID for the dataset we're saving,
512  # it's confusing to see it in the log message, even if the butler simply ignores it.
513  mergeDataId = patchRef.dataId.copy()
514  del mergeDataId["filter"]
515  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
516 
517  def writeMetadata(self, dataRefList):
518  """!
519  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
520  """
521  pass
522 
523 
524 class CullPeaksConfig(Config):
525  """!
526  \anchor CullPeaksConfig_
527 
528  \brief Configuration for culling garbage peaks after merging footprints.
529 
530  Peaks may also be culled after detection or during deblending; this configuration object
531  only deals with culling after merging Footprints.
532 
533  These cuts are based on three quantities:
534  - nBands: the number of bands in which the peak was detected
535  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
536  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
537 
538  The formula that identifie peaks to cull is:
539 
540  nBands < nBandsSufficient
541  AND (rank >= rankSufficient)
542  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
543 
544  To disable peak culling, simply set nBandsSufficient=1.
545  """
546 
547  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
548  doc="Always keep peaks detected in this many bands")
549  rankSufficient = RangeField(dtype=int, default=20, min=1,
550  doc="Always keep this many peaks in each family")
551  rankConsidered = RangeField(dtype=int, default=30, min=1,
552  doc=("Keep peaks with less than this rank that also match the "
553  "rankNormalizedConsidered condition."))
554  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
555  doc=("Keep peaks with less than this normalized rank that"
556  " also match the rankConsidered condition."))
557 
558 
560  """!
561  \anchor MergeDetectionsConfig_
562 
563  \brief Configuration parameters for the MergeDetectionsTask.
564  """
565  minNewPeak = Field(dtype=float, default=1,
566  doc="Minimum distance from closest peak to create a new one (in arcsec).")
567 
568  maxSamePeak = Field(dtype=float, default=0.3,
569  doc="When adding new catalogs to the merge, all peaks less than this distance "
570  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
571  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
572 
573  skyFilterName = Field(dtype=str, default="sky",
574  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
575  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
576  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
577 
578  def setDefaults(self):
579  MergeSourcesConfig.setDefaults(self)
580  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
581 
582 
583 
589 
590 
592  """!
593  \anchor MergeDetectionsTask_
594 
595  \brief Merge coadd detections from multiple bands.
596 
597  \section pipe_tasks_multiBand_Contents Contents
598 
599  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
600  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
601  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
602  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
603  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
604  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
605 
606  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
607 
608  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
609 
610  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
611  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
612  track of which band each source originates in.
613 
614  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
615  objects are culled as described in \ref CullPeaksConfig_.
616 
617  \par Inputs:
618  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
619  \par Outputs:
620  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
621  \par Data Unit:
622  tract, patch
623 
624  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
625 
626  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
627 
628  \copydoc \_\_init\_\_
629 
630  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
631 
632  \copydoc run
633 
634  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
635 
636  See \ref MergeDetectionsConfig_
637 
638  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
639 
640  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
641  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
642 
643  MergeDetectionsTask has no debug variables.
644 
645  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
646 
647  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
648  of the available bands.
649  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
650  chosen subset of filters.
651  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
652  and, eventually, perform forced photometry.
653  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
654  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
655  `--help` command line argument:
656  \code
657  mergeCoaddDetections.py --help
658  \endcode
659 
660  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
661  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
662  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
663  \code
664  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
665  \endcode
666  This will merge the HSC-I & -R band parent source catalogs and write the results to
667  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
668 
669  The next step in the multi-band processing procedure is
670  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
671  """
672  ConfigClass = MergeDetectionsConfig
673  _DefaultName = "mergeCoaddDetections"
674  inputDataset = "det"
675  outputDataset = "mergeDet"
676  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
677 
678  def __init__(self, butler=None, schema=None, **kwargs):
679  """!
680  \brief Initialize the merge detections task.
681 
682  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
683  merge the source catalogs.
684 
685  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
686  \param[in] schema the schema of the detection catalogs used as input to this one
687  \param[in] butler a butler used to read the input schema from disk, if schema is None
688  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
689 
690  The task will set its own self.schema attribute to the schema of the output merged catalog.
691  """
692  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
693  self.makeSubtask("skyObjects")
694  self.schema = self.getInputSchema(butler=butler, schema=schema)
695 
696  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
697  filterNames += [self.config.skyFilterName]
698  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
699 
700  def mergeCatalogs(self, catalogs, patchRef):
701  """!
702  \brief Merge multiple catalogs.
703 
704  After ordering the catalogs and filters in priority order,
705  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
706  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
707  garbage peaks detected around bright objects.
708 
709  \param[in] catalogs
710  \param[in] patchRef
711  \param[out] mergedList
712  """
713 
714  # Convert distance to tract coordinate
715  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
716  tractWcs = skyInfo.wcs
717  peakDistance = self.config.minNewPeak / tractWcs.pixelScale().asArcseconds()
718  samePeakDistance = self.config.maxSamePeak / tractWcs.pixelScale().asArcseconds()
719 
720  # Put catalogs, filters in priority order
721  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
722  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
723  if band in catalogs.keys()]
724 
725  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
726  self.schema, self.makeIdFactory(patchRef),
727  samePeakDistance)
728 
729  #
730  # Add extra sources that correspond to blank sky
731  #
732  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
733  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
734  if skySourceFootprints:
735  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
736  for foot in skySourceFootprints:
737  s = mergedList.addNew()
738  s.setFootprint(foot)
739  s.set(key, True)
740 
741  # Sort Peaks from brightest to faintest
742  for record in mergedList:
743  record.getFootprint().sortPeaks()
744  self.log.info("Merged to %d sources" % len(mergedList))
745  # Attempt to remove garbage peaks
746  self.cullPeaks(mergedList)
747  return mergedList
748 
749  def cullPeaks(self, catalog):
750  """!
751  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
752 
753  \param[in] catalog Source catalog
754  """
755  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
756  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
757  totalPeaks = 0
758  culledPeaks = 0
759  for parentSource in catalog:
760  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
761  # to it (which is easier than deleting as we iterate).
762  keptPeaks = parentSource.getFootprint().getPeaks()
763  oldPeaks = list(keptPeaks)
764  keptPeaks.clear()
765  familySize = len(oldPeaks)
766  totalPeaks += familySize
767  for rank, peak in enumerate(oldPeaks):
768  if ((rank < self.config.cullPeaks.rankSufficient) or
769  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
770  (rank < self.config.cullPeaks.rankConsidered and
771  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
772  keptPeaks.append(peak)
773  else:
774  culledPeaks += 1
775  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
776 
777  def getSchemaCatalogs(self):
778  """!
779  Return a dict of empty catalogs for each catalog dataset produced by this task.
780 
781  \param[out] dictionary of empty catalogs
782  """
783  mergeDet = afwTable.SourceCatalog(self.schema)
784  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
785  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
786  self.config.coaddName + "Coadd_peak": peak}
787 
788  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
789  """!
790  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
791 
792  \param mergedList The merged Footprints from all the input bands
793  \param skyInfo A description of the patch
794  \param seed Seed for the random number generator
795  """
796  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
797  detected = mask.getPlaneBitMask("DETECTED")
798  for s in mergedList:
799  s.getFootprint().spans.setMask(mask, detected)
800 
801  footprints = self.skyObjects.run(mask, seed)
802  if not footprints:
803  return footprints
804 
805  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
806  schema = self.merged.getPeakSchema()
807  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
808  converted = []
809  for oldFoot in footprints:
810  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
811  peak = oldFoot.getPeaks()[0]
812  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
813  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
814  newFoot.getPeaks()[0].set(mergeKey, True)
815  converted.append(newFoot)
816 
817  return converted
818 
819 
821  """!
822  \anchor MeasureMergedCoaddSourcesConfig_
823 
824  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
825  """
826  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
827  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
828  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
829  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
830  doPropagateFlags = Field(
831  dtype=bool, default=True,
832  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
833  )
834  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
835  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
836  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
837  doWriteMatchesDenormalized = Field(
838  dtype=bool,
839  default=False,
840  doc=("Write reference matches in denormalized format? "
841  "This format uses more disk space, but is more convenient to read."),
842  )
843  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
844  checkUnitsParseStrict = Field(
845  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
846  dtype=str,
847  default="raise",
848  )
849  doApCorr = Field(
850  dtype=bool,
851  default=True,
852  doc="Apply aperture corrections"
853  )
854  applyApCorr = ConfigurableField(
855  target=ApplyApCorrTask,
856  doc="Subtask to apply aperture corrections"
857  )
858  doRunCatalogCalculation = Field(
859  dtype=bool,
860  default=True,
861  doc='Run catalogCalculation task'
862  )
863  catalogCalculation = ConfigurableField(
864  target=CatalogCalculationTask,
865  doc="Subtask to run catalogCalculation plugins on catalog"
866  )
867 
868  def setDefaults(self):
869  Config.setDefaults(self)
870  self.deblend.propagateAllPeaks = True
871  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
872  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
873  'INEXACT_PSF']
874  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
875  'INEXACT_PSF']
876 
877 
883 
884 
885 class MeasureMergedCoaddSourcesTask(CmdLineTask):
886  """!
887  \anchor MeasureMergedCoaddSourcesTask_
888 
889  \brief Deblend sources from master catalog in each coadd seperately and measure.
890 
891  \section pipe_tasks_multiBand_Contents Contents
892 
893  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
894  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
895  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
896  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
897  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
898  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
899 
900  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
901 
902  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
903  measurement in each coadd.
904 
905  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
906  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
907  consistent set of child sources.
908 
909  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
910  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
911  flags are propagated to the coadd sources.
912 
913  Optionally, we can match the coadd sources to an external reference catalog.
914 
915  \par Inputs:
916  deepCoadd_mergeDet{tract,patch}: SourceCatalog
917  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
918  \par Outputs:
919  deepCoadd_meas{tract,patch,filter}: SourceCatalog
920  \par Data Unit:
921  tract, patch, filter
922 
923  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
924 
925  <DL>
926  <DT> \ref SourceDeblendTask_ "deblend"
927  <DD> Deblend all the sources from the master catalog.</DD>
928  <DT> \ref SingleFrameMeasurementTask_ "measurement"
929  <DD> Measure source properties of deblended sources.</DD>
930  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
931  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
932  not at the edge of the field and that have either not been deblended or are the children of deblended
933  sources</DD>
934  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
935  <DD> Propagate flags set in individual visits to the coadd.</DD>
936  <DT> \ref DirectMatchTask_ "match"
937  <DD> Match input sources to a reference catalog (optional).
938  </DD>
939  </DL>
940  These subtasks may be retargeted as required.
941 
942  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
943 
944  \copydoc \_\_init\_\_
945 
946  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
947 
948  \copydoc run
949 
950  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
951 
952  See \ref MeasureMergedCoaddSourcesConfig_
953 
954  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
955 
956  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
957  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
958  files.
959 
960  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
961  the various sub-tasks. See the documetation for individual sub-tasks for more information.
962 
963  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using MeasureMergedCoaddSourcesTask
964 
965  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
966  The next stage in the multi-band processing procedure will merge these measurements into a suitable
967  catalog for driving forced photometry.
968 
969  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds to be processed.
970  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
971  `--help` command line argument:
972  \code
973  measureCoaddSources.py --help
974  \endcode
975 
976  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
977  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
978  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
979  coadd as follows:
980  \code
981  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
982  \endcode
983  This will process the HSC-I band data. The results are written in
984  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
985 
986  It is also necessary to run
987  \code
988  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
989  \endcode
990  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
991  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
992  """
993  _DefaultName = "measureCoaddSources"
994  ConfigClass = MeasureMergedCoaddSourcesConfig
995  RunnerClass = ButlerInitializedTaskRunner
996  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
997  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
998 
999  @classmethod
1000  def _makeArgumentParser(cls):
1001  parser = ArgumentParser(name=cls._DefaultName)
1002  parser.add_id_argument("--id", "deepCoadd_calexp",
1003  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1004  ContainerClass=ExistingCoaddDataIdContainer)
1005  return parser
1006 
1007  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1008  """!
1009  \brief Initialize the task.
1010 
1011  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1012  \param[in] schema: the schema of the merged detection catalog used as input to this one
1013  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1014  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1015  catalog. May be None if the loader can be constructed from the butler argument or all steps
1016  requiring a reference catalog are disabled.
1017  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1018  catalog loader, if schema or peakSchema or refObjLoader is None
1019 
1020  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1021  This will include all fields from the input schema, as well as additional fields for all the
1022  measurements.
1023  """
1024  CmdLineTask.__init__(self, **kwargs)
1025  if schema is None:
1026  assert butler is not None, "Neither butler nor schema is defined"
1027  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1028  self.schemaMapper = afwTable.SchemaMapper(schema)
1029  self.schemaMapper.addMinimalSchema(schema)
1030  self.schema = self.schemaMapper.getOutputSchema()
1032  if self.config.doDeblend:
1033  if peakSchema is None:
1034  assert butler is not None, "Neither butler nor peakSchema is defined"
1035  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1036  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1037  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1038  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1039  if self.config.doMatchSources:
1040  if refObjLoader is None:
1041  assert butler is not None, "Neither butler nor refObjLoader is defined"
1042  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1043  if self.config.doPropagateFlags:
1044  self.makeSubtask("propagateFlags", schema=self.schema)
1045  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1046  if self.config.doApCorr:
1047  self.makeSubtask("applyApCorr", schema=self.schema)
1048  if self.config.doRunCatalogCalculation:
1049  self.makeSubtask("catalogCalculation", schema=self.schema)
1050 
1051  def run(self, patchRef):
1052  """!
1053  \brief Deblend and measure.
1054 
1055  \param[in] patchRef: Patch reference.
1056 
1057  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1058  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1059  Finally, write the deblended sources and measurements out.
1060  """
1061  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1062  sources = self.readSources(patchRef)
1063  if self.config.doDeblend:
1064  self.deblend.run(exposure, sources)
1065 
1066  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1067  # catalog is non-contiguous so can't extract column
1068  numBig = sum((s.get(bigKey) for s in sources))
1069  if numBig > 0:
1070  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1071  (patchRef.dataId, numBig))
1072 
1073  table = sources.getTable()
1074  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1075 
1076  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1077 
1078  if self.config.doApCorr:
1079  self.applyApCorr.run(
1080  catalog=sources,
1081  apCorrMap=exposure.getInfo().getApCorrMap()
1082  )
1083 
1084  if self.config.doRunCatalogCalculation:
1085  self.catalogCalculation.run(sources)
1086 
1087  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1088  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1089  includeDeblend=self.config.doDeblend)
1090  if self.config.doPropagateFlags:
1091  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1092  exposure.getWcs())
1093  if self.config.doMatchSources:
1094  self.writeMatches(patchRef, exposure, sources)
1095  self.write(patchRef, sources)
1096 
1097  def readSources(self, dataRef):
1098  """!
1099  \brief Read input sources.
1100 
1101  \param[in] dataRef: Data reference for catalog of merged detections
1102  \return List of sources in merged catalog
1103 
1104  We also need to add columns to hold the measurements we're about to make
1105  so we can measure in-place.
1106  """
1107  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1108  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1109  idFactory = self.makeIdFactory(dataRef)
1110  for s in merged:
1111  idFactory.notify(s.getId())
1112  table = afwTable.SourceTable.make(self.schema, idFactory)
1113  sources = afwTable.SourceCatalog(table)
1114  sources.extend(merged, self.schemaMapper)
1115  return sources
1116 
1117  def writeMatches(self, dataRef, exposure, sources):
1118  """!
1119  \brief Write matches of the sources to the astrometric reference catalog.
1120 
1121  We use the Wcs in the exposure to match sources.
1122 
1123  \param[in] dataRef: data reference
1124  \param[in] exposure: exposure with Wcs
1125  \param[in] sources: source catalog
1126  """
1127  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1128  if result.matches:
1129  matches = afwTable.packMatches(result.matches)
1130  matches.table.setMetadata(result.matchMeta)
1131  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1132  if self.config.doWriteMatchesDenormalized:
1133  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1134  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1135 
1136 
1137  def write(self, dataRef, sources):
1138  """!
1139  \brief Write the source catalog.
1140 
1141  \param[in] dataRef: data reference
1142  \param[in] sources: source catalog
1143  """
1144  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1145  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1146 
1147  def getExposureId(self, dataRef):
1148  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1149 
1150 
1151 
1152 
1154  """!
1155  \anchor MergeMeasurementsConfig_
1156 
1157  \brief Configuration parameters for the MergeMeasurementsTask
1158  """
1159  pseudoFilterList = ListField(dtype=str, default=["sky"],
1160  doc="Names of filters which may have no associated detection\n"
1161  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1162  snName = Field(dtype=str, default="base_PsfFlux",
1163  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1164  minSN = Field(dtype=float, default=10.,
1165  doc="If the S/N from the priority band is below this value (and the S/N "
1166  "is larger than minSNDiff compared to the priority band), use the band with "
1167  "the largest S/N as the reference band.")
1168  minSNDiff = Field(dtype=float, default=3.,
1169  doc="If the difference in S/N between another band and the priority band is larger "
1170  "than this value (and the S/N in the priority band is less than minSN) "
1171  "use the band with the largest S/N as the reference band")
1172  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1173  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1174  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1175 
1176 
1182 
1183 
1185  """!
1186  \anchor MergeMeasurementsTask_
1187 
1188  \brief Merge measurements from multiple bands
1189 
1190  \section pipe_tasks_multiBand_Contents Contents
1191 
1192  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1193  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1194  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1195  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1196  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1197  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1198 
1199  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1200 
1201  Command-line task that merges measurements from multiple bands.
1202 
1203  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1204  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1205  required to have centroid, shape and flux measurements in each band.
1206 
1207  \par Inputs:
1208  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1209  \par Outputs:
1210  deepCoadd_ref{tract,patch}: SourceCatalog
1211  \par Data Unit:
1212  tract, patch
1213 
1214  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1215 
1216  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1217 
1218  \copydoc \_\_init\_\_
1219 
1220  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1221 
1222  \copydoc run
1223 
1224  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1225 
1226  See \ref MergeMeasurementsConfig_
1227 
1228  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1229 
1230  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1231  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1232  files.
1233 
1234  MergeMeasurementsTask has no debug variables.
1235 
1236  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example of using MergeMeasurementsTask
1237 
1238  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1239  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1240  coadds and individual exposures.
1241  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1242  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1243  command line argument:
1244  \code
1245  mergeCoaddMeasurements.py --help
1246  \endcode
1247 
1248  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1249  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1250  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1251  as follows:
1252  \code
1253  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1254  \endcode
1255  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1256  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1257  """
1258  _DefaultName = "mergeCoaddMeasurements"
1259  ConfigClass = MergeMeasurementsConfig
1260  inputDataset = "meas"
1261  outputDataset = "ref"
1262  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1263 
1264  def __init__(self, butler=None, schema=None, **kwargs):
1265  """!
1266  Initialize the task.
1267 
1268  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1269  \param[in] schema: the schema of the detection catalogs used as input to this one
1270  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1271 
1272  The task will set its own self.schema attribute to the schema of the output merged catalog.
1273  """
1274  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1275  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1276  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1277  self.schemaMapper.addMinimalSchema(inputSchema, True)
1278  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1279  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1280  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1281 
1282  self.flagKeys = {}
1283  for band in self.config.priorityList:
1284  short = getShortFilterName(band)
1285  outputKey = self.schemaMapper.editOutputSchema().addField(
1286  "merge_measurement_%s" % short,
1287  type="Flag",
1288  doc="Flag field set if the measurements here are from the %s filter" % band
1289  )
1290  peakKey = inputSchema.find("merge_peak_%s" % short).key
1291  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1292  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1293  self.schema = self.schemaMapper.getOutputSchema()
1294 
1296  for filt in self.config.pseudoFilterList:
1297  try:
1298  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1299  except:
1300  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1301 
1302  self.badFlags = {}
1303  for flag in self.config.flags:
1304  try:
1305  self.badFlags[flag] = self.schema.find(flag).getKey()
1306  except KeyError as exc:
1307  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1308 
1309  def mergeCatalogs(self, catalogs, patchRef):
1310  """!
1311  Merge measurement catalogs to create a single reference catalog for forced photometry
1312 
1313  \param[in] catalogs: the catalogs to be merged
1314  \param[in] patchRef: patch reference for data
1315 
1316  For parent sources, we choose the first band in config.priorityList for which the
1317  merge_footprint flag for that band is is True.
1318 
1319  For child sources, the logic is the same, except that we use the merge_peak flags.
1320  """
1321  # Put catalogs, filters in priority order
1322  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1323  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1324 
1325  mergedCatalog = afwTable.SourceCatalog(self.schema)
1326  mergedCatalog.reserve(len(orderedCatalogs[0]))
1327 
1328  idKey = orderedCatalogs[0].table.getIdKey()
1329  for catalog in orderedCatalogs[1:]:
1330  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1331  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1332 
1333  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1334  # record for each band, in priority order.
1335  for orderedRecords in zip(*orderedCatalogs):
1336 
1337  maxSNRecord = None
1338  maxSNFlagKeys = None
1339  maxSN = 0.
1340  priorityRecord = None
1341  priorityFlagKeys = None
1342  prioritySN = 0.
1343  hasPseudoFilter = False
1344 
1345  # Now we iterate over those record-band pairs, keeping track of the priority and the
1346  # largest S/N band.
1347  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1348  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1349  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1350 
1351  if not (parent or child):
1352  for pseudoFilterKey in self.pseudoFilterKeys:
1353  if inputRecord.get(pseudoFilterKey):
1354  hasPseudoFilter = True
1355  priorityRecord = inputRecord
1356  priorityFlagKeys = flagKeys
1357  break
1358  if hasPseudoFilter:
1359  break
1360 
1361  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1362  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1363  sn = 0.
1364  else:
1365  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1366  if numpy.isnan(sn) or sn < 0.:
1367  sn = 0.
1368  if (parent or child) and priorityRecord is None:
1369  priorityRecord = inputRecord
1370  priorityFlagKeys = flagKeys
1371  prioritySN = sn
1372  if sn > maxSN:
1373  maxSNRecord = inputRecord
1374  maxSNFlagKeys = flagKeys
1375  maxSN = sn
1376 
1377  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1378  # the reference band instead. However, we only want to choose the highest S/N band if it is
1379  # significantly better than the priority band. Therefore, to choose a band other than the
1380  # priority, we require that the priority S/N is below the minimum threshold and that the
1381  # difference between the priority and highest S/N is larger than the difference threshold.
1382  #
1383  # For pseudo code objects we always choose the first band in the priority list.
1384  bestRecord = None
1385  bestFlagKeys = None
1386  if hasPseudoFilter:
1387  bestRecord = priorityRecord
1388  bestFlagKeys = priorityFlagKeys
1389  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1390  maxSNRecord is not None):
1391  bestRecord = maxSNRecord
1392  bestFlagKeys = maxSNFlagKeys
1393  elif priorityRecord is not None:
1394  bestRecord = priorityRecord
1395  bestFlagKeys = priorityFlagKeys
1396 
1397  if bestRecord is not None and bestFlagKeys is not None:
1398  outputRecord = mergedCatalog.addNew()
1399  outputRecord.assign(bestRecord, self.schemaMapper)
1400  outputRecord.set(bestFlagKeys.output, True)
1401  else: # if we didn't find any records
1402  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1403  inputRecord.getId())
1404 
1405  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1406  for inputCatalog in orderedCatalogs:
1407  if len(mergedCatalog) != len(inputCatalog):
1408  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1409  (len(mergedCatalog), len(orderedCatalogs)))
1410 
1411  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:788
Merge coadd detections from multiple bands.
Definition: multiBand.py:591
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:334
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:437
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:777
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:279
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:749
def run(self, patchRef)
Deblend and measure.
Definition: multiBand.py:1051
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:326
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:310
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:450
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1264
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:109
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:246
Merge measurements from multiple bands.
Definition: multiBand.py:1184
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:463
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:885
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1117
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:264
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1309
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:489
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1153
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1097
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:559
Configuration for merging sources.
Definition: multiBand.py:378
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:500
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:700
A base class for merging source catalogs.
Definition: multiBand.py:394
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1137
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:474
def scaleVariance(maskedImage, maskPlanes, log=None)
Scale the variance in a maskedImage.
Definition: coaddBase.py:280
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:820
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:249
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:517
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1007
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:678
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:352