lsst.pipe.tasks  15.0-7-g6bb3a066+2
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 import numpy
24 
25 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
26 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
27 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
28 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
30 from lsst.meas.deblender import SourceDeblendTask
31 from lsst.pipe.tasks.coaddBase import getSkyInfo
32 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
33 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
34 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
35 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
36 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
37 import lsst.afw.image as afwImage
38 import lsst.afw.table as afwTable
39 import lsst.afw.math as afwMath
40 import lsst.afw.geom as afwGeom
41 import lsst.afw.detection as afwDetect
42 from lsst.daf.base import PropertyList
43 
44 """
45 New dataset types:
46 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
47 * deepCoadd_mergeDet: merged detections (tract, patch)
48 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
49 * deepCoadd_ref: reference sources (tract, patch)
50 All of these have associated *_schema catalogs that require no data ID and hold no records.
51 
52 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
53 the mergeDet, meas, and ref dataset Footprints:
54 * deepCoadd_peak_schema
55 """
56 
57 
58 def _makeGetSchemaCatalogs(datasetSuffix):
59  """Construct a getSchemaCatalogs instance method
60 
61  These are identical for most of the classes here, so we'll consolidate
62  the code.
63 
64  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
65  """
66 
67  def getSchemaCatalogs(self):
68  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
69  src = afwTable.SourceCatalog(self.schema)
70  if hasattr(self, "algMetadata"):
71  src.getTable().setMetadata(self.algMetadata)
72  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
73  return getSchemaCatalogs
74 
75 
76 def _makeMakeIdFactory(datasetName):
77  """Construct a makeIdFactory instance method
78 
79  These are identical for all the classes here, so this consolidates
80  the code.
81 
82  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
83  """
84 
85  def makeIdFactory(self, dataRef):
86  """Return an IdFactory for setting the detection identifiers
87 
88  The actual parameters used in the IdFactory are provided by
89  the butler (through the provided data reference.
90  """
91  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
92  expId = int(dataRef.get(self.config.coaddName + datasetName))
93  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
94  return makeIdFactory
95 
96 
98  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
99  """
100  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
101  # and its the only way I could get it to work.
102  return afwImage.Filter(name).getFilterProperty().getName()
103 
104 
105 
106 
108  """!
109  \anchor DetectCoaddSourcesConfig_
110 
111  \brief Configuration parameters for the DetectCoaddSourcesTask
112  """
113  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
114  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
115  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
116  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
117  doInsertFakes = Field(dtype=bool, default=False,
118  doc="Run fake sources injection task")
119  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
120  doc="Injection of fake sources for testing "
121  "purposes (must be retargeted)")
122 
123  def setDefaults(self):
124  Config.setDefaults(self)
125  self.detection.thresholdType = "pixel_stdev"
126  self.detection.isotropicGrow = True
127  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
128  self.detection.reEstimateBackground = False
129  self.detection.background.useApprox = False
130  self.detection.background.binSize = 4096
131  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
132  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
133 
134 
140 
141 
142 class DetectCoaddSourcesTask(CmdLineTask):
143  """!
144  \anchor DetectCoaddSourcesTask_
145 
146  \brief Detect sources on a coadd
147 
148  \section pipe_tasks_multiBand_Contents Contents
149 
150  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
151  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
152  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
153  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
154  - \ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
155  - \ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
156 
157  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
158 
159  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
160 
161  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
162  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
163  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
164  propagate the full covariance matrix -- but it is simple and works well in practice.
165 
166  After scaling the variance plane, we detect sources and generate footprints by delegating to the \ref
167  SourceDetectionTask_ "detection" subtask.
168 
169  \par Inputs:
170  deepCoadd{tract,patch,filter}: ExposureF
171  \par Outputs:
172  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
173  \n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
174  exposure (ExposureF)
175  \n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
176  \par Data Unit:
177  tract, patch, filter
178 
179  DetectCoaddSourcesTask delegates most of its work to the \ref SourceDetectionTask_ "detection" subtask.
180  You can retarget this subtask if you wish.
181 
182  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
183 
184  \copydoc \_\_init\_\_
185 
186  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
187 
188  \copydoc run
189 
190  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
191 
192  See \ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
193 
194  \section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
195 
196  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
197  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
198  files.
199 
200  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
201  \ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
202  \ref SourceDetectionTask_ "SourceDetectionTask" for further information.
203 
204  \section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
205  of using DetectCoaddSourcesTask
206 
207  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
208  the task is to update the background, detect all sources in a single band and generate a set of parent
209  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
210  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
211  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
212  calling detectCoaddSources.py with the `--help` command line argument:
213  \code
214  detectCoaddSources.py --help
215  \endcode
216 
217  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
218  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
219  steps 1 - 4 at \ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
220  \code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
222  \endcode
223  that will process the HSC-I band data. The results are written to
224  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
225 
226  It is also necessary to run:
227  \code
228  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
229  \endcode
230  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
231  processing procedure: \ref MergeDetectionsTask_ "MergeDetectionsTask".
232  """
233  _DefaultName = "detectCoaddSources"
234  ConfigClass = DetectCoaddSourcesConfig
235  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
236  makeIdFactory = _makeMakeIdFactory("CoaddId")
237 
238  @classmethod
239  def _makeArgumentParser(cls):
240  parser = ArgumentParser(name=cls._DefaultName)
241  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
242  ContainerClass=ExistingCoaddDataIdContainer)
243  return parser
244 
245  def __init__(self, schema=None, **kwargs):
246  """!
247  \brief Initialize the task. Create the \ref SourceDetectionTask_ "detection" subtask.
248 
249  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
250 
251  \param[in] schema: initial schema for the output catalog, modified-in place to include all
252  fields set by this task. If None, the source minimal schema will be used.
253  \param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
254  """
255  CmdLineTask.__init__(self, **kwargs)
256  if schema is None:
257  schema = afwTable.SourceTable.makeMinimalSchema()
258  if self.config.doInsertFakes:
259  self.makeSubtask("insertFakes")
260  self.schema = schema
261  self.makeSubtask("detection", schema=self.schema)
262  if self.config.doScaleVariance:
263  self.makeSubtask("scaleVariance")
264 
265  def run(self, patchRef):
266  """!
267  \brief Run detection on a coadd.
268 
269  Invokes \ref runDetection and then uses \ref write to output the
270  results.
271 
272  \param[in] patchRef: data reference for patch
273  """
274  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
275  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
276  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
277  self.write(exposure, results, patchRef)
278  return results
279 
280  def runDetection(self, exposure, idFactory, expId):
281  """!
282  \brief Run detection on an exposure.
283 
284  First scale the variance plane to match the observed variance
285  using \ref ScaleVarianceTask. Then invoke the \ref SourceDetectionTask_ "detection" subtask to
286  detect sources.
287 
288  \param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
289  depending on configuration).
290  \param[in] idFactory: IdFactory to set source identifiers
291  \param[in] expId: Exposure identifier (integer) for RNG seed
292 
293  \return a pipe.base.Struct with fields
294  - sources: catalog of detections
295  - backgrounds: list of backgrounds
296  """
297  if self.config.doScaleVariance:
298  varScale = self.scaleVariance.run(exposure.maskedImage)
299  exposure.getMetadata().add("variance_scale", varScale)
300  backgrounds = afwMath.BackgroundList()
301  if self.config.doInsertFakes:
302  self.insertFakes.run(exposure, background=backgrounds)
303  table = afwTable.SourceTable.make(self.schema, idFactory)
304  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
305  sources = detections.sources
306  fpSets = detections.fpSets
307  if hasattr(fpSets, "background") and fpSets.background:
308  for bg in fpSets.background:
309  backgrounds.append(bg)
310  return Struct(sources=sources, backgrounds=backgrounds)
311 
312  def write(self, exposure, results, patchRef):
313  """!
314  \brief Write out results from runDetection.
315 
316  \param[in] exposure: Exposure to write out
317  \param[in] results: Struct returned from runDetection
318  \param[in] patchRef: data reference for patch
319  """
320  coaddName = self.config.coaddName + "Coadd"
321  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
322  patchRef.put(results.sources, coaddName + "_det")
323  patchRef.put(exposure, coaddName + "_calexp")
324 
325 
326 
327 
328 class MergeSourcesRunner(TaskRunner):
329  """!
330  \anchor MergeSourcesRunner_
331 
332  \brief Task runner for the \ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
333  requires a list of dataRefs rather than a single dataRef.
334  """
335 
336  def makeTask(self, parsedCmd=None, args=None):
337  """!
338  \brief Provide a butler to the Task constructor.
339 
340  \param[in] parsedCmd the parsed command
341  \param[in] args tuple of a list of data references and kwargs (un-used)
342  \throws RuntimeError if both parsedCmd & args are None
343  """
344  if parsedCmd is not None:
345  butler = parsedCmd.butler
346  elif args is not None:
347  dataRefList, kwargs = args
348  butler = dataRefList[0].getButler()
349  else:
350  raise RuntimeError("Neither parsedCmd or args specified")
351  return self.TaskClass(config=self.config, log=self.log, butler=butler)
352 
353  @staticmethod
354  def getTargetList(parsedCmd, **kwargs):
355  """!
356  \brief Provide a list of patch references for each patch.
357 
358  The patch references within the list will have different filters.
359 
360  \param[in] parsedCmd the parsed command
361  \param **kwargs key word arguments (unused)
362  \throws RuntimeError if multiple references are provided for the same combination of tract, patch and
363  filter
364  """
365  refList = {} # Will index this as refList[tract][patch][filter] = ref
366  for ref in parsedCmd.id.refList:
367  tract = ref.dataId["tract"]
368  patch = ref.dataId["patch"]
369  filter = ref.dataId["filter"]
370  if tract not in refList:
371  refList[tract] = {}
372  if patch not in refList[tract]:
373  refList[tract][patch] = {}
374  if filter in refList[tract][patch]:
375  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
376  refList[tract][patch][filter] = ref
377  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
378 
379 
380 class MergeSourcesConfig(Config):
381  """!
382  \anchor MergeSourcesConfig_
383 
384  \brief Configuration for merging sources.
385  """
386  priorityList = ListField(dtype=str, default=[],
387  doc="Priority-ordered list of bands for the merge.")
388  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
389 
390  def validate(self):
391  Config.validate(self)
392  if len(self.priorityList) == 0:
393  raise RuntimeError("No priority list provided")
394 
395 
396 class MergeSourcesTask(CmdLineTask):
397  """!
398  \anchor MergeSourcesTask_
399 
400  \brief A base class for merging source catalogs.
401 
402  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
403  so similar that it makes sense to re-use the code, in the form of this abstract base class.
404 
405  NB: Do not use this class directly. Instead use one of the child classes that inherit from
406  MergeSourcesTask such as \ref MergeDetectionsTask_ "MergeDetectionsTask" or \ref MergeMeasurementsTask_
407  "MergeMeasurementsTask"
408 
409  Sub-classes should set the following class variables:
410  * `_DefaultName`: name of Task
411  * `inputDataset`: name of dataset to read
412  * `outputDataset`: name of dataset to write
413  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
414 
415  In addition, sub-classes must implement the mergeCatalogs method.
416  """
417  _DefaultName = None
418  ConfigClass = MergeSourcesConfig
419  RunnerClass = MergeSourcesRunner
420  inputDataset = None
421  outputDataset = None
422  getSchemaCatalogs = None
423 
424  @classmethod
425  def _makeArgumentParser(cls):
426  """!
427  \brief Create a suitable ArgumentParser.
428 
429  We will use the ArgumentParser to get a provide a list of data
430  references for patches; the RunnerClass will sort them into lists
431  of data references for the same patch
432  """
433  parser = ArgumentParser(name=cls._DefaultName)
434  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
435  ContainerClass=ExistingCoaddDataIdContainer,
436  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
437  return parser
438 
439  def getInputSchema(self, butler=None, schema=None):
440  """!
441  \brief Obtain the input schema either directly or froma butler reference.
442 
443  \param[in] butler butler reference to obtain the input schema from
444  \param[in] schema the input schema
445  """
446  if schema is None:
447  assert butler is not None, "Neither butler nor schema specified"
448  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
449  immediate=True).schema
450  return schema
451 
452  def __init__(self, butler=None, schema=None, **kwargs):
453  """!
454  \brief Initialize the task.
455 
456  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
457  \param[in] schema the schema of the detection catalogs used as input to this one
458  \param[in] butler a butler used to read the input schema from disk, if schema is None
459 
460  Derived classes should use the getInputSchema() method to handle the additional
461  arguments and retreive the actual input schema.
462  """
463  CmdLineTask.__init__(self, **kwargs)
464 
465  def run(self, patchRefList):
466  """!
467  \brief Merge coadd sources from multiple bands. Calls \ref mergeCatalogs which must be defined in
468  subclasses that inherit from MergeSourcesTask.
469 
470  \param[in] patchRefList list of data references for each filter
471  """
472  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
473  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
474  self.write(patchRefList[0], mergedCatalog)
475 
476  def readCatalog(self, patchRef):
477  """!
478  \brief Read input catalog.
479 
480  We read the input dataset provided by the 'inputDataset'
481  class variable.
482 
483  \param[in] patchRef data reference for patch
484  \return tuple consisting of the filter name and the catalog
485  """
486  filterName = patchRef.dataId["filter"]
487  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
488  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
489  return filterName, catalog
490 
491  def mergeCatalogs(self, catalogs, patchRef):
492  """!
493  \brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
494  MergeSourcesTask.
495 
496  \param[in] catalogs dict mapping filter name to source catalog
497 
498  \return merged catalog
499  """
500  raise NotImplementedError()
501 
502  def write(self, patchRef, catalog):
503  """!
504  \brief Write the output.
505 
506  \param[in] patchRef data reference for patch
507  \param[in] catalog catalog
508 
509  We write as the dataset provided by the 'outputDataset'
510  class variable.
511  """
512  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
513  # since the filter isn't actually part of the data ID for the dataset we're saving,
514  # it's confusing to see it in the log message, even if the butler simply ignores it.
515  mergeDataId = patchRef.dataId.copy()
516  del mergeDataId["filter"]
517  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
518 
519  def writeMetadata(self, dataRefList):
520  """!
521  \brief No metadata to write, and not sure how to write it for a list of dataRefs.
522  """
523  pass
524 
525 
526 class CullPeaksConfig(Config):
527  """!
528  \anchor CullPeaksConfig_
529 
530  \brief Configuration for culling garbage peaks after merging footprints.
531 
532  Peaks may also be culled after detection or during deblending; this configuration object
533  only deals with culling after merging Footprints.
534 
535  These cuts are based on three quantities:
536  - nBands: the number of bands in which the peak was detected
537  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
538  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
539 
540  The formula that identifie peaks to cull is:
541 
542  nBands < nBandsSufficient
543  AND (rank >= rankSufficient)
544  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
545 
546  To disable peak culling, simply set nBandsSufficient=1.
547  """
548 
549  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
550  doc="Always keep peaks detected in this many bands")
551  rankSufficient = RangeField(dtype=int, default=20, min=1,
552  doc="Always keep this many peaks in each family")
553  rankConsidered = RangeField(dtype=int, default=30, min=1,
554  doc=("Keep peaks with less than this rank that also match the "
555  "rankNormalizedConsidered condition."))
556  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
557  doc=("Keep peaks with less than this normalized rank that"
558  " also match the rankConsidered condition."))
559 
560 
562  """!
563  \anchor MergeDetectionsConfig_
564 
565  \brief Configuration parameters for the MergeDetectionsTask.
566  """
567  minNewPeak = Field(dtype=float, default=1,
568  doc="Minimum distance from closest peak to create a new one (in arcsec).")
569 
570  maxSamePeak = Field(dtype=float, default=0.3,
571  doc="When adding new catalogs to the merge, all peaks less than this distance "
572  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
573  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
574 
575  skyFilterName = Field(dtype=str, default="sky",
576  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
577  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
578  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
579 
580  def setDefaults(self):
581  MergeSourcesConfig.setDefaults(self)
582  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
583 
584 
585 
591 
592 
594  """!
595  \anchor MergeDetectionsTask_
596 
597  \brief Merge coadd detections from multiple bands.
598 
599  \section pipe_tasks_multiBand_Contents Contents
600 
601  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
602  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Init
603  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Run
604  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Config
605  - \ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
606  - \ref pipe_tasks_multiband_MergeDetectionsTask_Example
607 
608  \section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
609 
610  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
611 
612  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
613  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
614  track of which band each source originates in.
615 
616  The catalog merge is performed by \ref getMergedSourceCatalog. Spurious peaks detected around bright
617  objects are culled as described in \ref CullPeaksConfig_.
618 
619  \par Inputs:
620  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
621  \par Outputs:
622  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
623  \par Data Unit:
624  tract, patch
625 
626  MergeDetectionsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
627 
628  \section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
629 
630  \copydoc \_\_init\_\_
631 
632  \section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
633 
634  \copydoc run
635 
636  \section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
637 
638  See \ref MergeDetectionsConfig_
639 
640  \section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
641 
642  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a flag \c -d
643  to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py files.
644 
645  MergeDetectionsTask has no debug variables.
646 
647  \section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
648 
649  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
650  of the available bands.
651  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
652  chosen subset of filters.
653  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
654  and, eventually, perform forced photometry.
655  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
656  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
657  `--help` command line argument:
658  \code
659  mergeCoaddDetections.py --help
660  \endcode
661 
662  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
663  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
664  step 5 at \ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
665  \code
666  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
667  \endcode
668  This will merge the HSC-I & -R band parent source catalogs and write the results to
669  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
670 
671  The next step in the multi-band processing procedure is
672  \ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
673  """
674  ConfigClass = MergeDetectionsConfig
675  _DefaultName = "mergeCoaddDetections"
676  inputDataset = "det"
677  outputDataset = "mergeDet"
678  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
679 
680  def __init__(self, butler=None, schema=None, **kwargs):
681  """!
682  \brief Initialize the merge detections task.
683 
684  A \ref FootprintMergeList_ "FootprintMergeList" will be used to
685  merge the source catalogs.
686 
687  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
688  \param[in] schema the schema of the detection catalogs used as input to this one
689  \param[in] butler a butler used to read the input schema from disk, if schema is None
690  \param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
691 
692  The task will set its own self.schema attribute to the schema of the output merged catalog.
693  """
694  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
695  self.makeSubtask("skyObjects")
696  self.schema = self.getInputSchema(butler=butler, schema=schema)
697 
698  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
699  filterNames += [self.config.skyFilterName]
700  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
701 
702  def mergeCatalogs(self, catalogs, patchRef):
703  """!
704  \brief Merge multiple catalogs.
705 
706  After ordering the catalogs and filters in priority order,
707  \ref getMergedSourceCatalog of the \ref FootprintMergeList_ "FootprintMergeList" created by
708  \ref \_\_init\_\_ is used to perform the actual merging. Finally, \ref cullPeaks is used to remove
709  garbage peaks detected around bright objects.
710 
711  \param[in] catalogs
712  \param[in] patchRef
713  \param[out] mergedList
714  """
715 
716  # Convert distance to tract coordinate
717  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
718  tractWcs = skyInfo.wcs
719  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
720  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
721 
722  # Put catalogs, filters in priority order
723  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
724  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
725  if band in catalogs.keys()]
726 
727  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
728  self.schema, self.makeIdFactory(patchRef),
729  samePeakDistance)
730 
731  #
732  # Add extra sources that correspond to blank sky
733  #
734  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
735  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
736  if skySourceFootprints:
737  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
738  for foot in skySourceFootprints:
739  s = mergedList.addNew()
740  s.setFootprint(foot)
741  s.set(key, True)
742 
743  # Sort Peaks from brightest to faintest
744  for record in mergedList:
745  record.getFootprint().sortPeaks()
746  self.log.info("Merged to %d sources" % len(mergedList))
747  # Attempt to remove garbage peaks
748  self.cullPeaks(mergedList)
749  return mergedList
750 
751  def cullPeaks(self, catalog):
752  """!
753  \brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
754 
755  \param[in] catalog Source catalog
756  """
757  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
758  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
759  totalPeaks = 0
760  culledPeaks = 0
761  for parentSource in catalog:
762  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
763  # to it (which is easier than deleting as we iterate).
764  keptPeaks = parentSource.getFootprint().getPeaks()
765  oldPeaks = list(keptPeaks)
766  keptPeaks.clear()
767  familySize = len(oldPeaks)
768  totalPeaks += familySize
769  for rank, peak in enumerate(oldPeaks):
770  if ((rank < self.config.cullPeaks.rankSufficient) or
771  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
772  (rank < self.config.cullPeaks.rankConsidered and
773  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
774  keptPeaks.append(peak)
775  else:
776  culledPeaks += 1
777  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
778 
779  def getSchemaCatalogs(self):
780  """!
781  Return a dict of empty catalogs for each catalog dataset produced by this task.
782 
783  \param[out] dictionary of empty catalogs
784  """
785  mergeDet = afwTable.SourceCatalog(self.schema)
786  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
787  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
788  self.config.coaddName + "Coadd_peak": peak}
789 
790  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
791  """!
792  \brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
793 
794  \param mergedList The merged Footprints from all the input bands
795  \param skyInfo A description of the patch
796  \param seed Seed for the random number generator
797  """
798  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
799  detected = mask.getPlaneBitMask("DETECTED")
800  for s in mergedList:
801  s.getFootprint().spans.setMask(mask, detected)
802 
803  footprints = self.skyObjects.run(mask, seed)
804  if not footprints:
805  return footprints
806 
807  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
808  schema = self.merged.getPeakSchema()
809  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
810  converted = []
811  for oldFoot in footprints:
812  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
813  peak = oldFoot.getPeaks()[0]
814  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
815  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
816  newFoot.getPeaks()[0].set(mergeKey, True)
817  converted.append(newFoot)
818 
819  return converted
820 
821 
823  """!
824  \anchor MeasureMergedCoaddSourcesConfig_
825 
826  \brief Configuration parameters for the MeasureMergedCoaddSourcesTask
827  """
828  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
829  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
830  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
831  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
832  doPropagateFlags = Field(
833  dtype=bool, default=True,
834  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
835  )
836  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
837  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
838  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
839  doWriteMatchesDenormalized = Field(
840  dtype=bool,
841  default=False,
842  doc=("Write reference matches in denormalized format? "
843  "This format uses more disk space, but is more convenient to read."),
844  )
845  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
846  checkUnitsParseStrict = Field(
847  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
848  dtype=str,
849  default="raise",
850  )
851  doApCorr = Field(
852  dtype=bool,
853  default=True,
854  doc="Apply aperture corrections"
855  )
856  applyApCorr = ConfigurableField(
857  target=ApplyApCorrTask,
858  doc="Subtask to apply aperture corrections"
859  )
860  doRunCatalogCalculation = Field(
861  dtype=bool,
862  default=True,
863  doc='Run catalogCalculation task'
864  )
865  catalogCalculation = ConfigurableField(
866  target=CatalogCalculationTask,
867  doc="Subtask to run catalogCalculation plugins on catalog"
868  )
869 
870  def setDefaults(self):
871  Config.setDefaults(self)
872  self.deblend.propagateAllPeaks = True
873  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
874  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
875  'INEXACT_PSF']
876  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
877  'INEXACT_PSF']
878 
879 
885 
886 
887 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
888  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
889  @staticmethod
890  def getTargetList(parsedCmd, **kwargs):
891  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
892 
893 
894 class MeasureMergedCoaddSourcesTask(CmdLineTask):
895  """!
896  \anchor MeasureMergedCoaddSourcesTask_
897 
898  \brief Deblend sources from master catalog in each coadd seperately and measure.
899 
900  \section pipe_tasks_multiBand_Contents Contents
901 
902  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
903  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
904  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
905  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
906  - \ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
907  - \ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
908 
909  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
910 
911  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
912  measurement in each coadd.
913 
914  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
915  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
916  consistent set of child sources.
917 
918  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
919  properties are measured and the \c is-primary flag (indicating sources with no children) is set. Visit
920  flags are propagated to the coadd sources.
921 
922  Optionally, we can match the coadd sources to an external reference catalog.
923 
924  \par Inputs:
925  deepCoadd_mergeDet{tract,patch}: SourceCatalog
926  \n deepCoadd_calexp{tract,patch,filter}: ExposureF
927  \par Outputs:
928  deepCoadd_meas{tract,patch,filter}: SourceCatalog
929  \par Data Unit:
930  tract, patch, filter
931 
932  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
933 
934  <DL>
935  <DT> \ref SourceDeblendTask_ "deblend"
936  <DD> Deblend all the sources from the master catalog.</DD>
937  <DT> \ref SingleFrameMeasurementTask_ "measurement"
938  <DD> Measure source properties of deblended sources.</DD>
939  <DT> \ref SetPrimaryFlagsTask_ "setPrimaryFlags"
940  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
941  not at the edge of the field and that have either not been deblended or are the children of deblended
942  sources</DD>
943  <DT> \ref PropagateVisitFlagsTask_ "propagateFlags"
944  <DD> Propagate flags set in individual visits to the coadd.</DD>
945  <DT> \ref DirectMatchTask_ "match"
946  <DD> Match input sources to a reference catalog (optional).
947  </DD>
948  </DL>
949  These subtasks may be retargeted as required.
950 
951  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
952 
953  \copydoc \_\_init\_\_
954 
955  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
956 
957  \copydoc run
958 
959  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
960 
961  See \ref MeasureMergedCoaddSourcesConfig_
962 
963  \section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
964 
965  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
966  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
967  files.
968 
969  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
970  the various sub-tasks. See the documetation for individual sub-tasks for more information.
971 
972  \section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
973  MeasureMergedCoaddSourcesTask
974 
975  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
976  The next stage in the multi-band processing procedure will merge these measurements into a suitable
977  catalog for driving forced photometry.
978 
979  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
980  to be processed.
981  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
982  `--help` command line argument:
983  \code
984  measureCoaddSources.py --help
985  \endcode
986 
987  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
988  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
989  step 6 at \ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
990  coadd as follows:
991  \code
992  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
993  \endcode
994  This will process the HSC-I band data. The results are written in
995  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
996 
997  It is also necessary to run
998  \code
999  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
1000  \endcode
1001  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1002  procedure: \ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1003  """
1004  _DefaultName = "measureCoaddSources"
1005  ConfigClass = MeasureMergedCoaddSourcesConfig
1006  RunnerClass = MeasureMergedCoaddSourcesRunner
1007  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1008  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1009 
1010  @classmethod
1011  def _makeArgumentParser(cls):
1012  parser = ArgumentParser(name=cls._DefaultName)
1013  parser.add_id_argument("--id", "deepCoadd_calexp",
1014  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1015  ContainerClass=ExistingCoaddDataIdContainer)
1016  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
1017  return parser
1018 
1019  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1020  """!
1021  \brief Initialize the task.
1022 
1023  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1024  \param[in] schema: the schema of the merged detection catalog used as input to this one
1025  \param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1026  \param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1027  catalog. May be None if the loader can be constructed from the butler argument or all steps
1028  requiring a reference catalog are disabled.
1029  \param[in] butler: a butler used to read the input schemas from disk or construct the reference
1030  catalog loader, if schema or peakSchema or refObjLoader is None
1031 
1032  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1033  This will include all fields from the input schema, as well as additional fields for all the
1034  measurements.
1035  """
1036  CmdLineTask.__init__(self, **kwargs)
1037  if schema is None:
1038  assert butler is not None, "Neither butler nor schema is defined"
1039  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1040  self.schemaMapper = afwTable.SchemaMapper(schema)
1041  self.schemaMapper.addMinimalSchema(schema)
1042  self.schema = self.schemaMapper.getOutputSchema()
1044  if self.config.doDeblend:
1045  if peakSchema is None:
1046  assert butler is not None, "Neither butler nor peakSchema is defined"
1047  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1048  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1049  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1050  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1051  if self.config.doMatchSources:
1052  if refObjLoader is None:
1053  assert butler is not None, "Neither butler nor refObjLoader is defined"
1054  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1055  if self.config.doPropagateFlags:
1056  self.makeSubtask("propagateFlags", schema=self.schema)
1057  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1058  if self.config.doApCorr:
1059  self.makeSubtask("applyApCorr", schema=self.schema)
1060  if self.config.doRunCatalogCalculation:
1061  self.makeSubtask("catalogCalculation", schema=self.schema)
1062 
1063  def run(self, patchRef, psfCache=100):
1064  """!
1065  \brief Deblend and measure.
1066 
1067  \param[in] patchRef: Patch reference.
1068 
1069  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1070  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1071  Finally, write the deblended sources and measurements out.
1072  """
1073  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1074  exposure.getPsf().setCacheCapacity(psfCache)
1075  sources = self.readSources(patchRef)
1076  if self.config.doDeblend:
1077  self.deblend.run(exposure, sources)
1078 
1079  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1080  # catalog is non-contiguous so can't extract column
1081  numBig = sum((s.get(bigKey) for s in sources))
1082  if numBig > 0:
1083  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1084  (patchRef.dataId, numBig))
1085 
1086  table = sources.getTable()
1087  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1088 
1089  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1090 
1091  if self.config.doApCorr:
1092  self.applyApCorr.run(
1093  catalog=sources,
1094  apCorrMap=exposure.getInfo().getApCorrMap()
1095  )
1096 
1097  if self.config.doRunCatalogCalculation:
1098  self.catalogCalculation.run(sources)
1099 
1100  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1101  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1102  includeDeblend=self.config.doDeblend)
1103  if self.config.doPropagateFlags:
1104  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1105  exposure.getWcs())
1106  if self.config.doMatchSources:
1107  self.writeMatches(patchRef, exposure, sources)
1108  self.write(patchRef, sources)
1109 
1110  def readSources(self, dataRef):
1111  """!
1112  \brief Read input sources.
1113 
1114  \param[in] dataRef: Data reference for catalog of merged detections
1115  \return List of sources in merged catalog
1116 
1117  We also need to add columns to hold the measurements we're about to make
1118  so we can measure in-place.
1119  """
1120  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1121  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1122  idFactory = self.makeIdFactory(dataRef)
1123  for s in merged:
1124  idFactory.notify(s.getId())
1125  table = afwTable.SourceTable.make(self.schema, idFactory)
1126  sources = afwTable.SourceCatalog(table)
1127  sources.extend(merged, self.schemaMapper)
1128  return sources
1129 
1130  def writeMatches(self, dataRef, exposure, sources):
1131  """!
1132  \brief Write matches of the sources to the astrometric reference catalog.
1133 
1134  We use the Wcs in the exposure to match sources.
1135 
1136  \param[in] dataRef: data reference
1137  \param[in] exposure: exposure with Wcs
1138  \param[in] sources: source catalog
1139  """
1140  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1141  if result.matches:
1142  matches = afwTable.packMatches(result.matches)
1143  matches.table.setMetadata(result.matchMeta)
1144  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1145  if self.config.doWriteMatchesDenormalized:
1146  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1147  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1148 
1149  def write(self, dataRef, sources):
1150  """!
1151  \brief Write the source catalog.
1152 
1153  \param[in] dataRef: data reference
1154  \param[in] sources: source catalog
1155  """
1156  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1157  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1158 
1159  def getExposureId(self, dataRef):
1160  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1161 
1162 
1164  """!
1165  \anchor MergeMeasurementsConfig_
1166 
1167  \brief Configuration parameters for the MergeMeasurementsTask
1168  """
1169  pseudoFilterList = ListField(dtype=str, default=["sky"],
1170  doc="Names of filters which may have no associated detection\n"
1171  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1172  snName = Field(dtype=str, default="base_PsfFlux",
1173  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1174  minSN = Field(dtype=float, default=10.,
1175  doc="If the S/N from the priority band is below this value (and the S/N "
1176  "is larger than minSNDiff compared to the priority band), use the band with "
1177  "the largest S/N as the reference band.")
1178  minSNDiff = Field(dtype=float, default=3.,
1179  doc="If the difference in S/N between another band and the priority band is larger "
1180  "than this value (and the S/N in the priority band is less than minSN) "
1181  "use the band with the largest S/N as the reference band")
1182  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1183  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1184  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1185 
1186 
1192 
1193 
1195  """!
1196  \anchor MergeMeasurementsTask_
1197 
1198  \brief Merge measurements from multiple bands
1199 
1200  \section pipe_tasks_multiBand_Contents Contents
1201 
1202  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1203  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1204  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1205  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1206  - \ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1207  - \ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1208 
1209  \section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1210 
1211  Command-line task that merges measurements from multiple bands.
1212 
1213  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1214  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1215  required to have centroid, shape and flux measurements in each band.
1216 
1217  \par Inputs:
1218  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1219  \par Outputs:
1220  deepCoadd_ref{tract,patch}: SourceCatalog
1221  \par Data Unit:
1222  tract, patch
1223 
1224  MergeMeasurementsTask subclasses \ref MergeSourcesTask_ "MergeSourcesTask".
1225 
1226  \section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1227 
1228  \copydoc \_\_init\_\_
1229 
1230  \section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1231 
1232  \copydoc run
1233 
1234  \section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1235 
1236  See \ref MergeMeasurementsConfig_
1237 
1238  \section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1239 
1240  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
1241  flag \c -d to import \b debug.py from your \c PYTHONPATH; see \ref baseDebug for more about \b debug.py
1242  files.
1243 
1244  MergeMeasurementsTask has no debug variables.
1245 
1246  \section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1247  of using MergeMeasurementsTask
1248 
1249  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1250  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1251  coadds and individual exposures.
1252  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1253  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1254  command line argument:
1255  \code
1256  mergeCoaddMeasurements.py --help
1257  \endcode
1258 
1259  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1260  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1261  step 7 at \ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1262  as follows:
1263  \code
1264  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1265  \endcode
1266  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1267  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1268  """
1269  _DefaultName = "mergeCoaddMeasurements"
1270  ConfigClass = MergeMeasurementsConfig
1271  inputDataset = "meas"
1272  outputDataset = "ref"
1273  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1274 
1275  def __init__(self, butler=None, schema=None, **kwargs):
1276  """!
1277  Initialize the task.
1278 
1279  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1280  \param[in] schema: the schema of the detection catalogs used as input to this one
1281  \param[in] butler: a butler used to read the input schema from disk, if schema is None
1282 
1283  The task will set its own self.schema attribute to the schema of the output merged catalog.
1284  """
1285  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1286  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1287  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1288  self.schemaMapper.addMinimalSchema(inputSchema, True)
1289  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1290  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1291  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1292 
1293  self.flagKeys = {}
1294  for band in self.config.priorityList:
1295  short = getShortFilterName(band)
1296  outputKey = self.schemaMapper.editOutputSchema().addField(
1297  "merge_measurement_%s" % short,
1298  type="Flag",
1299  doc="Flag field set if the measurements here are from the %s filter" % band
1300  )
1301  peakKey = inputSchema.find("merge_peak_%s" % short).key
1302  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1303  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1304  self.schema = self.schemaMapper.getOutputSchema()
1305 
1307  for filt in self.config.pseudoFilterList:
1308  try:
1309  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1310  except:
1311  self.log.warn("merge_peak is not set for pseudo-filter %s" % filt)
1312 
1313  self.badFlags = {}
1314  for flag in self.config.flags:
1315  try:
1316  self.badFlags[flag] = self.schema.find(flag).getKey()
1317  except KeyError as exc:
1318  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1319 
1320  def mergeCatalogs(self, catalogs, patchRef):
1321  """!
1322  Merge measurement catalogs to create a single reference catalog for forced photometry
1323 
1324  \param[in] catalogs: the catalogs to be merged
1325  \param[in] patchRef: patch reference for data
1326 
1327  For parent sources, we choose the first band in config.priorityList for which the
1328  merge_footprint flag for that band is is True.
1329 
1330  For child sources, the logic is the same, except that we use the merge_peak flags.
1331  """
1332  # Put catalogs, filters in priority order
1333  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1334  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1335 
1336  mergedCatalog = afwTable.SourceCatalog(self.schema)
1337  mergedCatalog.reserve(len(orderedCatalogs[0]))
1338 
1339  idKey = orderedCatalogs[0].table.getIdKey()
1340  for catalog in orderedCatalogs[1:]:
1341  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1342  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1343 
1344  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1345  # record for each band, in priority order.
1346  for orderedRecords in zip(*orderedCatalogs):
1347 
1348  maxSNRecord = None
1349  maxSNFlagKeys = None
1350  maxSN = 0.
1351  priorityRecord = None
1352  priorityFlagKeys = None
1353  prioritySN = 0.
1354  hasPseudoFilter = False
1355 
1356  # Now we iterate over those record-band pairs, keeping track of the priority and the
1357  # largest S/N band.
1358  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1359  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1360  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1361 
1362  if not (parent or child):
1363  for pseudoFilterKey in self.pseudoFilterKeys:
1364  if inputRecord.get(pseudoFilterKey):
1365  hasPseudoFilter = True
1366  priorityRecord = inputRecord
1367  priorityFlagKeys = flagKeys
1368  break
1369  if hasPseudoFilter:
1370  break
1371 
1372  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1373  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1374  sn = 0.
1375  else:
1376  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1377  if numpy.isnan(sn) or sn < 0.:
1378  sn = 0.
1379  if (parent or child) and priorityRecord is None:
1380  priorityRecord = inputRecord
1381  priorityFlagKeys = flagKeys
1382  prioritySN = sn
1383  if sn > maxSN:
1384  maxSNRecord = inputRecord
1385  maxSNFlagKeys = flagKeys
1386  maxSN = sn
1387 
1388  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1389  # the reference band instead. However, we only want to choose the highest S/N band if it is
1390  # significantly better than the priority band. Therefore, to choose a band other than the
1391  # priority, we require that the priority S/N is below the minimum threshold and that the
1392  # difference between the priority and highest S/N is larger than the difference threshold.
1393  #
1394  # For pseudo code objects we always choose the first band in the priority list.
1395  bestRecord = None
1396  bestFlagKeys = None
1397  if hasPseudoFilter:
1398  bestRecord = priorityRecord
1399  bestFlagKeys = priorityFlagKeys
1400  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1401  maxSNRecord is not None):
1402  bestRecord = maxSNRecord
1403  bestFlagKeys = maxSNFlagKeys
1404  elif priorityRecord is not None:
1405  bestRecord = priorityRecord
1406  bestFlagKeys = priorityFlagKeys
1407 
1408  if bestRecord is not None and bestFlagKeys is not None:
1409  outputRecord = mergedCatalog.addNew()
1410  outputRecord.assign(bestRecord, self.schemaMapper)
1411  outputRecord.set(bestFlagKeys.output, True)
1412  else: # if we didn't find any records
1413  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1414  inputRecord.getId())
1415 
1416  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1417  for inputCatalog in orderedCatalogs:
1418  if len(mergedCatalog) != len(inputCatalog):
1419  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1420  (len(mergedCatalog), len(orderedCatalogs)))
1421 
1422  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:790
Merge coadd detections from multiple bands.
Definition: multiBand.py:593
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:336
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:439
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:779
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:280
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:751
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:328
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:312
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:452
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1275
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:107
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:245
Merge measurements from multiple bands.
Definition: multiBand.py:1194
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:465
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:894
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1130
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:265
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1320
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:491
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1163
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1110
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:561
Configuration for merging sources.
Definition: multiBand.py:380
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:502
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:702
def run(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:1063
A base class for merging source catalogs.
Definition: multiBand.py:396
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1149
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:476
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:822
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:255
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:519
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1019
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:680
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:354