lsst.pipe.tasks  15.0-16-g6f0eb036+9
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 import numpy
24 
25 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
26 from lsst.pipe.base import CmdLineTask, Struct, TaskRunner, ArgumentParser, ButlerInitializedTaskRunner
27 from lsst.pex.config import Config, Field, ListField, ConfigurableField, RangeField, ConfigField
28 from lsst.meas.algorithms import DynamicDetectionTask, SkyObjectsTask
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
30 from lsst.meas.deblender import SourceDeblendTask
31 from lsst.pipe.tasks.coaddBase import getSkyInfo
32 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
33 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
34 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
35 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
36 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
37 import lsst.afw.image as afwImage
38 import lsst.afw.table as afwTable
39 import lsst.afw.math as afwMath
40 import lsst.afw.detection as afwDetect
41 from lsst.daf.base import PropertyList
42 
43 """
44 New dataset types:
45 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
46 * deepCoadd_mergeDet: merged detections (tract, patch)
47 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
48 * deepCoadd_ref: reference sources (tract, patch)
49 All of these have associated *_schema catalogs that require no data ID and hold no records.
50 
51 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
52 the mergeDet, meas, and ref dataset Footprints:
53 * deepCoadd_peak_schema
54 """
55 
56 
57 def _makeGetSchemaCatalogs(datasetSuffix):
58  """Construct a getSchemaCatalogs instance method
59 
60  These are identical for most of the classes here, so we'll consolidate
61  the code.
62 
63  datasetSuffix: Suffix of dataset name, e.g., "src" for "deepCoadd_src"
64  """
65 
66  def getSchemaCatalogs(self):
67  """Return a dict of empty catalogs for each catalog dataset produced by this task."""
68  src = afwTable.SourceCatalog(self.schema)
69  if hasattr(self, "algMetadata"):
70  src.getTable().setMetadata(self.algMetadata)
71  return {self.config.coaddName + "Coadd_" + datasetSuffix: src}
72  return getSchemaCatalogs
73 
74 
75 def _makeMakeIdFactory(datasetName):
76  """Construct a makeIdFactory instance method
77 
78  These are identical for all the classes here, so this consolidates
79  the code.
80 
81  datasetName: Dataset name without the coadd name prefix, e.g., "CoaddId" for "deepCoaddId"
82  """
83 
84  def makeIdFactory(self, dataRef):
85  """Return an IdFactory for setting the detection identifiers
86 
87  The actual parameters used in the IdFactory are provided by
88  the butler (through the provided data reference.
89  """
90  expBits = dataRef.get(self.config.coaddName + datasetName + "_bits")
91  expId = int(dataRef.get(self.config.coaddName + datasetName))
92  return afwTable.IdFactory.makeSource(expId, 64 - expBits)
93  return makeIdFactory
94 
95 
97  """Given a longer, camera-specific filter name (e.g. "HSC-I") return its shorthand name ("i").
98  """
99  # I'm not sure if this is the way this is supposed to be implemented, but it seems to work,
100  # and its the only way I could get it to work.
101  return afwImage.Filter(name).getFilterProperty().getName()
102 
103 
104 
105 
107  """!
108  @anchor DetectCoaddSourcesConfig_
109 
110  @brief Configuration parameters for the DetectCoaddSourcesTask
111  """
112  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
113  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
114  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
115  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
116  doInsertFakes = Field(dtype=bool, default=False,
117  doc="Run fake sources injection task")
118  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
119  doc="Injection of fake sources for testing "
120  "purposes (must be retargeted)")
121 
122  def setDefaults(self):
123  Config.setDefaults(self)
124  self.detection.thresholdType = "pixel_stdev"
125  self.detection.isotropicGrow = True
126  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
127  self.detection.reEstimateBackground = False
128  self.detection.background.useApprox = False
129  self.detection.background.binSize = 4096
130  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
131  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
132 
133 
139 
140 
141 class DetectCoaddSourcesTask(CmdLineTask):
142  """!
143  @anchor DetectCoaddSourcesTask_
144 
145  @brief Detect sources on a coadd
146 
147  @section pipe_tasks_multiBand_Contents Contents
148 
149  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
150  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
151  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
152  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
153  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
154  - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
155 
156  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
157 
158  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
159 
160  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
161  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
162  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
163  propagate the full covariance matrix -- but it is simple and works well in practice.
164 
165  After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref
166  SourceDetectionTask_ "detection" subtask.
167 
168  @par Inputs:
169  deepCoadd{tract,patch,filter}: ExposureF
170  @par Outputs:
171  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
172  @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
173  exposure (ExposureF)
174  @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
175  @par Data Unit:
176  tract, patch, filter
177 
178  DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask.
179  You can retarget this subtask if you wish.
180 
181  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
182 
183  @copydoc \_\_init\_\_
184 
185  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
186 
187  @copydoc run
188 
189  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
190 
191  See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
192 
193  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
194 
195  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
196  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
197  files.
198 
199  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
200  @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
201  @ref SourceDetectionTask_ "SourceDetectionTask" for further information.
202 
203  @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
204  of using DetectCoaddSourcesTask
205 
206  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
207  the task is to update the background, detect all sources in a single band and generate a set of parent
208  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
209  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
210  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
211  calling detectCoaddSources.py with the `--help` command line argument:
212  @code
213  detectCoaddSources.py --help
214  @endcode
215 
216  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
217  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
218  steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
219  @code
220  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
221  @endcode
222  that will process the HSC-I band data. The results are written to
223  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
224 
225  It is also necessary to run:
226  @code
227  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
228  @endcode
229  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
230  processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask".
231  """
232  _DefaultName = "detectCoaddSources"
233  ConfigClass = DetectCoaddSourcesConfig
234  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
235  makeIdFactory = _makeMakeIdFactory("CoaddId")
236 
237  @classmethod
238  def _makeArgumentParser(cls):
239  parser = ArgumentParser(name=cls._DefaultName)
240  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
241  ContainerClass=ExistingCoaddDataIdContainer)
242  return parser
243 
244  def __init__(self, schema=None, **kwargs):
245  """!
246  @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask.
247 
248  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
249 
250  @param[in] schema: initial schema for the output catalog, modified-in place to include all
251  fields set by this task. If None, the source minimal schema will be used.
252  @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
253  """
254  CmdLineTask.__init__(self, **kwargs)
255  if schema is None:
256  schema = afwTable.SourceTable.makeMinimalSchema()
257  if self.config.doInsertFakes:
258  self.makeSubtask("insertFakes")
259  self.schema = schema
260  self.makeSubtask("detection", schema=self.schema)
261  if self.config.doScaleVariance:
262  self.makeSubtask("scaleVariance")
263 
264  def run(self, patchRef):
265  """!
266  @brief Run detection on a coadd.
267 
268  Invokes @ref runDetection and then uses @ref write to output the
269  results.
270 
271  @param[in] patchRef: data reference for patch
272  """
273  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
274  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
275  results = self.runDetection(exposure, self.makeIdFactory(patchRef), expId=expId)
276  self.write(exposure, results, patchRef)
277  return results
278 
279  def runDetection(self, exposure, idFactory, expId):
280  """!
281  @brief Run detection on an exposure.
282 
283  First scale the variance plane to match the observed variance
284  using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to
285  detect sources.
286 
287  @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
288  depending on configuration).
289  @param[in] idFactory: IdFactory to set source identifiers
290  @param[in] expId: Exposure identifier (integer) for RNG seed
291 
292  @return a pipe.base.Struct with fields
293  - sources: catalog of detections
294  - backgrounds: list of backgrounds
295  """
296  if self.config.doScaleVariance:
297  varScale = self.scaleVariance.run(exposure.maskedImage)
298  exposure.getMetadata().add("variance_scale", varScale)
299  backgrounds = afwMath.BackgroundList()
300  if self.config.doInsertFakes:
301  self.insertFakes.run(exposure, background=backgrounds)
302  table = afwTable.SourceTable.make(self.schema, idFactory)
303  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
304  sources = detections.sources
305  fpSets = detections.fpSets
306  if hasattr(fpSets, "background") and fpSets.background:
307  for bg in fpSets.background:
308  backgrounds.append(bg)
309  return Struct(sources=sources, backgrounds=backgrounds)
310 
311  def write(self, exposure, results, patchRef):
312  """!
313  @brief Write out results from runDetection.
314 
315  @param[in] exposure: Exposure to write out
316  @param[in] results: Struct returned from runDetection
317  @param[in] patchRef: data reference for patch
318  """
319  coaddName = self.config.coaddName + "Coadd"
320  patchRef.put(results.backgrounds, coaddName + "_calexp_background")
321  patchRef.put(results.sources, coaddName + "_det")
322  patchRef.put(exposure, coaddName + "_calexp")
323 
324 
325 
326 
327 class MergeSourcesRunner(TaskRunner):
328  """!
329  @anchor MergeSourcesRunner_
330 
331  @brief Task runner for the @ref MergeSourcesTask_ "MergeSourcesTask". Required because the run method
332  requires a list of dataRefs rather than a single dataRef.
333  """
334 
335  def makeTask(self, parsedCmd=None, args=None):
336  """!
337  @brief Provide a butler to the Task constructor.
338 
339  @param[in] parsedCmd the parsed command
340  @param[in] args tuple of a list of data references and kwargs (un-used)
341  @throws RuntimeError if both parsedCmd & args are None
342  """
343  if parsedCmd is not None:
344  butler = parsedCmd.butler
345  elif args is not None:
346  dataRefList, kwargs = args
347  butler = dataRefList[0].getButler()
348  else:
349  raise RuntimeError("Neither parsedCmd or args specified")
350  return self.TaskClass(config=self.config, log=self.log, butler=butler)
351 
352  @staticmethod
353  def getTargetList(parsedCmd, **kwargs):
354  """!
355  @brief Provide a list of patch references for each patch.
356 
357  The patch references within the list will have different filters.
358 
359  @param[in] parsedCmd the parsed command
360  @param **kwargs key word arguments (unused)
361  @throws RuntimeError if multiple references are provided for the same combination of tract, patch and
362  filter
363  """
364  refList = {} # Will index this as refList[tract][patch][filter] = ref
365  for ref in parsedCmd.id.refList:
366  tract = ref.dataId["tract"]
367  patch = ref.dataId["patch"]
368  filter = ref.dataId["filter"]
369  if tract not in refList:
370  refList[tract] = {}
371  if patch not in refList[tract]:
372  refList[tract][patch] = {}
373  if filter in refList[tract][patch]:
374  raise RuntimeError("Multiple versions of %s" % (ref.dataId,))
375  refList[tract][patch][filter] = ref
376  return [(list(p.values()), kwargs) for t in refList.values() for p in t.values()]
377 
378 
379 class MergeSourcesConfig(Config):
380  """!
381  @anchor MergeSourcesConfig_
382 
383  @brief Configuration for merging sources.
384  """
385  priorityList = ListField(dtype=str, default=[],
386  doc="Priority-ordered list of bands for the merge.")
387  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
388 
389  def validate(self):
390  Config.validate(self)
391  if len(self.priorityList) == 0:
392  raise RuntimeError("No priority list provided")
393 
394 
395 class MergeSourcesTask(CmdLineTask):
396  """!
397  @anchor MergeSourcesTask_
398 
399  @brief A base class for merging source catalogs.
400 
401  Merging detections (MergeDetectionsTask) and merging measurements (MergeMeasurementsTask) are
402  so similar that it makes sense to re-use the code, in the form of this abstract base class.
403 
404  NB: Do not use this class directly. Instead use one of the child classes that inherit from
405  MergeSourcesTask such as @ref MergeDetectionsTask_ "MergeDetectionsTask" or @ref MergeMeasurementsTask_
406  "MergeMeasurementsTask"
407 
408  Sub-classes should set the following class variables:
409  * `_DefaultName`: name of Task
410  * `inputDataset`: name of dataset to read
411  * `outputDataset`: name of dataset to write
412  * `getSchemaCatalogs` to the result of `_makeGetSchemaCatalogs(outputDataset)`
413 
414  In addition, sub-classes must implement the mergeCatalogs method.
415  """
416  _DefaultName = None
417  ConfigClass = MergeSourcesConfig
418  RunnerClass = MergeSourcesRunner
419  inputDataset = None
420  outputDataset = None
421  getSchemaCatalogs = None
422 
423  @classmethod
424  def _makeArgumentParser(cls):
425  """!
426  @brief Create a suitable ArgumentParser.
427 
428  We will use the ArgumentParser to get a provide a list of data
429  references for patches; the RunnerClass will sort them into lists
430  of data references for the same patch
431  """
432  parser = ArgumentParser(name=cls._DefaultName)
433  parser.add_id_argument("--id", "deepCoadd_" + cls.inputDataset,
434  ContainerClass=ExistingCoaddDataIdContainer,
435  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i")
436  return parser
437 
438  def getInputSchema(self, butler=None, schema=None):
439  """!
440  @brief Obtain the input schema either directly or froma butler reference.
441 
442  @param[in] butler butler reference to obtain the input schema from
443  @param[in] schema the input schema
444  """
445  if schema is None:
446  assert butler is not None, "Neither butler nor schema specified"
447  schema = butler.get(self.config.coaddName + "Coadd_" + self.inputDataset + "_schema",
448  immediate=True).schema
449  return schema
450 
451  def __init__(self, butler=None, schema=None, **kwargs):
452  """!
453  @brief Initialize the task.
454 
455  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
456  @param[in] schema the schema of the detection catalogs used as input to this one
457  @param[in] butler a butler used to read the input schema from disk, if schema is None
458 
459  Derived classes should use the getInputSchema() method to handle the additional
460  arguments and retreive the actual input schema.
461  """
462  CmdLineTask.__init__(self, **kwargs)
463 
464  def run(self, patchRefList):
465  """!
466  @brief Merge coadd sources from multiple bands. Calls @ref mergeCatalogs which must be defined in
467  subclasses that inherit from MergeSourcesTask.
468 
469  @param[in] patchRefList list of data references for each filter
470  """
471  catalogs = dict(self.readCatalog(patchRef) for patchRef in patchRefList)
472  mergedCatalog = self.mergeCatalogs(catalogs, patchRefList[0])
473  self.write(patchRefList[0], mergedCatalog)
474 
475  def readCatalog(self, patchRef):
476  """!
477  @brief Read input catalog.
478 
479  We read the input dataset provided by the 'inputDataset'
480  class variable.
481 
482  @param[in] patchRef data reference for patch
483  @return tuple consisting of the filter name and the catalog
484  """
485  filterName = patchRef.dataId["filter"]
486  catalog = patchRef.get(self.config.coaddName + "Coadd_" + self.inputDataset, immediate=True)
487  self.log.info("Read %d sources for filter %s: %s" % (len(catalog), filterName, patchRef.dataId))
488  return filterName, catalog
489 
490  def mergeCatalogs(self, catalogs, patchRef):
491  """!
492  @brief Merge multiple catalogs. This function must be defined in all subclasses that inherit from
493  MergeSourcesTask.
494 
495  @param[in] catalogs dict mapping filter name to source catalog
496 
497  @return merged catalog
498  """
499  raise NotImplementedError()
500 
501  def write(self, patchRef, catalog):
502  """!
503  @brief Write the output.
504 
505  @param[in] patchRef data reference for patch
506  @param[in] catalog catalog
507 
508  We write as the dataset provided by the 'outputDataset'
509  class variable.
510  """
511  patchRef.put(catalog, self.config.coaddName + "Coadd_" + self.outputDataset)
512  # since the filter isn't actually part of the data ID for the dataset we're saving,
513  # it's confusing to see it in the log message, even if the butler simply ignores it.
514  mergeDataId = patchRef.dataId.copy()
515  del mergeDataId["filter"]
516  self.log.info("Wrote merged catalog: %s" % (mergeDataId,))
517 
518  def writeMetadata(self, dataRefList):
519  """!
520  @brief No metadata to write, and not sure how to write it for a list of dataRefs.
521  """
522  pass
523 
524 
525 class CullPeaksConfig(Config):
526  """!
527  @anchor CullPeaksConfig_
528 
529  @brief Configuration for culling garbage peaks after merging footprints.
530 
531  Peaks may also be culled after detection or during deblending; this configuration object
532  only deals with culling after merging Footprints.
533 
534  These cuts are based on three quantities:
535  - nBands: the number of bands in which the peak was detected
536  - peakRank: the position of the peak within its family, sorted from brightest to faintest.
537  - peakRankNormalized: the peak rank divided by the total number of peaks in the family.
538 
539  The formula that identifie peaks to cull is:
540 
541  nBands < nBandsSufficient
542  AND (rank >= rankSufficient)
543  AND (rank >= rankConsider OR rank >= rankNormalizedConsider)
544 
545  To disable peak culling, simply set nBandsSufficient=1.
546  """
547 
548  nBandsSufficient = RangeField(dtype=int, default=2, min=1,
549  doc="Always keep peaks detected in this many bands")
550  rankSufficient = RangeField(dtype=int, default=20, min=1,
551  doc="Always keep this many peaks in each family")
552  rankConsidered = RangeField(dtype=int, default=30, min=1,
553  doc=("Keep peaks with less than this rank that also match the "
554  "rankNormalizedConsidered condition."))
555  rankNormalizedConsidered = RangeField(dtype=float, default=0.7, min=0.0,
556  doc=("Keep peaks with less than this normalized rank that"
557  " also match the rankConsidered condition."))
558 
559 
561  """!
562  @anchor MergeDetectionsConfig_
563 
564  @brief Configuration parameters for the MergeDetectionsTask.
565  """
566  minNewPeak = Field(dtype=float, default=1,
567  doc="Minimum distance from closest peak to create a new one (in arcsec).")
568 
569  maxSamePeak = Field(dtype=float, default=0.3,
570  doc="When adding new catalogs to the merge, all peaks less than this distance "
571  " (in arcsec) to an existing peak will be flagged as detected in that catalog.")
572  cullPeaks = ConfigField(dtype=CullPeaksConfig, doc="Configuration for how to cull peaks.")
573 
574  skyFilterName = Field(dtype=str, default="sky",
575  doc="Name of `filter' used to label sky objects (e.g. flag merge_peak_sky is set)\n"
576  "(N.b. should be in MergeMeasurementsConfig.pseudoFilterList)")
577  skyObjects = ConfigurableField(target=SkyObjectsTask, doc="Generate sky objects")
578 
579  def setDefaults(self):
580  MergeSourcesConfig.setDefaults(self)
581  self.skyObjects.avoidMask = ["DETECTED"] # Nothing else is available in our custom mask
582 
583 
584 
590 
591 
593  """!
594  @anchor MergeDetectionsTask_
595 
596  @brief Merge coadd detections from multiple bands.
597 
598  @section pipe_tasks_multiBand_Contents Contents
599 
600  - @ref pipe_tasks_multiBand_MergeDetectionsTask_Purpose
601  - @ref pipe_tasks_multiBand_MergeDetectionsTask_Init
602  - @ref pipe_tasks_multiBand_MergeDetectionsTask_Run
603  - @ref pipe_tasks_multiBand_MergeDetectionsTask_Config
604  - @ref pipe_tasks_multiBand_MergeDetectionsTask_Debug
605  - @ref pipe_tasks_multiband_MergeDetectionsTask_Example
606 
607  @section pipe_tasks_multiBand_MergeDetectionsTask_Purpose Description
608 
609  Command-line task that merges sources detected in coadds of exposures obtained with different filters.
610 
611  To perform photometry consistently across coadds in multiple filter bands, we create a master catalog of
612  sources from all bands by merging the sources (peaks & footprints) detected in each coadd, while keeping
613  track of which band each source originates in.
614 
615  The catalog merge is performed by @ref getMergedSourceCatalog. Spurious peaks detected around bright
616  objects are culled as described in @ref CullPeaksConfig_.
617 
618  @par Inputs:
619  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
620  @par Outputs:
621  deepCoadd_mergeDet{tract,patch}: SourceCatalog (only parent Footprints)
622  @par Data Unit:
623  tract, patch
624 
625  MergeDetectionsTask subclasses @ref MergeSourcesTask_ "MergeSourcesTask".
626 
627  @section pipe_tasks_multiBand_MergeDetectionsTask_Init Task initialisation
628 
629  @copydoc \_\_init\_\_
630 
631  @section pipe_tasks_multiBand_MergeDetectionsTask_Run Invoking the Task
632 
633  @copydoc run
634 
635  @section pipe_tasks_multiBand_MergeDetectionsTask_Config Configuration parameters
636 
637  See @ref MergeDetectionsConfig_
638 
639  @section pipe_tasks_multiBand_MergeDetectionsTask_Debug Debug variables
640 
641  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag @c -d
642  to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files.
643 
644  MergeDetectionsTask has no debug variables.
645 
646  @section pipe_tasks_multiband_MergeDetectionsTask_Example A complete example of using MergeDetectionsTask
647 
648  MergeDetectionsTask is meant to be run after detecting sources in coadds generated for the chosen subset
649  of the available bands.
650  The purpose of the task is to merge sources (peaks & footprints) detected in the coadds generated from the
651  chosen subset of filters.
652  Subsequent tasks in the multi-band processing procedure will deblend the generated master list of sources
653  and, eventually, perform forced photometry.
654  Command-line usage of MergeDetectionsTask expects data references for all the coadds to be processed.
655  A list of the available optional arguments can be obtained by calling mergeCoaddDetections.py with the
656  `--help` command line argument:
657  @code
658  mergeCoaddDetections.py --help
659  @endcode
660 
661  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
662  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
663  step 5 at @ref pipeTasks_multiBand, one may merge the catalogs of sources from each coadd as follows:
664  @code
665  mergeCoaddDetections.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
666  @endcode
667  This will merge the HSC-I & -R band parent source catalogs and write the results to
668  `$CI_HSC_DIR/DATA/deepCoadd-results/merged/0/5,4/mergeDet-0-5,4.fits`.
669 
670  The next step in the multi-band processing procedure is
671  @ref MeasureMergedCoaddSourcesTask_ "MeasureMergedCoaddSourcesTask"
672  """
673  ConfigClass = MergeDetectionsConfig
674  _DefaultName = "mergeCoaddDetections"
675  inputDataset = "det"
676  outputDataset = "mergeDet"
677  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
678 
679  def __init__(self, butler=None, schema=None, **kwargs):
680  """!
681  @brief Initialize the merge detections task.
682 
683  A @ref FootprintMergeList_ "FootprintMergeList" will be used to
684  merge the source catalogs.
685 
686  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
687  @param[in] schema the schema of the detection catalogs used as input to this one
688  @param[in] butler a butler used to read the input schema from disk, if schema is None
689  @param[in] **kwargs keyword arguments to be passed to MergeSourcesTask.__init__
690 
691  The task will set its own self.schema attribute to the schema of the output merged catalog.
692  """
693  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
694  self.makeSubtask("skyObjects")
695  self.schema = self.getInputSchema(butler=butler, schema=schema)
696 
697  filterNames = [getShortFilterName(name) for name in self.config.priorityList]
698  filterNames += [self.config.skyFilterName]
699  self.merged = afwDetect.FootprintMergeList(self.schema, filterNames)
700 
701  def mergeCatalogs(self, catalogs, patchRef):
702  """!
703  @brief Merge multiple catalogs.
704 
705  After ordering the catalogs and filters in priority order,
706  @ref getMergedSourceCatalog of the @ref FootprintMergeList_ "FootprintMergeList" created by
707  @ref \_\_init\_\_ is used to perform the actual merging. Finally, @ref cullPeaks is used to remove
708  garbage peaks detected around bright objects.
709 
710  @param[in] catalogs
711  @param[in] patchRef
712  @param[out] mergedList
713  """
714 
715  # Convert distance to tract coordinate
716  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
717  tractWcs = skyInfo.wcs
718  peakDistance = self.config.minNewPeak / tractWcs.getPixelScale().asArcseconds()
719  samePeakDistance = self.config.maxSamePeak / tractWcs.getPixelScale().asArcseconds()
720 
721  # Put catalogs, filters in priority order
722  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
723  orderedBands = [getShortFilterName(band) for band in self.config.priorityList
724  if band in catalogs.keys()]
725 
726  mergedList = self.merged.getMergedSourceCatalog(orderedCatalogs, orderedBands, peakDistance,
727  self.schema, self.makeIdFactory(patchRef),
728  samePeakDistance)
729 
730  #
731  # Add extra sources that correspond to blank sky
732  #
733  skySeed = patchRef.get(self.config.coaddName + "MergedCoaddId")
734  skySourceFootprints = self.getSkySourceFootprints(mergedList, skyInfo, skySeed)
735  if skySourceFootprints:
736  key = mergedList.schema.find("merge_footprint_%s" % self.config.skyFilterName).key
737  for foot in skySourceFootprints:
738  s = mergedList.addNew()
739  s.setFootprint(foot)
740  s.set(key, True)
741 
742  # Sort Peaks from brightest to faintest
743  for record in mergedList:
744  record.getFootprint().sortPeaks()
745  self.log.info("Merged to %d sources" % len(mergedList))
746  # Attempt to remove garbage peaks
747  self.cullPeaks(mergedList)
748  return mergedList
749 
750  def cullPeaks(self, catalog):
751  """!
752  @brief Attempt to remove garbage peaks (mostly on the outskirts of large blends).
753 
754  @param[in] catalog Source catalog
755  """
756  keys = [item.key for item in self.merged.getPeakSchema().extract("merge_peak_*").values()]
757  assert len(keys) > 0, "Error finding flags that associate peaks with their detection bands."
758  totalPeaks = 0
759  culledPeaks = 0
760  for parentSource in catalog:
761  # Make a list copy so we can clear the attached PeakCatalog and append the ones we're keeping
762  # to it (which is easier than deleting as we iterate).
763  keptPeaks = parentSource.getFootprint().getPeaks()
764  oldPeaks = list(keptPeaks)
765  keptPeaks.clear()
766  familySize = len(oldPeaks)
767  totalPeaks += familySize
768  for rank, peak in enumerate(oldPeaks):
769  if ((rank < self.config.cullPeaks.rankSufficient) or
770  (sum([peak.get(k) for k in keys]) >= self.config.cullPeaks.nBandsSufficient) or
771  (rank < self.config.cullPeaks.rankConsidered and
772  rank < self.config.cullPeaks.rankNormalizedConsidered * familySize)):
773  keptPeaks.append(peak)
774  else:
775  culledPeaks += 1
776  self.log.info("Culled %d of %d peaks" % (culledPeaks, totalPeaks))
777 
778  def getSchemaCatalogs(self):
779  """!
780  Return a dict of empty catalogs for each catalog dataset produced by this task.
781 
782  @param[out] dictionary of empty catalogs
783  """
784  mergeDet = afwTable.SourceCatalog(self.schema)
785  peak = afwDetect.PeakCatalog(self.merged.getPeakSchema())
786  return {self.config.coaddName + "Coadd_mergeDet": mergeDet,
787  self.config.coaddName + "Coadd_peak": peak}
788 
789  def getSkySourceFootprints(self, mergedList, skyInfo, seed):
790  """!
791  @brief Return a list of Footprints of sky objects which don't overlap with anything in mergedList
792 
793  @param mergedList The merged Footprints from all the input bands
794  @param skyInfo A description of the patch
795  @param seed Seed for the random number generator
796  """
797  mask = afwImage.Mask(skyInfo.patchInfo.getOuterBBox())
798  detected = mask.getPlaneBitMask("DETECTED")
799  for s in mergedList:
800  s.getFootprint().spans.setMask(mask, detected)
801 
802  footprints = self.skyObjects.run(mask, seed)
803  if not footprints:
804  return footprints
805 
806  # Need to convert the peak catalog's schema so we can set the "merge_peak_<skyFilterName>" flags
807  schema = self.merged.getPeakSchema()
808  mergeKey = schema.find("merge_peak_%s" % self.config.skyFilterName).key
809  converted = []
810  for oldFoot in footprints:
811  assert len(oldFoot.getPeaks()) == 1, "Should be a single peak only"
812  peak = oldFoot.getPeaks()[0]
813  newFoot = afwDetect.Footprint(oldFoot.spans, schema)
814  newFoot.addPeak(peak.getFx(), peak.getFy(), peak.getPeakValue())
815  newFoot.getPeaks()[0].set(mergeKey, True)
816  converted.append(newFoot)
817 
818  return converted
819 
820 
822  """!
823  @anchor MeasureMergedCoaddSourcesConfig_
824 
825  @brief Configuration parameters for the MeasureMergedCoaddSourcesTask
826  """
827  doDeblend = Field(dtype=bool, default=True, doc="Deblend sources?")
828  deblend = ConfigurableField(target=SourceDeblendTask, doc="Deblend sources")
829  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
830  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
831  doPropagateFlags = Field(
832  dtype=bool, default=True,
833  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
834  )
835  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
836  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
837  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
838  doWriteMatchesDenormalized = Field(
839  dtype=bool,
840  default=False,
841  doc=("Write reference matches in denormalized format? "
842  "This format uses more disk space, but is more convenient to read."),
843  )
844  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
845  checkUnitsParseStrict = Field(
846  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
847  dtype=str,
848  default="raise",
849  )
850  doApCorr = Field(
851  dtype=bool,
852  default=True,
853  doc="Apply aperture corrections"
854  )
855  applyApCorr = ConfigurableField(
856  target=ApplyApCorrTask,
857  doc="Subtask to apply aperture corrections"
858  )
859  doRunCatalogCalculation = Field(
860  dtype=bool,
861  default=True,
862  doc='Run catalogCalculation task'
863  )
864  catalogCalculation = ConfigurableField(
865  target=CatalogCalculationTask,
866  doc="Subtask to run catalogCalculation plugins on catalog"
867  )
868 
869  def setDefaults(self):
870  Config.setDefaults(self)
871  self.deblend.propagateAllPeaks = True
872  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
873  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
874  'INEXACT_PSF']
875  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
876  'INEXACT_PSF']
877 
878 
884 
885 
886 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
887  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
888  @staticmethod
889  def getTargetList(parsedCmd, **kwargs):
890  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
891 
892 
893 class MeasureMergedCoaddSourcesTask(CmdLineTask):
894  """!
895  @anchor MeasureMergedCoaddSourcesTask_
896 
897  @brief Deblend sources from master catalog in each coadd seperately and measure.
898 
899  @section pipe_tasks_multiBand_Contents Contents
900 
901  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
902  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
903  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
904  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
905  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
906  - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
907 
908  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
909 
910  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
911  measurement in each coadd.
912 
913  Given a master input catalog of sources (peaks and footprints), deblend and measure each source on the
914  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
915  consistent set of child sources.
916 
917  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
918  properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit
919  flags are propagated to the coadd sources.
920 
921  Optionally, we can match the coadd sources to an external reference catalog.
922 
923  @par Inputs:
924  deepCoadd_mergeDet{tract,patch}: SourceCatalog
925  @n deepCoadd_calexp{tract,patch,filter}: ExposureF
926  @par Outputs:
927  deepCoadd_meas{tract,patch,filter}: SourceCatalog
928  @par Data Unit:
929  tract, patch, filter
930 
931  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
932 
933  <DL>
934  <DT> @ref SourceDeblendTask_ "deblend"
935  <DD> Deblend all the sources from the master catalog.</DD>
936  <DT> @ref SingleFrameMeasurementTask_ "measurement"
937  <DD> Measure source properties of deblended sources.</DD>
938  <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags"
939  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
940  not at the edge of the field and that have either not been deblended or are the children of deblended
941  sources</DD>
942  <DT> @ref PropagateVisitFlagsTask_ "propagateFlags"
943  <DD> Propagate flags set in individual visits to the coadd.</DD>
944  <DT> @ref DirectMatchTask_ "match"
945  <DD> Match input sources to a reference catalog (optional).
946  </DD>
947  </DL>
948  These subtasks may be retargeted as required.
949 
950  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
951 
952  @copydoc \_\_init\_\_
953 
954  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
955 
956  @copydoc run
957 
958  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
959 
960  See @ref MeasureMergedCoaddSourcesConfig_
961 
962  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
963 
964  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
965  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
966  files.
967 
968  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
969  the various sub-tasks. See the documetation for individual sub-tasks for more information.
970 
971  @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
972  MeasureMergedCoaddSourcesTask
973 
974  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
975  The next stage in the multi-band processing procedure will merge these measurements into a suitable
976  catalog for driving forced photometry.
977 
978  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
979  to be processed.
980  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
981  `--help` command line argument:
982  @code
983  measureCoaddSources.py --help
984  @endcode
985 
986  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
987  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
988  step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
989  coadd as follows:
990  @code
991  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
992  @endcode
993  This will process the HSC-I band data. The results are written in
994  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
995 
996  It is also necessary to run
997  @code
998  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
999  @endcode
1000  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
1001  procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask".
1002  """
1003  _DefaultName = "measureCoaddSources"
1004  ConfigClass = MeasureMergedCoaddSourcesConfig
1005  RunnerClass = MeasureMergedCoaddSourcesRunner
1006  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
1007  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
1008 
1009  @classmethod
1010  def _makeArgumentParser(cls):
1011  parser = ArgumentParser(name=cls._DefaultName)
1012  parser.add_id_argument("--id", "deepCoadd_calexp",
1013  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
1014  ContainerClass=ExistingCoaddDataIdContainer)
1015  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
1016  return parser
1017 
1018  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
1019  """!
1020  @brief Initialize the task.
1021 
1022  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
1023  @param[in] schema: the schema of the merged detection catalog used as input to this one
1024  @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
1025  @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
1026  catalog. May be None if the loader can be constructed from the butler argument or all steps
1027  requiring a reference catalog are disabled.
1028  @param[in] butler: a butler used to read the input schemas from disk or construct the reference
1029  catalog loader, if schema or peakSchema or refObjLoader is None
1030 
1031  The task will set its own self.schema attribute to the schema of the output measurement catalog.
1032  This will include all fields from the input schema, as well as additional fields for all the
1033  measurements.
1034  """
1035  CmdLineTask.__init__(self, **kwargs)
1036  if schema is None:
1037  assert butler is not None, "Neither butler nor schema is defined"
1038  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
1039  self.schemaMapper = afwTable.SchemaMapper(schema)
1040  self.schemaMapper.addMinimalSchema(schema)
1041  self.schema = self.schemaMapper.getOutputSchema()
1043  if self.config.doDeblend:
1044  if peakSchema is None:
1045  assert butler is not None, "Neither butler nor peakSchema is defined"
1046  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
1047  self.makeSubtask("deblend", schema=self.schema, peakSchema=peakSchema)
1048  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
1049  self.makeSubtask("setPrimaryFlags", schema=self.schema)
1050  if self.config.doMatchSources:
1051  if refObjLoader is None:
1052  assert butler is not None, "Neither butler nor refObjLoader is defined"
1053  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
1054  if self.config.doPropagateFlags:
1055  self.makeSubtask("propagateFlags", schema=self.schema)
1056  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
1057  if self.config.doApCorr:
1058  self.makeSubtask("applyApCorr", schema=self.schema)
1059  if self.config.doRunCatalogCalculation:
1060  self.makeSubtask("catalogCalculation", schema=self.schema)
1061 
1062  def run(self, patchRef, psfCache=100):
1063  """!
1064  @brief Deblend and measure.
1065 
1066  @param[in] patchRef: Patch reference.
1067 
1068  Deblend each source in every coadd and measure. Set 'is-primary' and related flags. Propagate flags
1069  from individual visits. Optionally match the sources to a reference catalog and write the matches.
1070  Finally, write the deblended sources and measurements out.
1071  """
1072  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
1073  exposure.getPsf().setCacheCapacity(psfCache)
1074  sources = self.readSources(patchRef)
1075  if self.config.doDeblend:
1076  self.deblend.run(exposure, sources)
1077 
1078  bigKey = sources.schema["deblend_parentTooBig"].asKey()
1079  # catalog is non-contiguous so can't extract column
1080  numBig = sum((s.get(bigKey) for s in sources))
1081  if numBig > 0:
1082  self.log.warn("Patch %s contains %d large footprints that were not deblended" %
1083  (patchRef.dataId, numBig))
1084 
1085  table = sources.getTable()
1086  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
1087 
1088  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
1089 
1090  if self.config.doApCorr:
1091  self.applyApCorr.run(
1092  catalog=sources,
1093  apCorrMap=exposure.getInfo().getApCorrMap()
1094  )
1095 
1096  # TODO DM-11568: this contiguous check-and-copy could go away if we
1097  # reserve enough space during SourceDetection and/or SourceDeblend.
1098  # NOTE: sourceSelectors require contiguous catalogs, so ensure
1099  # contiguity now, so views are preserved from here on.
1100  if not sources.isContiguous():
1101  sources = sources.copy(deep=True)
1102 
1103  if self.config.doRunCatalogCalculation:
1104  self.catalogCalculation.run(sources)
1105 
1106  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
1107  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1108  includeDeblend=self.config.doDeblend)
1109  if self.config.doPropagateFlags:
1110  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
1111  exposure.getWcs())
1112  if self.config.doMatchSources:
1113  self.writeMatches(patchRef, exposure, sources)
1114  self.write(patchRef, sources)
1115 
1116  def readSources(self, dataRef):
1117  """!
1118  @brief Read input sources.
1119 
1120  @param[in] dataRef: Data reference for catalog of merged detections
1121  @return List of sources in merged catalog
1122 
1123  We also need to add columns to hold the measurements we're about to make
1124  so we can measure in-place.
1125  """
1126  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
1127  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1128  idFactory = self.makeIdFactory(dataRef)
1129  for s in merged:
1130  idFactory.notify(s.getId())
1131  table = afwTable.SourceTable.make(self.schema, idFactory)
1132  sources = afwTable.SourceCatalog(table)
1133  sources.extend(merged, self.schemaMapper)
1134  return sources
1135 
1136  def writeMatches(self, dataRef, exposure, sources):
1137  """!
1138  @brief Write matches of the sources to the astrometric reference catalog.
1139 
1140  We use the Wcs in the exposure to match sources.
1141 
1142  @param[in] dataRef: data reference
1143  @param[in] exposure: exposure with Wcs
1144  @param[in] sources: source catalog
1145  """
1146  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
1147  if result.matches:
1148  matches = afwTable.packMatches(result.matches)
1149  matches.table.setMetadata(result.matchMeta)
1150  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
1151  if self.config.doWriteMatchesDenormalized:
1152  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
1153  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1154 
1155  def write(self, dataRef, sources):
1156  """!
1157  @brief Write the source catalog.
1158 
1159  @param[in] dataRef: data reference
1160  @param[in] sources: source catalog
1161  """
1162  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1163  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1164 
1165  def getExposureId(self, dataRef):
1166  return int(dataRef.get(self.config.coaddName + "CoaddId"))
1167 
1168 
1170  """!
1171  @anchor MergeMeasurementsConfig_
1172 
1173  @brief Configuration parameters for the MergeMeasurementsTask
1174  """
1175  pseudoFilterList = ListField(dtype=str, default=["sky"],
1176  doc="Names of filters which may have no associated detection\n"
1177  "(N.b. should include MergeDetectionsConfig.skyFilterName)")
1178  snName = Field(dtype=str, default="base_PsfFlux",
1179  doc="Name of flux measurement for calculating the S/N when choosing the reference band.")
1180  minSN = Field(dtype=float, default=10.,
1181  doc="If the S/N from the priority band is below this value (and the S/N "
1182  "is larger than minSNDiff compared to the priority band), use the band with "
1183  "the largest S/N as the reference band.")
1184  minSNDiff = Field(dtype=float, default=3.,
1185  doc="If the difference in S/N between another band and the priority band is larger "
1186  "than this value (and the S/N in the priority band is less than minSN) "
1187  "use the band with the largest S/N as the reference band")
1188  flags = ListField(dtype=str, doc="Require that these flags, if available, are not set",
1189  default=["base_PixelFlags_flag_interpolatedCenter", "base_PsfFlux_flag",
1190  "ext_photometryKron_KronFlux_flag", "modelfit_CModel_flag", ])
1191 
1192 
1198 
1199 
1201  """!
1202  @anchor MergeMeasurementsTask_
1203 
1204  @brief Merge measurements from multiple bands
1205 
1206  @section pipe_tasks_multiBand_Contents Contents
1207 
1208  - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Purpose
1209  - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Initialize
1210  - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Run
1211  - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Config
1212  - @ref pipe_tasks_multiBand_MergeMeasurementsTask_Debug
1213  - @ref pipe_tasks_multiband_MergeMeasurementsTask_Example
1214 
1215  @section pipe_tasks_multiBand_MergeMeasurementsTask_Purpose Description
1216 
1217  Command-line task that merges measurements from multiple bands.
1218 
1219  Combines consistent (i.e. with the same peaks and footprints) catalogs of sources from multiple filter
1220  bands to construct a unified catalog that is suitable for driving forced photometry. Every source is
1221  required to have centroid, shape and flux measurements in each band.
1222 
1223  @par Inputs:
1224  deepCoadd_meas{tract,patch,filter}: SourceCatalog
1225  @par Outputs:
1226  deepCoadd_ref{tract,patch}: SourceCatalog
1227  @par Data Unit:
1228  tract, patch
1229 
1230  MergeMeasurementsTask subclasses @ref MergeSourcesTask_ "MergeSourcesTask".
1231 
1232  @section pipe_tasks_multiBand_MergeMeasurementsTask_Initialize Task initialization
1233 
1234  @copydoc \_\_init\_\_
1235 
1236  @section pipe_tasks_multiBand_MergeMeasurementsTask_Run Invoking the Task
1237 
1238  @copydoc run
1239 
1240  @section pipe_tasks_multiBand_MergeMeasurementsTask_Config Configuration parameters
1241 
1242  See @ref MergeMeasurementsConfig_
1243 
1244  @section pipe_tasks_multiBand_MergeMeasurementsTask_Debug Debug variables
1245 
1246  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
1247  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
1248  files.
1249 
1250  MergeMeasurementsTask has no debug variables.
1251 
1252  @section pipe_tasks_multiband_MergeMeasurementsTask_Example A complete example
1253  of using MergeMeasurementsTask
1254 
1255  MergeMeasurementsTask is meant to be run after deblending & measuring sources in every band.
1256  The purpose of the task is to generate a catalog of sources suitable for driving forced photometry in
1257  coadds and individual exposures.
1258  Command-line usage of MergeMeasurementsTask expects a data reference to the coadds to be processed. A list
1259  of the available optional arguments can be obtained by calling mergeCoaddMeasurements.py with the `--help`
1260  command line argument:
1261  @code
1262  mergeCoaddMeasurements.py --help
1263  @endcode
1264 
1265  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
1266  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
1267  step 7 at @ref pipeTasks_multiBand, one may merge the catalogs generated after deblending and measuring
1268  as follows:
1269  @code
1270  mergeCoaddMeasurements.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I^HSC-R
1271  @endcode
1272  This will merge the HSC-I & HSC-R band catalogs. The results are written in
1273  `$CI_HSC_DIR/DATA/deepCoadd-results/`.
1274  """
1275  _DefaultName = "mergeCoaddMeasurements"
1276  ConfigClass = MergeMeasurementsConfig
1277  inputDataset = "meas"
1278  outputDataset = "ref"
1279  getSchemaCatalogs = _makeGetSchemaCatalogs("ref")
1280 
1281  def __init__(self, butler=None, schema=None, **kwargs):
1282  """!
1283  Initialize the task.
1284 
1285  Additional keyword arguments (forwarded to MergeSourcesTask.__init__):
1286  @param[in] schema: the schema of the detection catalogs used as input to this one
1287  @param[in] butler: a butler used to read the input schema from disk, if schema is None
1288 
1289  The task will set its own self.schema attribute to the schema of the output merged catalog.
1290  """
1291  MergeSourcesTask.__init__(self, butler=butler, schema=schema, **kwargs)
1292  inputSchema = self.getInputSchema(butler=butler, schema=schema)
1293  self.schemaMapper = afwTable.SchemaMapper(inputSchema, True)
1294  self.schemaMapper.addMinimalSchema(inputSchema, True)
1295  self.fluxKey = inputSchema.find(self.config.snName + "_flux").getKey()
1296  self.fluxErrKey = inputSchema.find(self.config.snName + "_fluxSigma").getKey()
1297  self.fluxFlagKey = inputSchema.find(self.config.snName + "_flag").getKey()
1298 
1299  self.flagKeys = {}
1300  for band in self.config.priorityList:
1301  short = getShortFilterName(band)
1302  outputKey = self.schemaMapper.editOutputSchema().addField(
1303  "merge_measurement_%s" % short,
1304  type="Flag",
1305  doc="Flag field set if the measurements here are from the %s filter" % band
1306  )
1307  peakKey = inputSchema.find("merge_peak_%s" % short).key
1308  footprintKey = inputSchema.find("merge_footprint_%s" % short).key
1309  self.flagKeys[band] = Struct(peak=peakKey, footprint=footprintKey, output=outputKey)
1310  self.schema = self.schemaMapper.getOutputSchema()
1311 
1313  for filt in self.config.pseudoFilterList:
1314  try:
1315  self.pseudoFilterKeys.append(self.schema.find("merge_peak_%s" % filt).getKey())
1316  except Exception as e:
1317  self.log.warn("merge_peak is not set for pseudo-filter %s: %s" % (filt, e))
1318 
1319  self.badFlags = {}
1320  for flag in self.config.flags:
1321  try:
1322  self.badFlags[flag] = self.schema.find(flag).getKey()
1323  except KeyError as exc:
1324  self.log.warn("Can't find flag %s in schema: %s" % (flag, exc,))
1325 
1326  def mergeCatalogs(self, catalogs, patchRef):
1327  """!
1328  Merge measurement catalogs to create a single reference catalog for forced photometry
1329 
1330  @param[in] catalogs: the catalogs to be merged
1331  @param[in] patchRef: patch reference for data
1332 
1333  For parent sources, we choose the first band in config.priorityList for which the
1334  merge_footprint flag for that band is is True.
1335 
1336  For child sources, the logic is the same, except that we use the merge_peak flags.
1337  """
1338  # Put catalogs, filters in priority order
1339  orderedCatalogs = [catalogs[band] for band in self.config.priorityList if band in catalogs.keys()]
1340  orderedKeys = [self.flagKeys[band] for band in self.config.priorityList if band in catalogs.keys()]
1341 
1342  mergedCatalog = afwTable.SourceCatalog(self.schema)
1343  mergedCatalog.reserve(len(orderedCatalogs[0]))
1344 
1345  idKey = orderedCatalogs[0].table.getIdKey()
1346  for catalog in orderedCatalogs[1:]:
1347  if numpy.any(orderedCatalogs[0].get(idKey) != catalog.get(idKey)):
1348  raise ValueError("Error in inputs to MergeCoaddMeasurements: source IDs do not match")
1349 
1350  # This first zip iterates over all the catalogs simultaneously, yielding a sequence of one
1351  # record for each band, in priority order.
1352  for orderedRecords in zip(*orderedCatalogs):
1353 
1354  maxSNRecord = None
1355  maxSNFlagKeys = None
1356  maxSN = 0.
1357  priorityRecord = None
1358  priorityFlagKeys = None
1359  prioritySN = 0.
1360  hasPseudoFilter = False
1361 
1362  # Now we iterate over those record-band pairs, keeping track of the priority and the
1363  # largest S/N band.
1364  for inputRecord, flagKeys in zip(orderedRecords, orderedKeys):
1365  parent = (inputRecord.getParent() == 0 and inputRecord.get(flagKeys.footprint))
1366  child = (inputRecord.getParent() != 0 and inputRecord.get(flagKeys.peak))
1367 
1368  if not (parent or child):
1369  for pseudoFilterKey in self.pseudoFilterKeys:
1370  if inputRecord.get(pseudoFilterKey):
1371  hasPseudoFilter = True
1372  priorityRecord = inputRecord
1373  priorityFlagKeys = flagKeys
1374  break
1375  if hasPseudoFilter:
1376  break
1377 
1378  isBad = any(inputRecord.get(flag) for flag in self.badFlags)
1379  if isBad or inputRecord.get(self.fluxFlagKey) or inputRecord.get(self.fluxErrKey) == 0:
1380  sn = 0.
1381  else:
1382  sn = inputRecord.get(self.fluxKey)/inputRecord.get(self.fluxErrKey)
1383  if numpy.isnan(sn) or sn < 0.:
1384  sn = 0.
1385  if (parent or child) and priorityRecord is None:
1386  priorityRecord = inputRecord
1387  priorityFlagKeys = flagKeys
1388  prioritySN = sn
1389  if sn > maxSN:
1390  maxSNRecord = inputRecord
1391  maxSNFlagKeys = flagKeys
1392  maxSN = sn
1393 
1394  # If the priority band has a low S/N we would like to choose the band with the highest S/N as
1395  # the reference band instead. However, we only want to choose the highest S/N band if it is
1396  # significantly better than the priority band. Therefore, to choose a band other than the
1397  # priority, we require that the priority S/N is below the minimum threshold and that the
1398  # difference between the priority and highest S/N is larger than the difference threshold.
1399  #
1400  # For pseudo code objects we always choose the first band in the priority list.
1401  bestRecord = None
1402  bestFlagKeys = None
1403  if hasPseudoFilter:
1404  bestRecord = priorityRecord
1405  bestFlagKeys = priorityFlagKeys
1406  elif (prioritySN < self.config.minSN and (maxSN - prioritySN) > self.config.minSNDiff and
1407  maxSNRecord is not None):
1408  bestRecord = maxSNRecord
1409  bestFlagKeys = maxSNFlagKeys
1410  elif priorityRecord is not None:
1411  bestRecord = priorityRecord
1412  bestFlagKeys = priorityFlagKeys
1413 
1414  if bestRecord is not None and bestFlagKeys is not None:
1415  outputRecord = mergedCatalog.addNew()
1416  outputRecord.assign(bestRecord, self.schemaMapper)
1417  outputRecord.set(bestFlagKeys.output, True)
1418  else: # if we didn't find any records
1419  raise ValueError("Error in inputs to MergeCoaddMeasurements: no valid reference for %s" %
1420  inputRecord.getId())
1421 
1422  # more checking for sane inputs, since zip silently iterates over the smallest sequence
1423  for inputCatalog in orderedCatalogs:
1424  if len(mergedCatalog) != len(inputCatalog):
1425  raise ValueError("Mismatch between catalog sizes: %s != %s" %
1426  (len(mergedCatalog), len(orderedCatalogs)))
1427 
1428  return mergedCatalog
def getSkySourceFootprints(self, mergedList, skyInfo, seed)
Return a list of Footprints of sky objects which don&#39;t overlap with anything in mergedList.
Definition: multiBand.py:789
Merge coadd detections from multiple bands.
Definition: multiBand.py:592
def makeTask(self, parsedCmd=None, args=None)
Provide a butler to the Task constructor.
Definition: multiBand.py:335
def getInputSchema(self, butler=None, schema=None)
Obtain the input schema either directly or froma butler reference.
Definition: multiBand.py:438
def getSchemaCatalogs(self)
Return a dict of empty catalogs for each catalog dataset produced by this task.
Definition: multiBand.py:778
def runDetection(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:279
def cullPeaks(self, catalog)
Attempt to remove garbage peaks (mostly on the outskirts of large blends).
Definition: multiBand.py:750
Task runner for the MergeSourcesTask. Required because the run method requires a list of dataRefs rat...
Definition: multiBand.py:327
def write(self, exposure, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:311
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:451
def __init__(self, butler=None, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:1281
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:106
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:244
Merge measurements from multiple bands.
Definition: multiBand.py:1200
def run(self, patchRefList)
Merge coadd sources from multiple bands.
Definition: multiBand.py:464
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:893
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1136
def run(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:264
def mergeCatalogs(self, catalogs, patchRef)
Merge measurement catalogs to create a single reference catalog for forced photometry.
Definition: multiBand.py:1326
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:490
Configuration parameters for the MergeMeasurementsTask.
Definition: multiBand.py:1169
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1116
Configuration parameters for the MergeDetectionsTask.
Definition: multiBand.py:560
Configuration for merging sources.
Definition: multiBand.py:379
def write(self, patchRef, catalog)
Write the output.
Definition: multiBand.py:501
def mergeCatalogs(self, catalogs, patchRef)
Merge multiple catalogs.
Definition: multiBand.py:701
def run(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:1062
A base class for merging source catalogs.
Definition: multiBand.py:395
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1155
def readCatalog(self, patchRef)
Read input catalog.
Definition: multiBand.py:475
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:821
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:253
def writeMetadata(self, dataRefList)
No metadata to write, and not sure how to write it for a list of dataRefs.
Definition: multiBand.py:518
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:1018
def __init__(self, butler=None, schema=None, kwargs)
Initialize the merge detections task.
Definition: multiBand.py:679
def getTargetList(parsedCmd, kwargs)
Provide a list of patch references for each patch.
Definition: multiBand.py:353