lsst.pipe.tasks  16.0-36-g07840cb1
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25  PipelineTask, InitOutputDatasetField, InputDatasetField, OutputDatasetField,
26  QuantumConfig)
27 from lsst.pex.config import Config, Field, ConfigurableField
28 from lsst.meas.algorithms import DynamicDetectionTask
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
30 from lsst.meas.deblender import SourceDeblendTask, MultibandDeblendTask
31 from lsst.pipe.tasks.coaddBase import getSkyInfo
32 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
33 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
34 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
35 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
36 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
37 import lsst.afw.image as afwImage
38 import lsst.afw.table as afwTable
39 import lsst.afw.math as afwMath
40 from lsst.daf.base import PropertyList
41 
42 from .mergeDetections import MergeDetectionsConfig, MergeDetectionsTask # noqa: F401
43 from .mergeMeasurements import MergeMeasurementsConfig, MergeMeasurementsTask # noqa: F401
44 from .multiBandUtils import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs # noqa: F401
45 from .multiBandUtils import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory # noqa: F401
46 
47 
48 """
49 New dataset types:
50 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
51 * deepCoadd_mergeDet: merged detections (tract, patch)
52 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
53 * deepCoadd_ref: reference sources (tract, patch)
54 All of these have associated *_schema catalogs that require no data ID and hold no records.
55 
56 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
57 the mergeDet, meas, and ref dataset Footprints:
58 * deepCoadd_peak_schema
59 """
60 
61 
62 
63 
65  """!
66  @anchor DetectCoaddSourcesConfig_
67 
68  @brief Configuration parameters for the DetectCoaddSourcesTask
69  """
70  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
71  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
72  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
73  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
74  doInsertFakes = Field(dtype=bool, default=False,
75  doc="Run fake sources injection task")
76  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
77  doc="Injection of fake sources for testing "
78  "purposes (must be retargeted)")
79  detectionSchema = InitOutputDatasetField(
80  doc="Schema of the detection catalog",
81  name="{}Coadd_det_schema",
82  storageClass="SourceCatalog",
83  )
84  exposure = InputDatasetField(
85  doc="Exposure on which detections are to be performed",
86  name="deepCoadd",
87  scalar=True,
88  storageClass="Exposure",
89  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
90  )
91  outputBackgrounds = OutputDatasetField(
92  doc="Output Backgrounds used in detection",
93  name="{}Coadd_calexp_background",
94  scalar=True,
95  storageClass="Background",
96  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
97  )
98  outputSources = OutputDatasetField(
99  doc="Detected sources catalog",
100  name="{}Coadd_det",
101  scalar=True,
102  storageClass="SourceCatalog",
103  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
104  )
105  outputExposure = OutputDatasetField(
106  doc="Exposure post detection",
107  name="{}Coadd_calexp",
108  scalar=True,
109  storageClass="Exposure",
110  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
111  )
112  quantum = QuantumConfig(
113  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
114  )
115 
116  def setDefaults(self):
117  Config.setDefaults(self)
118  self.detection.thresholdType = "pixel_stdev"
119  self.detection.isotropicGrow = True
120  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
121  self.detection.reEstimateBackground = False
122  self.detection.background.useApprox = False
123  self.detection.background.binSize = 4096
124  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
125  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
126 
127 
133 
134 
135 class DetectCoaddSourcesTask(PipelineTask, CmdLineTask):
136  r"""!
137  @anchor DetectCoaddSourcesTask_
138 
139  @brief Detect sources on a coadd
140 
141  @section pipe_tasks_multiBand_Contents Contents
142 
143  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
144  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
145  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
146  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
147  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
148  - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
149 
150  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
151 
152  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
153 
154  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
155  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
156  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
157  propagate the full covariance matrix -- but it is simple and works well in practice.
158 
159  After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref
160  SourceDetectionTask_ "detection" subtask.
161 
162  @par Inputs:
163  deepCoadd{tract,patch,filter}: ExposureF
164  @par Outputs:
165  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
166  @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
167  exposure (ExposureF)
168  @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
169  @par Data Unit:
170  tract, patch, filter
171 
172  DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask.
173  You can retarget this subtask if you wish.
174 
175  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
176 
177  @copydoc \_\_init\_\_
178 
179  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
180 
181  @copydoc run
182 
183  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
184 
185  See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
186 
187  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
188 
189  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
190  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
191  files.
192 
193  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
194  @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
195  @ref SourceDetectionTask_ "SourceDetectionTask" for further information.
196 
197  @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
198  of using DetectCoaddSourcesTask
199 
200  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
201  the task is to update the background, detect all sources in a single band and generate a set of parent
202  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
203  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
204  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
205  calling detectCoaddSources.py with the `--help` command line argument:
206  @code
207  detectCoaddSources.py --help
208  @endcode
209 
210  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
211  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
212  steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
213  @code
214  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
215  @endcode
216  that will process the HSC-I band data. The results are written to
217  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
218 
219  It is also necessary to run:
220  @code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
222  @endcode
223  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
224  processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask".
225  """
226  _DefaultName = "detectCoaddSources"
227  ConfigClass = DetectCoaddSourcesConfig
228  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
229  makeIdFactory = _makeMakeIdFactory("CoaddId")
230 
231  @classmethod
232  def _makeArgumentParser(cls):
233  parser = ArgumentParser(name=cls._DefaultName)
234  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
235  ContainerClass=ExistingCoaddDataIdContainer)
236  return parser
237 
238  @classmethod
239  def getOutputDatasetTypes(cls, config):
240  coaddName = config.coaddName
241  for name in ("outputBackgrounds", "outputSources", "outputExposure"):
242  attr = getattr(config, name)
243  setattr(attr, "name", attr.name.format(coaddName))
244  outputTypeDict = super().getOutputDatasetTypes(config)
245  return outputTypeDict
246 
247  @classmethod
248  def getInitOutputDatasetTypes(cls, config):
249  coaddName = config.coaddName
250  attr = config.detectionSchema
251  setattr(attr, "name", attr.name.format(coaddName))
252  output = super().getInitOutputDatasetTypes(config)
253  return output
254 
255  def __init__(self, schema=None, **kwargs):
256  """!
257  @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask.
258 
259  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
260 
261  @param[in] schema: initial schema for the output catalog, modified-in place to include all
262  fields set by this task. If None, the source minimal schema will be used.
263  @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
264  """
265  # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree
266  # call structure has been reviewed carefully to be sure super will work as intended.
267  super().__init__(**kwargs)
268  if schema is None:
269  schema = afwTable.SourceTable.makeMinimalSchema()
270  if self.config.doInsertFakes:
271  self.makeSubtask("insertFakes")
272  self.schema = schema
273  self.makeSubtask("detection", schema=self.schema)
274  if self.config.doScaleVariance:
275  self.makeSubtask("scaleVariance")
276 
278  return {"detectionSchema": afwTable.SourceCatalog(self.schema)}
279 
280  def runDataRef(self, patchRef):
281  """!
282  @brief Run detection on a coadd.
283 
284  Invokes @ref run and then uses @ref write to output the
285  results.
286 
287  @param[in] patchRef: data reference for patch
288  """
289  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
290  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
291  results = self.run(exposure, self.makeIdFactory(patchRef), expId=expId)
292  self.write(results, patchRef)
293  return results
294 
295  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds):
296  # FINDME: DM-15843 needs to come back and address these next two lines with a final solution
297  inputData["idFactory"] = afwTable.IdFactory.makeSimple()
298  inputData["expId"] = 0
299  return self.run(**inputData)
300 
301  def run(self, exposure, idFactory, expId):
302  """!
303  @brief Run detection on an exposure.
304 
305  First scale the variance plane to match the observed variance
306  using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to
307  detect sources.
308 
309  @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
310  depending on configuration).
311  @param[in] idFactory: IdFactory to set source identifiers
312  @param[in] expId: Exposure identifier (integer) for RNG seed
313 
314  @return a pipe.base.Struct with fields
315  - sources: catalog of detections
316  - backgrounds: list of backgrounds
317  """
318  if self.config.doScaleVariance:
319  varScale = self.scaleVariance.run(exposure.maskedImage)
320  exposure.getMetadata().add("variance_scale", varScale)
321  backgrounds = afwMath.BackgroundList()
322  if self.config.doInsertFakes:
323  self.insertFakes.run(exposure, background=backgrounds)
324  table = afwTable.SourceTable.make(self.schema, idFactory)
325  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
326  sources = detections.sources
327  fpSets = detections.fpSets
328  if hasattr(fpSets, "background") and fpSets.background:
329  for bg in fpSets.background:
330  backgrounds.append(bg)
331  return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
332 
333  def write(self, results, patchRef):
334  """!
335  @brief Write out results from runDetection.
336 
337  @param[in] exposure: Exposure to write out
338  @param[in] results: Struct returned from runDetection
339  @param[in] patchRef: data reference for patch
340  """
341  coaddName = self.config.coaddName + "Coadd"
342  patchRef.put(results.outputBackgrounds, coaddName + "_calexp_background")
343  patchRef.put(results.outputSources, coaddName + "_det")
344  patchRef.put(results.outputExposure, coaddName + "_calexp")
345 
346 
347 
348 
350  """DeblendCoaddSourcesConfig
351 
352  Configuration parameters for the `DeblendCoaddSourcesTask`.
353  """
354  singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
355  doc="Deblend sources separately in each band")
356  multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
357  doc="Deblend sources simultaneously across bands")
358  simultaneous = Field(dtype=bool, default=False, doc="Simultaneously deblend all bands?")
359  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
360 
361  def setDefaults(self):
362  Config.setDefaults(self)
363  self.singleBandDeblend.propagateAllPeaks = True
364 
365 
367  """Task runner for the `MergeSourcesTask`
368 
369  Required because the run method requires a list of
370  dataRefs rather than a single dataRef.
371  """
372  @staticmethod
373  def getTargetList(parsedCmd, **kwargs):
374  """Provide a list of patch references for each patch, tract, filter combo.
375 
376  Parameters
377  ----------
378  parsedCmd:
379  The parsed command
380  kwargs:
381  Keyword arguments passed to the task
382 
383  Returns
384  -------
385  targetList: list
386  List of tuples, where each tuple is a (dataRef, kwargs) pair.
387  """
388  refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
389  kwargs["psfCache"] = parsedCmd.psfCache
390  return [(list(p.values()), kwargs) for t in refDict.values() for p in t.values()]
391 
392 
393 class DeblendCoaddSourcesTask(CmdLineTask):
394  """Deblend the sources in a merged catalog
395 
396  Deblend sources from master catalog in each coadd.
397  This can either be done separately in each band using the HSC-SDSS deblender
398  (`DeblendCoaddSourcesTask.config.simultaneous==False`)
399  or use SCARLET to simultaneously fit the blend in all bands
400  (`DeblendCoaddSourcesTask.config.simultaneous==True`).
401  The task will set its own `self.schema` atribute to the `Schema` of the
402  output deblended catalog.
403  This will include all fields from the input `Schema`, as well as additional fields
404  from the deblender.
405 
406  `pipe.tasks.multiband.DeblendCoaddSourcesTask Description
407  ---------------------------------------------------------
408  `
409 
410  Parameters
411  ----------
412  butler: `Butler`
413  Butler used to read the input schemas from disk or
414  construct the reference catalog loader, if `schema` or `peakSchema` or
415  schema: `Schema`
416  The schema of the merged detection catalog as an input to this task.
417  peakSchema: `Schema`
418  The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog
419  """
420  ConfigClass = DeblendCoaddSourcesConfig
421  RunnerClass = DeblendCoaddSourcesRunner
422  _DefaultName = "deblendCoaddSources"
423  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
424 
425  @classmethod
426  def _makeArgumentParser(cls):
427  parser = ArgumentParser(name=cls._DefaultName)
428  parser.add_id_argument("--id", "deepCoadd_calexp",
429  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
430  ContainerClass=ExistingCoaddDataIdContainer)
431  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
432  return parser
433 
434  def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
435  CmdLineTask.__init__(self, **kwargs)
436  if schema is None:
437  assert butler is not None, "Neither butler nor schema is defined"
438  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
439  self.schemaMapper = afwTable.SchemaMapper(schema)
440  self.schemaMapper.addMinimalSchema(schema)
441  self.schema = self.schemaMapper.getOutputSchema()
442  if peakSchema is None:
443  assert butler is not None, "Neither butler nor peakSchema is defined"
444  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
445 
446  if self.config.simultaneous:
447  self.makeSubtask("multiBandDeblend", schema=self.schema, peakSchema=peakSchema)
448  else:
449  self.makeSubtask("singleBandDeblend", schema=self.schema, peakSchema=peakSchema)
450 
451  def getSchemaCatalogs(self):
452  """Return a dict of empty catalogs for each catalog dataset produced by this task.
453 
454  Returns
455  -------
456  result: dict
457  Dictionary of empty catalogs, with catalog names as keys.
458  """
459  catalog = afwTable.SourceCatalog(self.schema)
460  return {self.config.coaddName + "Coadd_deblendedFlux": catalog,
461  self.config.coaddName + "Coadd_deblendedModel": catalog}
462 
463  def runDataRef(self, patchRefList, psfCache=100):
464  """Deblend the patch
465 
466  Deblend each source simultaneously or separately
467  (depending on `DeblendCoaddSourcesTask.config.simultaneous`).
468  Set `is-primary` and related flags.
469  Propagate flags from individual visits.
470  Write the deblended sources out.
471 
472  Parameters
473  ----------
474  patchRefList: list
475  List of data references for each filter
476  """
477  if self.config.simultaneous:
478  # Use SCARLET to simultaneously deblend across filters
479  filters = []
480  exposures = []
481  for patchRef in patchRefList:
482  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
483  filters.append(patchRef.dataId["filter"])
484  exposures.append(exposure)
485  # The input sources are the same for all bands, since it is a merged catalog
486  sources = self.readSources(patchRef)
487  exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
488  fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
489  for n in range(len(patchRefList)):
490  self.write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
491  else:
492  # Use the singeband deblender to deblend each band separately
493  for patchRef in patchRefList:
494  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
495  exposure.getPsf().setCacheCapacity(psfCache)
496  sources = self.readSources(patchRef)
497  self.singleBandDeblend.run(exposure, sources)
498  self.write(patchRef, sources)
499 
500  def readSources(self, dataRef):
501  """Read merged catalog
502 
503  Read the catalog of merged detections and create a catalog
504  in a single band.
505 
506  Parameters
507  ----------
508  dataRef: data reference
509  Data reference for catalog of merged detections
510 
511  Returns
512  -------
513  sources: `SourceCatalog`
514  List of sources in merged catalog
515 
516  We also need to add columns to hold the measurements we're about to make
517  so we can measure in-place.
518  """
519  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
520  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
521  idFactory = self.makeIdFactory(dataRef)
522  for s in merged:
523  idFactory.notify(s.getId())
524  table = afwTable.SourceTable.make(self.schema, idFactory)
525  sources = afwTable.SourceCatalog(table)
526  sources.extend(merged, self.schemaMapper)
527  return sources
528 
529  def write(self, dataRef, flux_sources, template_sources=None):
530  """Write the source catalog(s)
531 
532  Parameters
533  ----------
534  dataRef: Data Reference
535  Reference to the output catalog.
536  flux_sources: `SourceCatalog`
537  Flux conserved sources to write to file.
538  If using the single band deblender, this is the catalog
539  generated.
540  template_sources: `SourceCatalog`
541  Source catalog using the multiband template models
542  as footprints.
543  """
544  # The multiband deblender does not have to conserve flux,
545  # so only write the flux conserved catalog if it exists
546  if flux_sources is not None:
547  assert not self.config.simultaneous or self.config.multiBandDeblend.conserveFlux
548  dataRef.put(flux_sources, self.config.coaddName + "Coadd_deblendedFlux")
549  # Only the multiband deblender has the option to output the
550  # template model catalog, which can optionally be used
551  # in MeasureMergedCoaddSources
552  if template_sources is not None:
553  assert self.config.multiBandDeblend.saveTemplates
554  dataRef.put(template_sources, self.config.coaddName + "Coadd_deblendedModel")
555  self.log.info("Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
556 
557  def writeMetadata(self, dataRefList):
558  """Write the metadata produced from processing the data.
559  Parameters
560  ----------
561  dataRefList
562  List of Butler data references used to write the metadata.
563  The metadata is written to dataset type `CmdLineTask._getMetadataName`.
564  """
565  for dataRef in dataRefList:
566  try:
567  metadataName = self._getMetadataName()
568  if metadataName is not None:
569  dataRef.put(self.getFullMetadata(), metadataName)
570  except Exception as e:
571  self.log.warn("Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
572 
573  def getExposureId(self, dataRef):
574  """Get the ExposureId from a data reference
575  """
576  return int(dataRef.get(self.config.coaddName + "CoaddId"))
577 
578 
580  """!
581  @anchor MeasureMergedCoaddSourcesConfig_
582 
583  @brief Configuration parameters for the MeasureMergedCoaddSourcesTask
584  """
585  inputCatalog = Field(dtype=str, default="deblendedFlux",
586  doc=("Name of the input catalog to use."
587  "If the single band deblender was used this should be 'deblendedFlux."
588  "If the multi-band deblender was used this should be 'deblendedModel."
589  "If no deblending was performed this should be 'mergeDet'"))
590  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
591  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
592  doPropagateFlags = Field(
593  dtype=bool, default=True,
594  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
595  )
596  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
597  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
598  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
599  doWriteMatchesDenormalized = Field(
600  dtype=bool,
601  default=False,
602  doc=("Write reference matches in denormalized format? "
603  "This format uses more disk space, but is more convenient to read."),
604  )
605  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
606  checkUnitsParseStrict = Field(
607  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
608  dtype=str,
609  default="raise",
610  )
611  doApCorr = Field(
612  dtype=bool,
613  default=True,
614  doc="Apply aperture corrections"
615  )
616  applyApCorr = ConfigurableField(
617  target=ApplyApCorrTask,
618  doc="Subtask to apply aperture corrections"
619  )
620  doRunCatalogCalculation = Field(
621  dtype=bool,
622  default=True,
623  doc='Run catalogCalculation task'
624  )
625  catalogCalculation = ConfigurableField(
626  target=CatalogCalculationTask,
627  doc="Subtask to run catalogCalculation plugins on catalog"
628  )
629 
630  def setDefaults(self):
631  Config.setDefaults(self)
632  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
633  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
634  'INEXACT_PSF']
635  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
636  'INEXACT_PSF']
637 
638 
644 
645 
646 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
647  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
648  @staticmethod
649  def getTargetList(parsedCmd, **kwargs):
650  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
651 
652 
653 class MeasureMergedCoaddSourcesTask(CmdLineTask):
654  r"""!
655  @anchor MeasureMergedCoaddSourcesTask_
656 
657  @brief Deblend sources from master catalog in each coadd seperately and measure.
658 
659  @section pipe_tasks_multiBand_Contents Contents
660 
661  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
662  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
663  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
664  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
665  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
666  - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
667 
668  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
669 
670  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
671  measurement in each coadd.
672 
673  Given a master input catalog of sources (peaks and footprints) or deblender outputs
674  (including a HeavyFootprint in each band), measure each source on the
675  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
676  consistent set of child sources.
677 
678  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
679  properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit
680  flags are propagated to the coadd sources.
681 
682  Optionally, we can match the coadd sources to an external reference catalog.
683 
684  @par Inputs:
685  deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog
686  @n deepCoadd_calexp{tract,patch,filter}: ExposureF
687  @par Outputs:
688  deepCoadd_meas{tract,patch,filter}: SourceCatalog
689  @par Data Unit:
690  tract, patch, filter
691 
692  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
693 
694  <DL>
695  <DT> @ref SingleFrameMeasurementTask_ "measurement"
696  <DD> Measure source properties of deblended sources.</DD>
697  <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags"
698  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
699  not at the edge of the field and that have either not been deblended or are the children of deblended
700  sources</DD>
701  <DT> @ref PropagateVisitFlagsTask_ "propagateFlags"
702  <DD> Propagate flags set in individual visits to the coadd.</DD>
703  <DT> @ref DirectMatchTask_ "match"
704  <DD> Match input sources to a reference catalog (optional).
705  </DD>
706  </DL>
707  These subtasks may be retargeted as required.
708 
709  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
710 
711  @copydoc \_\_init\_\_
712 
713  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
714 
715  @copydoc run
716 
717  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
718 
719  See @ref MeasureMergedCoaddSourcesConfig_
720 
721  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
722 
723  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
724  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
725  files.
726 
727  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
728  the various sub-tasks. See the documetation for individual sub-tasks for more information.
729 
730  @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
731  MeasureMergedCoaddSourcesTask
732 
733  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
734  The next stage in the multi-band processing procedure will merge these measurements into a suitable
735  catalog for driving forced photometry.
736 
737  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
738  to be processed.
739  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
740  `--help` command line argument:
741  @code
742  measureCoaddSources.py --help
743  @endcode
744 
745  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
746  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
747  step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
748  coadd as follows:
749  @code
750  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
751  @endcode
752  This will process the HSC-I band data. The results are written in
753  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
754 
755  It is also necessary to run
756  @code
757  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
758  @endcode
759  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
760  procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask".
761  """
762  _DefaultName = "measureCoaddSources"
763  ConfigClass = MeasureMergedCoaddSourcesConfig
764  RunnerClass = MeasureMergedCoaddSourcesRunner
765  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
766  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
767 
768  @classmethod
769  def _makeArgumentParser(cls):
770  parser = ArgumentParser(name=cls._DefaultName)
771  parser.add_id_argument("--id", "deepCoadd_calexp",
772  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
773  ContainerClass=ExistingCoaddDataIdContainer)
774  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
775  return parser
776 
777  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
778  """!
779  @brief Initialize the task.
780 
781  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
782  @param[in] schema: the schema of the merged detection catalog used as input to this one
783  @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
784  @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
785  catalog. May be None if the loader can be constructed from the butler argument or all steps
786  requiring a reference catalog are disabled.
787  @param[in] butler: a butler used to read the input schemas from disk or construct the reference
788  catalog loader, if schema or peakSchema or refObjLoader is None
789 
790  The task will set its own self.schema attribute to the schema of the output measurement catalog.
791  This will include all fields from the input schema, as well as additional fields for all the
792  measurements.
793  """
794  CmdLineTask.__init__(self, **kwargs)
795  self.deblended = self.config.inputCatalog.startswith("deblended")
796  self.inputCatalog = "Coadd_" + self.config.inputCatalog
797  if schema is None:
798  assert butler is not None, "Neither butler nor schema is defined"
799  schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema", immediate=True).schema
800  self.schemaMapper = afwTable.SchemaMapper(schema)
801  self.schemaMapper.addMinimalSchema(schema)
802  self.schema = self.schemaMapper.getOutputSchema()
804  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
805  self.makeSubtask("setPrimaryFlags", schema=self.schema)
806  if self.config.doMatchSources:
807  if refObjLoader is None:
808  assert butler is not None, "Neither butler nor refObjLoader is defined"
809  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
810  if self.config.doPropagateFlags:
811  self.makeSubtask("propagateFlags", schema=self.schema)
812  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
813  if self.config.doApCorr:
814  self.makeSubtask("applyApCorr", schema=self.schema)
815  if self.config.doRunCatalogCalculation:
816  self.makeSubtask("catalogCalculation", schema=self.schema)
817 
818  def runDataRef(self, patchRef, psfCache=100):
819  """!
820  @brief Deblend and measure.
821 
822  @param[in] patchRef: Patch reference.
823 
824  Set 'is-primary' and related flags. Propagate flags
825  from individual visits. Optionally match the sources to a reference catalog and write the matches.
826  Finally, write the deblended sources and measurements out.
827  """
828  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
829  exposure.getPsf().setCacheCapacity(psfCache)
830  sources = self.readSources(patchRef)
831  table = sources.getTable()
832  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
833 
834  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
835 
836  if self.config.doApCorr:
837  self.applyApCorr.run(
838  catalog=sources,
839  apCorrMap=exposure.getInfo().getApCorrMap()
840  )
841 
842  # TODO DM-11568: this contiguous check-and-copy could go away if we
843  # reserve enough space during SourceDetection and/or SourceDeblend.
844  # NOTE: sourceSelectors require contiguous catalogs, so ensure
845  # contiguity now, so views are preserved from here on.
846  if not sources.isContiguous():
847  sources = sources.copy(deep=True)
848 
849  if self.config.doRunCatalogCalculation:
850  self.catalogCalculation.run(sources)
851 
852  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
853  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
854  includeDeblend=self.deblended)
855  if self.config.doPropagateFlags:
856  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
857  exposure.getWcs())
858  if self.config.doMatchSources:
859  self.writeMatches(patchRef, exposure, sources)
860  self.write(patchRef, sources)
861 
862  def readSources(self, dataRef):
863  """!
864  @brief Read input sources.
865 
866  @param[in] dataRef: Data reference for catalog of merged detections
867  @return List of sources in merged catalog
868 
869  We also need to add columns to hold the measurements we're about to make
870  so we can measure in-place.
871  """
872  merged = dataRef.get(self.config.coaddName + self.inputCatalog, immediate=True)
873  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
874  idFactory = self.makeIdFactory(dataRef)
875  for s in merged:
876  idFactory.notify(s.getId())
877  table = afwTable.SourceTable.make(self.schema, idFactory)
878  sources = afwTable.SourceCatalog(table)
879  sources.extend(merged, self.schemaMapper)
880  return sources
881 
882  def writeMatches(self, dataRef, exposure, sources):
883  """!
884  @brief Write matches of the sources to the astrometric reference catalog.
885 
886  We use the Wcs in the exposure to match sources.
887 
888  @param[in] dataRef: data reference
889  @param[in] exposure: exposure with Wcs
890  @param[in] sources: source catalog
891  """
892  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
893  if result.matches:
894  matches = afwTable.packMatches(result.matches)
895  matches.table.setMetadata(result.matchMeta)
896  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
897  if self.config.doWriteMatchesDenormalized:
898  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
899  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
900 
901  def write(self, dataRef, sources):
902  """!
903  @brief Write the source catalog.
904 
905  @param[in] dataRef: data reference
906  @param[in] sources: source catalog
907  """
908  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
909  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
910 
911  def getExposureId(self, dataRef):
912  return int(dataRef.get(self.config.coaddName + "CoaddId"))
def runDataRef(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:280
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:64
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds)
Definition: multiBand.py:295
def runDataRef(self, patchRefList, psfCache=100)
Definition: multiBand.py:463
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:255
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:653
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:882
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
Definition: multiBand.py:434
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:862
def write(self, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:333
def write(self, dataRef, flux_sources, template_sources=None)
Definition: multiBand.py:529
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:901
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:579
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:226
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:777
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:301
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:818