lsst.pipe.tasks  16.0-29-gdc8abbdf+4
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25  PipelineTask, InitOutputDatasetField, InputDatasetField, OutputDatasetField,
26  QuantumConfig)
27 from lsst.pex.config import Config, Field, ConfigurableField
28 from lsst.meas.algorithms import DynamicDetectionTask
29 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
30 from lsst.meas.deblender import SourceDeblendTask, MultibandDeblendTask
31 from lsst.pipe.tasks.coaddBase import getSkyInfo
32 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
33 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
34 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
35 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
36 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
37 import lsst.afw.image as afwImage
38 import lsst.afw.table as afwTable
39 import lsst.afw.math as afwMath
40 from lsst.daf.base import PropertyList
41 
42 from .mergeDetections import MergeDetectionsConfig, MergeDetectionsTask # noqa: F401
43 from .mergeMeasurements import MergeMeasurementsConfig, MergeMeasurementsTask # noqa: F401
44 from .multiBandUtils import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs # noqa: F401
45 from .multiBandUtils import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory # noqa: F401
46 
47 
48 """
49 New dataset types:
50 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
51 * deepCoadd_mergeDet: merged detections (tract, patch)
52 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
53 * deepCoadd_ref: reference sources (tract, patch)
54 All of these have associated *_schema catalogs that require no data ID and hold no records.
55 
56 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
57 the mergeDet, meas, and ref dataset Footprints:
58 * deepCoadd_peak_schema
59 """
60 
61 
62 
63 
65  """!
66  @anchor DetectCoaddSourcesConfig_
67 
68  @brief Configuration parameters for the DetectCoaddSourcesTask
69  """
70  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
71  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
72  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
73  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
74  doInsertFakes = Field(dtype=bool, default=False,
75  doc="Run fake sources injection task")
76  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
77  doc="Injection of fake sources for testing "
78  "purposes (must be retargeted)")
79  detectionSchema = InitOutputDatasetField(
80  doc="Schema of the detection catalog",
81  name="{}Coadd_det_schema",
82  storageClass="SourceCatalog",
83  )
84  exposure = InputDatasetField(
85  doc="Exposure on which detections are to be performed",
86  name="deepCoadd",
87  scalar=True,
88  storageClass="Exposure",
89  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
90  )
91  outputBackgrounds = OutputDatasetField(
92  doc="Output Backgrounds used in detection",
93  name="{}Coadd_calexp_background",
94  scalar=True,
95  storageClass="Background",
96  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
97  )
98  outputSources = OutputDatasetField(
99  doc="Detected sources catalog",
100  name="{}Coadd_det",
101  scalar=True,
102  storageClass="SourceCatalog",
103  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
104  )
105  outputExposure = OutputDatasetField(
106  doc="Exposure post detection",
107  name="{}Coadd_calexp",
108  scalar=True,
109  storageClass="Exposure",
110  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
111  )
112  quantum = QuantumConfig(
113  units=("Tract", "Patch", "AbstractFilter", "SkyMap")
114  )
115 
116  def setDefaults(self):
117  Config.setDefaults(self)
118  self.detection.thresholdType = "pixel_stdev"
119  self.detection.isotropicGrow = True
120  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
121  self.detection.reEstimateBackground = False
122  self.detection.background.useApprox = False
123  self.detection.background.binSize = 4096
124  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
125  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
126 
127 
133 
134 
135 class DetectCoaddSourcesTask(PipelineTask, CmdLineTask):
136  r"""!
137  @anchor DetectCoaddSourcesTask_
138 
139  @brief Detect sources on a coadd
140 
141  @section pipe_tasks_multiBand_Contents Contents
142 
143  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
144  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
145  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
146  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
147  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
148  - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
149 
150  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
151 
152  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
153 
154  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
155  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
156  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
157  propagate the full covariance matrix -- but it is simple and works well in practice.
158 
159  After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref
160  SourceDetectionTask_ "detection" subtask.
161 
162  @par Inputs:
163  deepCoadd{tract,patch,filter}: ExposureF
164  @par Outputs:
165  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
166  @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
167  exposure (ExposureF)
168  @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
169  @par Data Unit:
170  tract, patch, filter
171 
172  DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask.
173  You can retarget this subtask if you wish.
174 
175  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
176 
177  @copydoc \_\_init\_\_
178 
179  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
180 
181  @copydoc run
182 
183  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
184 
185  See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
186 
187  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
188 
189  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
190  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
191  files.
192 
193  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
194  @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
195  @ref SourceDetectionTask_ "SourceDetectionTask" for further information.
196 
197  @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
198  of using DetectCoaddSourcesTask
199 
200  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
201  the task is to update the background, detect all sources in a single band and generate a set of parent
202  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
203  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
204  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
205  calling detectCoaddSources.py with the `--help` command line argument:
206  @code
207  detectCoaddSources.py --help
208  @endcode
209 
210  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
211  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
212  steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
213  @code
214  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
215  @endcode
216  that will process the HSC-I band data. The results are written to
217  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
218 
219  It is also necessary to run:
220  @code
221  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
222  @endcode
223  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
224  processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask".
225  """
226  _DefaultName = "detectCoaddSources"
227  ConfigClass = DetectCoaddSourcesConfig
228  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
229  makeIdFactory = _makeMakeIdFactory("CoaddId")
230 
231  @classmethod
232  def _makeArgumentParser(cls):
233  parser = ArgumentParser(name=cls._DefaultName)
234  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
235  ContainerClass=ExistingCoaddDataIdContainer)
236  return parser
237 
238  @classmethod
239  def getOutputDatasetTypes(cls, config):
240  coaddName = config.coaddName
241  for name in ("outputBackgrounds", "outputSources", "outputExposure"):
242  attr = getattr(config, name)
243  setattr(attr, "name", attr.name.format(coaddName))
244  outputTypeDict = super().getOutputDatasetTypes(config)
245  return outputTypeDict
246 
247  @classmethod
248  def getInitOutputDatasetTypes(cls, config):
249  coaddName = config.coaddName
250  attr = config.detectionSchema
251  setattr(attr, "name", attr.name.format(coaddName))
252  output = super().getInitOutputDatasetTypes(config)
253  print(output)
254  return output
255 
256  def __init__(self, schema=None, **kwargs):
257  """!
258  @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask.
259 
260  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
261 
262  @param[in] schema: initial schema for the output catalog, modified-in place to include all
263  fields set by this task. If None, the source minimal schema will be used.
264  @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
265  """
266  # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree
267  # call structure has been reviewed carefully to be sure super will work as intended.
268  super().__init__(**kwargs)
269  if schema is None:
270  schema = afwTable.SourceTable.makeMinimalSchema()
271  if self.config.doInsertFakes:
272  self.makeSubtask("insertFakes")
273  self.schema = schema
274  self.makeSubtask("detection", schema=self.schema)
275  if self.config.doScaleVariance:
276  self.makeSubtask("scaleVariance")
277 
279  return {"detectionSchema": afwTable.SourceCatalog(self.schema)}
280 
281  def runDataRef(self, patchRef):
282  """!
283  @brief Run detection on a coadd.
284 
285  Invokes @ref run and then uses @ref write to output the
286  results.
287 
288  @param[in] patchRef: data reference for patch
289  """
290  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
291  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
292  results = self.run(exposure, self.makeIdFactory(patchRef), expId=expId)
293  self.write(results, patchRef)
294  return results
295 
296  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds):
297  # FINDME: DM-15843 needs to come back and address these next two lines with a final solution
298  inputData["idFactory"] = afwTable.IdFactory.makeSimple()
299  inputData["expId"] = 0
300  return self.run(**inputData)
301 
302  def run(self, exposure, idFactory, expId):
303  """!
304  @brief Run detection on an exposure.
305 
306  First scale the variance plane to match the observed variance
307  using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to
308  detect sources.
309 
310  @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
311  depending on configuration).
312  @param[in] idFactory: IdFactory to set source identifiers
313  @param[in] expId: Exposure identifier (integer) for RNG seed
314 
315  @return a pipe.base.Struct with fields
316  - sources: catalog of detections
317  - backgrounds: list of backgrounds
318  """
319  if self.config.doScaleVariance:
320  varScale = self.scaleVariance.run(exposure.maskedImage)
321  exposure.getMetadata().add("variance_scale", varScale)
322  backgrounds = afwMath.BackgroundList()
323  if self.config.doInsertFakes:
324  self.insertFakes.run(exposure, background=backgrounds)
325  table = afwTable.SourceTable.make(self.schema, idFactory)
326  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
327  sources = detections.sources
328  fpSets = detections.fpSets
329  if hasattr(fpSets, "background") and fpSets.background:
330  for bg in fpSets.background:
331  backgrounds.append(bg)
332  return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
333 
334  def write(self, results, patchRef):
335  """!
336  @brief Write out results from runDetection.
337 
338  @param[in] exposure: Exposure to write out
339  @param[in] results: Struct returned from runDetection
340  @param[in] patchRef: data reference for patch
341  """
342  coaddName = self.config.coaddName + "Coadd"
343  patchRef.put(results.outputBackgrounds, coaddName + "_calexp_background")
344  patchRef.put(results.outputSources, coaddName + "_det")
345  patchRef.put(results.outputExposure, coaddName + "_calexp")
346 
347 
348 
349 
351  """DeblendCoaddSourcesConfig
352 
353  Configuration parameters for the `DeblendCoaddSourcesTask`.
354  """
355  singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
356  doc="Deblend sources separately in each band")
357  multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
358  doc="Deblend sources simultaneously across bands")
359  simultaneous = Field(dtype=bool, default=False, doc="Simultaneously deblend all bands?")
360  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
361 
362  def setDefaults(self):
363  Config.setDefaults(self)
364  self.singleBandDeblend.propagateAllPeaks = True
365 
366 
368  """Task runner for the `MergeSourcesTask`
369 
370  Required because the run method requires a list of
371  dataRefs rather than a single dataRef.
372  """
373  @staticmethod
374  def getTargetList(parsedCmd, **kwargs):
375  """Provide a list of patch references for each patch, tract, filter combo.
376 
377  Parameters
378  ----------
379  parsedCmd:
380  The parsed command
381  kwargs:
382  Keyword arguments passed to the task
383 
384  Returns
385  -------
386  targetList: list
387  List of tuples, where each tuple is a (dataRef, kwargs) pair.
388  """
389  refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
390  kwargs["psfCache"] = parsedCmd.psfCache
391  return [(list(p.values()), kwargs) for t in refDict.values() for p in t.values()]
392 
393 
394 class DeblendCoaddSourcesTask(CmdLineTask):
395  """Deblend the sources in a merged catalog
396 
397  Deblend sources from master catalog in each coadd.
398  This can either be done separately in each band using the HSC-SDSS deblender
399  (`DeblendCoaddSourcesTask.config.simultaneous==False`)
400  or use SCARLET to simultaneously fit the blend in all bands
401  (`DeblendCoaddSourcesTask.config.simultaneous==True`).
402  The task will set its own `self.schema` atribute to the `Schema` of the
403  output deblended catalog.
404  This will include all fields from the input `Schema`, as well as additional fields
405  from the deblender.
406 
407  `pipe.tasks.multiband.DeblendCoaddSourcesTask Description
408  ---------------------------------------------------------
409  `
410 
411  Parameters
412  ----------
413  butler: `Butler`
414  Butler used to read the input schemas from disk or
415  construct the reference catalog loader, if `schema` or `peakSchema` or
416  schema: `Schema`
417  The schema of the merged detection catalog as an input to this task.
418  peakSchema: `Schema`
419  The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog
420  """
421  ConfigClass = DeblendCoaddSourcesConfig
422  RunnerClass = DeblendCoaddSourcesRunner
423  _DefaultName = "deblendCoaddSources"
424  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
425 
426  @classmethod
427  def _makeArgumentParser(cls):
428  parser = ArgumentParser(name=cls._DefaultName)
429  parser.add_id_argument("--id", "deepCoadd_calexp",
430  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
431  ContainerClass=ExistingCoaddDataIdContainer)
432  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
433  return parser
434 
435  def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
436  CmdLineTask.__init__(self, **kwargs)
437  if schema is None:
438  assert butler is not None, "Neither butler nor schema is defined"
439  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
440  self.schemaMapper = afwTable.SchemaMapper(schema)
441  self.schemaMapper.addMinimalSchema(schema)
442  self.schema = self.schemaMapper.getOutputSchema()
443  if peakSchema is None:
444  assert butler is not None, "Neither butler nor peakSchema is defined"
445  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
446 
447  if self.config.simultaneous:
448  self.makeSubtask("multiBandDeblend", schema=self.schema, peakSchema=peakSchema)
449  else:
450  self.makeSubtask("singleBandDeblend", schema=self.schema, peakSchema=peakSchema)
451 
452  def getSchemaCatalogs(self):
453  """Return a dict of empty catalogs for each catalog dataset produced by this task.
454 
455  Returns
456  -------
457  result: dict
458  Dictionary of empty catalogs, with catalog names as keys.
459  """
460  catalog = afwTable.SourceCatalog(self.schema)
461  return {self.config.coaddName + "Coadd_deblendedFlux": catalog,
462  self.config.coaddName + "Coadd_deblendedModel": catalog}
463 
464  def runDataRef(self, patchRefList, psfCache=100):
465  """Deblend the patch
466 
467  Deblend each source simultaneously or separately
468  (depending on `DeblendCoaddSourcesTask.config.simultaneous`).
469  Set `is-primary` and related flags.
470  Propagate flags from individual visits.
471  Write the deblended sources out.
472 
473  Parameters
474  ----------
475  patchRefList: list
476  List of data references for each filter
477  """
478  if self.config.simultaneous:
479  # Use SCARLET to simultaneously deblend across filters
480  filters = []
481  exposures = []
482  for patchRef in patchRefList:
483  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
484  filters.append(patchRef.dataId["filter"])
485  exposures.append(exposure)
486  # The input sources are the same for all bands, since it is a merged catalog
487  sources = self.readSources(patchRef)
488  exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
489  fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
490  for n in range(len(patchRefList)):
491  self.write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
492  else:
493  # Use the singeband deblender to deblend each band separately
494  for patchRef in patchRefList:
495  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
496  exposure.getPsf().setCacheCapacity(psfCache)
497  sources = self.readSources(patchRef)
498  self.singleBandDeblend.run(exposure, sources)
499  self.write(patchRef, sources)
500 
501  def readSources(self, dataRef):
502  """Read merged catalog
503 
504  Read the catalog of merged detections and create a catalog
505  in a single band.
506 
507  Parameters
508  ----------
509  dataRef: data reference
510  Data reference for catalog of merged detections
511 
512  Returns
513  -------
514  sources: `SourceCatalog`
515  List of sources in merged catalog
516 
517  We also need to add columns to hold the measurements we're about to make
518  so we can measure in-place.
519  """
520  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
521  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
522  idFactory = self.makeIdFactory(dataRef)
523  for s in merged:
524  idFactory.notify(s.getId())
525  table = afwTable.SourceTable.make(self.schema, idFactory)
526  sources = afwTable.SourceCatalog(table)
527  sources.extend(merged, self.schemaMapper)
528  return sources
529 
530  def write(self, dataRef, flux_sources, template_sources=None):
531  """Write the source catalog(s)
532 
533  Parameters
534  ----------
535  dataRef: Data Reference
536  Reference to the output catalog.
537  flux_sources: `SourceCatalog`
538  Flux conserved sources to write to file.
539  If using the single band deblender, this is the catalog
540  generated.
541  template_sources: `SourceCatalog`
542  Source catalog using the multiband template models
543  as footprints.
544  """
545  # The multiband deblender does not have to conserve flux,
546  # so only write the flux conserved catalog if it exists
547  if flux_sources is not None:
548  assert not self.config.simultaneous or self.config.multiBandDeblend.conserveFlux
549  dataRef.put(flux_sources, self.config.coaddName + "Coadd_deblendedFlux")
550  # Only the multiband deblender has the option to output the
551  # template model catalog, which can optionally be used
552  # in MeasureMergedCoaddSources
553  if template_sources is not None:
554  assert self.config.multiBandDeblend.saveTemplates
555  dataRef.put(template_sources, self.config.coaddName + "Coadd_deblendedModel")
556  self.log.info("Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
557 
558  def writeMetadata(self, dataRefList):
559  """Write the metadata produced from processing the data.
560  Parameters
561  ----------
562  dataRefList
563  List of Butler data references used to write the metadata.
564  The metadata is written to dataset type `CmdLineTask._getMetadataName`.
565  """
566  for dataRef in dataRefList:
567  try:
568  metadataName = self._getMetadataName()
569  if metadataName is not None:
570  dataRef.put(self.getFullMetadata(), metadataName)
571  except Exception as e:
572  self.log.warn("Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
573 
574  def getExposureId(self, dataRef):
575  """Get the ExposureId from a data reference
576  """
577  return int(dataRef.get(self.config.coaddName + "CoaddId"))
578 
579 
581  """!
582  @anchor MeasureMergedCoaddSourcesConfig_
583 
584  @brief Configuration parameters for the MeasureMergedCoaddSourcesTask
585  """
586  inputCatalog = Field(dtype=str, default="deblendedFlux",
587  doc=("Name of the input catalog to use."
588  "If the single band deblender was used this should be 'deblendedFlux."
589  "If the multi-band deblender was used this should be 'deblendedModel."
590  "If no deblending was performed this should be 'mergeDet'"))
591  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
592  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
593  doPropagateFlags = Field(
594  dtype=bool, default=True,
595  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
596  )
597  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
598  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
599  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
600  doWriteMatchesDenormalized = Field(
601  dtype=bool,
602  default=False,
603  doc=("Write reference matches in denormalized format? "
604  "This format uses more disk space, but is more convenient to read."),
605  )
606  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
607  checkUnitsParseStrict = Field(
608  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
609  dtype=str,
610  default="raise",
611  )
612  doApCorr = Field(
613  dtype=bool,
614  default=True,
615  doc="Apply aperture corrections"
616  )
617  applyApCorr = ConfigurableField(
618  target=ApplyApCorrTask,
619  doc="Subtask to apply aperture corrections"
620  )
621  doRunCatalogCalculation = Field(
622  dtype=bool,
623  default=True,
624  doc='Run catalogCalculation task'
625  )
626  catalogCalculation = ConfigurableField(
627  target=CatalogCalculationTask,
628  doc="Subtask to run catalogCalculation plugins on catalog"
629  )
630 
631  def setDefaults(self):
632  Config.setDefaults(self)
633  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
634  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
635  'INEXACT_PSF']
636  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
637  'INEXACT_PSF']
638 
639 
645 
646 
647 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
648  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
649  @staticmethod
650  def getTargetList(parsedCmd, **kwargs):
651  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
652 
653 
654 class MeasureMergedCoaddSourcesTask(CmdLineTask):
655  r"""!
656  @anchor MeasureMergedCoaddSourcesTask_
657 
658  @brief Deblend sources from master catalog in each coadd seperately and measure.
659 
660  @section pipe_tasks_multiBand_Contents Contents
661 
662  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
663  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
664  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
665  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
666  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
667  - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
668 
669  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
670 
671  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
672  measurement in each coadd.
673 
674  Given a master input catalog of sources (peaks and footprints) or deblender outputs
675  (including a HeavyFootprint in each band), measure each source on the
676  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
677  consistent set of child sources.
678 
679  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
680  properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit
681  flags are propagated to the coadd sources.
682 
683  Optionally, we can match the coadd sources to an external reference catalog.
684 
685  @par Inputs:
686  deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog
687  @n deepCoadd_calexp{tract,patch,filter}: ExposureF
688  @par Outputs:
689  deepCoadd_meas{tract,patch,filter}: SourceCatalog
690  @par Data Unit:
691  tract, patch, filter
692 
693  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
694 
695  <DL>
696  <DT> @ref SingleFrameMeasurementTask_ "measurement"
697  <DD> Measure source properties of deblended sources.</DD>
698  <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags"
699  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
700  not at the edge of the field and that have either not been deblended or are the children of deblended
701  sources</DD>
702  <DT> @ref PropagateVisitFlagsTask_ "propagateFlags"
703  <DD> Propagate flags set in individual visits to the coadd.</DD>
704  <DT> @ref DirectMatchTask_ "match"
705  <DD> Match input sources to a reference catalog (optional).
706  </DD>
707  </DL>
708  These subtasks may be retargeted as required.
709 
710  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
711 
712  @copydoc \_\_init\_\_
713 
714  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
715 
716  @copydoc run
717 
718  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
719 
720  See @ref MeasureMergedCoaddSourcesConfig_
721 
722  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
723 
724  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
725  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
726  files.
727 
728  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
729  the various sub-tasks. See the documetation for individual sub-tasks for more information.
730 
731  @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
732  MeasureMergedCoaddSourcesTask
733 
734  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
735  The next stage in the multi-band processing procedure will merge these measurements into a suitable
736  catalog for driving forced photometry.
737 
738  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
739  to be processed.
740  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
741  `--help` command line argument:
742  @code
743  measureCoaddSources.py --help
744  @endcode
745 
746  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
747  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
748  step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
749  coadd as follows:
750  @code
751  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
752  @endcode
753  This will process the HSC-I band data. The results are written in
754  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
755 
756  It is also necessary to run
757  @code
758  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
759  @endcode
760  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
761  procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask".
762  """
763  _DefaultName = "measureCoaddSources"
764  ConfigClass = MeasureMergedCoaddSourcesConfig
765  RunnerClass = MeasureMergedCoaddSourcesRunner
766  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
767  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
768 
769  @classmethod
770  def _makeArgumentParser(cls):
771  parser = ArgumentParser(name=cls._DefaultName)
772  parser.add_id_argument("--id", "deepCoadd_calexp",
773  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
774  ContainerClass=ExistingCoaddDataIdContainer)
775  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
776  return parser
777 
778  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, **kwargs):
779  """!
780  @brief Initialize the task.
781 
782  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
783  @param[in] schema: the schema of the merged detection catalog used as input to this one
784  @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
785  @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
786  catalog. May be None if the loader can be constructed from the butler argument or all steps
787  requiring a reference catalog are disabled.
788  @param[in] butler: a butler used to read the input schemas from disk or construct the reference
789  catalog loader, if schema or peakSchema or refObjLoader is None
790 
791  The task will set its own self.schema attribute to the schema of the output measurement catalog.
792  This will include all fields from the input schema, as well as additional fields for all the
793  measurements.
794  """
795  CmdLineTask.__init__(self, **kwargs)
796  self.deblended = self.config.inputCatalog.startswith("deblended")
797  self.inputCatalog = "Coadd_" + self.config.inputCatalog
798  if schema is None:
799  assert butler is not None, "Neither butler nor schema is defined"
800  schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema", immediate=True).schema
801  self.schemaMapper = afwTable.SchemaMapper(schema)
802  self.schemaMapper.addMinimalSchema(schema)
803  self.schema = self.schemaMapper.getOutputSchema()
805  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
806  self.makeSubtask("setPrimaryFlags", schema=self.schema)
807  if self.config.doMatchSources:
808  if refObjLoader is None:
809  assert butler is not None, "Neither butler nor refObjLoader is defined"
810  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
811  if self.config.doPropagateFlags:
812  self.makeSubtask("propagateFlags", schema=self.schema)
813  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
814  if self.config.doApCorr:
815  self.makeSubtask("applyApCorr", schema=self.schema)
816  if self.config.doRunCatalogCalculation:
817  self.makeSubtask("catalogCalculation", schema=self.schema)
818 
819  def runDataRef(self, patchRef, psfCache=100):
820  """!
821  @brief Deblend and measure.
822 
823  @param[in] patchRef: Patch reference.
824 
825  Set 'is-primary' and related flags. Propagate flags
826  from individual visits. Optionally match the sources to a reference catalog and write the matches.
827  Finally, write the deblended sources and measurements out.
828  """
829  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
830  exposure.getPsf().setCacheCapacity(psfCache)
831  sources = self.readSources(patchRef)
832  table = sources.getTable()
833  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
834 
835  self.measurement.run(sources, exposure, exposureId=self.getExposureId(patchRef))
836 
837  if self.config.doApCorr:
838  self.applyApCorr.run(
839  catalog=sources,
840  apCorrMap=exposure.getInfo().getApCorrMap()
841  )
842 
843  # TODO DM-11568: this contiguous check-and-copy could go away if we
844  # reserve enough space during SourceDetection and/or SourceDeblend.
845  # NOTE: sourceSelectors require contiguous catalogs, so ensure
846  # contiguity now, so views are preserved from here on.
847  if not sources.isContiguous():
848  sources = sources.copy(deep=True)
849 
850  if self.config.doRunCatalogCalculation:
851  self.catalogCalculation.run(sources)
852 
853  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
854  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
855  includeDeblend=self.deblended)
856  if self.config.doPropagateFlags:
857  self.propagateFlags.run(patchRef.getButler(), sources, self.propagateFlags.getCcdInputs(exposure),
858  exposure.getWcs())
859  if self.config.doMatchSources:
860  self.writeMatches(patchRef, exposure, sources)
861  self.write(patchRef, sources)
862 
863  def readSources(self, dataRef):
864  """!
865  @brief Read input sources.
866 
867  @param[in] dataRef: Data reference for catalog of merged detections
868  @return List of sources in merged catalog
869 
870  We also need to add columns to hold the measurements we're about to make
871  so we can measure in-place.
872  """
873  merged = dataRef.get(self.config.coaddName + self.inputCatalog, immediate=True)
874  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
875  idFactory = self.makeIdFactory(dataRef)
876  for s in merged:
877  idFactory.notify(s.getId())
878  table = afwTable.SourceTable.make(self.schema, idFactory)
879  sources = afwTable.SourceCatalog(table)
880  sources.extend(merged, self.schemaMapper)
881  return sources
882 
883  def writeMatches(self, dataRef, exposure, sources):
884  """!
885  @brief Write matches of the sources to the astrometric reference catalog.
886 
887  We use the Wcs in the exposure to match sources.
888 
889  @param[in] dataRef: data reference
890  @param[in] exposure: exposure with Wcs
891  @param[in] sources: source catalog
892  """
893  result = self.match.run(sources, exposure.getInfo().getFilter().getName())
894  if result.matches:
895  matches = afwTable.packMatches(result.matches)
896  matches.table.setMetadata(result.matchMeta)
897  dataRef.put(matches, self.config.coaddName + "Coadd_measMatch")
898  if self.config.doWriteMatchesDenormalized:
899  denormMatches = denormalizeMatches(result.matches, result.matchMeta)
900  dataRef.put(denormMatches, self.config.coaddName + "Coadd_measMatchFull")
901 
902  def write(self, dataRef, sources):
903  """!
904  @brief Write the source catalog.
905 
906  @param[in] dataRef: data reference
907  @param[in] sources: source catalog
908  """
909  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
910  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
911 
912  def getExposureId(self, dataRef):
913  return int(dataRef.get(self.config.coaddName + "CoaddId"))
def runDataRef(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:281
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:64
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds)
Definition: multiBand.py:296
def runDataRef(self, patchRefList, psfCache=100)
Definition: multiBand.py:464
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:256
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:654
def writeMatches(self, dataRef, exposure, sources)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:883
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
Definition: multiBand.py:435
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:863
def write(self, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:334
def write(self, dataRef, flux_sources, template_sources=None)
Definition: multiBand.py:530
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:902
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:580
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:253
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, kwargs)
Initialize the task.
Definition: multiBand.py:778
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:302
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:819