lsst.pipe.tasks  16.0-53-ga4affb86
multiBand.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from lsst.coadd.utils.coaddDataIdContainer import ExistingCoaddDataIdContainer
24 from lsst.pipe.base import (CmdLineTask, Struct, ArgumentParser, ButlerInitializedTaskRunner,
25  PipelineTask, PipelineTaskConfig, InitInputDatasetField,
26  InitOutputDatasetField, InputDatasetField, OutputDatasetField,
27  QuantumConfig)
28 from lsst.pex.config import Config, Field, ConfigurableField
29 from lsst.meas.algorithms import DynamicDetectionTask, ReferenceObjectLoader
30 from lsst.meas.base import SingleFrameMeasurementTask, ApplyApCorrTask, CatalogCalculationTask
31 from lsst.meas.deblender import SourceDeblendTask, MultibandDeblendTask
32 from lsst.pipe.tasks.coaddBase import getSkyInfo
33 from lsst.pipe.tasks.scaleVariance import ScaleVarianceTask
34 from lsst.meas.astrom import DirectMatchTask, denormalizeMatches
35 from lsst.pipe.tasks.fakes import BaseFakeSourcesTask
36 from lsst.pipe.tasks.setPrimaryFlags import SetPrimaryFlagsTask
37 from lsst.pipe.tasks.propagateVisitFlags import PropagateVisitFlagsTask
38 import lsst.afw.image as afwImage
39 import lsst.afw.table as afwTable
40 import lsst.afw.math as afwMath
41 from lsst.daf.base import PropertyList
42 
43 from .mergeDetections import MergeDetectionsConfig, MergeDetectionsTask # noqa: F401
44 from .mergeMeasurements import MergeMeasurementsConfig, MergeMeasurementsTask # noqa: F401
45 from .multiBandUtils import MergeSourcesRunner, CullPeaksConfig, _makeGetSchemaCatalogs # noqa: F401
46 from .multiBandUtils import getInputSchema, getShortFilterName, readCatalog, _makeMakeIdFactory # noqa: F401
47 from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesSingleConfig # noqa: F401
48 from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesSingleTask # noqa: F401
49 from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiConfig # noqa: F401
50 from .deblendCoaddSourcesPipeline import DeblendCoaddSourcesMultiTask # noqa: F401
51 
52 
53 """
54 New set types:
55 * deepCoadd_det: detections from what used to be processCoadd (tract, patch, filter)
56 * deepCoadd_mergeDet: merged detections (tract, patch)
57 * deepCoadd_meas: measurements of merged detections (tract, patch, filter)
58 * deepCoadd_ref: reference sources (tract, patch)
59 All of these have associated *_schema catalogs that require no data ID and hold no records.
60 
61 In addition, we have a schema-only dataset, which saves the schema for the PeakRecords in
62 the mergeDet, meas, and ref dataset Footprints:
63 * deepCoadd_peak_schema
64 """
65 
66 
67 
68 
70  """!
71  @anchor DetectCoaddSourcesConfig_
72 
73  @brief Configuration parameters for the DetectCoaddSourcesTask
74  """
75  doScaleVariance = Field(dtype=bool, default=True, doc="Scale variance plane using empirical noise?")
76  scaleVariance = ConfigurableField(target=ScaleVarianceTask, doc="Variance rescaling")
77  detection = ConfigurableField(target=DynamicDetectionTask, doc="Source detection")
78  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
79  doInsertFakes = Field(dtype=bool, default=False,
80  doc="Run fake sources injection task")
81  insertFakes = ConfigurableField(target=BaseFakeSourcesTask,
82  doc="Injection of fake sources for testing "
83  "purposes (must be retargeted)")
84  detectionSchema = InitOutputDatasetField(
85  doc="Schema of the detection catalog",
86  name="{}Coadd_det_schema",
87  storageClass="SourceCatalog",
88  )
89  exposure = InputDatasetField(
90  doc="Exposure on which detections are to be performed",
91  name="deepCoadd",
92  scalar=True,
93  storageClass="ExposureF",
94  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
95  )
96  outputBackgrounds = OutputDatasetField(
97  doc="Output Backgrounds used in detection",
98  name="{}Coadd_calexp_background",
99  scalar=True,
100  storageClass="Background",
101  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
102  )
103  outputSources = OutputDatasetField(
104  doc="Detected sources catalog",
105  name="{}Coadd_det",
106  scalar=True,
107  storageClass="SourceCatalog",
108  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
109  )
110  outputExposure = OutputDatasetField(
111  doc="Exposure post detection",
112  name="{}Coadd_calexp",
113  scalar=True,
114  storageClass="ExposureF",
115  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
116  )
117  quantum = QuantumConfig(
118  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
119  )
120 
121  def setDefaults(self):
122  Config.setDefaults(self)
123  self.detection.thresholdType = "pixel_stdev"
124  self.detection.isotropicGrow = True
125  # Coadds are made from background-subtracted CCDs, so any background subtraction should be very basic
126  self.detection.reEstimateBackground = False
127  self.detection.background.useApprox = False
128  self.detection.background.binSize = 4096
129  self.detection.background.undersampleStyle = 'REDUCE_INTERP_ORDER'
130  self.detection.doTempWideBackground = True # Suppress large footprints that overwhelm the deblender
131 
132 
138 
139 
140 class DetectCoaddSourcesTask(PipelineTask, CmdLineTask):
141  r"""!
142  @anchor DetectCoaddSourcesTask_
143 
144  @brief Detect sources on a coadd
145 
146  @section pipe_tasks_multiBand_Contents Contents
147 
148  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose
149  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize
150  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Run
151  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Config
152  - @ref pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug
153  - @ref pipe_tasks_multiband_DetectCoaddSourcesTask_Example
154 
155  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Purpose Description
156 
157  Command-line task that detects sources on a coadd of exposures obtained with a single filter.
158 
159  Coadding individual visits requires each exposure to be warped. This introduces covariance in the noise
160  properties across pixels. Before detection, we correct the coadd variance by scaling the variance plane
161  in the coadd to match the observed variance. This is an approximate approach -- strictly, we should
162  propagate the full covariance matrix -- but it is simple and works well in practice.
163 
164  After scaling the variance plane, we detect sources and generate footprints by delegating to the @ref
165  SourceDetectionTask_ "detection" subtask.
166 
167  @par Inputs:
168  deepCoadd{tract,patch,filter}: ExposureF
169  @par Outputs:
170  deepCoadd_det{tract,patch,filter}: SourceCatalog (only parent Footprints)
171  @n deepCoadd_calexp{tract,patch,filter}: Variance scaled, background-subtracted input
172  exposure (ExposureF)
173  @n deepCoadd_calexp_background{tract,patch,filter}: BackgroundList
174  @par Data Unit:
175  tract, patch, filter
176 
177  DetectCoaddSourcesTask delegates most of its work to the @ref SourceDetectionTask_ "detection" subtask.
178  You can retarget this subtask if you wish.
179 
180  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Initialize Task initialization
181 
182  @copydoc \_\_init\_\_
183 
184  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Run Invoking the Task
185 
186  @copydoc run
187 
188  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Config Configuration parameters
189 
190  See @ref DetectCoaddSourcesConfig_ "DetectSourcesConfig"
191 
192  @section pipe_tasks_multiBand_DetectCoaddSourcesTask_Debug Debug variables
193 
194  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
195  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
196  files.
197 
198  DetectCoaddSourcesTask has no debug variables of its own because it relegates all the work to
199  @ref SourceDetectionTask_ "SourceDetectionTask"; see the documetation for
200  @ref SourceDetectionTask_ "SourceDetectionTask" for further information.
201 
202  @section pipe_tasks_multiband_DetectCoaddSourcesTask_Example A complete example
203  of using DetectCoaddSourcesTask
204 
205  DetectCoaddSourcesTask is meant to be run after assembling a coadded image in a given band. The purpose of
206  the task is to update the background, detect all sources in a single band and generate a set of parent
207  footprints. Subsequent tasks in the multi-band processing procedure will merge sources across bands and,
208  eventually, perform forced photometry. Command-line usage of DetectCoaddSourcesTask expects a data
209  reference to the coadd to be processed. A list of the available optional arguments can be obtained by
210  calling detectCoaddSources.py with the `--help` command line argument:
211  @code
212  detectCoaddSources.py --help
213  @endcode
214 
215  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
216  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has followed
217  steps 1 - 4 at @ref pipeTasks_multiBand, one may detect all the sources in each coadd as follows:
218  @code
219  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
220  @endcode
221  that will process the HSC-I band data. The results are written to
222  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I`.
223 
224  It is also necessary to run:
225  @code
226  detectCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
227  @endcode
228  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
229  processing procedure: @ref MergeDetectionsTask_ "MergeDetectionsTask".
230  """
231  _DefaultName = "detectCoaddSources"
232  ConfigClass = DetectCoaddSourcesConfig
233  getSchemaCatalogs = _makeGetSchemaCatalogs("det")
234  makeIdFactory = _makeMakeIdFactory("CoaddId")
235 
236  @classmethod
237  def _makeArgumentParser(cls):
238  parser = ArgumentParser(name=cls._DefaultName)
239  parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
240  ContainerClass=ExistingCoaddDataIdContainer)
241  return parser
242 
243  @classmethod
244  def getOutputDatasetTypes(cls, config):
245  coaddName = config.coaddName
246  for name in ("outputBackgrounds", "outputSources", "outputExposure"):
247  attr = getattr(config, name)
248  setattr(attr, "name", attr.name.format(coaddName))
249  outputTypeDict = super().getOutputDatasetTypes(config)
250  return outputTypeDict
251 
252  @classmethod
253  def getInitOutputDatasetTypes(cls, config):
254  coaddName = config.coaddName
255  attr = config.detectionSchema
256  setattr(attr, "name", attr.name.format(coaddName))
257  output = super().getInitOutputDatasetTypes(config)
258  return output
259 
260  def __init__(self, schema=None, **kwargs):
261  """!
262  @brief Initialize the task. Create the @ref SourceDetectionTask_ "detection" subtask.
263 
264  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
265 
266  @param[in] schema: initial schema for the output catalog, modified-in place to include all
267  fields set by this task. If None, the source minimal schema will be used.
268  @param[in] **kwargs: keyword arguments to be passed to lsst.pipe.base.task.Task.__init__
269  """
270  # N.B. Super is used here to handle the multiple inheritance of PipelineTasks, the init tree
271  # call structure has been reviewed carefully to be sure super will work as intended.
272  super().__init__(**kwargs)
273  if schema is None:
274  schema = afwTable.SourceTable.makeMinimalSchema()
275  if self.config.doInsertFakes:
276  self.makeSubtask("insertFakes")
277  self.schema = schema
278  self.makeSubtask("detection", schema=self.schema)
279  if self.config.doScaleVariance:
280  self.makeSubtask("scaleVariance")
281 
283  return {"detectionSchema": afwTable.SourceCatalog(self.schema)}
284 
285  def runDataRef(self, patchRef):
286  """!
287  @brief Run detection on a coadd.
288 
289  Invokes @ref run and then uses @ref write to output the
290  results.
291 
292  @param[in] patchRef: data reference for patch
293  """
294  exposure = patchRef.get(self.config.coaddName + "Coadd", immediate=True)
295  expId = int(patchRef.get(self.config.coaddName + "CoaddId"))
296  results = self.run(exposure, self.makeIdFactory(patchRef), expId=expId)
297  self.write(results, patchRef)
298  return results
299 
300  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
301  # FINDME: DM-15843 needs to come back and address these next two lines with a final solution
302  inputData["idFactory"] = afwTable.IdFactory.makeSimple()
303  inputData["expId"] = 0
304  return self.run(**inputData)
305 
306  def run(self, exposure, idFactory, expId):
307  """!
308  @brief Run detection on an exposure.
309 
310  First scale the variance plane to match the observed variance
311  using @ref ScaleVarianceTask. Then invoke the @ref SourceDetectionTask_ "detection" subtask to
312  detect sources.
313 
314  @param[in,out] exposure: Exposure on which to detect (may be backround-subtracted and scaled,
315  depending on configuration).
316  @param[in] idFactory: IdFactory to set source identifiers
317  @param[in] expId: Exposure identifier (integer) for RNG seed
318 
319  @return a pipe.base.Struct with fields
320  - sources: catalog of detections
321  - backgrounds: list of backgrounds
322  """
323  if self.config.doScaleVariance:
324  varScale = self.scaleVariance.run(exposure.maskedImage)
325  exposure.getMetadata().add("variance_scale", varScale)
326  backgrounds = afwMath.BackgroundList()
327  if self.config.doInsertFakes:
328  self.insertFakes.run(exposure, background=backgrounds)
329  table = afwTable.SourceTable.make(self.schema, idFactory)
330  detections = self.detection.makeSourceCatalog(table, exposure, expId=expId)
331  sources = detections.sources
332  fpSets = detections.fpSets
333  if hasattr(fpSets, "background") and fpSets.background:
334  for bg in fpSets.background:
335  backgrounds.append(bg)
336  return Struct(outputSources=sources, outputBackgrounds=backgrounds, outputExposure=exposure)
337 
338  def write(self, results, patchRef):
339  """!
340  @brief Write out results from runDetection.
341 
342  @param[in] exposure: Exposure to write out
343  @param[in] results: Struct returned from runDetection
344  @param[in] patchRef: data reference for patch
345  """
346  coaddName = self.config.coaddName + "Coadd"
347  patchRef.put(results.outputBackgrounds, coaddName + "_calexp_background")
348  patchRef.put(results.outputSources, coaddName + "_det")
349  patchRef.put(results.outputExposure, coaddName + "_calexp")
350 
351 
352 
353 
355  """DeblendCoaddSourcesConfig
356 
357  Configuration parameters for the `DeblendCoaddSourcesTask`.
358  """
359  singleBandDeblend = ConfigurableField(target=SourceDeblendTask,
360  doc="Deblend sources separately in each band")
361  multiBandDeblend = ConfigurableField(target=MultibandDeblendTask,
362  doc="Deblend sources simultaneously across bands")
363  simultaneous = Field(dtype=bool, default=False, doc="Simultaneously deblend all bands?")
364  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
365 
366  def setDefaults(self):
367  Config.setDefaults(self)
368  self.singleBandDeblend.propagateAllPeaks = True
369 
370 
372  """Task runner for the `MergeSourcesTask`
373 
374  Required because the run method requires a list of
375  dataRefs rather than a single dataRef.
376  """
377  @staticmethod
378  def getTargetList(parsedCmd, **kwargs):
379  """Provide a list of patch references for each patch, tract, filter combo.
380 
381  Parameters
382  ----------
383  parsedCmd:
384  The parsed command
385  kwargs:
386  Keyword arguments passed to the task
387 
388  Returns
389  -------
390  targetList: list
391  List of tuples, where each tuple is a (dataRef, kwargs) pair.
392  """
393  refDict = MergeSourcesRunner.buildRefDict(parsedCmd)
394  kwargs["psfCache"] = parsedCmd.psfCache
395  return [(list(p.values()), kwargs) for t in refDict.values() for p in t.values()]
396 
397 
398 class DeblendCoaddSourcesTask(CmdLineTask):
399  """Deblend the sources in a merged catalog
400 
401  Deblend sources from master catalog in each coadd.
402  This can either be done separately in each band using the HSC-SDSS deblender
403  (`DeblendCoaddSourcesTask.config.simultaneous==False`)
404  or use SCARLET to simultaneously fit the blend in all bands
405  (`DeblendCoaddSourcesTask.config.simultaneous==True`).
406  The task will set its own `self.schema` atribute to the `Schema` of the
407  output deblended catalog.
408  This will include all fields from the input `Schema`, as well as additional fields
409  from the deblender.
410 
411  `pipe.tasks.multiband.DeblendCoaddSourcesTask Description
412  ---------------------------------------------------------
413  `
414 
415  Parameters
416  ----------
417  butler: `Butler`
418  Butler used to read the input schemas from disk or
419  construct the reference catalog loader, if `schema` or `peakSchema` or
420  schema: `Schema`
421  The schema of the merged detection catalog as an input to this task.
422  peakSchema: `Schema`
423  The schema of the `PeakRecord`s in the `Footprint`s in the merged detection catalog
424  """
425  ConfigClass = DeblendCoaddSourcesConfig
426  RunnerClass = DeblendCoaddSourcesRunner
427  _DefaultName = "deblendCoaddSources"
428  makeIdFactory = _makeMakeIdFactory("MergedCoaddId")
429 
430  @classmethod
431  def _makeArgumentParser(cls):
432  parser = ArgumentParser(name=cls._DefaultName)
433  parser.add_id_argument("--id", "deepCoadd_calexp",
434  help="data ID, e.g. --id tract=12345 patch=1,2 filter=g^r^i",
435  ContainerClass=ExistingCoaddDataIdContainer)
436  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
437  return parser
438 
439  def __init__(self, butler=None, schema=None, peakSchema=None, **kwargs):
440  CmdLineTask.__init__(self, **kwargs)
441  if schema is None:
442  assert butler is not None, "Neither butler nor schema is defined"
443  schema = butler.get(self.config.coaddName + "Coadd_mergeDet_schema", immediate=True).schema
444  self.schemaMapper = afwTable.SchemaMapper(schema)
445  self.schemaMapper.addMinimalSchema(schema)
446  self.schema = self.schemaMapper.getOutputSchema()
447  if peakSchema is None:
448  assert butler is not None, "Neither butler nor peakSchema is defined"
449  peakSchema = butler.get(self.config.coaddName + "Coadd_peak_schema", immediate=True).schema
450 
451  if self.config.simultaneous:
452  self.makeSubtask("multiBandDeblend", schema=self.schema, peakSchema=peakSchema)
453  else:
454  self.makeSubtask("singleBandDeblend", schema=self.schema, peakSchema=peakSchema)
455 
456  def getSchemaCatalogs(self):
457  """Return a dict of empty catalogs for each catalog dataset produced by this task.
458 
459  Returns
460  -------
461  result: dict
462  Dictionary of empty catalogs, with catalog names as keys.
463  """
464  catalog = afwTable.SourceCatalog(self.schema)
465  return {self.config.coaddName + "Coadd_deblendedFlux": catalog,
466  self.config.coaddName + "Coadd_deblendedModel": catalog}
467 
468  def runDataRef(self, patchRefList, psfCache=100):
469  """Deblend the patch
470 
471  Deblend each source simultaneously or separately
472  (depending on `DeblendCoaddSourcesTask.config.simultaneous`).
473  Set `is-primary` and related flags.
474  Propagate flags from individual visits.
475  Write the deblended sources out.
476 
477  Parameters
478  ----------
479  patchRefList: list
480  List of data references for each filter
481  """
482  if self.config.simultaneous:
483  # Use SCARLET to simultaneously deblend across filters
484  filters = []
485  exposures = []
486  for patchRef in patchRefList:
487  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
488  filters.append(patchRef.dataId["filter"])
489  exposures.append(exposure)
490  # The input sources are the same for all bands, since it is a merged catalog
491  sources = self.readSources(patchRef)
492  exposure = afwImage.MultibandExposure.fromExposures(filters, exposures)
493  fluxCatalogs, templateCatalogs = self.multiBandDeblend.run(exposure, sources)
494  for n in range(len(patchRefList)):
495  self.write(patchRefList[n], fluxCatalogs[filters[n]], templateCatalogs[filters[n]])
496  else:
497  # Use the singeband deblender to deblend each band separately
498  for patchRef in patchRefList:
499  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
500  exposure.getPsf().setCacheCapacity(psfCache)
501  sources = self.readSources(patchRef)
502  self.singleBandDeblend.run(exposure, sources)
503  self.write(patchRef, sources)
504 
505  def readSources(self, dataRef):
506  """Read merged catalog
507 
508  Read the catalog of merged detections and create a catalog
509  in a single band.
510 
511  Parameters
512  ----------
513  dataRef: data reference
514  Data reference for catalog of merged detections
515 
516  Returns
517  -------
518  sources: `SourceCatalog`
519  List of sources in merged catalog
520 
521  We also need to add columns to hold the measurements we're about to make
522  so we can measure in-place.
523  """
524  merged = dataRef.get(self.config.coaddName + "Coadd_mergeDet", immediate=True)
525  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
526  idFactory = self.makeIdFactory(dataRef)
527  for s in merged:
528  idFactory.notify(s.getId())
529  table = afwTable.SourceTable.make(self.schema, idFactory)
530  sources = afwTable.SourceCatalog(table)
531  sources.extend(merged, self.schemaMapper)
532  return sources
533 
534  def write(self, dataRef, flux_sources, template_sources=None):
535  """Write the source catalog(s)
536 
537  Parameters
538  ----------
539  dataRef: Data Reference
540  Reference to the output catalog.
541  flux_sources: `SourceCatalog`
542  Flux conserved sources to write to file.
543  If using the single band deblender, this is the catalog
544  generated.
545  template_sources: `SourceCatalog`
546  Source catalog using the multiband template models
547  as footprints.
548  """
549  # The multiband deblender does not have to conserve flux,
550  # so only write the flux conserved catalog if it exists
551  if flux_sources is not None:
552  assert not self.config.simultaneous or self.config.multiBandDeblend.conserveFlux
553  dataRef.put(flux_sources, self.config.coaddName + "Coadd_deblendedFlux")
554  # Only the multiband deblender has the option to output the
555  # template model catalog, which can optionally be used
556  # in MeasureMergedCoaddSources
557  if template_sources is not None:
558  assert self.config.multiBandDeblend.saveTemplates
559  dataRef.put(template_sources, self.config.coaddName + "Coadd_deblendedModel")
560  self.log.info("Wrote %d sources: %s" % (len(flux_sources), dataRef.dataId))
561 
562  def writeMetadata(self, dataRefList):
563  """Write the metadata produced from processing the data.
564  Parameters
565  ----------
566  dataRefList
567  List of Butler data references used to write the metadata.
568  The metadata is written to dataset type `CmdLineTask._getMetadataName`.
569  """
570  for dataRef in dataRefList:
571  try:
572  metadataName = self._getMetadataName()
573  if metadataName is not None:
574  dataRef.put(self.getFullMetadata(), metadataName)
575  except Exception as e:
576  self.log.warn("Could not persist metadata for dataId=%s: %s", dataRef.dataId, e)
577 
578  def getExposureId(self, dataRef):
579  """Get the ExposureId from a data reference
580  """
581  return int(dataRef.get(self.config.coaddName + "CoaddId"))
582 
583 
584 class MeasureMergedCoaddSourcesConfig(PipelineTaskConfig):
585  """!
586  @anchor MeasureMergedCoaddSourcesConfig_
587 
588  @brief Configuration parameters for the MeasureMergedCoaddSourcesTask
589  """
590  inputCatalog = Field(dtype=str, default="deblendedFlux",
591  doc=("Name of the input catalog to use."
592  "If the single band deblender was used this should be 'deblendedFlux."
593  "If the multi-band deblender was used this should be 'deblendedModel."
594  "If no deblending was performed this should be 'mergeDet'"))
595  measurement = ConfigurableField(target=SingleFrameMeasurementTask, doc="Source measurement")
596  setPrimaryFlags = ConfigurableField(target=SetPrimaryFlagsTask, doc="Set flags for primary tract/patch")
597  doPropagateFlags = Field(
598  dtype=bool, default=True,
599  doc="Whether to match sources to CCD catalogs to propagate flags (to e.g. identify PSF stars)"
600  )
601  propagateFlags = ConfigurableField(target=PropagateVisitFlagsTask, doc="Propagate visit flags to coadd")
602  doMatchSources = Field(dtype=bool, default=True, doc="Match sources to reference catalog?")
603  match = ConfigurableField(target=DirectMatchTask, doc="Matching to reference catalog")
604  doWriteMatchesDenormalized = Field(
605  dtype=bool,
606  default=False,
607  doc=("Write reference matches in denormalized format? "
608  "This format uses more disk space, but is more convenient to read."),
609  )
610  coaddName = Field(dtype=str, default="deep", doc="Name of coadd")
611  psfCache = Field(dtype=int, default=100, doc="Size of psfCache")
612  checkUnitsParseStrict = Field(
613  doc="Strictness of Astropy unit compatibility check, can be 'raise', 'warn' or 'silent'",
614  dtype=str,
615  default="raise",
616  )
617  doApCorr = Field(
618  dtype=bool,
619  default=True,
620  doc="Apply aperture corrections"
621  )
622  applyApCorr = ConfigurableField(
623  target=ApplyApCorrTask,
624  doc="Subtask to apply aperture corrections"
625  )
626  doRunCatalogCalculation = Field(
627  dtype=bool,
628  default=True,
629  doc='Run catalogCalculation task'
630  )
631  catalogCalculation = ConfigurableField(
632  target=CatalogCalculationTask,
633  doc="Subtask to run catalogCalculation plugins on catalog"
634  )
635  inputSchema = InitInputDatasetField(
636  doc="Input schema for measure merged task produced by a deblender or detection task",
637  nameTemplate="{inputCoaddName}Coadd_deblendedFlux_schema",
638  storageClass="SourceCatalog"
639  )
640  outputSchema = InitOutputDatasetField(
641  doc="Output schema after all new fields are added by task",
642  nameTemplate="{inputCoaddName}Coadd_meas_schema",
643  storageClass="SourceCatalog"
644  )
645  refCat = InputDatasetField(
646  doc="Reference catalog used to match measured sources against known sources",
647  name="ref_cat",
648  storageClass="SimpleCatalog",
649  dimensions=("SkyPix",),
650  manualLoad=True
651  )
652  exposure = InputDatasetField(
653  doc="Input coadd image",
654  nameTemplate="{inputCoaddName}Coadd_calexp",
655  scalar=True,
656  storageClass="ExposureF",
657  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap")
658  )
659  skyMap = InputDatasetField(
660  doc="SkyMap to use in processing",
661  nameTemplate="{inputCoaddName}Coadd_skyMap",
662  storageClass="SkyMap",
663  dimensions=("SkyMap",),
664  scalar=True
665  )
666  visitCatalogs = InputDatasetField(
667  doc="Source catalogs for visits which overlap input tract, patch, abstract_filter. Will be "
668  "further filtered in the task for the purpose of propagating flags from image calibration "
669  "and characterization to codd objects",
670  name="src",
671  dimensions=("Instrument", "Visit", "Detector"),
672  storageClass="SourceCatalog"
673  )
674  intakeCatalog = InputDatasetField(
675  doc=("Name of the input catalog to use."
676  "If the single band deblender was used this should be 'deblendedFlux."
677  "If the multi-band deblender was used this should be 'deblendedModel, "
678  "or deblendedFlux if the multiband deblender was configured to output "
679  "deblended flux catalogs. If no deblending was performed this should "
680  "be 'mergeDet'"),
681  nameTemplate="{inputCoaddName}Coadd_deblendedFlux",
682  storageClass="SourceCatalog",
683  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap"),
684  scalar=True
685  )
686  outputSources = OutputDatasetField(
687  doc="Source catalog containing all the measurement information generated in this task",
688  nameTemplate="{outputCoaddName}Coadd_meas",
689  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap"),
690  storageClass="SourceCatalog",
691  scalar=True
692  )
693  matchResult = OutputDatasetField(
694  doc="Match catalog produced by configured matcher, optional on doMatchSources",
695  nameTemplate="{outputCoaddName}Coadd_measMatch",
696  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap"),
697  storageClass="Catalog",
698  scalar=True
699  )
700  denormMatches = OutputDatasetField(
701  doc="Denormalized Match catalog produced by configured matcher, optional on "
702  "doWriteMatchesDenormalized",
703  nameTemplate="{outputCoaddName}Coadd_measMatchFull",
704  dimensions=("Tract", "Patch", "AbstractFilter", "SkyMap"),
705  storageClass="Catalog",
706  scalar=True
707  )
708 
709  @property
710  def refObjLoader(self):
711  return self.match.refObjLoader
712 
713  def setDefaults(self):
714  super().setDefaults()
715  self.formatTemplateNames({"inputCoaddName": "deep", "outputCoaddName": "deep"})
716  self.quantum.dimensions = ("Tract", "Patch", "AbstractFilter", "SkyMap")
717  self.measurement.plugins.names |= ['base_InputCount', 'base_Variance']
718  self.measurement.plugins['base_PixelFlags'].masksFpAnywhere = ['CLIPPED', 'SENSOR_EDGE',
719  'INEXACT_PSF']
720  self.measurement.plugins['base_PixelFlags'].masksFpCenter = ['CLIPPED', 'SENSOR_EDGE',
721  'INEXACT_PSF']
722 
723 
729 
730 
731 class MeasureMergedCoaddSourcesRunner(ButlerInitializedTaskRunner):
732  """Get the psfCache setting into MeasureMergedCoaddSourcesTask"""
733  @staticmethod
734  def getTargetList(parsedCmd, **kwargs):
735  return ButlerInitializedTaskRunner.getTargetList(parsedCmd, psfCache=parsedCmd.psfCache)
736 
737 
738 class MeasureMergedCoaddSourcesTask(PipelineTask, CmdLineTask):
739  r"""!
740  @anchor MeasureMergedCoaddSourcesTask_
741 
742  @brief Deblend sources from master catalog in each coadd seperately and measure.
743 
744  @section pipe_tasks_multiBand_Contents Contents
745 
746  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose
747  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize
748  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run
749  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config
750  - @ref pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug
751  - @ref pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example
752 
753  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Purpose Description
754 
755  Command-line task that uses peaks and footprints from a master catalog to perform deblending and
756  measurement in each coadd.
757 
758  Given a master input catalog of sources (peaks and footprints) or deblender outputs
759  (including a HeavyFootprint in each band), measure each source on the
760  coadd. Repeating this procedure with the same master catalog across multiple coadds will generate a
761  consistent set of child sources.
762 
763  The deblender retains all peaks and deblends any missing peaks (dropouts in that band) as PSFs. Source
764  properties are measured and the @c is-primary flag (indicating sources with no children) is set. Visit
765  flags are propagated to the coadd sources.
766 
767  Optionally, we can match the coadd sources to an external reference catalog.
768 
769  @par Inputs:
770  deepCoadd_mergeDet{tract,patch} or deepCoadd_deblend{tract,patch}: SourceCatalog
771  @n deepCoadd_calexp{tract,patch,filter}: ExposureF
772  @par Outputs:
773  deepCoadd_meas{tract,patch,filter}: SourceCatalog
774  @par Data Unit:
775  tract, patch, filter
776 
777  MeasureMergedCoaddSourcesTask delegates most of its work to a set of sub-tasks:
778 
779  <DL>
780  <DT> @ref SingleFrameMeasurementTask_ "measurement"
781  <DD> Measure source properties of deblended sources.</DD>
782  <DT> @ref SetPrimaryFlagsTask_ "setPrimaryFlags"
783  <DD> Set flag 'is-primary' as well as related flags on sources. 'is-primary' is set for sources that are
784  not at the edge of the field and that have either not been deblended or are the children of deblended
785  sources</DD>
786  <DT> @ref PropagateVisitFlagsTask_ "propagateFlags"
787  <DD> Propagate flags set in individual visits to the coadd.</DD>
788  <DT> @ref DirectMatchTask_ "match"
789  <DD> Match input sources to a reference catalog (optional).
790  </DD>
791  </DL>
792  These subtasks may be retargeted as required.
793 
794  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Initialize Task initialization
795 
796  @copydoc \_\_init\_\_
797 
798  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Run Invoking the Task
799 
800  @copydoc run
801 
802  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Config Configuration parameters
803 
804  See @ref MeasureMergedCoaddSourcesConfig_
805 
806  @section pipe_tasks_multiBand_MeasureMergedCoaddSourcesTask_Debug Debug variables
807 
808  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a
809  flag @c -d to import @b debug.py from your @c PYTHONPATH; see @ref baseDebug for more about @b debug.py
810  files.
811 
812  MeasureMergedCoaddSourcesTask has no debug variables of its own because it delegates all the work to
813  the various sub-tasks. See the documetation for individual sub-tasks for more information.
814 
815  @section pipe_tasks_multiband_MeasureMergedCoaddSourcesTask_Example A complete example of using
816  MeasureMergedCoaddSourcesTask
817 
818  After MeasureMergedCoaddSourcesTask has been run on multiple coadds, we have a set of per-band catalogs.
819  The next stage in the multi-band processing procedure will merge these measurements into a suitable
820  catalog for driving forced photometry.
821 
822  Command-line usage of MeasureMergedCoaddSourcesTask expects a data reference to the coadds
823  to be processed.
824  A list of the available optional arguments can be obtained by calling measureCoaddSources.py with the
825  `--help` command line argument:
826  @code
827  measureCoaddSources.py --help
828  @endcode
829 
830  To demonstrate usage of the DetectCoaddSourcesTask in the larger context of multi-band processing, we
831  will process HSC data in the [ci_hsc](https://github.com/lsst/ci_hsc) package. Assuming one has finished
832  step 6 at @ref pipeTasks_multiBand, one may perform deblending and measure sources in the HSC-I band
833  coadd as follows:
834  @code
835  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-I
836  @endcode
837  This will process the HSC-I band data. The results are written in
838  `$CI_HSC_DIR/DATA/deepCoadd-results/HSC-I/0/5,4/meas-HSC-I-0-5,4.fits
839 
840  It is also necessary to run
841  @code
842  measureCoaddSources.py $CI_HSC_DIR/DATA --id patch=5,4 tract=0 filter=HSC-R
843  @endcode
844  to generate the sources catalogs for the HSC-R band required by the next step in the multi-band
845  procedure: @ref MergeMeasurementsTask_ "MergeMeasurementsTask".
846  """
847  _DefaultName = "measureCoaddSources"
848  ConfigClass = MeasureMergedCoaddSourcesConfig
849  RunnerClass = MeasureMergedCoaddSourcesRunner
850  getSchemaCatalogs = _makeGetSchemaCatalogs("meas")
851  makeIdFactory = _makeMakeIdFactory("MergedCoaddId") # The IDs we already have are of this type
852 
853  @classmethod
854  def _makeArgumentParser(cls):
855  parser = ArgumentParser(name=cls._DefaultName)
856  parser.add_id_argument("--id", "deepCoadd_calexp",
857  help="data ID, e.g. --id tract=12345 patch=1,2 filter=r",
858  ContainerClass=ExistingCoaddDataIdContainer)
859  parser.add_argument("--psfCache", type=int, default=100, help="Size of CoaddPsf cache")
860  return parser
861 
862  def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None,
863  **kwargs):
864  """!
865  @brief Initialize the task.
866 
867  Keyword arguments (in addition to those forwarded to CmdLineTask.__init__):
868  @param[in] schema: the schema of the merged detection catalog used as input to this one
869  @param[in] peakSchema: the schema of the PeakRecords in the Footprints in the merged detection catalog
870  @param[in] refObjLoader: an instance of LoadReferenceObjectsTasks that supplies an external reference
871  catalog. May be None if the loader can be constructed from the butler argument or all steps
872  requiring a reference catalog are disabled.
873  @param[in] butler: a butler used to read the input schemas from disk or construct the reference
874  catalog loader, if schema or peakSchema or refObjLoader is None
875 
876  The task will set its own self.schema attribute to the schema of the output measurement catalog.
877  This will include all fields from the input schema, as well as additional fields for all the
878  measurements.
879  """
880  super().__init__(**kwargs)
881  self.deblended = self.config.inputCatalog.startswith("deblended")
882  self.inputCatalog = "Coadd_" + self.config.inputCatalog
883  if initInputs is not None:
884  schema = initInputs['inputSchema'].schema
885  if schema is None:
886  assert butler is not None, "Neither butler nor schema is defined"
887  schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema", immediate=True).schema
888  self.schemaMapper = afwTable.SchemaMapper(schema)
889  self.schemaMapper.addMinimalSchema(schema)
890  self.schema = self.schemaMapper.getOutputSchema()
892  self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
893  self.makeSubtask("setPrimaryFlags", schema=self.schema)
894  if self.config.doMatchSources:
895  self.makeSubtask("match", butler=butler, refObjLoader=refObjLoader)
896  if self.config.doPropagateFlags:
897  self.makeSubtask("propagateFlags", schema=self.schema)
898  self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
899  if self.config.doApCorr:
900  self.makeSubtask("applyApCorr", schema=self.schema)
901  if self.config.doRunCatalogCalculation:
902  self.makeSubtask("catalogCalculation", schema=self.schema)
903 
904  @classmethod
905  def getInputDatasetTypes(cls, config):
906  inputDatasetTypes = super().getInputDatasetTypes(config)
907  if not config.doPropagateFlags:
908  inputDatasetTypes.pop("visitCatalogs")
909  return inputDatasetTypes
910 
911  @classmethod
912  def getOutputDatasetTypes(cls, config):
913  outputDatasetTypes = super().getOutputDatasetTypes(config)
914  if config.doMatchSources is False:
915  outputDatasetTypes.pop("matchResult")
916  if config.doWriteMatchesDenormalized is False:
917  outputDatasetTypes.pop("denormMatches")
918  return outputDatasetTypes
919 
921  return {"outputSchema": afwTable.SourceCatalog(self.schema)}
922 
923  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
924  refObjLoader = ReferenceObjectLoader(inputDataIds['refCat'], butler,
925  config=self.config.refObjLoader, log=self.log)
926  self.match.setRefObjLoader(refObjLoader)
927 
928  # Set psfcache
929  # move this to run after gen2 deprecation
930  inputData['exposure'].getPsf().setCacheCapacity(self.config.psfCache)
931 
932  # Transform inputCatalog
933  idFactory = afwTable.IdFactory.makeSimple()
934  table = afwTable.SourceTable.make(self.schema, idFactory)
935  sources = afwTable.SourceCatalog(table)
936  sources.extend(inputData.pop('intakeCatalog'), self.schemaMapper)
937  table = sources.getTable()
938  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
939  inputData['sources'] = sources
940 
941  inputData['exposureId'] = 0
942 
943  skyMap = inputData.pop('skyMap')
944  tractNumber = inputDataIds['intakeCatalog']['tract']
945  tractInfo = skyMap[tractNumber]
946  patchInfo = tractInfo.getPatchInfo(inputDataIds['intakeCatalog']['patch'])
947  skyInfo = Struct(
948  skyMap=skyMap,
949  tractInfo=tractInfo,
950  patchInfo=patchInfo,
951  wcs=tractInfo.getWcs(),
952  bbox=patchInfo.getOuterBBox()
953  )
954  inputData['skyInfo'] = skyInfo
955 
956  if self.config.doPropagateFlags:
957  # Filter out any visit catalog that is not coadd inputs
958  ccdInputs = inputData['exposure'].getInfo().getCoaddInputs().ccds
959  visitKey = ccdInputs.schema.find("visit").key
960  ccdKey = ccdInputs.schema.find("ccd").key
961  inputVisitIds = set()
962  ccdRecordsWcs = {}
963  for ccdRecord in ccdInputs:
964  visit = ccdRecord.get(visitKey)
965  ccd = ccdRecord.get(ccdKey)
966  inputVisitIds.add((visit, ccd))
967  ccdRecordsWcs[(visit, ccd)] = ccdRecord.getWcs()
968 
969  inputCatalogsToKeep = []
970  inputCatalogWcsUpdate = []
971  for i, dataId in enumerate(inputDataIds['visitCatalogs']):
972  key = (dataId['visit'], dataId['detector'])
973  if key in inputVisitIds:
974  inputCatalogsToKeep.append(inputData['visitCatalogs'][i])
975  inputCatalogWcsUpdate.append(ccdRecordsWcs[key])
976  inputData['visitCatalogs'] = inputCatalogsToKeep
977  inputData['wcsUpdates'] = inputCatalogWcsUpdate
978  inputData['ccdInputs'] = ccdInputs
979 
980  return self.run(**inputData)
981 
982  def runDataRef(self, patchRef, psfCache=100):
983  """!
984  @brief Deblend and measure.
985 
986  @param[in] patchRef: Patch reference.
987 
988  Set 'is-primary' and related flags. Propagate flags
989  from individual visits. Optionally match the sources to a reference catalog and write the matches.
990  Finally, write the deblended sources and measurements out.
991  """
992  exposure = patchRef.get(self.config.coaddName + "Coadd_calexp", immediate=True)
993  exposure.getPsf().setCacheCapacity(psfCache)
994  sources = self.readSources(patchRef)
995  table = sources.getTable()
996  table.setMetadata(self.algMetadata) # Capture algorithm metadata to write out to the source catalog.
997  skyInfo = getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)
998 
999  results = self.run(exposure=exposure, sources=sources,
1000  ccdInputs=self.propagateFlags.getCcdInputs(exposure),
1001  skyInfo=skyInfo, butler=patchRef.getButler(),
1002  exposureId=self.getExposureId(patchRef))
1003 
1004  if self.config.doMatchSources:
1005  self.writeMatches(patchRef, results)
1006  self.write(patchRef, results.outputSources)
1007 
1008  def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None,
1009  butler=None):
1010  """Run measurement algorithms on the input exposure, and optionally populate the
1011  resulting catalog with extra information.
1012 
1013  Parameters
1014  ----------
1015  exposure : `lsst.afw.exposure.Exposure`
1016  The input exposure on which measurements are to be performed
1017  sources : `lsst.afw.table.SourceCatalog`
1018  A catalog built from the results of merged detections, or
1019  deblender outputs.
1020  skyInfo : `lsst.pipe.base.Struct`
1021  A struct containing information about the position of the input exposure within
1022  a `SkyMap`, the `SkyMap`, its `Wcs`, and its bounding box
1023  exposureId : `int` or `bytes`
1024  packed unique number or bytes unique to the input exposure
1025  ccdInputs : `lsst.afw.table.ExposureCatalog`
1026  Catalog containing information on the individual visits which went into making
1027  the exposure
1028  visitCatalogs : list of `lsst.afw.table.SourceCatalogs` or `None`
1029  A list of source catalogs corresponding to measurements made on the individual
1030  visits which went into the input exposure. If None and butler is `None` then
1031  the task cannot propagate visit flags to the output catalog.
1032  wcsUpdates : list of `lsst.afw.geom.SkyWcs` or `None`
1033  If visitCatalogs is not `None` this should be a list of wcs objects which correspond
1034  to the input visits. Used to put all coordinates to common system. If `None` and
1035  butler is `None` then the task cannot propagate visit flags to the output catalog.
1036  butler : `lsst.daf.butler.Butler` or `lsst.daf.persistence.Butler`
1037  Either a gen2 or gen3 butler used to load visit catalogs
1038 
1039  Returns
1040  -------
1041  results : `lsst.pipe.base.Struct`
1042  Results of running measurement task. Will contain the catalog in the
1043  sources attribute. Optionally will have results of matching to a
1044  reference catalog in the matchResults attribute, and denormalized
1045  matches in the denormMatches attribute.
1046  """
1047  self.measurement.run(sources, exposure, exposureId=exposureId)
1048 
1049  if self.config.doApCorr:
1050  self.applyApCorr.run(
1051  catalog=sources,
1052  apCorrMap=exposure.getInfo().getApCorrMap()
1053  )
1054 
1055  # TODO DM-11568: this contiguous check-and-copy could go away if we
1056  # reserve enough space during SourceDetection and/or SourceDeblend.
1057  # NOTE: sourceSelectors require contiguous catalogs, so ensure
1058  # contiguity now, so views are preserved from here on.
1059  if not sources.isContiguous():
1060  sources = sources.copy(deep=True)
1061 
1062  if self.config.doRunCatalogCalculation:
1063  self.catalogCalculation.run(sources)
1064 
1065  self.setPrimaryFlags.run(sources, skyInfo.skyMap, skyInfo.tractInfo, skyInfo.patchInfo,
1066  includeDeblend=self.deblended)
1067  if self.config.doPropagateFlags:
1068  self.propagateFlags.run(butler, sources, ccdInputs, exposure.getWcs(), visitCatalogs, wcsUpdates)
1069 
1070  results = Struct()
1071 
1072  if self.config.doMatchSources:
1073  matchResult = self.match.run(sources, exposure.getInfo().getFilter().getName())
1074  matches = afwTable.packMatches(matchResult.matches)
1075  matches.table.setMetadata(matchResult.matchMeta)
1076  results.matchResult = matches
1077  if self.config.doWriteMatchesDenormalized:
1078  results.denormMatches = denormalizeMatches(matchResult.matches,
1079  matchResult.matchMeta)
1080 
1081  results.outputSources = sources
1082  return results
1083 
1084  def readSources(self, dataRef):
1085  """!
1086  @brief Read input sources.
1087 
1088  @param[in] dataRef: Data reference for catalog of merged detections
1089  @return List of sources in merged catalog
1090 
1091  We also need to add columns to hold the measurements we're about to make
1092  so we can measure in-place.
1093  """
1094  merged = dataRef.get(self.config.coaddName + self.inputCatalog, immediate=True)
1095  self.log.info("Read %d detections: %s" % (len(merged), dataRef.dataId))
1096  idFactory = self.makeIdFactory(dataRef)
1097  for s in merged:
1098  idFactory.notify(s.getId())
1099  table = afwTable.SourceTable.make(self.schema, idFactory)
1100  sources = afwTable.SourceCatalog(table)
1101  sources.extend(merged, self.schemaMapper)
1102  return sources
1103 
1104  def writeMatches(self, dataRef, results):
1105  """!
1106  @brief Write matches of the sources to the astrometric reference catalog.
1107 
1108  @param[in] dataRef: data reference
1109  @param[in] results: results struct from run method
1110  """
1111  if hasattr(results, "matchResult"):
1112  dataRef.put(results.matchResult, self.config.coaddName + "Coadd_measMatch")
1113  if hasattr(results, "denormMatches"):
1114  dataRef.put(results.denormMatches, self.config.coaddName + "Coadd_measMatchFull")
1115 
1116  def write(self, dataRef, sources):
1117  """!
1118  @brief Write the source catalog.
1119 
1120  @param[in] dataRef: data reference
1121  @param[in] sources: source catalog
1122  """
1123  dataRef.put(sources, self.config.coaddName + "Coadd_meas")
1124  self.log.info("Wrote %d sources: %s" % (len(sources), dataRef.dataId))
1125 
1126  def getExposureId(self, dataRef):
1127  return int(dataRef.get(self.config.coaddName + "CoaddId"))
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: multiBand.py:923
def runDataRef(self, patchRef)
Run detection on a coadd.
Definition: multiBand.py:285
Configuration parameters for the DetectCoaddSourcesTask.
Definition: multiBand.py:69
def runDataRef(self, patchRefList, psfCache=100)
Definition: multiBand.py:468
def __init__(self, schema=None, kwargs)
Initialize the task.
Definition: multiBand.py:260
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
Definition: multiBand.py:300
Deblend sources from master catalog in each coadd seperately and measure.
Definition: multiBand.py:738
def __init__(self, butler=None, schema=None, peakSchema=None, kwargs)
Definition: multiBand.py:439
def writeMatches(self, dataRef, results)
Write matches of the sources to the astrometric reference catalog.
Definition: multiBand.py:1104
def run(self, exposure, sources, skyInfo, exposureId, ccdInputs=None, visitCatalogs=None, wcsUpdates=None, butler=None)
Definition: multiBand.py:1009
def readSources(self, dataRef)
Read input sources.
Definition: multiBand.py:1084
def write(self, results, patchRef)
Write out results from runDetection.
Definition: multiBand.py:338
def write(self, dataRef, flux_sources, template_sources=None)
Definition: multiBand.py:534
def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None, initInputs=None, kwargs)
Initialize the task.
Definition: multiBand.py:863
def write(self, dataRef, sources)
Write the source catalog.
Definition: multiBand.py:1116
Configuration parameters for the MeasureMergedCoaddSourcesTask.
Definition: multiBand.py:584
def getSkyInfo(coaddName, patchRef)
Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded...
Definition: coaddBase.py:231
def run(self, exposure, idFactory, expId)
Run detection on an exposure.
Definition: multiBand.py:306
def runDataRef(self, patchRef, psfCache=100)
Deblend and measure.
Definition: multiBand.py:982