lsst.meas.base  22.0.1-16-g7290ee3+1e924434c1
forcedPhotCcd.py
Go to the documentation of this file.
1 # This file is part of meas_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import logging
24 import pandas as pd
25 import numpy as np
26 
27 import lsst.pex.config
29 import lsst.pipe.base
30 import lsst.geom
31 import lsst.afw.geom
32 import lsst.afw.image
33 import lsst.afw.table
34 import lsst.sphgeom
35 
36 from lsst.pipe.base import PipelineTaskConnections
37 import lsst.pipe.base.connectionTypes as cT
38 
39 import lsst.pipe.base as pipeBase
40 from lsst.skymap import BaseSkyMap
41 
42 from .references import MultiBandReferencesTask
43 from .forcedMeasurement import ForcedMeasurementTask
44 from .applyApCorr import ApplyApCorrTask
45 from .catalogCalculation import CatalogCalculationTask
46 
47 try:
48  from lsst.meas.mosaic import applyMosaicResults
49 except ImportError:
50  applyMosaicResults = None
51 
52 __all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract",
53  "ForcedPhotCcdFromDataFrameTask", "ForcedPhotCcdFromDataFrameConfig")
54 
55 
56 class PerTractCcdDataIdContainer(pipeBase.DataIdContainer):
57  """A data ID container which combines raw data IDs with a tract.
58 
59  Notes
60  -----
61  Required because we need to add "tract" to the raw data ID keys (defined as
62  whatever we use for ``src``) when no tract is provided (so that the user is
63  not required to know which tracts are spanned by the raw data ID).
64 
65  This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
66  being measured using the detection information, a set of reference
67  catalogs, from the set of coadds which intersect with the calexp. It needs
68  the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
69  decide what set of coadds to use. The references from the tract whose
70  patches intersect with the calexp are used.
71  """
72 
73  def makeDataRefList(self, namespace):
74  """Make self.refList from self.idList
75  """
76  if self.datasetType is None:
77  raise RuntimeError("Must call setDatasetType first")
78  log = logging.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
79  skymap = None
80  visitTract = collections.defaultdict(set) # Set of tracts for each visit
81  visitRefs = collections.defaultdict(list) # List of data references for each visit
82  for dataId in self.idList:
83  if "tract" not in dataId:
84  # Discover which tracts the data overlaps
85  log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
86  if skymap is None:
87  skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
88 
89  for ref in namespace.butler.subset("calexp", dataId=dataId):
90  if not ref.datasetExists("calexp"):
91  continue
92 
93  visit = ref.dataId["visit"]
94  visitRefs[visit].append(ref)
95 
96  md = ref.get("calexp_md", immediate=True)
97  wcs = lsst.afw.geom.makeSkyWcs(md)
99  # Going with just the nearest tract. Since we're throwing all tracts for the visit
100  # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
101  tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
102  if imageOverlapsTract(tract, wcs, box):
103  visitTract[visit].add(tract.getId())
104  else:
105  self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
106 
107  # Ensure all components of a visit are kept together by putting them all in the same set of tracts
108  for visit, tractSet in visitTract.items():
109  for ref in visitRefs[visit]:
110  for tract in tractSet:
111  self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
112  dataId=ref.dataId, tract=tract))
113  if visitTract:
114  tractCounter = collections.Counter()
115  for tractSet in visitTract.values():
116  tractCounter.update(tractSet)
117  log.info("Number of visits for each tract: %s", dict(tractCounter))
118 
119 
120 def imageOverlapsTract(tract, imageWcs, imageBox):
121  """Return whether the given bounding box overlaps the tract given a WCS.
122 
123  Parameters
124  ----------
125  tract : `lsst.skymap.TractInfo`
126  TractInfo specifying a tract.
127  imageWcs : `lsst.afw.geom.SkyWcs`
128  World coordinate system for the image.
129  imageBox : `lsst.geom.Box2I`
130  Bounding box for the image.
131 
132  Returns
133  -------
134  overlap : `bool`
135  `True` if the bounding box overlaps the tract; `False` otherwise.
136  """
137  tractPoly = tract.getOuterSkyPolygon()
138 
139  imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners()
140  try:
141  imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
142  except lsst.pex.exceptions.LsstCppException as e:
143  # Protecting ourselves from awful Wcs solutions in input images
144  if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException)
145  and not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
146  raise
147  return False
148 
149  imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners])
150  return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
151 
152 
153 class ForcedPhotCcdConnections(PipelineTaskConnections,
154  dimensions=("instrument", "visit", "detector", "skymap", "tract"),
155  defaultTemplates={"inputCoaddName": "deep",
156  "inputName": "calexp"}):
157  inputSchema = cT.InitInput(
158  doc="Schema for the input measurement catalogs.",
159  name="{inputCoaddName}Coadd_ref_schema",
160  storageClass="SourceCatalog",
161  )
162  outputSchema = cT.InitOutput(
163  doc="Schema for the output forced measurement catalogs.",
164  name="forced_src_schema",
165  storageClass="SourceCatalog",
166  )
167  exposure = cT.Input(
168  doc="Input exposure to perform photometry on.",
169  name="{inputName}",
170  storageClass="ExposureF",
171  dimensions=["instrument", "visit", "detector"],
172  )
173  refCat = cT.Input(
174  doc="Catalog of shapes and positions at which to force photometry.",
175  name="{inputCoaddName}Coadd_ref",
176  storageClass="SourceCatalog",
177  dimensions=["skymap", "tract", "patch"],
178  multiple=True,
179  deferLoad=True,
180  )
181  skyMap = cT.Input(
182  doc="SkyMap dataset that defines the coordinate system of the reference catalog.",
183  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
184  storageClass="SkyMap",
185  dimensions=["skymap"],
186  )
187  measCat = cT.Output(
188  doc="Output forced photometry catalog.",
189  name="forced_src",
190  storageClass="SourceCatalog",
191  dimensions=["instrument", "visit", "detector", "skymap", "tract"],
192  )
193 
194 
195 class ForcedPhotCcdConfig(pipeBase.PipelineTaskConfig,
196  pipelineConnections=ForcedPhotCcdConnections):
197  """Config class for forced measurement driver task."""
198  references = lsst.pex.config.ConfigurableField(
199  target=MultiBandReferencesTask,
200  doc="subtask to retrieve reference source catalog"
201  )
202  measurement = lsst.pex.config.ConfigurableField(
203  target=ForcedMeasurementTask,
204  doc="subtask to do forced measurement"
205  )
206  coaddName = lsst.pex.config.Field(
207  doc="coadd name: typically one of deep or goodSeeing",
208  dtype=str,
209  default="deep",
210  )
211  doApCorr = lsst.pex.config.Field(
212  dtype=bool,
213  default=True,
214  doc="Run subtask to apply aperture corrections"
215  )
216  applyApCorr = lsst.pex.config.ConfigurableField(
217  target=ApplyApCorrTask,
218  doc="Subtask to apply aperture corrections"
219  )
220  catalogCalculation = lsst.pex.config.ConfigurableField(
221  target=CatalogCalculationTask,
222  doc="Subtask to run catalogCalculation plugins on catalog"
223  )
224  doApplyUberCal = lsst.pex.config.Field(
225  dtype=bool,
226  doc="Apply meas_mosaic ubercal results to input calexps?",
227  default=False,
228  deprecated="Deprecated by DM-23352; use doApplyExternalPhotoCalib and doApplyExternalSkyWcs instead",
229  )
230  doApplyExternalPhotoCalib = lsst.pex.config.Field(
231  dtype=bool,
232  default=False,
233  doc=("Whether to apply external photometric calibration via an "
234  "`lsst.afw.image.PhotoCalib` object. Uses the "
235  "``externalPhotoCalibName`` field to determine which calibration "
236  "to load."),
237  )
238  doApplyExternalSkyWcs = lsst.pex.config.Field(
239  dtype=bool,
240  default=False,
241  doc=("Whether to apply external astrometric calibration via an "
242  "`lsst.afw.geom.SkyWcs` object. Uses ``externalSkyWcsName`` "
243  "field to determine which calibration to load."),
244  )
245  doApplySkyCorr = lsst.pex.config.Field(
246  dtype=bool,
247  default=False,
248  doc="Apply sky correction?",
249  )
250  includePhotoCalibVar = lsst.pex.config.Field(
251  dtype=bool,
252  default=False,
253  doc="Add photometric calibration variance to warp variance plane?",
254  )
255  externalPhotoCalibName = lsst.pex.config.ChoiceField(
256  dtype=str,
257  doc=("Type of external PhotoCalib if ``doApplyExternalPhotoCalib`` is True. "
258  "Unused for Gen3 middleware."),
259  default="jointcal",
260  allowed={
261  "jointcal": "Use jointcal_photoCalib",
262  "fgcm": "Use fgcm_photoCalib",
263  "fgcm_tract": "Use fgcm_tract_photoCalib"
264  },
265  )
266  externalSkyWcsName = lsst.pex.config.ChoiceField(
267  dtype=str,
268  doc="Type of external SkyWcs if ``doApplyExternalSkyWcs`` is True. Unused for Gen3 middleware.",
269  default="jointcal",
270  allowed={
271  "jointcal": "Use jointcal_wcs"
272  },
273  )
274 
275  def setDefaults(self):
276  # Docstring inherited.
277  # Make catalogCalculation a no-op by default as no modelFlux is setup by default in
278  # ForcedMeasurementTask
279  super().setDefaults()
280  self.measurement.plugins.names |= ['base_LocalPhotoCalib', 'base_LocalWcs']
281  self.catalogCalculation.plugins.names = []
282 
283 
284 class ForcedPhotCcdTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
285  """A command-line driver for performing forced measurement on CCD images.
286 
287  Parameters
288  ----------
289  butler : `lsst.daf.persistence.butler.Butler`, optional
290  A Butler which will be passed to the references subtask to allow it to
291  load its schema from disk. Optional, but must be specified if
292  ``refSchema`` is not; if both are specified, ``refSchema`` takes
293  precedence.
294  refSchema : `lsst.afw.table.Schema`, optional
295  The schema of the reference catalog, passed to the constructor of the
296  references subtask. Optional, but must be specified if ``butler`` is
297  not; if both are specified, ``refSchema`` takes precedence.
298  **kwds
299  Keyword arguments are passed to the supertask constructor.
300 
301  Notes
302  -----
303  The `runDataRef` method takes a `~lsst.daf.persistence.ButlerDataRef` argument
304  that corresponds to a single CCD. This should contain the data ID keys that
305  correspond to the ``forced_src`` dataset (the output dataset for this
306  task), which are typically all those used to specify the ``calexp`` dataset
307  (``visit``, ``raft``, ``sensor`` for LSST data) as well as a coadd tract.
308  The tract is used to look up the appropriate coadd measurement catalogs to
309  use as references (e.g. ``deepCoadd_src``; see
310  :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
311  information). While the tract must be given as part of the dataRef, the
312  patches are determined automatically from the bounding box and WCS of the
313  calexp to be measured, and the filter used to fetch references is set via
314  the ``filter`` option in the configuration of
315  :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
316  """
317 
318  ConfigClass = ForcedPhotCcdConfig
319  RunnerClass = pipeBase.ButlerInitializedTaskRunner
320  _DefaultName = "forcedPhotCcd"
321  dataPrefix = ""
322 
323  def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
324  super().__init__(**kwds)
325 
326  if initInputs is not None:
327  refSchema = initInputs['inputSchema'].schema
328 
329  self.makeSubtask("references", butler=butler, schema=refSchema)
330  if refSchema is None:
331  refSchema = self.references.schema
332  self.makeSubtask("measurement", refSchema=refSchema)
333  # It is necessary to get the schema internal to the forced measurement task until such a time
334  # that the schema is not owned by the measurement task, but is passed in by an external caller
335  if self.config.doApCorr:
336  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
337  self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
338  self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
339 
340  def runQuantum(self, butlerQC, inputRefs, outputRefs):
341  inputs = butlerQC.get(inputRefs)
342 
343  tract = butlerQC.quantum.dataId['tract']
344  skyMap = inputs.pop("skyMap")
345  inputs['refWcs'] = skyMap[tract].getWcs()
346 
347  inputs['refCat'] = self.mergeAndFilterReferences(inputs['exposure'], inputs['refCat'],
348  inputs['refWcs'])
349 
350  inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
351  inputs['exposure'],
352  inputs['refCat'], inputs['refWcs'],
353  "visit_detector")
354  self.attachFootprints(inputs['measCat'], inputs['refCat'], inputs['exposure'], inputs['refWcs'])
355  # TODO: apply external calibrations (DM-17062)
356  outputs = self.run(**inputs)
357  butlerQC.put(outputs, outputRefs)
358 
359  def mergeAndFilterReferences(self, exposure, refCats, refWcs):
360  """Filter reference catalog so that all sources are within the
361  boundaries of the exposure.
362 
363  Parameters
364  ----------
365  exposure : `lsst.afw.image.exposure.Exposure`
366  Exposure to generate the catalog for.
367  refCats : sequence of `lsst.daf.butler.DeferredDatasetHandle`
368  Handles for catalogs of shapes and positions at which to force
369  photometry.
370  refWcs : `lsst.afw.image.SkyWcs`
371  Reference world coordinate system.
372 
373  Returns
374  -------
375  refSources : `lsst.afw.table.SourceCatalog`
376  Filtered catalog of forced sources to measure.
377 
378  Notes
379  -----
380  Filtering the reference catalog is currently handled by Gen2
381  specific methods. To function for Gen3, this method copies
382  code segments to do the filtering and transformation. The
383  majority of this code is based on the methods of
384  lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
385 
386  """
387 
388  # Step 1: Determine bounds of the exposure photometry will
389  # be performed on.
390  expWcs = exposure.getWcs()
391  expRegion = exposure.getBBox(lsst.afw.image.PARENT)
392  expBBox = lsst.geom.Box2D(expRegion)
393  expBoxCorners = expBBox.getCorners()
394  expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
395  corner in expBoxCorners]
396  expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
397 
398  # Step 2: Filter out reference catalog sources that are
399  # not contained within the exposure boundaries, or whose
400  # parents are not within the exposure boundaries. Note
401  # that within a single input refCat, the parents always
402  # appear before the children.
403  mergedRefCat = None
404  for refCat in refCats:
405  refCat = refCat.get()
406  if mergedRefCat is None:
407  mergedRefCat = lsst.afw.table.SourceCatalog(refCat.table)
408  containedIds = {0} # zero as a parent ID means "this is a parent"
409  for record in refCat:
410  if expPolygon.contains(record.getCoord().getVector()) and record.getParent() in containedIds:
411  record.setFootprint(record.getFootprint().transform(refWcs, expWcs, expRegion))
412  mergedRefCat.append(record)
413  containedIds.add(record.getId())
414  if mergedRefCat is None:
415  raise RuntimeError("No reference objects for forced photometry.")
416  mergedRefCat.sort(lsst.afw.table.SourceTable.getParentKey())
417  return mergedRefCat
418 
419  def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName):
420  """Generate a measurement catalog for Gen3.
421 
422  Parameters
423  ----------
424  exposureDataId : `DataId`
425  Butler dataId for this exposure.
426  exposure : `lsst.afw.image.exposure.Exposure`
427  Exposure to generate the catalog for.
428  refCat : `lsst.afw.table.SourceCatalog`
429  Catalog of shapes and positions at which to force photometry.
430  refWcs : `lsst.afw.image.SkyWcs`
431  Reference world coordinate system.
432  idPackerName : `str`
433  Type of ID packer to construct from the registry.
434 
435  Returns
436  -------
437  measCat : `lsst.afw.table.SourceCatalog`
438  Catalog of forced sources to measure.
439  expId : `int`
440  Unique binary id associated with the input exposure
441  """
442  expId, expBits = exposureDataId.pack(idPackerName, returnMaxBits=True)
443  idFactory = lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
444 
445  measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
446  idFactory=idFactory)
447  return measCat, expId
448 
449  def runDataRef(self, dataRef, psfCache=None):
450  """Perform forced measurement on a single exposure.
451 
452  Parameters
453  ----------
454  dataRef : `lsst.daf.persistence.ButlerDataRef`
455  Passed to the ``references`` subtask to obtain the reference WCS,
456  the ``getExposure`` method (implemented by derived classes) to
457  read the measurment image, and the ``fetchReferences`` method to
458  get the exposure and load the reference catalog (see
459  :lsst-task`lsst.meas.base.references.CoaddSrcReferencesTask`).
460  Refer to derived class documentation for details of the datasets
461  and data ID keys which are used.
462  psfCache : `int`, optional
463  Size of PSF cache, or `None`. The size of the PSF cache can have
464  a significant effect upon the runtime for complicated PSF models.
465 
466  Notes
467  -----
468  Sources are generated with ``generateMeasCat`` in the ``measurement``
469  subtask. These are passed to ``measurement``'s ``run`` method, which
470  fills the source catalog with the forced measurement results. The
471  sources are then passed to the ``writeOutputs`` method (implemented by
472  derived classes) which writes the outputs.
473  """
474  refWcs = self.references.getWcs(dataRef)
475  exposure = self.getExposure(dataRef)
476  if psfCache is not None:
477  exposure.getPsf().setCacheSize(psfCache)
478  refCat = self.fetchReferences(dataRef, exposure)
479 
480  measCat = self.measurement.generateMeasCat(exposure, refCat, refWcs,
481  idFactory=self.makeIdFactory(dataRef))
482  self.log.info("Performing forced measurement on %s", dataRef.dataId)
483  self.attachFootprints(measCat, refCat, exposure, refWcs)
484 
485  exposureId = self.getExposureId(dataRef)
486 
487  forcedPhotResult = self.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
488 
489  self.writeOutput(dataRef, forcedPhotResult.measCat)
490 
491  def run(self, measCat, exposure, refCat, refWcs, exposureId=None):
492  """Perform forced measurement on a single exposure.
493 
494  Parameters
495  ----------
496  measCat : `lsst.afw.table.SourceCatalog`
497  The measurement catalog, based on the sources listed in the
498  reference catalog.
499  exposure : `lsst.afw.image.Exposure`
500  The measurement image upon which to perform forced detection.
501  refCat : `lsst.afw.table.SourceCatalog`
502  The reference catalog of sources to measure.
503  refWcs : `lsst.afw.image.SkyWcs`
504  The WCS for the references.
505  exposureId : `int`
506  Optional unique exposureId used for random seed in measurement
507  task.
508 
509  Returns
510  -------
511  result : `lsst.pipe.base.Struct`
512  Structure with fields:
513 
514  ``measCat``
515  Catalog of forced measurement results
516  (`lsst.afw.table.SourceCatalog`).
517  """
518  self.measurement.run(measCat, exposure, refCat, refWcs, exposureId=exposureId)
519  if self.config.doApCorr:
520  self.applyApCorr.run(
521  catalog=measCat,
522  apCorrMap=exposure.getInfo().getApCorrMap()
523  )
524  self.catalogCalculation.run(measCat)
525 
526  return pipeBase.Struct(measCat=measCat)
527 
528  def makeIdFactory(self, dataRef):
529  """Create an object that generates globally unique source IDs.
530 
531  Source IDs are created based on a per-CCD ID and the ID of the CCD
532  itself.
533 
534  Parameters
535  ----------
536  dataRef : `lsst.daf.persistence.ButlerDataRef`
537  Butler data reference. The ``ccdExposureId_bits`` and
538  ``ccdExposureId`` datasets are accessed. The data ID must have the
539  keys that correspond to ``ccdExposureId``, which are generally the
540  same as those that correspond to ``calexp`` (``visit``, ``raft``,
541  ``sensor`` for LSST data).
542  """
543  expBits = dataRef.get("ccdExposureId_bits")
544  expId = int(dataRef.get("ccdExposureId"))
545  return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
546 
547  def getExposureId(self, dataRef):
548  return int(dataRef.get("ccdExposureId", immediate=True))
549 
550  def fetchReferences(self, dataRef, exposure):
551  """Get sources that overlap the exposure.
552 
553  Parameters
554  ----------
555  dataRef : `lsst.daf.persistence.ButlerDataRef`
556  Butler data reference corresponding to the image to be measured;
557  should have ``tract``, ``patch``, and ``filter`` keys.
558  exposure : `lsst.afw.image.Exposure`
559  The image to be measured (used only to obtain a WCS and bounding
560  box).
561 
562  Returns
563  -------
564  referencs : `lsst.afw.table.SourceCatalog`
565  Catalog of sources that overlap the exposure
566 
567  Notes
568  -----
569  The returned catalog is sorted by ID and guarantees that all included
570  children have their parent included and that all Footprints are valid.
571 
572  All work is delegated to the references subtask; see
573  :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
574  for information about the default behavior.
575  """
576  references = lsst.afw.table.SourceCatalog(self.references.schema)
577  badParents = set()
578  unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
579  for record in unfiltered:
580  if record.getFootprint() is None or record.getFootprint().getArea() == 0:
581  if record.getParent() != 0:
582  self.log.warning("Skipping reference %s (child of %s) with bad Footprint",
583  record.getId(), record.getParent())
584  else:
585  self.log.warning("Skipping reference parent %s with bad Footprint", record.getId())
586  badParents.add(record.getId())
587  elif record.getParent() not in badParents:
588  references.append(record)
589  # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
590  references.sort(lsst.afw.table.SourceTable.getParentKey())
591  return references
592 
593  def attachFootprints(self, sources, refCat, exposure, refWcs):
594  r"""Attach footprints to blank sources prior to measurements.
595 
596  Notes
597  -----
598  `~lsst.afw.detection.Footprint`\ s for forced photometry must be in the
599  pixel coordinate system of the image being measured, while the actual
600  detections may start out in a different coordinate system.
601 
602  Subclasses of this class must implement this method to define how
603  those `~lsst.afw.detection.Footprint`\ s should be generated.
604 
605  This default implementation transforms the
606  `~lsst.afw.detection.Footprint`\ s from the reference catalog from the
607  reference WCS to the exposure's WcS, which downgrades
608  `lsst.afw.detection.heavyFootprint.HeavyFootprint`\ s into regular
609  `~lsst.afw.detection.Footprint`\ s, destroying deblend information.
610  """
611  return self.measurement.attachTransformedFootprints(sources, refCat, exposure, refWcs)
612 
613  def getExposure(self, dataRef):
614  """Read input exposure for measurement.
615 
616  Parameters
617  ----------
618  dataRef : `lsst.daf.persistence.ButlerDataRef`
619  Butler data reference.
620  """
621  exposure = dataRef.get(self.dataPrefix + "calexp", immediate=True)
622 
623  if self.config.doApplyExternalPhotoCalib:
624  source = f"{self.config.externalPhotoCalibName}_photoCalib"
625  self.log.info("Applying external photoCalib from %s", source)
626  photoCalib = dataRef.get(source)
627  exposure.setPhotoCalib(photoCalib) # No need for calibrateImage; having the photoCalib suffices
628 
629  if self.config.doApplyExternalSkyWcs:
630  source = f"{self.config.externalSkyWcsName}_wcs"
631  self.log.info("Applying external skyWcs from %s", source)
632  skyWcs = dataRef.get(source)
633  exposure.setWcs(skyWcs)
634 
635  if self.config.doApplySkyCorr:
636  self.log.info("Apply sky correction")
637  skyCorr = dataRef.get("skyCorr")
638  exposure.maskedImage -= skyCorr.getImage()
639 
640  return exposure
641 
642  def writeOutput(self, dataRef, sources):
643  """Write forced source table
644 
645  Parameters
646  ----------
647  dataRef : `lsst.daf.persistence.ButlerDataRef`
648  Butler data reference. The forced_src dataset (with
649  self.dataPrefix prepended) is all that will be modified.
650  sources : `lsst.afw.table.SourceCatalog`
651  Catalog of sources to save.
652  """
653  dataRef.put(sources, self.dataPrefix + "forced_src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
654 
655  def getSchemaCatalogs(self):
656  """The schema catalogs that will be used by this task.
657 
658  Returns
659  -------
660  schemaCatalogs : `dict`
661  Dictionary mapping dataset type to schema catalog.
662 
663  Notes
664  -----
665  There is only one schema for each type of forced measurement. The
666  dataset type for this measurement is defined in the mapper.
667  """
668  catalog = lsst.afw.table.SourceCatalog(self.measurement.schema)
669  catalog.getTable().setMetadata(self.measurement.algMetadata)
670  datasetType = self.dataPrefix + "forced_src"
671  return {datasetType: catalog}
672 
673  def _getConfigName(self):
674  # Documented in superclass.
675  return self.dataPrefix + "forcedPhotCcd_config"
676 
677  def _getMetadataName(self):
678  # Documented in superclass
679  return self.dataPrefix + "forcedPhotCcd_metadata"
680 
681  @classmethod
682  def _makeArgumentParser(cls):
683  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
684  parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
685  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
686  ContainerClass=PerTractCcdDataIdContainer)
687  return parser
688 
689 
690 class ForcedPhotCcdFromDataFrameConnections(PipelineTaskConnections,
691  dimensions=("instrument", "visit", "detector", "skymap", "tract"),
692  defaultTemplates={"inputCoaddName": "goodSeeing",
693  "inputName": "calexp"}):
694  refCat = cT.Input(
695  doc="Catalog of positions at which to force photometry.",
696  name="{inputCoaddName}Diff_fullDiaObjTable",
697  storageClass="DataFrame",
698  dimensions=["skymap", "tract", "patch"],
699  multiple=True,
700  deferLoad=True,
701  )
702  exposure = cT.Input(
703  doc="Input exposure to perform photometry on.",
704  name="{inputName}",
705  storageClass="ExposureF",
706  dimensions=["instrument", "visit", "detector"],
707  )
708  measCat = cT.Output(
709  doc="Output forced photometry catalog.",
710  name="forced_src_diaObject",
711  storageClass="SourceCatalog",
712  dimensions=["instrument", "visit", "detector", "skymap", "tract"],
713  )
714  outputSchema = cT.InitOutput(
715  doc="Schema for the output forced measurement catalogs.",
716  name="forced_src_diaObject_schema",
717  storageClass="SourceCatalog",
718  )
719 
720 
721 class ForcedPhotCcdFromDataFrameConfig(ForcedPhotCcdConfig,
722  pipelineConnections=ForcedPhotCcdFromDataFrameConnections):
723  def setDefaults(self):
724  self.measurement.doReplaceWithNoise = False
725  self.measurement.plugins = ["base_TransformedCentroidFromCoord", "base_PsfFlux"]
726  self.measurement.copyColumns = {'id': 'diaObjectId', 'coord_ra': 'coord_ra', 'coord_dec': 'coord_dec'}
727  self.measurement.slots.centroid = "base_TransformedCentroidFromCoord"
728  self.measurement.slots.psfFlux = "base_PsfFlux"
729  self.measurement.slots.shape = None
730  self.catalogCalculation.plugins.names = []
731 
732 
733 class ForcedPhotCcdFromDataFrameTask(ForcedPhotCcdTask):
734  """Force Photometry on a per-detector exposure with coords from a DataFrame
735 
736  Uses input from a DataFrame instead of SourceCatalog
737  like the base class ForcedPhotCcd does.
738  Writes out a SourceCatalog so that the downstream WriteForcedSourceTableTask
739  can be reused with output from this Task.
740  """
741  _DefaultName = "forcedPhotCcdFromDataFrame"
742  ConfigClass = ForcedPhotCcdFromDataFrameConfig
743 
744  def __init__(self, butler=None, refSchema=None, initInputs=None, **kwds):
745  # Parent's init assumes that we have a reference schema; Cannot reuse
746  pipeBase.PipelineTask.__init__(self, **kwds)
747 
748  self.makeSubtask("measurement", refSchema=lsst.afw.table.SourceTable.makeMinimalSchema())
749 
750  if self.config.doApCorr:
751  self.makeSubtask("applyApCorr", schema=self.measurement.schema)
752  self.makeSubtask('catalogCalculation', schema=self.measurement.schema)
753  self.outputSchema = lsst.afw.table.SourceCatalog(self.measurement.schema)
754 
755  def runQuantum(self, butlerQC, inputRefs, outputRefs):
756  inputs = butlerQC.get(inputRefs)
757  self.log.info("Filtering ref cats: %s", ','.join([str(i.dataId) for i in inputs['refCat']]))
758  refCat = self.df2RefCat([i.get(parameters={"columns": ['diaObjectId', 'ra', 'decl']})
759  for i in inputs['refCat']],
760  inputs['exposure'].getBBox(), inputs['exposure'].getWcs())
761  inputs['refCat'] = refCat
762  inputs['refWcs'] = inputs['exposure'].getWcs()
763  inputs['measCat'], inputs['exposureId'] = self.generateMeasCat(inputRefs.exposure.dataId,
764  inputs['exposure'], inputs['refCat'],
765  inputs['refWcs'],
766  "visit_detector")
767  outputs = self.run(**inputs)
768  butlerQC.put(outputs, outputRefs)
769 
770  def df2RefCat(self, dfList, exposureBBox, exposureWcs):
771  """Convert list of DataFrames to reference catalog
772 
773  Concatenate list of DataFrames presumably from multiple patches and
774  downselect rows that overlap the exposureBBox using the exposureWcs.
775 
776  Parameters
777  ----------
778  dfList : `list` of `pandas.DataFrame`
779  Each element containst diaObjects with ra/decl position in degrees
780  Columns 'diaObjectId', 'ra', 'decl' are expected
781  exposureBBox : `lsst.geom.Box2I`
782  Bounding box on which to select rows that overlap
783  exposureWcs : `lsst.afw.geom.SkyWcs`
784  World coordinate system to convert sky coords in ref cat to
785  pixel coords with which to compare with exposureBBox
786 
787  Returns
788  -------
789  refCat : `lsst.afw.table.SourceTable`
790  Source Catalog with minimal schema that overlaps exposureBBox
791  """
792  df = pd.concat(dfList)
793  # translate ra/decl coords in dataframe to detector pixel coords
794  # to down select rows that overlap the detector bbox
795  mapping = exposureWcs.getTransform().getMapping()
796  x, y = mapping.applyInverse(np.array(df[['ra', 'decl']].values*2*np.pi/360).T)
797  inBBox = exposureBBox.contains(x, y)
798  refCat = self.df2SourceCat(df[inBBox])
799  return refCat
800 
801  def df2SourceCat(self, df):
802  """Create minimal schema SourceCatalog from a pandas DataFrame.
803 
804  The forced measurement subtask expects this as input.
805 
806  Parameters
807  ----------
808  df : `pandas.DataFrame`
809  DiaObjects with locations and ids.
810 
811  Returns
812  -------
813  outputCatalog : `lsst.afw.table.SourceTable`
814  Output catalog with minimal schema.
815  """
817  outputCatalog = lsst.afw.table.SourceCatalog(schema)
818  outputCatalog.reserve(len(df))
819 
820  for id, diaObjectId, ra, decl in df[['diaObjectId', 'ra', 'decl']].itertuples():
821  outputRecord = outputCatalog.addNew()
822  outputRecord.setId(diaObjectId)
823  outputRecord.setCoord(lsst.geom.SpherePoint(ra, decl, lsst.geom.degrees))
824  return outputCatalog
static std::shared_ptr< IdFactory > makeSource(RecordId expId, int reserved)
static Key< RecordId > getParentKey()
static Schema makeMinimalSchema()
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)
std::shared_ptr< SkyWcs > makeSkyWcs(daf::base::PropertySet &metadata, bool strip=false)
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
def imageOverlapsTract(tract, imageWcs, imageBox)
def attachFootprints(self, sources, refCat, exposure, refWcs, dataRef)
def writeOutput(self, dataRef, sources)
def fetchReferences(self, dataRef, exposure)