lsst.meas.base  16.0-19-g283fd30+1
forcedPhotCcd.py
Go to the documentation of this file.
1 # This file is part of meas_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 
24 import lsst.pex.config
26 from lsst.log import Log
27 import lsst.pipe.base
28 import lsst.geom
29 import lsst.afw.geom
30 import lsst.afw.image
31 import lsst.afw.table
32 import lsst.sphgeom
33 
34 from .forcedPhotImage import ForcedPhotImageTask, ForcedPhotImageConfig
35 
36 try:
37  from lsst.meas.mosaic import applyMosaicResults
38 except ImportError:
39  applyMosaicResults = None
40 
41 __all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask", "imageOverlapsTract")
42 
43 
44 class PerTractCcdDataIdContainer(lsst.pipe.base.DataIdContainer):
45  """A data ID container which combines raw data IDs with a tract.
46 
47  Notes
48  -----
49  Required because we need to add "tract" to the raw data ID keys (defined as
50  whatever we use for ``src``) when no tract is provided (so that the user is
51  not required to know which tracts are spanned by the raw data ID).
52 
53  This subclass of `~lsst.pipe.base.DataIdContainer` assumes that a calexp is
54  being measured using the detection information, a set of reference
55  catalogs, from the set of coadds which intersect with the calexp. It needs
56  the calexp id (e.g. visit, raft, sensor), but is also uses the tract to
57  decide what set of coadds to use. The references from the tract whose
58  patches intersect with the calexp are used.
59  """
60 
61  def makeDataRefList(self, namespace):
62  """Make self.refList from self.idList
63  """
64  if self.datasetType is None:
65  raise RuntimeError("Must call setDatasetType first")
66  log = Log.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
67  skymap = None
68  visitTract = collections.defaultdict(set) # Set of tracts for each visit
69  visitRefs = collections.defaultdict(list) # List of data references for each visit
70  for dataId in self.idList:
71  if "tract" not in dataId:
72  # Discover which tracts the data overlaps
73  log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
74  if skymap is None:
75  skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
76 
77  for ref in namespace.butler.subset("calexp", dataId=dataId):
78  if not ref.datasetExists("calexp"):
79  continue
80 
81  visit = ref.dataId["visit"]
82  visitRefs[visit].append(ref)
83 
84  md = ref.get("calexp_md", immediate=True)
85  wcs = lsst.afw.geom.makeSkyWcs(md)
87  # Going with just the nearest tract. Since we're throwing all tracts for the visit
88  # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
89  tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
90  if imageOverlapsTract(tract, wcs, box):
91  visitTract[visit].add(tract.getId())
92  else:
93  self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
94 
95  # Ensure all components of a visit are kept together by putting them all in the same set of tracts
96  for visit, tractSet in visitTract.items():
97  for ref in visitRefs[visit]:
98  for tract in tractSet:
99  self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
100  dataId=ref.dataId, tract=tract))
101  if visitTract:
102  tractCounter = collections.Counter()
103  for tractSet in visitTract.values():
104  tractCounter.update(tractSet)
105  log.info("Number of visits for each tract: %s", dict(tractCounter))
106 
107 
108 def imageOverlapsTract(tract, imageWcs, imageBox):
109  """Return whether the given bounding box overlaps the tract given a WCS.
110 
111  Parameters
112  ----------
113  tract : `lsst.skymap.TractInfo`
114  TractInfo specifying a tract.
115  imageWcs : `lsst.afw.geom.SkyWcs`
116  World coordinate system for the image.
117  imageBox : `lsst.geom.Box2I`
118  Bounding box for the image.
119 
120  Returns
121  -------
122  overlap : `bool`
123  `True` if the bounding box overlaps the tract; `False` otherwise.
124  """
125  tractPoly = tract.getOuterSkyPolygon()
126 
127  imagePixelCorners = lsst.geom.Box2D(imageBox).getCorners()
128  try:
129  imageSkyCorners = imageWcs.pixelToSky(imagePixelCorners)
130  except lsst.pex.exceptions.LsstCppException as e:
131  # Protecting ourselves from awful Wcs solutions in input images
132  if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException) and
133  not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
134  raise
135  return False
136 
137  imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageSkyCorners])
138  return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
139 
140 
142  doApplyUberCal = lsst.pex.config.Field(
143  dtype=bool,
144  doc="Apply meas_mosaic ubercal results to input calexps?",
145  default=False
146  )
147 
148  def setDefaults(self):
149  super().setDefaults()
150 
151  # Override the Gen3 datasets from ForcedPhotImageTaskConfig.
152  # When these nameTemplate values are used in ForcedPhotCoadd,
153  # there is a coadd name that may change, depending on the
154  # coadd type. For CCD forced photometry, there will likely
155  # only ever be a single calexp type, but for consistency, use
156  # the nameTemplate with nothing to substitute.
157  self.outputSchema.nameTemplate = "forced_src_schema"
158  self.exposure.nameTemplate = "{inputName}"
159  self.exposure.dimensions = ["Instrument", "Visit", "Detector"]
160  self.measCat.nameTemplate = "forced_src"
161  self.measCat.dimensions = ["Instrument", "Visit", "Detector", "SkyMap", "Tract"]
162 
163  self.formatTemplateNames({"inputName": "calexp",
164  "inputCoaddName": "deep"})
165  self.quantum.dimensions = ("Instrument", "Visit", "Detector", "SkyMap", "Tract")
166 
167 
169  """A command-line driver for performing forced measurement on CCD images.
170 
171  Notes
172  -----
173  This task is a subclass of
174  :lsst-task:`lsst.meas.base.forcedPhotImage.ForcedPhotImageTask` which is
175  specifically for doing forced measurement on a single CCD exposure, using
176  as a reference catalog the detections which were made on overlapping
177  coadds.
178 
179  The `run` method (inherited from `ForcedPhotImageTask`) takes a
180  `~lsst.daf.persistence.ButlerDataRef` argument that corresponds to a single
181  CCD. This should contain the data ID keys that correspond to the
182  ``forced_src`` dataset (the output dataset for this task), which are
183  typically all those used to specify the ``calexp`` dataset (``visit``,
184  ``raft``, ``sensor`` for LSST data) as well as a coadd tract. The tract is
185  used to look up the appropriate coadd measurement catalogs to use as
186  references (e.g. ``deepCoadd_src``; see
187  :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask` for more
188  information). While the tract must be given as part of the dataRef, the
189  patches are determined automatically from the bounding box and WCS of the
190  calexp to be measured, and the filter used to fetch references is set via
191  the ``filter`` option in the configuration of
192  :lsst-task:`lsst.meas.base.references.BaseReferencesTask`).
193 
194  In addition to the `run` method, `ForcedPhotCcdTask` overrides several
195  methods of `ForcedPhotImageTask` to specialize it for single-CCD
196  processing, including `~ForcedPhotImageTask.makeIdFactory`,
197  `~ForcedPhotImageTask.fetchReferences`, and
198  `~ForcedPhotImageTask.getExposure`. None of these should be called
199  directly by the user, though it may be useful to override them further in
200  subclasses.
201  """
202 
203  ConfigClass = ForcedPhotCcdConfig
204  RunnerClass = lsst.pipe.base.ButlerInitializedTaskRunner
205  _DefaultName = "forcedPhotCcd"
206  dataPrefix = ""
207 
208  def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
209  inputData['refCat'] = self.filterReferences(inputData['exposure'],
210  inputData['refCat'], inputData['refWcs'])
211  inputData['measCat'] = self.generateMeasCat(inputDataIds['exposure'],
212  inputData['exposure'],
213  inputData['refCat'], inputData['refWcs'],
214  "VisitDetector", butler)
215 
216  return self.run(**inputData)
217 
218  def filterReferences(self, exposure, refCat, refWcs):
219  """Filter reference catalog so that all sources are within the
220  boundaries of the exposure.
221 
222  Parameters
223  ----------
224  exposure : `lsst.afw.image.exposure.Exposure`
225  Exposure to generate the catalog for.
226  refCat : `lsst.afw.table.SourceCatalog`
227  Catalog of shapes and positions at which to force photometry.
228  refWcs : `lsst.afw.image.SkyWcs`
229  Reference world coordinate system.
230 
231  Returns
232  -------
233  refSources : `lsst.afw.table.SourceCatalog`
234  Filtered catalog of forced sources to measure.
235 
236  Notes
237  -----
238  Filtering the reference catalog is currently handled by Gen2
239  specific methods. To function for Gen3, this method copies
240  code segments to do the filtering and transformation. The
241  majority of this code is based on the methods of
242  lsst.meas.algorithms.loadReferenceObjects.ReferenceObjectLoader
243 
244  """
245 
246  # Step 1: Determine bounds of the exposure photometry will
247  # be performed on.
248  expWcs = exposure.getWcs()
249  expRegion = exposure.getBBox(lsst.afw.image.PARENT)
250  expBBox = lsst.geom.Box2D(expRegion)
251  expBoxCorners = expBBox.getCorners()
252  expSkyCorners = [expWcs.pixelToSky(corner).getVector() for
253  corner in expBoxCorners]
254  expPolygon = lsst.sphgeom.ConvexPolygon(expSkyCorners)
255 
256  # Step 2: Filter out reference catalog sources that are
257  # not contained within the exposure boundaries.
258  sources = type(refCat)(refCat.table)
259  for record in refCat:
260  if expPolygon.contains(record.getCoord().getVector()):
261  sources.append(record)
262  refCatIdDict = {ref.getId(): ref.getParent() for ref in sources}
263 
264  # Step 3: Cull sources that do not have their parent
265  # source in the filtered catalog. Save two copies of each
266  # source.
267  refSources = type(refCat)(refCat.table)
268  for record in refCat:
269  if expPolygon.contains(record.getCoord().getVector()):
270  recordId = record.getId()
271  topId = recordId
272  while (topId > 0):
273  if topId in refCatIdDict:
274  topId = refCatIdDict[topId]
275  else:
276  break
277  if topId == 0:
278  refSources.append(record)
279 
280  # Step 4: Transform source footprints from the reference
281  # coordinates to the exposure coordinates.
282  for refRecord in refSources:
283  refRecord.setFootprint(refRecord.getFootprint().transform(refWcs,
284  expWcs, expRegion))
285  # Step 5: Replace reference catalog with filtered source list.
286  return refSources
287 
288  def makeIdFactory(self, dataRef):
289  """Create an object that generates globally unique source IDs.
290 
291  Source IDs are created based on a per-CCD ID and the ID of the CCD
292  itself.
293 
294  Parameters
295  ----------
296  dataRef : `lsst.daf.persistence.ButlerDataRef`
297  Butler data reference. The ``ccdExposureId_bits`` and
298  ``ccdExposureId`` datasets are accessed. The data ID must have the
299  keys that correspond to ``ccdExposureId``, which are generally the
300  same as those that correspond to ``calexp`` (``visit``, ``raft``,
301  ``sensor`` for LSST data).
302  """
303  expBits = dataRef.get("ccdExposureId_bits")
304  expId = int(dataRef.get("ccdExposureId"))
305  return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
306 
307  def getExposureId(self, dataRef):
308  return int(dataRef.get("ccdExposureId", immediate=True))
309 
310  def fetchReferences(self, dataRef, exposure):
311  """Get sources that overlap the exposure.
312 
313  Parameters
314  ----------
315  dataRef : `lsst.daf.persistence.ButlerDataRef`
316  Butler data reference corresponding to the image to be measured;
317  should have ``tract``, ``patch``, and ``filter`` keys.
318  exposure : `lsst.afw.image.Exposure`
319  The image to be measured (used only to obtain a WCS and bounding
320  box).
321 
322  Returns
323  -------
324  referencs : `lsst.afw.table.SourceCatalog`
325  Catalog of sources that overlap the exposure
326 
327  Notes
328  -----
329  The returned catalog is sorted by ID and guarantees that all included
330  children have their parent included and that all Footprints are valid.
331 
332  All work is delegated to the references subtask; see
333  :lsst-task:`lsst.meas.base.references.CoaddSrcReferencesTask`
334  for information about the default behavior.
335  """
336  references = lsst.afw.table.SourceCatalog(self.references.schema)
337  badParents = set()
338  unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
339  for record in unfiltered:
340  if record.getFootprint() is None or record.getFootprint().getArea() == 0:
341  if record.getParent() != 0:
342  self.log.warn("Skipping reference %s (child of %s) with bad Footprint",
343  record.getId(), record.getParent())
344  else:
345  self.log.warn("Skipping reference parent %s with bad Footprint", record.getId())
346  badParents.add(record.getId())
347  elif record.getParent() not in badParents:
348  references.append(record)
349  # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
350  references.sort(lsst.afw.table.SourceTable.getParentKey())
351  return references
352 
353  def getExposure(self, dataRef):
354  """Read input exposure for measurement.
355 
356  Parameters
357  ----------
358  dataRef : `lsst.daf.persistence.ButlerDataRef`
359  Butler data reference. Only the ``calexp`` dataset is used, unless
360  ``config.doApplyUberCal`` is `True`, in which case the
361  corresponding meas_mosaic outputs are used as well.
362  """
363  exposure = ForcedPhotImageTask.getExposure(self, dataRef)
364  if not self.config.doApplyUberCal:
365  return exposure
366  if applyMosaicResults is None:
367  raise RuntimeError(
368  "Cannot use improved calibrations for %s because meas_mosaic could not be imported."
369  % (dataRef.dataId,))
370  else:
371  applyMosaicResults(dataRef, calexp=exposure)
372  return exposure
373 
374  def _getConfigName(self):
375  # Documented in superclass.
376  return self.dataPrefix + "forcedPhotCcd_config"
377 
378  def _getMetadataName(self):
379  # Documented in superclass
380  return self.dataPrefix + "forcedPhotCcd_metadata"
381 
382  @classmethod
383  def _makeArgumentParser(cls):
384  parser = lsst.pipe.base.ArgumentParser(name=cls._DefaultName)
385  parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
386  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
387  ContainerClass=PerTractCcdDataIdContainer)
388  return parser
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler)
def run(self, measCat, exposure, refCat, refWcs, exposureId=None)
static std::shared_ptr< IdFactory > makeSource(RecordId expId, int reserved)
def generateMeasCat(self, exposureDataId, exposure, refCat, refWcs, idPackerName, butler)
def imageOverlapsTract(tract, imageWcs, imageBox)
std::shared_ptr< SkyWcs > makeSkyWcs(TransformPoint2ToPoint2 const &pixelsToFieldAngle, lsst::geom::Angle const &orientation, bool flipX, lsst::geom::SpherePoint const &boresight, std::string const &projection="TAN")
def filterReferences(self, exposure, refCat, refWcs)
static Key< RecordId > getParentKey()
def fetchReferences(self, dataRef, exposure)
lsst::geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)