lsst.meas.base  14.0-23-g2010ef9
forcedPhotCcd.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2015 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 import collections
24 
25 import lsst.pex.config
27 from lsst.log import Log
28 import lsst.pipe.base
29 import lsst.afw.geom
30 import lsst.afw.image
31 import lsst.afw.table
32 from lsst.geom import convexHull
33 
34 from .forcedPhotImage import ForcedPhotImageTask, ForcedPhotImageConfig
35 
36 try:
37  from lsst.meas.mosaic import applyMosaicResults
38 except ImportError:
39  applyMosaicResults = None
40 
41 __all__ = ("PerTractCcdDataIdContainer", "ForcedPhotCcdConfig", "ForcedPhotCcdTask")
42 
43 
44 class PerTractCcdDataIdContainer(lsst.pipe.base.DataIdContainer):
45  """A version of lsst.pipe.base.DataIdContainer that combines raw data IDs with a tract.
46 
47  Required because we need to add "tract" to the raw data ID keys (defined as whatever we
48  use for 'src') when no tract is provided (so that the user is not required to know
49  which tracts are spanned by the raw data ID).
50 
51  This IdContainer assumes that a calexp is being measured using the detection information,
52  a set of reference catalogs, from the set of coadds which intersect with the calexp.
53  It needs the calexp id (e.g. visit, raft, sensor), but is also uses the tract to decide
54  what set of coadds to use. The references from the tract whose patches intersect with
55  the calexp are used.
56  """
57 
58  def makeDataRefList(self, namespace):
59  """Make self.refList from self.idList
60  """
61  if self.datasetType is None:
62  raise RuntimeError("Must call setDatasetType first")
63  log = Log.getLogger("meas.base.forcedPhotCcd.PerTractCcdDataIdContainer")
64  skymap = None
65  visitTract = collections.defaultdict(set) # Set of tracts for each visit
66  visitRefs = collections.defaultdict(list) # List of data references for each visit
67  for dataId in self.idList:
68  if "tract" not in dataId:
69  # Discover which tracts the data overlaps
70  log.info("Reading WCS for components of dataId=%s to determine tracts", dict(dataId))
71  if skymap is None:
72  skymap = namespace.butler.get(namespace.config.coaddName + "Coadd_skyMap")
73 
74  for ref in namespace.butler.subset("calexp", dataId=dataId):
75  if not ref.datasetExists("calexp"):
76  continue
77 
78  visit = ref.dataId["visit"]
79  visitRefs[visit].append(ref)
80 
81  md = ref.get("calexp_md", immediate=True)
82  wcs = lsst.afw.geom.makeSkyWcs(md)
84  # Going with just the nearest tract. Since we're throwing all tracts for the visit
85  # together, this shouldn't be a problem unless the tracts are much smaller than a CCD.
86  tract = skymap.findTract(wcs.pixelToSky(box.getCenter()))
87  if overlapsTract(tract, wcs, box):
88  visitTract[visit].add(tract.getId())
89  else:
90  self.refList.extend(ref for ref in namespace.butler.subset(self.datasetType, dataId=dataId))
91 
92  # Ensure all components of a visit are kept together by putting them all in the same set of tracts
93  for visit, tractSet in visitTract.items():
94  for ref in visitRefs[visit]:
95  for tract in tractSet:
96  self.refList.append(namespace.butler.dataRef(datasetType=self.datasetType,
97  dataId=ref.dataId, tract=tract))
98  if visitTract:
99  tractCounter = collections.Counter()
100  for tractSet in visitTract.values():
101  tractCounter.update(tractSet)
102  log.info("Number of visits for each tract: %s", dict(tractCounter))
103 
104 
105 def overlapsTract(tract, imageWcs, imageBox):
106  """Return whether the image (specified by Wcs and bounding box) overlaps the tract
107 
108  @param tract: TractInfo specifying a tract
109  @param imageWcs: Wcs for image
110  @param imageBox: Bounding box for image
111  @return bool
112  """
113  tractWcs = tract.getWcs()
114  tractCorners = [tractWcs.pixelToSky(lsst.afw.geom.Point2D(coord)).getVector() for
115  coord in tract.getBBox().getCorners()]
116  tractPoly = convexHull(tractCorners)
117 
118  try:
119  imageCorners = [imageWcs.pixelToSky(lsst.afw.geom.Point2D(pix)) for pix in imageBox.getCorners()]
120  except lsst.pex.exceptions.LsstCppException as e:
121  # Protecting ourselves from awful Wcs solutions in input images
122  if (not isinstance(e.message, lsst.pex.exceptions.DomainErrorException) and
123  not isinstance(e.message, lsst.pex.exceptions.RuntimeErrorException)):
124  raise
125  return False
126 
127  imagePoly = convexHull([coord.getVector() for coord in imageCorners])
128  if imagePoly is None:
129  return False
130  return tractPoly.intersects(imagePoly) # "intersects" also covers "contains" or "is contained by"
131 
132 
134  doApplyUberCal = lsst.pex.config.Field(
135  dtype=bool,
136  doc="Apply meas_mosaic ubercal results to input calexps?",
137  default=False
138  )
139 
140 
146 
147 
149  """!A command-line driver for performing forced measurement on CCD images
150 
151  This task is a subclass of ForcedPhotImageTask which is specifically for doing forced
152  measurement on a single CCD exposure, using as a reference catalog the detections which
153  were made on overlapping coadds.
154 
155  The run method (inherited from ForcedPhotImageTask) takes a lsst.daf.persistence.ButlerDataRef
156  argument that corresponds to a single CCD. This should contain the data ID keys that correspond to
157  the "forced_src" dataset (the output dataset for ForcedPhotCcdTask), which are typically all those
158  used to specify the "calexp" dataset (e.g. visit, raft, sensor for LSST data) as well as a coadd
159  tract. The tract is used to look up the appropriate coadd measurement catalogs to use as references
160  (e.g. deepCoadd_src; see CoaddSrcReferencesTask for more information). While the tract must be given
161  as part of the dataRef, the patches are determined automatically from the bounding box and WCS of the
162  calexp to be measured, and the filter used to fetch references is set via config
163  (BaseReferencesConfig.filter).
164 
165  In addition to the run method, ForcedPhotCcdTask overrides several methods of ForcedPhotImageTask
166  to specialize it for single-CCD processing, including makeIdFactory(), fetchReferences(), and
167  getExposure(). None of these should be called directly by the user, though it may be useful
168  to override them further in subclasses.
169  """
170 
171  ConfigClass = ForcedPhotCcdConfig
172  RunnerClass = lsst.pipe.base.ButlerInitializedTaskRunner
173  _DefaultName = "forcedPhotCcd"
174  dataPrefix = ""
175 
176  def makeIdFactory(self, dataRef):
177  """Create an object that generates globally unique source IDs from per-CCD IDs and the CCD ID.
178 
179  @param dataRef Data reference from butler. The "ccdExposureId_bits" and "ccdExposureId"
180  datasets are accessed. The data ID must have the keys that correspond
181  to ccdExposureId, which is generally the same that correspond to "calexp"
182  (e.g. visit, raft, sensor for LSST data).
183  """
184  expBits = dataRef.get("ccdExposureId_bits")
185  expId = int(dataRef.get("ccdExposureId"))
186  return lsst.afw.table.IdFactory.makeSource(expId, 64 - expBits)
187 
188  def getExposureId(self, dataRef):
189  return int(dataRef.get("ccdExposureId", immediate=True))
190 
191  def fetchReferences(self, dataRef, exposure):
192  """Return a SourceCatalog of sources which overlap the exposure.
193 
194  The returned catalog is sorted by ID and guarantees that all included children have their
195  parent included and that all Footprints are valid.
196 
197  @param dataRef Data reference from butler corresponding to the image to be measured;
198  should have tract, patch, and filter keys.
199  @param exposure lsst.afw.image.Exposure to be measured (used only to obtain a Wcs and
200  bounding box).
201 
202  All work is delegated to the references subtask; see CoaddSrcReferencesTask for information
203  about the default behavior.
204  """
205  references = lsst.afw.table.SourceCatalog(self.references.schema)
206  badParents = set()
207  unfiltered = self.references.fetchInBox(dataRef, exposure.getBBox(), exposure.getWcs())
208  for record in unfiltered:
209  if record.getFootprint() is None or record.getFootprint().getArea() == 0:
210  if record.getParent() != 0:
211  self.log.warn("Skipping reference %s (child of %s) with bad Footprint",
212  record.getId(), record.getParent())
213  else:
214  self.log.warn("Skipping reference parent %s with bad Footprint", record.getId())
215  badParents.add(record.getId())
216  elif record.getParent() not in badParents:
217  references.append(record)
218  # catalog must be sorted by parent ID for lsst.afw.table.getChildren to work
219  references.sort(lsst.afw.table.SourceTable.getParentKey())
220  return references
221 
222  def getExposure(self, dataRef):
223  """Read input exposure to measure
224 
225  @param dataRef Data reference from butler. Only the 'calexp' dataset is used,
226  unless config.doApplyUberCal is true, in which case the corresponding
227  meas_mosaic outputs are used as well.
228  """
229  exposure = ForcedPhotImageTask.getExposure(self, dataRef)
230  if not self.config.doApplyUberCal:
231  return exposure
232  if applyMosaicResults is None:
233  raise RuntimeError(
234  "Cannot use improved calibrations for %s because meas_mosaic could not be imported."
235  % (dataRef.dataId,))
236  else:
237  applyMosaicResults(dataRef, calexp=exposure)
238  return exposure
239 
240  def _getConfigName(self):
241  """!Return the name of the config dataset. Forces config comparison from run-to-run
242  """
243  return self.dataPrefix + "forcedPhotCcd_config"
244 
245  def _getMetadataName(self):
246  """!Return the name of the metadata dataset. Forced metadata to be saved
247  """
248  return self.dataPrefix + "forcedPhotCcd_metadata"
249 
250  @classmethod
251  def _makeArgumentParser(cls):
252  parser = lsst.pipe.base.ArgumentParser(name=cls._DefaultName)
253  parser.add_id_argument("--id", "forced_src", help="data ID with raw CCD keys [+ tract optionally], "
254  "e.g. --id visit=12345 ccd=1,2 [tract=0]",
255  ContainerClass=PerTractCcdDataIdContainer)
256  return parser
A base class for command-line forced measurement drivers.
A command-line driver for performing forced measurement on CCD images.
static std::shared_ptr< IdFactory > makeSource(RecordId expId, int reserved)
geom::Box2I bboxFromMetadata(daf::base::PropertySet &metadata)
std::shared_ptr< SkyWcs > makeSkyWcs(Point2D const &crpix, SpherePoint const &crval, Eigen::Matrix2d const &cdMatrix, std::string const &projection="TAN")
def overlapsTract(tract, imageWcs, imageBox)
static Key< RecordId > getParentKey()
def fetchReferences(self, dataRef, exposure)
Config class for forced measurement driver task.