lsst.meas.extensions.astrometryNet  master-g43362ee2f3+7
 All Classes Namespaces Files Functions Variables Properties Groups Pages
anetAstrometry.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 
24 __all__ = ["ANetAstrometryConfig", "ANetAstrometryTask", "showAstrometry"]
25 
26 from builtins import input
27 from builtins import zip
28 from builtins import range
29 from contextlib import contextmanager
30 
31 import numpy as np
32 
33 import lsstDebug
34 import lsst.pex.exceptions
35 import lsst.afw.geom as afwGeom
36 from lsst.afw.cameraGeom import TAN_PIXELS
37 from lsst.afw.table import Point2DKey, CovarianceMatrix2fKey
38 import lsst.pex.config as pexConfig
39 import lsst.pipe.base as pipeBase
40 from lsst.meas.astrom import displayAstrometry
41 from lsst.meas.astrom.sip import makeCreateWcsWithSip
42 from .anetBasicAstrometry import ANetBasicAstrometryTask
43 
44 
45 class ANetAstrometryConfig(pexConfig.Config):
46  solver = pexConfig.ConfigurableField(
47  target=ANetBasicAstrometryTask,
48  doc="Basic astrometry solver",
49  )
50  forceKnownWcs = pexConfig.Field(dtype=bool, doc=(
51  "Assume that the input image's WCS is correct, without comparing it to any external reality." +
52  " (In contrast to using Astrometry.net). NOTE, if you set this, you probably also want to" +
53  " un-set 'solver.calculateSip'; otherwise we'll still try to find a TAN-SIP WCS starting " +
54  " from the existing WCS"), default=False)
55  rejectThresh = pexConfig.RangeField(dtype=float, default=3.0, doc="Rejection threshold for Wcs fitting",
56  min=0.0, inclusiveMin=False)
57  rejectIter = pexConfig.RangeField(dtype=int, default=3, doc="Rejection iterations for Wcs fitting",
58  min=0)
59 
60  @property
61  def refObjLoader(self):
62  """An alias, for a uniform interface with the standard AstrometryTask"""
63  return self.solver
64 
65  # \addtogroup LSST_task_documentation
66  # \{
67  # \page measAstrom_anetAstrometryTask
68  # \ref ANetAstrometryTask_ "ANetAstrometryTask"
69  # Use astrometry.net to match input sources with a reference catalog and solve for the Wcs
70  # \}
71 
72 
73 class ANetAstrometryTask(pipeBase.Task):
74  """!Use astrometry.net to match input sources with a reference catalog and solve for the Wcs
75 
76  @anchor ANetAstrometryTask_
77 
78  The actual matching and solving is done by the 'solver'; this Task
79  serves as a wrapper for taking into account the known optical distortion.
80 
81  \section pipe_tasks_astrometry_Contents Contents
82 
83  - \ref pipe_tasks_astrometry_Purpose
84  - \ref pipe_tasks_astrometry_Initialize
85  - \ref pipe_tasks_astrometry_IO
86  - \ref pipe_tasks_astrometry_Config
87  - \ref pipe_tasks_astrometry_Debug
88  - \ref pipe_tasks_astrometry_Example
89 
90  \section pipe_tasks_astrometry_Purpose Description
91 
92  \copybrief ANetAstrometryTask
93 
94  \section pipe_tasks_astrometry_Initialize Task initialisation
95 
96  \copydoc \_\_init\_\_
97 
98  \section pipe_tasks_astrometry_IO Invoking the Task
99 
100  \copydoc run
101 
102  \section pipe_tasks_astrometry_Config Configuration parameters
103 
104  See \ref ANetAstrometryConfig
105 
106  \section pipe_tasks_astrometry_Debug Debug variables
107 
108  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
109  flag \c -d to import \b debug.py from your \c PYTHONPATH;
110  see \ref baseDebug for more about \b debug.py files.
111 
112  The available variables in ANetAstrometryTask are:
113  <DL>
114  <DT> \c display
115  <DD> If True call showAstrometry while iterating ANetAstrometryConfig.rejectIter times,
116  and also after converging; and call displayAstrometry after applying the distortion correction.
117  <DT> \c frame
118  <DD> ds9 frame to use in showAstrometry and displayAstrometry
119  <DT> \c pause
120  <DD> Pause after showAstrometry and displayAstrometry?
121  </DL>
122 
123  \section pipe_tasks_astrometry_Example A complete example of using ANetAstrometryTask
124 
125  See \ref pipe_tasks_photocal_Example.
126 
127  To investigate the \ref pipe_tasks_astrometry_Debug, put something like
128  \code{.py}
129  import lsstDebug
130  def DebugInfo(name):
131  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
132  if name in ("lsst.pipe.tasks.anetAstrometry", "lsst.pipe.tasks.anetBasicAstrometry"):
133  di.display = 1
134  di.frame = 1
135  di.pause = True
136 
137  return di
138 
139  lsstDebug.Info = DebugInfo
140  \endcode
141  into your debug.py file and run photoCalTask.py with the \c --debug flag.
142  """
143  ConfigClass = ANetAstrometryConfig
144 
145  def __init__(self, schema, refObjLoader=None, **kwds):
146  """!Create the astrometric calibration task. Most arguments are simply passed onto pipe.base.Task.
147 
148  \param schema An lsst::afw::table::Schema used to create the output lsst.afw.table.SourceCatalog
149  \param refObjLoader The AstrometryTask constructor requires a refObjLoader. In order to make this
150  task retargettable for AstrometryTask it needs to take the same arguments. This argument will be
151  ignored since it uses its own internal loader.
152  \param **kwds keyword arguments to be passed to the lsst.pipe.base.task.Task constructor
153 
154  A centroid field "centroid.distorted" (used internally during the Task's operation)
155  will be added to the schema.
156  """
157  pipeBase.Task.__init__(self, **kwds)
158  self.distortedName = "astrom_distorted"
159  self.centroidXKey = schema.addField(self.distortedName + "_x", type="D",
160  doc="centroid distorted for astrometry solver")
161  self.centroidYKey = schema.addField(self.distortedName + "_y", type="D",
162  doc="centroid distorted for astrometry solver")
163  self.centroidXErrKey = schema.addField(self.distortedName + "_xSigma", type="F",
164  doc="centroid distorted err for astrometry solver")
165  self.centroidYErrKey = schema.addField(self.distortedName + "_ySigma", type="F",
166  doc="centroid distorted err for astrometry solver")
167  self.centroidFlagKey = schema.addField(self.distortedName + "_flag", type="Flag",
168  doc="centroid distorted flag astrometry solver")
169  self.centroidKey = Point2DKey(self.centroidXKey, self.centroidYKey)
170  self.centroidErrKey = CovarianceMatrix2fKey((self.centroidXErrKey, self.centroidYErrKey))
171  # postpone making the solver subtask because it may not be needed and is expensive to create
172  self.solver = None
173 
174  @pipeBase.timeMethod
175  def run(self, exposure, sourceCat):
176  """!Load reference objects, match sources and optionally fit a WCS
177 
178  This is a thin layer around solve or loadAndMatch, depending on config.forceKnownWcs
179 
180  @param[in,out] exposure exposure whose WCS is to be fit
181  The following are read only:
182  - bbox
183  - calib (may be absent)
184  - filter (may be unset)
185  - detector (if wcs is pure tangent; may be absent)
186  The following are updated:
187  - wcs (the initial value is used as an initial guess, and is required)
188  @param[in] sourceCat catalog of sourceCat detected on the exposure (an lsst.afw.table.SourceCatalog)
189  @return an lsst.pipe.base.Struct with these fields:
190  - refCat reference object catalog of objects that overlap the exposure (with some margin)
191  (an lsst::afw::table::SimpleCatalog)
192  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
193  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
194  """
195  if self.config.forceKnownWcs:
196  return self.loadAndMatch(exposure=exposure, sourceCat=sourceCat)
197  else:
198  return self.solve(exposure=exposure, sourceCat=sourceCat)
199 
200  @pipeBase.timeMethod
201  def solve(self, exposure, sourceCat):
202  """!Match with reference sources and calculate an astrometric solution
203 
204  \param[in,out] exposure Exposure to calibrate; wcs is updated
205  \param[in] sourceCat catalog of measured sources (an lsst.afw.table.SourceCatalog)
206  \return a pipeBase.Struct with fields:
207  - refCat reference object catalog of objects that overlap the exposure (with some margin)
208  (an lsst::afw::table::SimpleCatalog)
209  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
210  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
211 
212  The reference catalog actually used is up to the implementation
213  of the solver; it will be manifested in the returned matches as
214  a list of lsst.afw.table.ReferenceMatch objects (\em i.e. of lsst.afw.table.Match with
215  \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord ---
216  the reference object and matched object respectively).
217 
218  \note
219  The input sources have the centroid slot moved to a new column "centroid.distorted"
220  which has the positions corrected for any known optical distortion;
221  the 'solver' (which is instantiated in the 'astrometry' member)
222  should therefore simply use the centroids provided by calling
223  afw.table.Source.getCentroid() on the individual source records. This column \em must
224  be present in the sources table.
225 
226  \note ignores config.forceKnownWcs
227  """
228  with self.distortionContext(sourceCat=sourceCat, exposure=exposure) as bbox:
229  results = self._astrometry(sourceCat=sourceCat, exposure=exposure, bbox=bbox)
230 
231  if results.matches:
232  self.refitWcs(sourceCat=sourceCat, exposure=exposure, matches=results.matches)
233 
234  return results
235 
236  @pipeBase.timeMethod
237  def distort(self, sourceCat, exposure):
238  """!Calculate distorted source positions
239 
240  CCD images are often affected by optical distortion that makes
241  the astrometric solution higher order than linear. Unfortunately,
242  most (all?) matching algorithms require that the distortion be
243  small or zero, and so it must be removed. We do this by calculating
244  (un-)distorted positions, based on a known optical distortion model
245  in the Ccd.
246 
247  The distortion correction moves sources, so we return the distorted bounding box.
248 
249  \param[in] exposure Exposure to process
250  \param[in,out] sourceCat SourceCatalog; getX() and getY() will be used as inputs,
251  with distorted points in "centroid.distorted" field.
252  \return bounding box of distorted exposure
253  """
254  detector = exposure.getDetector()
255  pixToTanXYTransform = None
256  if detector is None:
257  self.log.warn("No detector associated with exposure; assuming null distortion")
258  else:
259  tanSys = detector.makeCameraSys(TAN_PIXELS)
260  pixToTanXYTransform = detector.getTransformMap().get(tanSys)
261 
262  if pixToTanXYTransform is None:
263  self.log.info("Null distortion correction")
264  for s in sourceCat:
265  s.set(self.centroidKey, s.getCentroid())
266  s.set(self.centroidErrKey, s.getCentroidErr())
267  s.set(self.centroidFlagKey, s.getCentroidFlag())
268  return exposure.getBBox()
269 
270  # Distort source positions
271  self.log.info("Applying distortion correction")
272  for s in sourceCat:
273  centroid = pixToTanXYTransform.forwardTransform(s.getCentroid())
274  s.set(self.centroidKey, centroid)
275  s.set(self.centroidErrKey, s.getCentroidErr())
276  s.set(self.centroidFlagKey, s.getCentroidFlag())
277 
278  # Get distorted image size so that astrometry_net does not clip.
279  bboxD = afwGeom.Box2D()
280  for corner in detector.getCorners(TAN_PIXELS):
281  bboxD.include(corner)
282 
283  if lsstDebug.Info(__name__).display:
284  frame = lsstDebug.Info(__name__).frame
285  pause = lsstDebug.Info(__name__).pause
286  displayAstrometry(sourceCat=sourceCat, distortedCentroidKey=self.centroidKey,
287  exposure=exposure, frame=frame, pause=pause)
288 
289  return afwGeom.Box2I(bboxD)
290 
291  @contextmanager
292  def distortionContext(self, sourceCat, exposure):
293  """!Context manager that applies and removes distortion
294 
295  We move the "centroid" definition in the catalog table to
296  point to the distorted positions. This is undone on exit
297  from the context.
298 
299  The input Wcs is taken to refer to the coordinate system
300  with the distortion correction applied, and hence no shift
301  is required when the sources are distorted. However, after
302  Wcs fitting, the Wcs is in the distorted frame so when the
303  distortion correction is removed, the Wcs needs to be
304  shifted to compensate.
305 
306  \param sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
307  \param exposure Exposure holding Wcs, an lsst.afw.image.ExposureF or D
308  \return bounding box of distorted exposure
309  """
310  # Apply distortion, if not already present in the exposure's WCS
311  if exposure.getWcs().hasDistortion():
312  yield exposure.getBBox()
313  else:
314  bbox = self.distort(sourceCat=sourceCat, exposure=exposure)
315  oldCentroidName = sourceCat.table.getCentroidDefinition()
316  sourceCat.table.defineCentroid(self.distortedName)
317  try:
318  yield bbox # Execute 'with' block, providing bbox to 'as' variable
319  finally:
320  # Un-apply distortion
321  sourceCat.table.defineCentroid(oldCentroidName)
322  x0, y0 = exposure.getXY0()
323  wcs = exposure.getWcs()
324  if wcs:
325  wcs.shiftReferencePixel(-bbox.getMinX() + x0, -bbox.getMinY() + y0)
326 
327  @pipeBase.timeMethod
328  def loadAndMatch(self, exposure, sourceCat, bbox=None):
329  """!Load reference objects overlapping an exposure and match to sources detected on that exposure
330 
331  @param[in] exposure exposure whose WCS is to be fit
332  @param[in] sourceCat catalog of sourceCat detected on the exposure (an lsst.afw.table.SourceCatalog)
333  @param[in] bbox bounding box go use for finding reference objects; if None, use exposure's bbox
334 
335  @return an lsst.pipe.base.Struct with these fields:
336  - refCat reference object catalog of objects that overlap the exposure (with some margin)
337  (an lsst::afw::table::SimpleCatalog)
338  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
339  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
340 
341  @note ignores config.forceKnownWcs
342  """
343  with self.distortionContext(sourceCat=sourceCat, exposure=exposure) as bbox:
344  if not self.solver:
345  self.makeSubtask("solver")
346 
347  astrom = self.solver.useKnownWcs(
348  sourceCat=sourceCat,
349  exposure=exposure,
350  bbox=bbox,
351  calculateSip=False,
352  )
353 
354  if astrom is None or astrom.getWcs() is None:
355  raise RuntimeError("Unable to solve astrometry")
356 
357  matches = astrom.getMatches()
358  matchMeta = astrom.getMatchMetadata()
359  if matches is None or len(matches) == 0:
360  raise RuntimeError("No astrometric matches")
361  self.log.info("%d astrometric matches" % (len(matches)))
362 
363  if self._display:
364  frame = lsstDebug.Info(__name__).frame
365  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
366  frame=frame, pause=False)
367 
368  return pipeBase.Struct(
369  refCat=astrom.refCat,
370  matches=matches,
371  matchMeta=matchMeta,
372  )
373 
374  @pipeBase.timeMethod
375  def _astrometry(self, sourceCat, exposure, bbox=None):
376  """!Solve astrometry to produce WCS
377 
378  \param[in] sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
379  \param[in,out] exposure Exposure to process, an lsst.afw.image.ExposureF or D; wcs is updated
380  \param[in] bbox Bounding box, or None to use exposure
381  \return a pipe.base.Struct with fields:
382  - refCat reference object catalog of objects that overlap the exposure (with some margin)
383  (an lsst::afw::table::SimpleCatalog)
384  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
385  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
386  """
387  self.log.info("Solving astrometry")
388  if bbox is None:
389  bbox = exposure.getBBox()
390 
391  if not self.solver:
392  self.makeSubtask("solver")
393 
394  astrom = self.solver.determineWcs(sourceCat=sourceCat, exposure=exposure, bbox=bbox)
395 
396  if astrom is None or astrom.getWcs() is None:
397  raise RuntimeError("Unable to solve astrometry")
398 
399  matches = astrom.getMatches()
400  matchMeta = astrom.getMatchMetadata()
401  if matches is None or len(matches) == 0:
402  raise RuntimeError("No astrometric matches")
403  self.log.info("%d astrometric matches" % (len(matches)))
404 
405  # Note that this is the Wcs for the provided positions, which may be distorted
406  exposure.setWcs(astrom.getWcs())
407 
408  if self._display:
409  frame = lsstDebug.Info(__name__).frame
410  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
411  frame=frame, pause=False)
412 
413  return pipeBase.Struct(
414  refCat=astrom.refCat,
415  matches=matches,
416  matchMeta=matchMeta,
417  )
418 
419  @pipeBase.timeMethod
420  def refitWcs(self, sourceCat, exposure, matches):
421  """!A final Wcs solution after matching and removing distortion
422 
423  Specifically, fitting the non-linear part, since the linear
424  part has been provided by the matching engine.
425 
426  @param sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
427  @param exposure Exposure of interest, an lsst.afw.image.ExposureF or D
428  @param matches Astrometric matches, as a list of lsst.afw.table.ReferenceMatch
429 
430  @return the resolved-Wcs object, or None if config.solver.calculateSip is False.
431  """
432  sip = None
433  if self.config.solver.calculateSip:
434  self.log.info("Refitting WCS")
435  origMatches = matches
436  wcs = exposure.getWcs()
437 
438  import lsstDebug
439  display = lsstDebug.Info(__name__).display
440  frame = lsstDebug.Info(__name__).frame
441  pause = lsstDebug.Info(__name__).pause
442 
443  def fitWcs(initialWcs, title=None):
444  """!Do the WCS fitting and display of the results"""
445  sip = makeCreateWcsWithSip(matches, initialWcs, self.config.solver.sipOrder)
446  resultWcs = sip.getNewWcs()
447  if display:
448  showAstrometry(exposure, resultWcs, origMatches, matches, frame=frame,
449  title=title, pause=pause)
450  return resultWcs, sip.getScatterOnSky()
451 
452  numRejected = 0
453  try:
454  for i in range(self.config.rejectIter):
455  wcs, scatter = fitWcs(wcs, title="Iteration %d" % i)
456 
457  ref = np.array([wcs.skyToPixel(m.first.getCoord()) for m in matches])
458  src = np.array([m.second.getCentroid() for m in matches])
459  diff = ref - src
460  rms = diff.std()
461  trimmed = []
462  for d, m in zip(diff, matches):
463  if np.all(np.abs(d) < self.config.rejectThresh*rms):
464  trimmed.append(m)
465  else:
466  numRejected += 1
467  if len(matches) == len(trimmed):
468  break
469  matches = trimmed
470 
471  # Final fit after rejection iterations
472  wcs, scatter = fitWcs(wcs, title="Final astrometry")
473 
474  except lsst.pex.exceptions.LengthError as e:
475  self.log.warn("Unable to fit SIP: %s" % e)
476 
477  self.log.info("Astrometric scatter: %f arcsec (%s non-linear terms, %d matches, %d rejected)" %
478  (scatter.asArcseconds(), "with" if wcs.hasDistortion() else "without",
479  len(matches), numRejected))
480  exposure.setWcs(wcs)
481 
482  # Apply WCS to sources
483  for index, source in enumerate(sourceCat):
484  sky = wcs.pixelToSky(source.getX(), source.getY())
485  source.setCoord(sky)
486  else:
487  self.log.warn("Not calculating a SIP solution; matches may be suspect")
488 
489  if self._display:
490  frame = lsstDebug.Info(__name__).frame
491  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
492  frame=frame, pause=False)
493 
494  return sip
495 
496 
497 def showAstrometry(exposure, wcs, allMatches, useMatches, frame=0, title=None, pause=False):
498  """!Show results of astrometry fitting
499 
500  \param exposure Image to display
501  \param wcs Astrometric solution
502  \param allMatches List of all astrometric matches (including rejects)
503  \param useMatches List of used astrometric matches
504  \param frame Frame number for display
505  \param title Title for display
506  \param pause Pause to allow viewing of the display and optional debugging?
507 
508  - Matches are shown in yellow if used in the Wcs solution, otherwise red
509  - +: Detected objects
510  - x: Catalogue objects
511  """
512  import lsst.afw.display.ds9 as ds9
513  ds9.mtv(exposure, frame=frame, title=title)
514 
515  useIndices = set(m.second.getId() for m in useMatches)
516 
517  radii = []
518  with ds9.Buffering():
519  for i, m in enumerate(allMatches):
520  x, y = m.second.getX(), m.second.getY()
521  pix = wcs.skyToPixel(m.first.getCoord())
522 
523  isUsed = m.second.getId() in useIndices
524  if isUsed:
525  radii.append(np.hypot(pix[0] - x, pix[1] - y))
526 
527  color = ds9.YELLOW if isUsed else ds9.RED
528 
529  ds9.dot("+", x, y, size=10, frame=frame, ctype=color)
530  ds9.dot("x", pix[0], pix[1], size=10, frame=frame, ctype=color)
531 
532  radii = np.array(radii)
533  print("<dr> = %.4g +- %.4g pixels [%d/%d matches]" % (radii.mean(), radii.std(),
534  len(useMatches), len(allMatches)))
535 
536  if pause:
537  import sys
538  while True:
539  try:
540  reply = input("Debugging? [p]db [q]uit; any other key to continue... ").strip()
541  except EOFError:
542  reply = ""
543 
544  if len(reply) > 1:
545  reply = reply[0]
546  if reply == "p":
547  import pdb
548  pdb.set_trace()
549  elif reply == "q":
550  sys.exit(1)
551  else:
552  break
def distortionContext
Context manager that applies and removes distortion.
def loadAndMatch
Load reference objects overlapping an exposure and match to sources detected on that exposure...
def run
Load reference objects, match sources and optionally fit a WCS.
def solve
Match with reference sources and calculate an astrometric solution.
def refitWcs
A final Wcs solution after matching and removing distortion.
Use astrometry.net to match input sources with a reference catalog and solve for the Wcs...
def showAstrometry
Show results of astrometry fitting.