lsst.meas.extensions.astrometryNet  14.0-1-g013352c+34
anetAstrometry.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 # Copyright 2008-2016 AURA/LSST.
4 #
5 # This product includes software developed by the
6 # LSST Project (http://www.lsst.org/).
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <http://www.lsstcorp.org/LegalNotices/>.
21 #
22 from __future__ import absolute_import, division, print_function
23 
24 __all__ = ["ANetAstrometryConfig", "ANetAstrometryTask", "showAstrometry"]
25 
26 from builtins import input
27 from builtins import zip
28 from builtins import range
29 from contextlib import contextmanager
30 
31 import numpy as np
32 
33 import lsstDebug
35 import lsst.afw.geom as afwGeom
36 from lsst.afw.cameraGeom import PIXELS, TAN_PIXELS
37 from lsst.afw.table import Point2DKey, CovarianceMatrix2fKey
38 import lsst.pex.config as pexConfig
39 import lsst.pipe.base as pipeBase
40 from lsst.meas.astrom import displayAstrometry
41 from lsst.meas.astrom.sip import makeCreateWcsWithSip
42 from .anetBasicAstrometry import ANetBasicAstrometryTask
43 
44 
45 class ANetAstrometryConfig(pexConfig.Config):
46  solver = pexConfig.ConfigurableField(
47  target=ANetBasicAstrometryTask,
48  doc="Basic astrometry solver",
49  )
50  forceKnownWcs = pexConfig.Field(dtype=bool, doc=(
51  "Assume that the input image's WCS is correct, without comparing it to any external reality." +
52  " (In contrast to using Astrometry.net). NOTE, if you set this, you probably also want to" +
53  " un-set 'solver.calculateSip'; otherwise we'll still try to find a TAN-SIP WCS starting " +
54  " from the existing WCS"), default=False)
55  rejectThresh = pexConfig.RangeField(dtype=float, default=3.0, doc="Rejection threshold for Wcs fitting",
56  min=0.0, inclusiveMin=False)
57  rejectIter = pexConfig.RangeField(dtype=int, default=3, doc="Rejection iterations for Wcs fitting",
58  min=0)
59 
60  @property
61  def refObjLoader(self):
62  """An alias, for a uniform interface with the standard AstrometryTask"""
63  return self.solver
64 
65  # \addtogroup LSST_task_documentation
66  # \{
67  # \page measAstrom_anetAstrometryTask
68  # \ref ANetAstrometryTask_ "ANetAstrometryTask"
69  # Use astrometry.net to match input sources with a reference catalog and solve for the Wcs
70  # \}
71 
72 
73 class ANetAstrometryTask(pipeBase.Task):
74  """!Use astrometry.net to match input sources with a reference catalog and solve for the Wcs
75 
76  @anchor ANetAstrometryTask_
77 
78  The actual matching and solving is done by the 'solver'; this Task
79  serves as a wrapper for taking into account the known optical distortion.
80 
81  \section pipe_tasks_astrometry_Contents Contents
82 
83  - \ref pipe_tasks_astrometry_Purpose
84  - \ref pipe_tasks_astrometry_Initialize
85  - \ref pipe_tasks_astrometry_IO
86  - \ref pipe_tasks_astrometry_Config
87  - \ref pipe_tasks_astrometry_Debug
88  - \ref pipe_tasks_astrometry_Example
89 
90  \section pipe_tasks_astrometry_Purpose Description
91 
92  \copybrief ANetAstrometryTask
93 
94  \section pipe_tasks_astrometry_Initialize Task initialisation
95 
96  \copydoc \_\_init\_\_
97 
98  \section pipe_tasks_astrometry_IO Invoking the Task
99 
100  \copydoc run
101 
102  \section pipe_tasks_astrometry_Config Configuration parameters
103 
104  See \ref ANetAstrometryConfig
105 
106  \section pipe_tasks_astrometry_Debug Debug variables
107 
108  The \link lsst.pipe.base.cmdLineTask.CmdLineTask command line task\endlink interface supports a
109  flag \c -d to import \b debug.py from your \c PYTHONPATH;
110  see \ref baseDebug for more about \b debug.py files.
111 
112  The available variables in ANetAstrometryTask are:
113  <DL>
114  <DT> \c display
115  <DD> If True call showAstrometry while iterating ANetAstrometryConfig.rejectIter times,
116  and also after converging; and call displayAstrometry after applying the distortion correction.
117  <DT> \c frame
118  <DD> ds9 frame to use in showAstrometry and displayAstrometry
119  <DT> \c pause
120  <DD> Pause after showAstrometry and displayAstrometry?
121  </DL>
122 
123  \section pipe_tasks_astrometry_Example A complete example of using ANetAstrometryTask
124 
125  See \ref pipe_tasks_photocal_Example.
126 
127  To investigate the \ref pipe_tasks_astrometry_Debug, put something like
128  \code{.py}
129  import lsstDebug
130  def DebugInfo(name):
131  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
132  if name in ("lsst.pipe.tasks.anetAstrometry", "lsst.pipe.tasks.anetBasicAstrometry"):
133  di.display = 1
134  di.frame = 1
135  di.pause = True
136 
137  return di
138 
139  lsstDebug.Info = DebugInfo
140  \endcode
141  into your debug.py file and run photoCalTask.py with the \c --debug flag.
142  """
143  ConfigClass = ANetAstrometryConfig
144 
145  def __init__(self, schema, refObjLoader=None, **kwds):
146  """!Create the astrometric calibration task. Most arguments are simply passed onto pipe.base.Task.
147 
148  \param schema An lsst::afw::table::Schema used to create the output lsst.afw.table.SourceCatalog
149  \param refObjLoader The AstrometryTask constructor requires a refObjLoader. In order to make this
150  task retargettable for AstrometryTask it needs to take the same arguments. This argument will be
151  ignored since it uses its own internal loader.
152  \param **kwds keyword arguments to be passed to the lsst.pipe.base.task.Task constructor
153 
154  A centroid field "centroid.distorted" (used internally during the Task's operation)
155  will be added to the schema.
156  """
157  pipeBase.Task.__init__(self, **kwds)
158  self.distortedName = "astrom_distorted"
159  self.centroidXKey = schema.addField(self.distortedName + "_x", type="D",
160  doc="centroid distorted for astrometry solver")
161  self.centroidYKey = schema.addField(self.distortedName + "_y", type="D",
162  doc="centroid distorted for astrometry solver")
163  self.centroidXErrKey = schema.addField(self.distortedName + "_xSigma", type="F",
164  doc="centroid distorted err for astrometry solver")
165  self.centroidYErrKey = schema.addField(self.distortedName + "_ySigma", type="F",
166  doc="centroid distorted err for astrometry solver")
167  self.centroidFlagKey = schema.addField(self.distortedName + "_flag", type="Flag",
168  doc="centroid distorted flag astrometry solver")
170  self.centroidErrKey = CovarianceMatrix2fKey((self.centroidXErrKey, self.centroidYErrKey))
171  # postpone making the solver subtask because it may not be needed and is expensive to create
172  self.solver = None
173 
174  @pipeBase.timeMethod
175  def run(self, exposure, sourceCat):
176  """!Load reference objects, match sources and optionally fit a WCS
177 
178  This is a thin layer around solve or loadAndMatch, depending on config.forceKnownWcs
179 
180  @param[in,out] exposure exposure whose WCS is to be fit
181  The following are read only:
182  - bbox
183  - calib (may be absent)
184  - filter (may be unset)
185  - detector (if wcs is pure tangent; may be absent)
186  The following are updated:
187  - wcs (the initial value is used as an initial guess, and is required)
188  @param[in] sourceCat catalog of sourceCat detected on the exposure (an lsst.afw.table.SourceCatalog)
189  @return an lsst.pipe.base.Struct with these fields:
190  - refCat reference object catalog of objects that overlap the exposure (with some margin)
191  (an lsst::afw::table::SimpleCatalog)
192  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
193  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
194  """
195  if self.config.forceKnownWcs:
196  return self.loadAndMatch(exposure=exposure, sourceCat=sourceCat)
197  else:
198  return self.solve(exposure=exposure, sourceCat=sourceCat)
199 
200  @pipeBase.timeMethod
201  def solve(self, exposure, sourceCat):
202  """!Match with reference sources and calculate an astrometric solution
203 
204  \param[in,out] exposure Exposure to calibrate; wcs is updated
205  \param[in] sourceCat catalog of measured sources (an lsst.afw.table.SourceCatalog)
206  \return a pipeBase.Struct with fields:
207  - refCat reference object catalog of objects that overlap the exposure (with some margin)
208  (an lsst::afw::table::SimpleCatalog)
209  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
210  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
211 
212  The reference catalog actually used is up to the implementation
213  of the solver; it will be manifested in the returned matches as
214  a list of lsst.afw.table.ReferenceMatch objects (\em i.e. of lsst.afw.table.Match with
215  \c first being of type lsst.afw.table.SimpleRecord and \c second type lsst.afw.table.SourceRecord ---
216  the reference object and matched object respectively).
217 
218  \note
219  The input sources have the centroid slot moved to a new column "centroid.distorted"
220  which has the positions corrected for any known optical distortion;
221  the 'solver' (which is instantiated in the 'astrometry' member)
222  should therefore simply use the centroids provided by calling
223  afw.table.Source.getCentroid() on the individual source records. This column \em must
224  be present in the sources table.
225 
226  \note ignores config.forceKnownWcs
227  """
228  with self.distortionContext(sourceCat=sourceCat, exposure=exposure) as bbox:
229  results = self._astrometry(sourceCat=sourceCat, exposure=exposure, bbox=bbox)
230 
231  if results.matches:
232  self.refitWcs(sourceCat=sourceCat, exposure=exposure, matches=results.matches)
233 
234  return results
235 
236  @pipeBase.timeMethod
237  def distort(self, sourceCat, exposure):
238  """!Calculate distorted source positions
239 
240  CCD images are often affected by optical distortion that makes
241  the astrometric solution higher order than linear. Unfortunately,
242  most (all?) matching algorithms require that the distortion be
243  small or zero, and so it must be removed. We do this by calculating
244  (un-)distorted positions, based on a known optical distortion model
245  in the Ccd.
246 
247  The distortion correction moves sources, so we return the distorted bounding box.
248 
249  \param[in] exposure Exposure to process
250  \param[in,out] sourceCat SourceCatalog; getX() and getY() will be used as inputs,
251  with distorted points in "centroid.distorted" field.
252  \return bounding box of distorted exposure
253  """
254  detector = exposure.getDetector()
255  pixToTanXYTransform = None
256  if detector is None:
257  self.log.warn("No detector associated with exposure; assuming null distortion")
258  else:
259  pixToTanXYTransform = detector.getTransform(PIXELS, TAN_PIXELS)
260 
261  if pixToTanXYTransform is None:
262  self.log.info("Null distortion correction")
263  for s in sourceCat:
264  s.set(self.centroidKey, s.getCentroid())
265  s.set(self.centroidErrKey, s.getCentroidErr())
266  s.set(self.centroidFlagKey, s.getCentroidFlag())
267  return exposure.getBBox()
268 
269  # Distort source positions
270  self.log.info("Applying distortion correction")
271  for s in sourceCat:
272  centroid = pixToTanXYTransform.forwardTransform(s.getCentroid())
273  s.set(self.centroidKey, centroid)
274  s.set(self.centroidErrKey, s.getCentroidErr())
275  s.set(self.centroidFlagKey, s.getCentroidFlag())
276 
277  # Get distorted image size so that astrometry_net does not clip.
278  bboxD = afwGeom.Box2D()
279  for corner in detector.getCorners(TAN_PIXELS):
280  bboxD.include(corner)
281 
282  if lsstDebug.Info(__name__).display:
283  frame = lsstDebug.Info(__name__).frame
284  pause = lsstDebug.Info(__name__).pause
285  displayAstrometry(sourceCat=sourceCat, distortedCentroidKey=self.centroidKey,
286  exposure=exposure, frame=frame, pause=pause)
287 
288  return afwGeom.Box2I(bboxD)
289 
290  @contextmanager
291  def distortionContext(self, sourceCat, exposure):
292  """!Context manager that applies and removes distortion
293 
294  We move the "centroid" definition in the catalog table to
295  point to the distorted positions. This is undone on exit
296  from the context.
297 
298  The input Wcs is taken to refer to the coordinate system
299  with the distortion correction applied, and hence no shift
300  is required when the sources are distorted. However, after
301  Wcs fitting, the Wcs is in the distorted frame so when the
302  distortion correction is removed, the Wcs needs to be
303  shifted to compensate.
304 
305  \param sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
306  \param exposure Exposure holding Wcs, an lsst.afw.image.ExposureF or D
307  \return bounding box of distorted exposure
308  """
309  # Apply distortion, if not already present in the exposure's WCS
310  if exposure.getWcs().hasDistortion():
311  yield exposure.getBBox()
312  else:
313  bbox = self.distort(sourceCat=sourceCat, exposure=exposure)
314  oldCentroidName = sourceCat.table.getCentroidDefinition()
315  sourceCat.table.defineCentroid(self.distortedName)
316  try:
317  yield bbox # Execute 'with' block, providing bbox to 'as' variable
318  finally:
319  # Un-apply distortion
320  sourceCat.table.defineCentroid(oldCentroidName)
321  x0, y0 = exposure.getXY0()
322  wcs = exposure.getWcs()
323  if wcs:
324  wcs.shiftReferencePixel(-bbox.getMinX() + x0, -bbox.getMinY() + y0)
325 
326  @pipeBase.timeMethod
327  def loadAndMatch(self, exposure, sourceCat, bbox=None):
328  """!Load reference objects overlapping an exposure and match to sources detected on that exposure
329 
330  @param[in] exposure exposure whose WCS is to be fit
331  @param[in] sourceCat catalog of sourceCat detected on the exposure (an lsst.afw.table.SourceCatalog)
332  @param[in] bbox bounding box go use for finding reference objects; if None, use exposure's bbox
333 
334  @return an lsst.pipe.base.Struct with these fields:
335  - refCat reference object catalog of objects that overlap the exposure (with some margin)
336  (an lsst::afw::table::SimpleCatalog)
337  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
338  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
339 
340  @note ignores config.forceKnownWcs
341  """
342  with self.distortionContext(sourceCat=sourceCat, exposure=exposure) as bbox:
343  if not self.solver:
344  self.makeSubtask("solver")
345 
346  astrom = self.solver.useKnownWcs(
347  sourceCat=sourceCat,
348  exposure=exposure,
349  bbox=bbox,
350  calculateSip=False,
351  )
352 
353  if astrom is None or astrom.getWcs() is None:
354  raise RuntimeError("Unable to solve astrometry")
355 
356  matches = astrom.getMatches()
357  matchMeta = astrom.getMatchMetadata()
358  if matches is None or len(matches) == 0:
359  raise RuntimeError("No astrometric matches")
360  self.log.info("%d astrometric matches" % (len(matches)))
361 
362  if self._display:
363  frame = lsstDebug.Info(__name__).frame
364  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
365  frame=frame, pause=False)
366 
367  return pipeBase.Struct(
368  refCat=astrom.refCat,
369  matches=matches,
370  matchMeta=matchMeta,
371  )
372 
373  @pipeBase.timeMethod
374  def _astrometry(self, sourceCat, exposure, bbox=None):
375  """!Solve astrometry to produce WCS
376 
377  \param[in] sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
378  \param[in,out] exposure Exposure to process, an lsst.afw.image.ExposureF or D; wcs is updated
379  \param[in] bbox Bounding box, or None to use exposure
380  \return a pipe.base.Struct with fields:
381  - refCat reference object catalog of objects that overlap the exposure (with some margin)
382  (an lsst::afw::table::SimpleCatalog)
383  - matches astrometric matches, a list of lsst.afw.table.ReferenceMatch
384  - matchMeta metadata about the field (an lsst.daf.base.PropertyList)
385  """
386  self.log.info("Solving astrometry")
387  if bbox is None:
388  bbox = exposure.getBBox()
389 
390  if not self.solver:
391  self.makeSubtask("solver")
392 
393  astrom = self.solver.determineWcs(sourceCat=sourceCat, exposure=exposure, bbox=bbox)
394 
395  if astrom is None or astrom.getWcs() is None:
396  raise RuntimeError("Unable to solve astrometry")
397 
398  matches = astrom.getMatches()
399  matchMeta = astrom.getMatchMetadata()
400  if matches is None or len(matches) == 0:
401  raise RuntimeError("No astrometric matches")
402  self.log.info("%d astrometric matches" % (len(matches)))
403 
404  # Note that this is the Wcs for the provided positions, which may be distorted
405  exposure.setWcs(astrom.getWcs())
406 
407  if self._display:
408  frame = lsstDebug.Info(__name__).frame
409  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
410  frame=frame, pause=False)
411 
412  return pipeBase.Struct(
413  refCat=astrom.refCat,
414  matches=matches,
415  matchMeta=matchMeta,
416  )
417 
418  @pipeBase.timeMethod
419  def refitWcs(self, sourceCat, exposure, matches):
420  """!A final Wcs solution after matching and removing distortion
421 
422  Specifically, fitting the non-linear part, since the linear
423  part has been provided by the matching engine.
424 
425  @param sourceCat Sources on exposure, an lsst.afw.table.SourceCatalog
426  @param exposure Exposure of interest, an lsst.afw.image.ExposureF or D
427  @param matches Astrometric matches, as a list of lsst.afw.table.ReferenceMatch
428 
429  @return the resolved-Wcs object, or None if config.solver.calculateSip is False.
430  """
431  sip = None
432  if self.config.solver.calculateSip:
433  self.log.info("Refitting WCS")
434  origMatches = matches
435  wcs = exposure.getWcs()
436 
437  import lsstDebug
438  display = lsstDebug.Info(__name__).display
439  frame = lsstDebug.Info(__name__).frame
440  pause = lsstDebug.Info(__name__).pause
441 
442  def fitWcs(initialWcs, title=None):
443  """!Do the WCS fitting and display of the results"""
444  sip = makeCreateWcsWithSip(matches, initialWcs, self.config.solver.sipOrder)
445  resultWcs = sip.getNewWcs()
446  if display:
447  showAstrometry(exposure, resultWcs, origMatches, matches, frame=frame,
448  title=title, pause=pause)
449  return resultWcs, sip.getScatterOnSky()
450 
451  numRejected = 0
452  try:
453  for i in range(self.config.rejectIter):
454  wcs, scatter = fitWcs(wcs, title="Iteration %d" % i)
455 
456  ref = np.array([wcs.skyToPixel(m.first.getCoord()) for m in matches])
457  src = np.array([m.second.getCentroid() for m in matches])
458  diff = ref - src
459  rms = diff.std()
460  trimmed = []
461  for d, m in zip(diff, matches):
462  if np.all(np.abs(d) < self.config.rejectThresh*rms):
463  trimmed.append(m)
464  else:
465  numRejected += 1
466  if len(matches) == len(trimmed):
467  break
468  matches = trimmed
469 
470  # Final fit after rejection iterations
471  wcs, scatter = fitWcs(wcs, title="Final astrometry")
472 
474  self.log.warn("Unable to fit SIP: %s" % e)
475 
476  self.log.info("Astrometric scatter: %f arcsec (%s non-linear terms, %d matches, %d rejected)" %
477  (scatter.asArcseconds(), "with" if wcs.hasDistortion() else "without",
478  len(matches), numRejected))
479  exposure.setWcs(wcs)
480 
481  # Apply WCS to sources
482  for index, source in enumerate(sourceCat):
483  sky = wcs.pixelToSky(source.getX(), source.getY())
484  source.setCoord(sky)
485  else:
486  self.log.warn("Not calculating a SIP solution; matches may be suspect")
487 
488  if self._display:
489  frame = lsstDebug.Info(__name__).frame
490  displayAstrometry(exposure=exposure, sourceCat=sourceCat, matches=matches,
491  frame=frame, pause=False)
492 
493  return sip
494 
495 
496 def showAstrometry(exposure, wcs, allMatches, useMatches, frame=0, title=None, pause=False):
497  """!Show results of astrometry fitting
498 
499  \param exposure Image to display
500  \param wcs Astrometric solution
501  \param allMatches List of all astrometric matches (including rejects)
502  \param useMatches List of used astrometric matches
503  \param frame Frame number for display
504  \param title Title for display
505  \param pause Pause to allow viewing of the display and optional debugging?
506 
507  - Matches are shown in yellow if used in the Wcs solution, otherwise red
508  - +: Detected objects
509  - x: Catalogue objects
510  """
511  import lsst.afw.display.ds9 as ds9
512  ds9.mtv(exposure, frame=frame, title=title)
513 
514  useIndices = set(m.second.getId() for m in useMatches)
515 
516  radii = []
517  with ds9.Buffering():
518  for i, m in enumerate(allMatches):
519  x, y = m.second.getX(), m.second.getY()
520  pix = wcs.skyToPixel(m.first.getCoord())
521 
522  isUsed = m.second.getId() in useIndices
523  if isUsed:
524  radii.append(np.hypot(pix[0] - x, pix[1] - y))
525 
526  color = ds9.YELLOW if isUsed else ds9.RED
527 
528  ds9.dot("+", x, y, size=10, frame=frame, ctype=color)
529  ds9.dot("x", pix[0], pix[1], size=10, frame=frame, ctype=color)
530 
531  radii = np.array(radii)
532  print("<dr> = %.4g +- %.4g pixels [%d/%d matches]" % (radii.mean(), radii.std(),
533  len(useMatches), len(allMatches)))
534 
535  if pause:
536  import sys
537  while True:
538  try:
539  reply = input("Debugging? [p]db [q]uit; any other key to continue... ").strip()
540  except EOFError:
541  reply = ""
542 
543  if len(reply) > 1:
544  reply = reply[0]
545  if reply == "p":
546  import pdb
547  pdb.set_trace()
548  elif reply == "q":
549  sys.exit(1)
550  else:
551  break
def loadAndMatch(self, exposure, sourceCat, bbox=None)
Load reference objects overlapping an exposure and match to sources detected on that exposure...
def solve(self, exposure, sourceCat)
Match with reference sources and calculate an astrometric solution.
def distort(self, sourceCat, exposure)
Calculate distorted source positions.
def showAstrometry(exposure, wcs, allMatches, useMatches, frame=0, title=None, pause=False)
Show results of astrometry fitting.
def refitWcs(self, sourceCat, exposure, matches)
A final Wcs solution after matching and removing distortion.
def _astrometry(self, sourceCat, exposure, bbox=None)
Solve astrometry to produce WCS.
def run(self, exposure, sourceCat)
Load reference objects, match sources and optionally fit a WCS.
def distortionContext(self, sourceCat, exposure)
Context manager that applies and removes distortion.
def __init__(self, schema, refObjLoader=None, kwds)
Create the astrometric calibration task.
Use astrometry.net to match input sources with a reference catalog and solve for the Wcs...