lsst.jointcal  master-gc935ebf72c+13
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 
9 import lsst.utils
10 import lsst.pex.config as pexConfig
11 import lsst.pipe.base as pipeBase
12 import lsst.afw.image as afwImage
13 import lsst.afw.geom as afwGeom
14 import lsst.afw.coord as afwCoord
15 import lsst.pex.exceptions as pexExceptions
16 import lsst.afw.table
17 
18 from lsst.meas.extensions.astrometryNet import LoadAstrometryNetObjectsTask
19 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
20 
21 from .dataIds import PerTractCcdDataIdContainer
22 
23 import lsst.jointcal
24 
25 __all__ = ["JointcalConfig", "JointcalTask"]
26 
27 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
28 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
29 
30 
31 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
32  """Subclass of TaskRunner for jointcalTask
33 
34  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
35  extracted from the command line (whereas most CmdLineTasks' run methods take
36  single dataRef, are are called repeatedly). This class transforms the processed
37  arguments generated by the ArgumentParser into the arguments expected by
38  Jointcal.run().
39 
40  See pipeBase.TaskRunner for more information.
41  """
42 
43  @staticmethod
44  def getTargetList(parsedCmd, **kwargs):
45  """
46  Return a list of tuples per tract, each containing (dataRefs, kwargs).
47 
48  Jointcal operates on lists of dataRefs simultaneously.
49  """
50  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
51  kwargs['butler'] = parsedCmd.butler
52 
53  # organize data IDs by tract
54  refListDict = {}
55  for ref in parsedCmd.id.refList:
56  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
57  # we call run() once with each tract
58  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
59  return result
60 
61  def __call__(self, args):
62  """
63  @param args Arguments for Task.run()
64 
65  @return
66  - None if self.doReturnResults is False
67  - A pipe.base.Struct containing these fields if self.doReturnResults is True:
68  - dataRef: the provided data references, with update post-fit WCS's.
69  """
70  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
71  dataRefList, kwargs = args
72  butler = kwargs.pop('butler')
73  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
74  result = task.run(dataRefList, **kwargs)
75  if self.doReturnResults:
76  return pipeBase.Struct(result=result)
77 
78 
79 class JointcalConfig(pexConfig.Config):
80  """Config for jointcalTask"""
81 
82  doAstrometry = pexConfig.Field(
83  doc="Fit astrometry and write the fitted result.",
84  dtype=bool,
85  default=True
86  )
87  doPhotometry = pexConfig.Field(
88  doc="Fit photometry and write the fitted result.",
89  dtype=bool,
90  default=True
91  )
92  coaddName = pexConfig.Field(
93  doc="Type of coadd, typically deep or goodSeeing",
94  dtype=str,
95  default="deep"
96  )
97  posError = pexConfig.Field(
98  doc="Constant term for error on position (in pixel unit)",
99  dtype=float,
100  default=0.02,
101  )
102  # TODO: DM-6885 matchCut should be an afw.geom.Angle
103  matchCut = pexConfig.Field(
104  doc="Matching radius between fitted and reference stars (arcseconds)",
105  dtype=float,
106  default=3.0,
107  )
108  minMeasurements = pexConfig.Field(
109  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
110  dtype=int,
111  default=2,
112  )
113  polyOrder = pexConfig.Field(
114  doc="Polynomial order for fitting distorsion",
115  dtype=int,
116  default=3,
117  )
118  astrometryModel = pexConfig.ChoiceField(
119  doc="Type of model to fit to astrometry",
120  dtype=str,
121  default="simplePoly",
122  allowed={"simplePoly": "One polynomial per ccd",
123  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
124  )
125  astrometryRefObjLoader = pexConfig.ConfigurableField(
126  target=LoadAstrometryNetObjectsTask,
127  doc="Reference object loader for astrometric fit",
128  )
129  photometryRefObjLoader = pexConfig.ConfigurableField(
130  target=LoadAstrometryNetObjectsTask,
131  doc="Reference object loader for photometric fit",
132  )
133  sourceSelector = sourceSelectorRegistry.makeField(
134  doc="How to select sources for cross-matching",
135  default="astrometry"
136  )
137 
138  def setDefaults(self):
139  sourceSelector = self.sourceSelector["astrometry"]
140  sourceSelector.setDefaults()
141  # don't want to lose existing flags, just add to them.
142  sourceSelector.badFlags.extend(["slot_Shape_flag"])
143  # This should be used to set the FluxField value in jointcal::JointcalControl
144  sourceSelector.sourceFluxType = 'Calib'
145 
146 
147 class JointcalTask(pipeBase.CmdLineTask):
148  """Jointly astrometrically (photometrically later) calibrate a group of images."""
149 
150  ConfigClass = JointcalConfig
151  RunnerClass = JointcalRunner
152  _DefaultName = "jointcal"
153 
154  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
155  """
156  Instantiate a JointcalTask.
157 
158  Parameters
159  ----------
160  butler : lsst.daf.persistence.Butler
161  The butler is passed to the refObjLoader constructor in case it is
162  needed. Ignored if the refObjLoader argument provides a loader directly.
163  Used to initialize the astrometry and photometry refObjLoaders.
164  profile_jointcal : bool
165  set to True to profile different stages of this jointcal run.
166  """
167  pipeBase.CmdLineTask.__init__(self, **kwargs)
168  self.profile_jointcal = profile_jointcal
169  self.makeSubtask("sourceSelector")
170  self.makeSubtask('astrometryRefObjLoader', butler=butler)
171  self.makeSubtask('photometryRefObjLoader', butler=butler)
172 
173  # To hold various computed metrics for use by tests
174  self.metrics = {}
175 
176  # We don't need to persist config and metadata at this stage.
177  # In this way, we don't need to put a specific entry in the camera mapper policy file
178  def _getConfigName(self):
179  return None
180 
181  def _getMetadataName(self):
182  return None
183 
184  @classmethod
185  def _makeArgumentParser(cls):
186  """Create an argument parser"""
187  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
188  parser.add_argument("--profile_jointcal", default=False, action="store_true",
189  help="Profile steps of jointcal separately.")
190  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
191  ContainerClass=PerTractCcdDataIdContainer)
192  return parser
193 
194  def _build_ccdImage(self, dataRef, associations, jointcalControl):
195  """
196  Extract the necessary things from this dataRef to add a new ccdImage.
197 
198  Parameters
199  ----------
200  dataRef : lsst.daf.persistence.ButlerDataRef
201  dataRef to extract info from.
202  associations : lsst.jointcal.Associations
203  object to add the info to, to construct a new CcdImage
204  jointcalControl : jointcal.JointcalControl
205  control object for associations management
206 
207  Returns
208  ------
209  namedtuple
210  wcs : lsst.afw.image.TanWcs
211  the TAN WCS of this image, read from the calexp
212  key : namedtuple
213  a key to identify this dataRef by its visit and ccd ids
214  filter : str
215  this calexp's filter
216  """
217  visit = dataRef.dataId["visit"]
218  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
219  calexp = dataRef.get("calexp", immediate=True)
220  visitInfo = calexp.getInfo().getVisitInfo()
221  ccdname = calexp.getDetector().getId()
222 
223  calib = calexp.getCalib()
224  tanWcs = calexp.getWcs()
225  bbox = calexp.getBBox()
226  filt = calexp.getInfo().getFilter().getName()
227 
228  goodSrc = self.sourceSelector.selectSources(src)
229 
230  if len(goodSrc.sourceCat) == 0:
231  self.log.warn("no stars selected in ", visit, ccdname)
232  return tanWcs
233  self.log.info("%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
234  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, calib,
235  visit, ccdname, jointcalControl)
236 
237  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
238  Key = collections.namedtuple('Key', ('visit', 'ccd'))
239  return Result(tanWcs, Key(visit, ccdname), filt)
240 
241  @pipeBase.timeMethod
242  def run(self, dataRefs, profile_jointcal=False):
243  """
244  Jointly calibrate the astrometry and photometry across a set of images.
245 
246  Parameters
247  ----------
248  dataRefs : list of lsst.daf.persistence.ButlerDataRef
249  List of data references to the exposures to be fit.
250  profile_jointcal : bool
251  Profile the individual steps of jointcal.
252 
253  Returns
254  -------
255  pipe.base.Struct
256  struct containing:
257  * dataRefs: the provided data references that were fit (with updated WCSs)
258  * oldWcsList: the original WCS from each dataRef
259  * metrics: dictionary of internally-computed metrics for testing/validation.
260  """
261  if len(dataRefs) == 0:
262  raise ValueError('Need a list of data references!')
263 
264  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
265  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
266  associations = lsst.jointcal.Associations()
267 
268  visit_ccd_to_dataRef = {}
269  oldWcsList = []
270  filters = []
271  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
272  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
273  for ref in dataRefs:
274  result = self._build_ccdImage(ref, associations, jointcalControl)
275  oldWcsList.append(result.wcs)
276  visit_ccd_to_dataRef[result.key] = ref
277  filters.append(result.filter)
278  filters = collections.Counter(filters)
279 
280  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
281  commonTangentPoint = lsst.afw.coord.averageCoord(centers)
282  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition())
283  associations.setCommonTangentPoint(commonTangentPoint.getPosition())
284 
285  # Use external reference catalogs handled by LSST stack mechanism
286  # Get the bounding box overlapping all associated images
287  # ==> This is probably a bad idea to do it this way <== To be improved
288  bbox = associations.getRaDecBBox()
289  center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
290  corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
291  radius = center.angularSeparation(corner).asRadians()
292 
293  # Get astrometry_net_data path
294  anDir = lsst.utils.getPackageDir('astrometry_net_data')
295  if anDir is None:
296  raise RuntimeError("astrometry_net_data is not setup")
297 
298  # Determine a default filter associated with the catalog. See DM-9093
299  defaultFilter = filters.most_common(1)[0][0]
300  self.log.debug("Using %s band for reference flux", defaultFilter)
301 
302  # TODO: need a better way to get the tract.
303  tract = dataRefs[0].dataId['tract']
304 
305  if self.config.doAstrometry:
306  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
307  name="Astrometry",
308  refObjLoader=self.astrometryRefObjLoader,
309  fit_function=self._fit_astrometry,
310  profile_jointcal=profile_jointcal,
311  tract=tract)
312  else:
313  astrometry = Astrometry(None, None, None)
314 
315  if self.config.doPhotometry:
316  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
317  name="Photometry",
318  refObjLoader=self.photometryRefObjLoader,
319  fit_function=self._fit_photometry,
320  profile_jointcal=profile_jointcal,
321  tract=tract)
322  else:
323  photometry = Photometry(None, None)
324 
325  load_cat_prof_file = 'jointcal_write_results.prof' if profile_jointcal else ''
326  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
327  self._write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
328 
329  return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.metrics)
330 
331  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
332  name="", refObjLoader=None, fit_function=None, tract=None,
333  profile_jointcal=False, match_cut=3.0):
334  """Load reference catalog, perform the fit, and return the result.
335 
336  Parameters
337  ----------
338  associations : lsst.jointcal.Associations
339  The star/reference star associations to fit.
340  defaultFilter : str
341  filter to load from reference catalog.
342  center : lsst.afw.coord.Coord
343  Center of field to load from reference catalog.
344  radius : lsst.afw.geom.Angle
345  On-sky radius to load from reference catalog.
346  name : str
347  Name of thing being fit: "Astrometry" or "Photometry".
348  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
349  Reference object loader to load from for fit.
350  fit_function : function
351  function to call to perform fit (takes associations object).
352  tract : str
353  Name of tract currently being fit.
354  profile_jointcal : bool, optional
355  Separately profile the fitting step.
356  match_cut : float, optional
357  Radius in arcseconds to find cross-catalog matches to during
358  associations.associateCatalogs.
359 
360  Returns
361  -------
362  Result of `fit_function()`
363  """
364  self.log.info("====== Now processing %s...", name)
365  # TODO: this should not print "trying to invert a singular transformation:"
366  # if it does that, something's not right about the WCS...
367  associations.associateCatalogs(match_cut)
368  self.metrics['associated%sFittedStars' % name] = associations.fittedStarListSize()
369 
370  skyCircle = refObjLoader.loadSkyCircle(center,
371  afwGeom.Angle(radius, afwGeom.radians),
372  defaultFilter)
373  associations.collectRefStars(skyCircle.refCat,
374  self.config.matchCut*afwGeom.arcseconds,
375  skyCircle.fluxField)
376  self.metrics['collected%sRefStars' % name] = associations.refStarListSize()
377 
378  associations.selectFittedStars(self.config.minMeasurements)
379  self._check_star_lists(associations, name)
380  self.metrics['selected%sRefStars' % name] = associations.refStarListSize()
381  self.metrics['selected%sFittedStars' % name] = associations.fittedStarListSize()
382  self.metrics['selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
383 
384  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
385  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
386  result = fit_function(associations)
387  # TODO: not clear that this is really needed any longer?
388  # TODO: makeResTuple should at least be renamed, if we do want to keep that big data-dump around.
389  # Fill reference and measurement n-tuples for each tract
390  tupleName = "{}_res_{}.list".format(name, tract)
391  result.fit.makeResTuple(tupleName)
392 
393  return result
394 
395  def _check_star_lists(self, associations, name):
396  # TODO: these should be len(blah), but we need this properly wrapped first.
397  if associations.nCcdImagesValidForFit() == 0:
398  raise RuntimeError('No images in the ccdImageList!')
399  if associations.fittedStarListSize() == 0:
400  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
401  if associations.refStarListSize() == 0:
402  raise RuntimeError('No stars in the {} reference star list!'.format(name))
403 
404  def _fit_photometry(self, associations):
405  """
406  Fit the photometric data.
407 
408  Parameters
409  ----------
410  associations : lsst.jointcal.Associations
411  The star/reference star associations to fit.
412 
413  Returns
414  -------
415  namedtuple
416  fit : lsst.jointcal.PhotometryFit
417  The photometric fitter used to perform the fit.
418  model : lsst.jointcal.PhotometryModel
419  The photometric model that was fit.
420  """
421 
422  self.log.info("=== Starting photometric fitting...")
423  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
424 
425  fit = lsst.jointcal.PhotometryFit(associations, model)
426  fit.minimize("Model")
427  chi2 = fit.computeChi2()
428  self.log.info(str(chi2))
429  fit.minimize("Fluxes")
430  chi2 = fit.computeChi2()
431  self.log.info(str(chi2))
432  fit.minimize("Model Fluxes")
433  chi2 = fit.computeChi2()
434  self.log.info("Fit completed with %s", str(chi2))
435 
436  self.metrics['photometryFinalChi2'] = chi2.chi2
437  self.metrics['photometryFinalNdof'] = chi2.ndof
438  return Photometry(fit, model)
439 
440  def _fit_astrometry(self, associations):
441  """
442  Fit the astrometric data.
443 
444  Parameters
445  ----------
446  associations : lsst.jointcal.Associations
447  The star/reference star associations to fit.
448 
449  Returns
450  -------
451  namedtuple
452  fit : lsst.jointcal.AstrometryFit
453  The astrometric fitter used to perform the fit.
454  model : lsst.jointcal.AstrometryModel
455  The astrometric model that was fit.
456  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
457  The model for the sky to tangent plane projection that was used in the fit.
458  """
459 
460  self.log.info("=== Starting astrometric fitting...")
461 
462  associations.deprojectFittedStars()
463 
464  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
465  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
466  # them so carefully?
467  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
468 
469  if self.config.astrometryModel == "constrainedPoly":
470  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
471  sky_to_tan_projection, True, 0)
472  elif self.config.astrometryModel == "simplePoly":
473  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
474  sky_to_tan_projection,
475  True, 0, self.config.polyOrder)
476 
477  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
478  fit.minimize("Distortions")
479  chi2 = fit.computeChi2()
480  self.log.info(str(chi2))
481  fit.minimize("Positions")
482  chi2 = fit.computeChi2()
483  self.log.info(str(chi2))
484  fit.minimize("Distortions Positions")
485  chi2 = fit.computeChi2()
486  self.log.info(str(chi2))
487 
488  max_steps = 20
489  for i in range(max_steps):
490  r = fit.minimize("Distortions Positions", 5) # outliers removal at 5 sigma.
491  chi2 = fit.computeChi2()
492  self.log.info(str(chi2))
493  if r == 0:
494  self.log.debug("""fit has converged - no more outliers - redo minimixation\
495  one more time in case we have lost accuracy in rank update""")
496  # Redo minimization one more time in case we have lost accuracy in rank update
497  r = fit.minimize("Distortions Positions", 5) # outliers removal at 5 sigma.
498  chi2 = fit.computeChi2()
499  self.log.info("Fit completed with: %s", str(chi2))
500  break
501  elif r == 2:
502  self.log.warn("minimization failed")
503  elif r == 1:
504  self.log.warn("still some ouliers but chi2 increases - retry")
505  else:
506  break
507  self.log.error("unxepected return code from minimize")
508  else:
509  self.log.error("astrometry failed to converge after %d steps", max_steps)
510 
511  self.metrics['astrometryFinalChi2'] = chi2.chi2
512  self.metrics['astrometryFinalNdof'] = chi2.ndof
513 
514  return Astrometry(fit, model, sky_to_tan_projection)
515 
516  def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef):
517  """
518  Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef.
519 
520  Parameters
521  ----------
522  associations : lsst.jointcal.Associations
523  The star/reference star associations to fit.
524  astrom_model : lsst.jointcal.AstrometryModel
525  The astrometric model that was fit.
526  photom_model : lsst.jointcal.PhotometryModel
527  The photometric model that was fit.
528  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
529  dict of ccdImage identifiers to dataRefs that were fit
530  """
531 
532  ccdImageList = associations.getCcdImageList()
533  for ccdImage in ccdImageList:
534  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
535  ccd = ccdImage.ccdId
536  visit = ccdImage.visit
537  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
538  exp = afwImage.ExposureI(0, 0)
539  if self.config.doAstrometry:
540  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
541  tanSip = astrom_model.produceSipWcs(ccdImage)
542  tanWcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
543  exp.setWcs(tanWcs)
544  if self.config.doPhotometry:
545  self.log.info("Updating Calib for visit: %d, ccd: %d", visit, ccd)
546  # start with the original calib saved to the ccdImage
547  fluxMag0, fluxMag0Sigma = ccdImage.getCalib().getFluxMag0()
548  exp.getCalib().setFluxMag0(fluxMag0*photom_model.photomFactor(ccdImage), fluxMag0Sigma)
549  try:
550  dataRef.put(exp, 'wcs')
551  except pexExceptions.Exception as e:
552  self.log.fatal('Failed to write updated Wcs and Calib: %s', str(e))
553  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometric response model which has a single photometric factor per CcdImage.