lsst.jointcal  14.0-15-gc2c14a3+1
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.afw.coord as afwCoord
16 import lsst.pex.exceptions as pexExceptions
17 import lsst.afw.table
19 
20 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
21 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
22 
23 from .dataIds import PerTractCcdDataIdContainer
24 
25 import lsst.jointcal
26 from lsst.jointcal import MinimizeResult
27 
28 __all__ = ["JointcalConfig", "JointcalTask"]
29 
30 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
31 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
32 
33 
34 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
35  """Subclass of TaskRunner for jointcalTask
36 
37  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
38  extracted from the command line (whereas most CmdLineTasks' run methods take
39  single dataRef, are are called repeatedly). This class transforms the processed
40  arguments generated by the ArgumentParser into the arguments expected by
41  Jointcal.run().
42 
43  See pipeBase.TaskRunner for more information.
44  """
45 
46  @staticmethod
47  def getTargetList(parsedCmd, **kwargs):
48  """
49  Return a list of tuples per tract, each containing (dataRefs, kwargs).
50 
51  Jointcal operates on lists of dataRefs simultaneously.
52  """
53  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
54  kwargs['butler'] = parsedCmd.butler
55 
56  # organize data IDs by tract
57  refListDict = {}
58  for ref in parsedCmd.id.refList:
59  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
60  # we call run() once with each tract
61  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
62  return result
63 
64  def __call__(self, args):
65  """
66  @param args Arguments for Task.run()
67 
68  @return
69  - A pipe.base.Struct containing these fields if self.doReturnResults is False:
70  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
71  - A pipe.base.Struct containing these fields if self.doReturnResults is True:
72  - ``result``: the result of calling jointcal.run()
73  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
74  """
75  exitStatus = 0 # exit status for shell
76 
77  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
78  dataRefList, kwargs = args
79  butler = kwargs.pop('butler')
80  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
81  result = None
82  try:
83  result = task.run(dataRefList, **kwargs)
84  exitStatus = result.exitStatus
85  except Exception as e: # catch everything, sort it out later.
86  if self.doRaise:
87  raise e
88  else:
89  exitStatus = 1
90  eName = type(e).__name__
91  tract = dataRefList[0].dataId['tract']
92  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
93 
94  if self.doReturnResults:
95  return pipeBase.Struct(result=result, exitStatus=exitStatus)
96  else:
97  return pipeBase.Struct(exitStatus=exitStatus)
98 
99 
100 class JointcalConfig(pexConfig.Config):
101  """Config for jointcalTask"""
102 
103  doAstrometry = pexConfig.Field(
104  doc="Fit astrometry and write the fitted result.",
105  dtype=bool,
106  default=True
107  )
108  doPhotometry = pexConfig.Field(
109  doc="Fit photometry and write the fitted result.",
110  dtype=bool,
111  default=True
112  )
113  coaddName = pexConfig.Field(
114  doc="Type of coadd, typically deep or goodSeeing",
115  dtype=str,
116  default="deep"
117  )
118  posError = pexConfig.Field(
119  doc="Constant term for error on position (in pixel unit)",
120  dtype=float,
121  default=0.02,
122  )
123  # TODO: DM-6885 matchCut should be an afw.geom.Angle
124  matchCut = pexConfig.Field(
125  doc="Matching radius between fitted and reference stars (arcseconds)",
126  dtype=float,
127  default=3.0,
128  )
129  minMeasurements = pexConfig.Field(
130  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
131  dtype=int,
132  default=2,
133  )
134  astrometrySimpleDegree = pexConfig.Field(
135  doc="Polynomial degree for fitting the simple astrometry model.",
136  dtype=int,
137  default=3,
138  )
139  astrometryChipDegree = pexConfig.Field(
140  doc="Degree of the per-chip transform for the constrained astrometry model.",
141  dtype=int,
142  default=2,
143  )
144  astrometryVisitDegree = pexConfig.Field(
145  doc="Degree of the per-visit transform for the constrained astrometry model.",
146  dtype=int,
147  default=3,
148  )
149  astrometryModel = pexConfig.ChoiceField(
150  doc="Type of model to fit to astrometry",
151  dtype=str,
152  default="simplePoly",
153  allowed={"simplePoly": "One polynomial per ccd",
154  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
155  )
156  photometryModel = pexConfig.ChoiceField(
157  doc="Type of model to fit to photometry",
158  dtype=str,
159  default="simple",
160  allowed={"simple": "One constant zeropoint per ccd and visit",
161  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
162  )
163  photometryVisitDegree = pexConfig.Field(
164  doc="Degree of the per-visit polynomial transform for the constrained photometry model.",
165  dtype=int,
166  default=7,
167  )
168  astrometryRefObjLoader = pexConfig.ConfigurableField(
169  target=LoadIndexedReferenceObjectsTask,
170  doc="Reference object loader for astrometric fit",
171  )
172  photometryRefObjLoader = pexConfig.ConfigurableField(
173  target=LoadIndexedReferenceObjectsTask,
174  doc="Reference object loader for photometric fit",
175  )
176  sourceSelector = sourceSelectorRegistry.makeField(
177  doc="How to select sources for cross-matching",
178  default="astrometry"
179  )
180  writeChi2ContributionFiles = pexConfig.Field(
181  dtype=bool,
182  doc="Write initial/final fit files containing the contributions to chi2.",
183  default=False
184  )
185 
186  def setDefaults(self):
187  sourceSelector = self.sourceSelector["astrometry"]
188  sourceSelector.setDefaults()
189  # don't want to lose existing flags, just add to them.
190  sourceSelector.badFlags.extend(["slot_Shape_flag"])
191  # This should be used to set the FluxField value in jointcal::JointcalControl
192  sourceSelector.sourceFluxType = 'Calib'
193 
194 
195 class JointcalTask(pipeBase.CmdLineTask):
196  """Jointly astrometrically (photometrically later) calibrate a group of images."""
197 
198  ConfigClass = JointcalConfig
199  RunnerClass = JointcalRunner
200  _DefaultName = "jointcal"
201 
202  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
203  """
204  Instantiate a JointcalTask.
205 
206  Parameters
207  ----------
208  butler : lsst.daf.persistence.Butler
209  The butler is passed to the refObjLoader constructor in case it is
210  needed. Ignored if the refObjLoader argument provides a loader directly.
211  Used to initialize the astrometry and photometry refObjLoaders.
212  profile_jointcal : bool
213  set to True to profile different stages of this jointcal run.
214  """
215  pipeBase.CmdLineTask.__init__(self, **kwargs)
216  self.profile_jointcal = profile_jointcal
217  self.makeSubtask("sourceSelector")
218  if self.config.doAstrometry:
219  self.makeSubtask('astrometryRefObjLoader', butler=butler)
220  if self.config.doPhotometry:
221  self.makeSubtask('photometryRefObjLoader', butler=butler)
222 
223  # To hold various computed metrics for use by tests
224  self.metrics = {}
225 
226  # We don't need to persist config and metadata at this stage.
227  # In this way, we don't need to put a specific entry in the camera mapper policy file
228  def _getConfigName(self):
229  return None
230 
231  def _getMetadataName(self):
232  return None
233 
234  @classmethod
235  def _makeArgumentParser(cls):
236  """Create an argument parser"""
237  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
238  parser.add_argument("--profile_jointcal", default=False, action="store_true",
239  help="Profile steps of jointcal separately.")
240  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
241  ContainerClass=PerTractCcdDataIdContainer)
242  return parser
243 
244  def _build_ccdImage(self, dataRef, associations, jointcalControl):
245  """
246  Extract the necessary things from this dataRef to add a new ccdImage.
247 
248  Parameters
249  ----------
250  dataRef : lsst.daf.persistence.ButlerDataRef
251  dataRef to extract info from.
252  associations : lsst.jointcal.Associations
253  object to add the info to, to construct a new CcdImage
254  jointcalControl : jointcal.JointcalControl
255  control object for associations management
256 
257  Returns
258  ------
259  namedtuple
260  wcs : lsst.afw.image.TanWcs
261  the TAN WCS of this image, read from the calexp
262  key : namedtuple
263  a key to identify this dataRef by its visit and ccd ids
264  filter : str
265  this calexp's filter
266  """
267  if "visit" in dataRef.dataId.keys():
268  visit = dataRef.dataId["visit"]
269  else:
270  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
271 
272  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
273 
274  visitInfo = dataRef.get('calexp_visitInfo')
275  detector = dataRef.get('calexp_detector')
276  ccdId = detector.getId()
277  calib = dataRef.get('calexp_calib')
278  tanWcs = dataRef.get('calexp_wcs')
279  bbox = dataRef.get('calexp_bbox')
280  filt = dataRef.get('calexp_filter')
281  filterName = filt.getName()
282  fluxMag0 = calib.getFluxMag0()
283  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
284 
285  goodSrc = self.sourceSelector.selectSources(src)
286 
287  if len(goodSrc.sourceCat) == 0:
288  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
289  else:
290  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
291  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
292  visit, ccdId, jointcalControl)
293 
294  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
295  Key = collections.namedtuple('Key', ('visit', 'ccd'))
296  return Result(tanWcs, Key(visit, ccdId), filterName)
297 
298  @pipeBase.timeMethod
299  def run(self, dataRefs, profile_jointcal=False):
300  """
301  Jointly calibrate the astrometry and photometry across a set of images.
302 
303  Parameters
304  ----------
305  dataRefs : list of lsst.daf.persistence.ButlerDataRef
306  List of data references to the exposures to be fit.
307  profile_jointcal : bool
308  Profile the individual steps of jointcal.
309 
310  Returns
311  -------
312  pipe.base.Struct
313  struct containing:
314  * dataRefs: the provided data references that were fit (with updated WCSs)
315  * oldWcsList: the original WCS from each dataRef
316  * metrics: dictionary of internally-computed metrics for testing/validation.
317  """
318  if len(dataRefs) == 0:
319  raise ValueError('Need a non-empty list of data references!')
320 
321  exitStatus = 0 # exit status for shell
322 
323  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
324  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
325  associations = lsst.jointcal.Associations()
326 
327  visit_ccd_to_dataRef = {}
328  oldWcsList = []
329  filters = []
330  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
331  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
332  # We need the bounding-box of the focal plane for photometry visit models.
333  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
334  camera = dataRefs[0].get('camera', immediate=True)
335  self.focalPlaneBBox = camera.getFpBBox()
336  for ref in dataRefs:
337  result = self._build_ccdImage(ref, associations, jointcalControl)
338  oldWcsList.append(result.wcs)
339  visit_ccd_to_dataRef[result.key] = ref
340  filters.append(result.filter)
341  filters = collections.Counter(filters)
342 
343  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
344  commonTangentPoint = lsst.afw.coord.averageCoord(centers)
345  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition())
346  associations.setCommonTangentPoint(commonTangentPoint.getPosition())
347 
348  # Use external reference catalogs handled by LSST stack mechanism
349  # Get the bounding box overlapping all associated images
350  # ==> This is probably a bad idea to do it this way <== To be improved
351  bbox = associations.getRaDecBBox()
352  center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
353  corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
354  radius = center.angularSeparation(corner).asRadians()
355 
356  # Get astrometry_net_data path
357  anDir = lsst.utils.getPackageDir('astrometry_net_data')
358  if anDir is None:
359  raise RuntimeError("astrometry_net_data is not setup")
360 
361  # Determine a default filter associated with the catalog. See DM-9093
362  defaultFilter = filters.most_common(1)[0][0]
363  self.log.debug("Using %s band for reference flux", defaultFilter)
364 
365  # TODO: need a better way to get the tract.
366  tract = dataRefs[0].dataId['tract']
367 
368  if self.config.doAstrometry:
369  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
370  name="Astrometry",
371  refObjLoader=self.astrometryRefObjLoader,
372  fit_function=self._fit_astrometry,
373  profile_jointcal=profile_jointcal,
374  tract=tract)
375  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
376  else:
377  astrometry = Astrometry(None, None, None)
378 
379  if self.config.doPhotometry:
380  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
381  name="Photometry",
382  refObjLoader=self.photometryRefObjLoader,
383  fit_function=self._fit_photometry,
384  profile_jointcal=profile_jointcal,
385  tract=tract,
386  filters=filters,
387  reject_bad_fluxes=True)
388  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
389  else:
390  photometry = Photometry(None, None)
391 
392  return pipeBase.Struct(dataRefs=dataRefs,
393  oldWcsList=oldWcsList,
394  metrics=self.metrics,
395  exitStatus=exitStatus)
396 
397  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
398  name="", refObjLoader=None, filters=[], fit_function=None,
399  tract=None, profile_jointcal=False, match_cut=3.0,
400  reject_bad_fluxes=False):
401  """Load reference catalog, perform the fit, and return the result.
402 
403  Parameters
404  ----------
405  associations : lsst.jointcal.Associations
406  The star/reference star associations to fit.
407  defaultFilter : str
408  filter to load from reference catalog.
409  center : lsst.afw.coord.Coord
410  Center of field to load from reference catalog.
411  radius : lsst.afw.geom.Angle
412  On-sky radius to load from reference catalog.
413  name : str
414  Name of thing being fit: "Astrometry" or "Photometry".
415  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
416  Reference object loader to load from for fit.
417  filters : list of str, optional
418  List of filters to load from the reference catalog.
419  fit_function : function
420  function to call to perform fit (takes associations object).
421  tract : str
422  Name of tract currently being fit.
423  profile_jointcal : bool, optional
424  Separately profile the fitting step.
425  match_cut : float, optional
426  Radius in arcseconds to find cross-catalog matches to during
427  associations.associateCatalogs.
428  reject_bad_fluxes : bool, optional
429  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
430 
431  Returns
432  -------
433  Result of `fit_function()`
434  """
435  self.log.info("====== Now processing %s...", name)
436  # TODO: this should not print "trying to invert a singular transformation:"
437  # if it does that, something's not right about the WCS...
438  associations.associateCatalogs(match_cut)
439  self.metrics['associated%sFittedStars' % name] = associations.fittedStarListSize()
440 
441  skyCircle = refObjLoader.loadSkyCircle(center,
442  afwGeom.Angle(radius, afwGeom.radians),
443  defaultFilter)
444 
445  # Need memory contiguity to get reference filters as a vector.
446  if not skyCircle.refCat.isContiguous():
447  refCat = skyCircle.refCat.copy(deep=True)
448  else:
449  refCat = skyCircle.refCat
450 
451  # load the reference catalog fluxes.
452  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
453  refFluxes = {}
454  refFluxErrs = {}
455  for filt in filters:
456  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
457  refFluxes[filt] = refCat.get(filtKeys[0])
458  refFluxErrs[filt] = refCat.get(filtKeys[1])
459 
460  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
461  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
462  self.metrics['collected%sRefStars' % name] = associations.refStarListSize()
463 
464  associations.selectFittedStars(self.config.minMeasurements)
465  self._check_star_lists(associations, name)
466  self.metrics['selected%sRefStars' % name] = associations.refStarListSize()
467  self.metrics['selected%sFittedStars' % name] = associations.fittedStarListSize()
468  self.metrics['selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
469 
470  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
471  dataName = "{}_{}".format(tract, defaultFilter)
472  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
473  result = fit_function(associations, dataName)
474  # TODO DM-12446: turn this into a "butler save" somehow.
475  # Save reference and measurement chi2 contributions for this data
476  if self.config.writeChi2ContributionFiles:
477  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
478  result.fit.saveChi2Contributions(baseName)
479 
480  return result
481 
482  def _check_star_lists(self, associations, name):
483  # TODO: these should be len(blah), but we need this properly wrapped first.
484  if associations.nCcdImagesValidForFit() == 0:
485  raise RuntimeError('No images in the ccdImageList!')
486  if associations.fittedStarListSize() == 0:
487  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
488  if associations.refStarListSize() == 0:
489  raise RuntimeError('No stars in the {} reference star list!'.format(name))
490 
491  def _fit_photometry(self, associations, dataName=None):
492  """
493  Fit the photometric data.
494 
495  Parameters
496  ----------
497  associations : lsst.jointcal.Associations
498  The star/reference star associations to fit.
499  dataName : str
500  Name of the data being processed (e.g. "1234_HSC-Y"), for
501  identifying debugging files.
502 
503  Returns
504  -------
505  namedtuple
506  fit : lsst.jointcal.PhotometryFit
507  The photometric fitter used to perform the fit.
508  model : lsst.jointcal.PhotometryModel
509  The photometric model that was fit.
510  """
511  self.log.info("=== Starting photometric fitting...")
512 
513  # TODO: should use pex.config.RegistryField here (see DM-9195)
514  if self.config.photometryModel == "constrained":
515  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
516  self.focalPlaneBBox,
517  visitDegree=self.config.photometryVisitDegree)
518  elif self.config.photometryModel == "simple":
519  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
520 
521  fit = lsst.jointcal.PhotometryFit(associations, model)
522  chi2 = fit.computeChi2()
523  # TODO DM-12446: turn this into a "butler save" somehow.
524  # Save reference and measurement chi2 contributions for this data
525  if self.config.writeChi2ContributionFiles:
526  baseName = "Photometry_initial_chi2-{}.csv".format(dataName)
527  fit.saveChi2Contributions(baseName)
528 
529  if not np.isfinite(chi2.chi2):
530  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
531  self.log.info("Initialized: %s", str(chi2))
532  fit.minimize("Model")
533  chi2 = fit.computeChi2()
534  self.log.info(str(chi2))
535  fit.minimize("Fluxes")
536  chi2 = fit.computeChi2()
537  self.log.info(str(chi2))
538  fit.minimize("Model Fluxes")
539  chi2 = fit.computeChi2()
540  if not np.isfinite(chi2.chi2):
541  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
542  self.log.info("Fit prepared with %s", str(chi2))
543 
544  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
545 
546  self.metrics['photometryFinalChi2'] = chi2.chi2
547  self.metrics['photometryFinalNdof'] = chi2.ndof
548  return Photometry(fit, model)
549 
550  def _fit_astrometry(self, associations, dataName=None):
551  """
552  Fit the astrometric data.
553 
554  Parameters
555  ----------
556  associations : lsst.jointcal.Associations
557  The star/reference star associations to fit.
558  dataName : str
559  Name of the data being processed (e.g. "1234_HSC-Y"), for
560  identifying debugging files.
561 
562  Returns
563  -------
564  namedtuple
565  fit : lsst.jointcal.AstrometryFit
566  The astrometric fitter used to perform the fit.
567  model : lsst.jointcal.AstrometryModel
568  The astrometric model that was fit.
569  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
570  The model for the sky to tangent plane projection that was used in the fit.
571  """
572 
573  self.log.info("=== Starting astrometric fitting...")
574 
575  associations.deprojectFittedStars()
576 
577  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
578  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
579  # them so carefully?
580  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
581 
582  if self.config.astrometryModel == "constrainedPoly":
583  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
584  sky_to_tan_projection, True, 0,
585  chipDegree=self.config.astrometryChipDegree,
586  visitDegree=self.config.astrometryVisitDegree)
587  elif self.config.astrometryModel == "simplePoly":
588  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
589  sky_to_tan_projection,
590  True, 0, self.config.astrometrySimpleDegree)
591 
592  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
593  chi2 = fit.computeChi2()
594  # TODO DM-12446: turn this into a "butler save" somehow.
595  # Save reference and measurement chi2 contributions for this data
596  if self.config.writeChi2ContributionFiles:
597  baseName = "Astrometry_initial_chi2-{}.csv".format(dataName)
598  fit.saveChi2Contributions(baseName)
599 
600  if not np.isfinite(chi2.chi2):
601  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
602  self.log.info("Initialized: %s", str(chi2))
603  fit.minimize("Distortions")
604  chi2 = fit.computeChi2()
605  self.log.info(str(chi2))
606  fit.minimize("Positions")
607  chi2 = fit.computeChi2()
608  self.log.info(str(chi2))
609  fit.minimize("Distortions Positions")
610  chi2 = fit.computeChi2()
611  self.log.info(str(chi2))
612  if not np.isfinite(chi2.chi2):
613  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
614  self.log.info("Fit prepared with %s", str(chi2))
615 
616  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
617 
618  self.metrics['astrometryFinalChi2'] = chi2.chi2
619  self.metrics['astrometryFinalNdof'] = chi2.ndof
620 
621  return Astrometry(fit, model, sky_to_tan_projection)
622 
623  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
624  """Run fit.minimize up to max_steps times, returning the final chi2."""
625 
626  for i in range(max_steps):
627  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
628  chi2 = fit.computeChi2()
629  if not np.isfinite(chi2.chi2):
630  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
631  self.log.info(str(chi2))
632  if r == MinimizeResult.Converged:
633  self.log.debug("fit has converged - no more outliers - redo minimixation"
634  "one more time in case we have lost accuracy in rank update")
635  # Redo minimization one more time in case we have lost accuracy in rank update
636  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
637  chi2 = fit.computeChi2()
638  self.log.info("Fit completed with: %s", str(chi2))
639  break
640  elif r == MinimizeResult.Chi2Increased:
641  self.log.warn("still some ouliers but chi2 increases - retry")
642  elif r == MinimizeResult.Failed:
643  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
644  else:
645  raise RuntimeError("Unxepected return code from minimize().")
646  else:
647  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
648 
649  return chi2
650 
651  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
652  """
653  Write the fitted astrometric results to a new 'wcs' dataRef.
654 
655  Parameters
656  ----------
657  associations : lsst.jointcal.Associations
658  The star/reference star associations to fit.
659  model : lsst.jointcal.AstrometryModel
660  The astrometric model that was fit.
661  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
662  dict of ccdImage identifiers to dataRefs that were fit
663  """
664 
665  ccdImageList = associations.getCcdImageList()
666  for ccdImage in ccdImageList:
667  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
668  ccd = ccdImage.ccdId
669  visit = ccdImage.visit
670  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
671  exp = afwImage.ExposureI(0, 0)
672  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
673  tanSip = model.produceSipWcs(ccdImage)
674  tanWcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
675  exp.setWcs(tanWcs)
676  try:
677  dataRef.put(exp, 'wcs')
678  except pexExceptions.Exception as e:
679  self.log.fatal('Failed to write updated Wcs: %s', str(e))
680  raise e
681 
682  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
683  """
684  Write the fitted photometric results to a new 'photoCalib' dataRef.
685 
686  Parameters
687  ----------
688  associations : lsst.jointcal.Associations
689  The star/reference star associations to fit.
690  model : lsst.jointcal.PhotometryModel
691  The photoometric model that was fit.
692  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
693  dict of ccdImage identifiers to dataRefs that were fit
694  """
695 
696  ccdImageList = associations.getCcdImageList()
697  for ccdImage in ccdImageList:
698  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
699  ccd = ccdImage.ccdId
700  visit = ccdImage.visit
701  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
702  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
703  photoCalib = model.toPhotoCalib(ccdImage)
704  try:
705  dataRef.put(photoCalib, 'photoCalib')
706  except pexExceptions.Exception as e:
707  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
708  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:244
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:491
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:47
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:550
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:623
def _check_star_lists(self, associations, name)
Definition: jointcal.py:482
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:682
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:20
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:400
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:651
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:299
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:202