lsst.jointcal  14.0-19-g0e46020+5
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.afw.coord as afwCoord
16 import lsst.pex.exceptions as pexExceptions
17 import lsst.afw.table
19 from lsst.verify import Job, Measurement
20 
21 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
22 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
23 
24 from .dataIds import PerTractCcdDataIdContainer
25 
26 import lsst.jointcal
27 from lsst.jointcal import MinimizeResult
28 
29 __all__ = ["JointcalConfig", "JointcalTask"]
30 
31 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
32 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
33 
34 
35 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
36 def add_measurement(job, name, value):
37  meas = Measurement(job.metrics[name], value)
38  job.measurements.insert(meas)
39 
40 
41 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
42  """Subclass of TaskRunner for jointcalTask
43 
44  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
45  extracted from the command line (whereas most CmdLineTasks' run methods take
46  single dataRef, are are called repeatedly). This class transforms the processed
47  arguments generated by the ArgumentParser into the arguments expected by
48  Jointcal.run().
49 
50  See pipeBase.TaskRunner for more information.
51  """
52 
53  @staticmethod
54  def getTargetList(parsedCmd, **kwargs):
55  """
56  Return a list of tuples per tract, each containing (dataRefs, kwargs).
57 
58  Jointcal operates on lists of dataRefs simultaneously.
59  """
60  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
61  kwargs['butler'] = parsedCmd.butler
62 
63  # organize data IDs by tract
64  refListDict = {}
65  for ref in parsedCmd.id.refList:
66  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
67  # we call run() once with each tract
68  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
69  return result
70 
71  def __call__(self, args):
72  """
73  @param args Arguments for Task.run()
74 
75  @return
76  - A pipe.base.Struct containing these fields if self.doReturnResults is False:
77  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
78  - A pipe.base.Struct containing these fields if self.doReturnResults is True:
79  - ``result``: the result of calling jointcal.run()
80  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
81  """
82  exitStatus = 0 # exit status for shell
83 
84  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
85  dataRefList, kwargs = args
86  butler = kwargs.pop('butler')
87  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
88  result = None
89  try:
90  result = task.run(dataRefList, **kwargs)
91  exitStatus = result.exitStatus
92  job_path = butler.get('verify_job_filename')
93  result.job.write(job_path[0])
94  except Exception as e: # catch everything, sort it out later.
95  if self.doRaise:
96  raise e
97  else:
98  exitStatus = 1
99  eName = type(e).__name__
100  tract = dataRefList[0].dataId['tract']
101  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
102 
103  if self.doReturnResults:
104  return pipeBase.Struct(result=result, exitStatus=exitStatus)
105  else:
106  return pipeBase.Struct(exitStatus=exitStatus)
107 
108 
109 class JointcalConfig(pexConfig.Config):
110  """Config for jointcalTask"""
111 
112  doAstrometry = pexConfig.Field(
113  doc="Fit astrometry and write the fitted result.",
114  dtype=bool,
115  default=True
116  )
117  doPhotometry = pexConfig.Field(
118  doc="Fit photometry and write the fitted result.",
119  dtype=bool,
120  default=True
121  )
122  coaddName = pexConfig.Field(
123  doc="Type of coadd, typically deep or goodSeeing",
124  dtype=str,
125  default="deep"
126  )
127  posError = pexConfig.Field(
128  doc="Constant term for error on position (in pixel unit)",
129  dtype=float,
130  default=0.02,
131  )
132  # TODO: DM-6885 matchCut should be an afw.geom.Angle
133  matchCut = pexConfig.Field(
134  doc="Matching radius between fitted and reference stars (arcseconds)",
135  dtype=float,
136  default=3.0,
137  )
138  minMeasurements = pexConfig.Field(
139  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
140  dtype=int,
141  default=2,
142  )
143  astrometrySimpleDegree = pexConfig.Field(
144  doc="Polynomial degree for fitting the simple astrometry model.",
145  dtype=int,
146  default=3,
147  )
148  astrometryChipDegree = pexConfig.Field(
149  doc="Degree of the per-chip transform for the constrained astrometry model.",
150  dtype=int,
151  default=2,
152  )
153  astrometryVisitDegree = pexConfig.Field(
154  doc="Degree of the per-visit transform for the constrained astrometry model.",
155  dtype=int,
156  default=3,
157  )
158  useInputWcs = pexConfig.Field(
159  doc="Use the input calexp WCSs to initialize the astrometryModel.",
160  dtype=bool,
161  default=True,
162  )
163  astrometryModel = pexConfig.ChoiceField(
164  doc="Type of model to fit to astrometry",
165  dtype=str,
166  default="simplePoly",
167  allowed={"simplePoly": "One polynomial per ccd",
168  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
169  )
170  photometryModel = pexConfig.ChoiceField(
171  doc="Type of model to fit to photometry",
172  dtype=str,
173  default="simple",
174  allowed={"simple": "One constant zeropoint per ccd and visit",
175  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
176  )
177  photometryVisitDegree = pexConfig.Field(
178  doc="Degree of the per-visit polynomial transform for the constrained photometry model.",
179  dtype=int,
180  default=7,
181  )
182  astrometryRefObjLoader = pexConfig.ConfigurableField(
183  target=LoadIndexedReferenceObjectsTask,
184  doc="Reference object loader for astrometric fit",
185  )
186  photometryRefObjLoader = pexConfig.ConfigurableField(
187  target=LoadIndexedReferenceObjectsTask,
188  doc="Reference object loader for photometric fit",
189  )
190  sourceSelector = sourceSelectorRegistry.makeField(
191  doc="How to select sources for cross-matching",
192  default="astrometry"
193  )
194  writeChi2ContributionFiles = pexConfig.Field(
195  dtype=bool,
196  doc="Write initial/final fit files containing the contributions to chi2.",
197  default=False
198  )
199 
200  def setDefaults(self):
201  sourceSelector = self.sourceSelector["astrometry"]
202  sourceSelector.setDefaults()
203  # don't want to lose existing flags, just add to them.
204  sourceSelector.badFlags.extend(["slot_Shape_flag"])
205  # This should be used to set the FluxField value in jointcal::JointcalControl
206  sourceSelector.sourceFluxType = 'Calib'
207 
208 
209 class JointcalTask(pipeBase.CmdLineTask):
210  """Jointly astrometrically (photometrically later) calibrate a group of images."""
211 
212  ConfigClass = JointcalConfig
213  RunnerClass = JointcalRunner
214  _DefaultName = "jointcal"
215 
216  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
217  """
218  Instantiate a JointcalTask.
219 
220  Parameters
221  ----------
222  butler : lsst.daf.persistence.Butler
223  The butler is passed to the refObjLoader constructor in case it is
224  needed. Ignored if the refObjLoader argument provides a loader directly.
225  Used to initialize the astrometry and photometry refObjLoaders.
226  profile_jointcal : bool
227  set to True to profile different stages of this jointcal run.
228  """
229  pipeBase.CmdLineTask.__init__(self, **kwargs)
230  self.profile_jointcal = profile_jointcal
231  self.makeSubtask("sourceSelector")
232  if self.config.doAstrometry:
233  self.makeSubtask('astrometryRefObjLoader', butler=butler)
234  if self.config.doPhotometry:
235  self.makeSubtask('photometryRefObjLoader', butler=butler)
236 
237  # To hold various computed metrics for use by tests
238  self.job = Job.load_metrics_package(subset='jointcal')
239 
240  # We don't need to persist config and metadata at this stage.
241  # In this way, we don't need to put a specific entry in the camera mapper policy file
242  def _getConfigName(self):
243  return None
244 
245  def _getMetadataName(self):
246  return None
247 
248  @classmethod
249  def _makeArgumentParser(cls):
250  """Create an argument parser"""
251  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
252  parser.add_argument("--profile_jointcal", default=False, action="store_true",
253  help="Profile steps of jointcal separately.")
254  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
255  ContainerClass=PerTractCcdDataIdContainer)
256  return parser
257 
258  def _build_ccdImage(self, dataRef, associations, jointcalControl):
259  """
260  Extract the necessary things from this dataRef to add a new ccdImage.
261 
262  Parameters
263  ----------
264  dataRef : lsst.daf.persistence.ButlerDataRef
265  dataRef to extract info from.
266  associations : lsst.jointcal.Associations
267  object to add the info to, to construct a new CcdImage
268  jointcalControl : jointcal.JointcalControl
269  control object for associations management
270 
271  Returns
272  ------
273  namedtuple
274  wcs : lsst.afw.image.TanWcs
275  the TAN WCS of this image, read from the calexp
276  key : namedtuple
277  a key to identify this dataRef by its visit and ccd ids
278  filter : str
279  this calexp's filter
280  """
281  if "visit" in dataRef.dataId.keys():
282  visit = dataRef.dataId["visit"]
283  else:
284  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
285 
286  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
287 
288  visitInfo = dataRef.get('calexp_visitInfo')
289  detector = dataRef.get('calexp_detector')
290  ccdId = detector.getId()
291  calib = dataRef.get('calexp_calib')
292  tanWcs = dataRef.get('calexp_wcs')
293  bbox = dataRef.get('calexp_bbox')
294  filt = dataRef.get('calexp_filter')
295  filterName = filt.getName()
296  fluxMag0 = calib.getFluxMag0()
297  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
298 
299  goodSrc = self.sourceSelector.selectSources(src)
300 
301  if len(goodSrc.sourceCat) == 0:
302  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
303  else:
304  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
305  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
306  visit, ccdId, jointcalControl)
307 
308  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
309  Key = collections.namedtuple('Key', ('visit', 'ccd'))
310  return Result(tanWcs, Key(visit, ccdId), filterName)
311 
312  @pipeBase.timeMethod
313  def run(self, dataRefs, profile_jointcal=False):
314  """
315  Jointly calibrate the astrometry and photometry across a set of images.
316 
317  Parameters
318  ----------
319  dataRefs : list of lsst.daf.persistence.ButlerDataRef
320  List of data references to the exposures to be fit.
321  profile_jointcal : bool
322  Profile the individual steps of jointcal.
323 
324  Returns
325  -------
326  pipe.base.Struct
327  struct containing:
328  * dataRefs: the provided data references that were fit (with updated WCSs)
329  * oldWcsList: the original WCS from each dataRef
330  * metrics: dictionary of internally-computed metrics for testing/validation.
331  """
332  if len(dataRefs) == 0:
333  raise ValueError('Need a non-empty list of data references!')
334 
335  exitStatus = 0 # exit status for shell
336 
337  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
338  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
339  associations = lsst.jointcal.Associations()
340 
341  visit_ccd_to_dataRef = {}
342  oldWcsList = []
343  filters = []
344  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
345  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
346  # We need the bounding-box of the focal plane for photometry visit models.
347  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
348  camera = dataRefs[0].get('camera', immediate=True)
349  self.focalPlaneBBox = camera.getFpBBox()
350  for ref in dataRefs:
351  result = self._build_ccdImage(ref, associations, jointcalControl)
352  oldWcsList.append(result.wcs)
353  visit_ccd_to_dataRef[result.key] = ref
354  filters.append(result.filter)
355  filters = collections.Counter(filters)
356 
357  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
358  commonTangentPoint = lsst.afw.coord.averageCoord(centers)
359  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition())
360  associations.setCommonTangentPoint(commonTangentPoint.getPosition())
361 
362  # Use external reference catalogs handled by LSST stack mechanism
363  # Get the bounding box overlapping all associated images
364  # ==> This is probably a bad idea to do it this way <== To be improved
365  bbox = associations.getRaDecBBox()
366  center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
367  corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
368  radius = center.angularSeparation(corner).asRadians()
369 
370  # Get astrometry_net_data path
371  anDir = lsst.utils.getPackageDir('astrometry_net_data')
372  if anDir is None:
373  raise RuntimeError("astrometry_net_data is not setup")
374 
375  # Determine a default filter associated with the catalog. See DM-9093
376  defaultFilter = filters.most_common(1)[0][0]
377  self.log.debug("Using %s band for reference flux", defaultFilter)
378 
379  # TODO: need a better way to get the tract.
380  tract = dataRefs[0].dataId['tract']
381 
382  if self.config.doAstrometry:
383  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
384  name="astrometry",
385  refObjLoader=self.astrometryRefObjLoader,
386  fit_function=self._fit_astrometry,
387  profile_jointcal=profile_jointcal,
388  tract=tract)
389  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
390  else:
391  astrometry = Astrometry(None, None, None)
392 
393  if self.config.doPhotometry:
394  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
395  name="photometry",
396  refObjLoader=self.photometryRefObjLoader,
397  fit_function=self._fit_photometry,
398  profile_jointcal=profile_jointcal,
399  tract=tract,
400  filters=filters,
401  reject_bad_fluxes=True)
402  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
403  else:
404  photometry = Photometry(None, None)
405 
406  return pipeBase.Struct(dataRefs=dataRefs,
407  oldWcsList=oldWcsList,
408  job=self.job,
409  exitStatus=exitStatus)
410 
411  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
412  name="", refObjLoader=None, filters=[], fit_function=None,
413  tract=None, profile_jointcal=False, match_cut=3.0,
414  reject_bad_fluxes=False):
415  """Load reference catalog, perform the fit, and return the result.
416 
417  Parameters
418  ----------
419  associations : lsst.jointcal.Associations
420  The star/reference star associations to fit.
421  defaultFilter : str
422  filter to load from reference catalog.
423  center : lsst.afw.coord.Coord
424  Center of field to load from reference catalog.
425  radius : lsst.afw.geom.Angle
426  On-sky radius to load from reference catalog.
427  name : str
428  Name of thing being fit: "Astrometry" or "Photometry".
429  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
430  Reference object loader to load from for fit.
431  filters : list of str, optional
432  List of filters to load from the reference catalog.
433  fit_function : function
434  function to call to perform fit (takes associations object).
435  tract : str
436  Name of tract currently being fit.
437  profile_jointcal : bool, optional
438  Separately profile the fitting step.
439  match_cut : float, optional
440  Radius in arcseconds to find cross-catalog matches to during
441  associations.associateCatalogs.
442  reject_bad_fluxes : bool, optional
443  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
444 
445  Returns
446  -------
447  Result of `fit_function()`
448  """
449  self.log.info("====== Now processing %s...", name)
450  # TODO: this should not print "trying to invert a singular transformation:"
451  # if it does that, something's not right about the WCS...
452  associations.associateCatalogs(match_cut)
453  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
454  associations.fittedStarListSize())
455 
456  skyCircle = refObjLoader.loadSkyCircle(center,
457  afwGeom.Angle(radius, afwGeom.radians),
458  defaultFilter)
459 
460  # Need memory contiguity to get reference filters as a vector.
461  if not skyCircle.refCat.isContiguous():
462  refCat = skyCircle.refCat.copy(deep=True)
463  else:
464  refCat = skyCircle.refCat
465 
466  # load the reference catalog fluxes.
467  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
468  refFluxes = {}
469  refFluxErrs = {}
470  for filt in filters:
471  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
472  refFluxes[filt] = refCat.get(filtKeys[0])
473  refFluxErrs[filt] = refCat.get(filtKeys[1])
474 
475  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
476  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
477  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
478  associations.refStarListSize())
479 
480  associations.selectFittedStars(self.config.minMeasurements)
481  self._check_star_lists(associations, name)
482  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
483  associations.nFittedStarsWithAssociatedRefStar())
484  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
485  associations.fittedStarListSize())
486  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
487  associations.nCcdImagesValidForFit())
488 
489  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
490  dataName = "{}_{}".format(tract, defaultFilter)
491  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
492  result = fit_function(associations, dataName)
493  # TODO DM-12446: turn this into a "butler save" somehow.
494  # Save reference and measurement chi2 contributions for this data
495  if self.config.writeChi2ContributionFiles:
496  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
497  result.fit.saveChi2Contributions(baseName)
498 
499  return result
500 
501  def _check_star_lists(self, associations, name):
502  # TODO: these should be len(blah), but we need this properly wrapped first.
503  if associations.nCcdImagesValidForFit() == 0:
504  raise RuntimeError('No images in the ccdImageList!')
505  if associations.fittedStarListSize() == 0:
506  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
507  if associations.refStarListSize() == 0:
508  raise RuntimeError('No stars in the {} reference star list!'.format(name))
509 
510  def _fit_photometry(self, associations, dataName=None):
511  """
512  Fit the photometric data.
513 
514  Parameters
515  ----------
516  associations : lsst.jointcal.Associations
517  The star/reference star associations to fit.
518  dataName : str
519  Name of the data being processed (e.g. "1234_HSC-Y"), for
520  identifying debugging files.
521 
522  Returns
523  -------
524  namedtuple
525  fit : lsst.jointcal.PhotometryFit
526  The photometric fitter used to perform the fit.
527  model : lsst.jointcal.PhotometryModel
528  The photometric model that was fit.
529  """
530  self.log.info("=== Starting photometric fitting...")
531 
532  # TODO: should use pex.config.RegistryField here (see DM-9195)
533  if self.config.photometryModel == "constrained":
534  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
535  self.focalPlaneBBox,
536  visitDegree=self.config.photometryVisitDegree)
537  elif self.config.photometryModel == "simple":
538  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
539 
540  fit = lsst.jointcal.PhotometryFit(associations, model)
541  chi2 = fit.computeChi2()
542  # TODO DM-12446: turn this into a "butler save" somehow.
543  # Save reference and measurement chi2 contributions for this data
544  if self.config.writeChi2ContributionFiles:
545  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
546  fit.saveChi2Contributions(baseName)
547 
548  if not np.isfinite(chi2.chi2):
549  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
550  self.log.info("Initialized: %s", str(chi2))
551  fit.minimize("Model")
552  chi2 = fit.computeChi2()
553  self.log.info(str(chi2))
554  fit.minimize("Fluxes")
555  chi2 = fit.computeChi2()
556  self.log.info(str(chi2))
557  fit.minimize("Model Fluxes")
558  chi2 = fit.computeChi2()
559  if not np.isfinite(chi2.chi2):
560  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
561  self.log.info("Fit prepared with %s", str(chi2))
562 
563  model.freezeErrorTransform()
564  self.log.debug("Photometry error scales are frozen.")
565 
566  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
567 
568  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
569  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
570  return Photometry(fit, model)
571 
572  def _fit_astrometry(self, associations, dataName=None):
573  """
574  Fit the astrometric data.
575 
576  Parameters
577  ----------
578  associations : lsst.jointcal.Associations
579  The star/reference star associations to fit.
580  dataName : str
581  Name of the data being processed (e.g. "1234_HSC-Y"), for
582  identifying debugging files.
583 
584  Returns
585  -------
586  namedtuple
587  fit : lsst.jointcal.AstrometryFit
588  The astrometric fitter used to perform the fit.
589  model : lsst.jointcal.AstrometryModel
590  The astrometric model that was fit.
591  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
592  The model for the sky to tangent plane projection that was used in the fit.
593  """
594 
595  self.log.info("=== Starting astrometric fitting...")
596 
597  associations.deprojectFittedStars()
598 
599  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
600  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
601  # them so carefully?
602  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
603 
604  if self.config.astrometryModel == "constrainedPoly":
605  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
606  sky_to_tan_projection, self.config.useInputWcs, 0,
607  chipDegree=self.config.astrometryChipDegree,
608  visitDegree=self.config.astrometryVisitDegree)
609  elif self.config.astrometryModel == "simplePoly":
610  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
611  sky_to_tan_projection, self.config.useInputWcs, 0,
612  self.config.astrometrySimpleDegree)
613 
614  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
615  chi2 = fit.computeChi2()
616  # TODO DM-12446: turn this into a "butler save" somehow.
617  # Save reference and measurement chi2 contributions for this data
618  if self.config.writeChi2ContributionFiles:
619  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
620  fit.saveChi2Contributions(baseName)
621 
622  if not np.isfinite(chi2.chi2):
623  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
624  self.log.info("Initialized: %s", str(chi2))
625  fit.minimize("Distortions")
626  chi2 = fit.computeChi2()
627  self.log.info(str(chi2))
628  fit.minimize("Positions")
629  chi2 = fit.computeChi2()
630  self.log.info(str(chi2))
631  fit.minimize("Distortions Positions")
632  chi2 = fit.computeChi2()
633  self.log.info(str(chi2))
634  if not np.isfinite(chi2.chi2):
635  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
636  self.log.info("Fit prepared with %s", str(chi2))
637 
638  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
639 
640  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
641  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
642 
643  return Astrometry(fit, model, sky_to_tan_projection)
644 
645  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
646  """Run fit.minimize up to max_steps times, returning the final chi2."""
647 
648  for i in range(max_steps):
649  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
650  chi2 = fit.computeChi2()
651  if not np.isfinite(chi2.chi2):
652  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
653  self.log.info(str(chi2))
654  if r == MinimizeResult.Converged:
655  self.log.debug("fit has converged - no more outliers - redo minimixation"
656  "one more time in case we have lost accuracy in rank update")
657  # Redo minimization one more time in case we have lost accuracy in rank update
658  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
659  chi2 = fit.computeChi2()
660  self.log.info("Fit completed with: %s", str(chi2))
661  break
662  elif r == MinimizeResult.Chi2Increased:
663  self.log.warn("still some ouliers but chi2 increases - retry")
664  elif r == MinimizeResult.Failed:
665  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
666  else:
667  raise RuntimeError("Unxepected return code from minimize().")
668  else:
669  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
670 
671  return chi2
672 
673  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
674  """
675  Write the fitted astrometric results to a new 'wcs' dataRef.
676 
677  Parameters
678  ----------
679  associations : lsst.jointcal.Associations
680  The star/reference star associations to fit.
681  model : lsst.jointcal.AstrometryModel
682  The astrometric model that was fit.
683  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
684  dict of ccdImage identifiers to dataRefs that were fit
685  """
686 
687  ccdImageList = associations.getCcdImageList()
688  for ccdImage in ccdImageList:
689  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
690  ccd = ccdImage.ccdId
691  visit = ccdImage.visit
692  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
693  exp = afwImage.ExposureI(0, 0)
694  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
695  tanSip = model.produceSipWcs(ccdImage)
696  tanWcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
697  exp.setWcs(tanWcs)
698  try:
699  dataRef.put(exp, 'wcs')
700  except pexExceptions.Exception as e:
701  self.log.fatal('Failed to write updated Wcs: %s', str(e))
702  raise e
703 
704  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
705  """
706  Write the fitted photometric results to a new 'photoCalib' dataRef.
707 
708  Parameters
709  ----------
710  associations : lsst.jointcal.Associations
711  The star/reference star associations to fit.
712  model : lsst.jointcal.PhotometryModel
713  The photoometric model that was fit.
714  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
715  dict of ccdImage identifiers to dataRefs that were fit
716  """
717 
718  ccdImageList = associations.getCcdImageList()
719  for ccdImage in ccdImageList:
720  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
721  ccd = ccdImage.ccdId
722  visit = ccdImage.visit
723  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
724  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
725  photoCalib = model.toPhotoCalib(ccdImage)
726  try:
727  dataRef.put(photoCalib, 'photoCalib')
728  except pexExceptions.Exception as e:
729  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
730  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:258
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:510
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:54
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:572
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:645
def _check_star_lists(self, associations, name)
Definition: jointcal.py:501
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:704
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:36
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:414
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:673
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:313
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:216