lsst.jointcal  14.0-17-ge3cc87b+9
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.afw.coord as afwCoord
16 import lsst.pex.exceptions as pexExceptions
17 import lsst.afw.table
19 from lsst.verify import Job, Measurement
20 
21 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
22 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
23 
24 from .dataIds import PerTractCcdDataIdContainer
25 
26 import lsst.jointcal
27 from lsst.jointcal import MinimizeResult
28 
29 __all__ = ["JointcalConfig", "JointcalTask"]
30 
31 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
32 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
33 
34 
35 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
36 def add_measurement(job, name, value):
37  meas = Measurement(job.metrics[name], value)
38  job.measurements.insert(meas)
39 
40 
41 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
42  """Subclass of TaskRunner for jointcalTask
43 
44  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
45  extracted from the command line (whereas most CmdLineTasks' run methods take
46  single dataRef, are are called repeatedly). This class transforms the processed
47  arguments generated by the ArgumentParser into the arguments expected by
48  Jointcal.run().
49 
50  See pipeBase.TaskRunner for more information.
51  """
52 
53  @staticmethod
54  def getTargetList(parsedCmd, **kwargs):
55  """
56  Return a list of tuples per tract, each containing (dataRefs, kwargs).
57 
58  Jointcal operates on lists of dataRefs simultaneously.
59  """
60  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
61  kwargs['butler'] = parsedCmd.butler
62 
63  # organize data IDs by tract
64  refListDict = {}
65  for ref in parsedCmd.id.refList:
66  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
67  # we call run() once with each tract
68  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
69  return result
70 
71  def __call__(self, args):
72  """
73  @param args Arguments for Task.run()
74 
75  @return
76  - A pipe.base.Struct containing these fields if self.doReturnResults is False:
77  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
78  - A pipe.base.Struct containing these fields if self.doReturnResults is True:
79  - ``result``: the result of calling jointcal.run()
80  - ``exitStatus`: 0 if the task completed successfully, 1 otherwise.
81  """
82  exitStatus = 0 # exit status for shell
83 
84  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
85  dataRefList, kwargs = args
86  butler = kwargs.pop('butler')
87  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
88  result = None
89  try:
90  result = task.run(dataRefList, **kwargs)
91  exitStatus = result.exitStatus
92  job_path = butler.get('verify_job_filename')
93  result.job.write(job_path[0])
94  except Exception as e: # catch everything, sort it out later.
95  if self.doRaise:
96  raise e
97  else:
98  exitStatus = 1
99  eName = type(e).__name__
100  tract = dataRefList[0].dataId['tract']
101  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
102 
103  if self.doReturnResults:
104  return pipeBase.Struct(result=result, exitStatus=exitStatus)
105  else:
106  return pipeBase.Struct(exitStatus=exitStatus)
107 
108 
109 class JointcalConfig(pexConfig.Config):
110  """Config for jointcalTask"""
111 
112  doAstrometry = pexConfig.Field(
113  doc="Fit astrometry and write the fitted result.",
114  dtype=bool,
115  default=True
116  )
117  doPhotometry = pexConfig.Field(
118  doc="Fit photometry and write the fitted result.",
119  dtype=bool,
120  default=True
121  )
122  coaddName = pexConfig.Field(
123  doc="Type of coadd, typically deep or goodSeeing",
124  dtype=str,
125  default="deep"
126  )
127  posError = pexConfig.Field(
128  doc="Constant term for error on position (in pixel unit)",
129  dtype=float,
130  default=0.02,
131  )
132  # TODO: DM-6885 matchCut should be an afw.geom.Angle
133  matchCut = pexConfig.Field(
134  doc="Matching radius between fitted and reference stars (arcseconds)",
135  dtype=float,
136  default=3.0,
137  )
138  minMeasurements = pexConfig.Field(
139  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
140  dtype=int,
141  default=2,
142  )
143  astrometrySimpleDegree = pexConfig.Field(
144  doc="Polynomial degree for fitting the simple astrometry model.",
145  dtype=int,
146  default=3,
147  )
148  astrometryChipDegree = pexConfig.Field(
149  doc="Degree of the per-chip transform for the constrained astrometry model.",
150  dtype=int,
151  default=2,
152  )
153  astrometryVisitDegree = pexConfig.Field(
154  doc="Degree of the per-visit transform for the constrained astrometry model.",
155  dtype=int,
156  default=3,
157  )
158  useInputWcs = pexConfig.Field(
159  doc="Use the input calexp WCSs to initialize the astrometryModel.",
160  dtype=bool,
161  default=True,
162  )
163  astrometryModel = pexConfig.ChoiceField(
164  doc="Type of model to fit to astrometry",
165  dtype=str,
166  default="simplePoly",
167  allowed={"simplePoly": "One polynomial per ccd",
168  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
169  )
170  photometryModel = pexConfig.ChoiceField(
171  doc="Type of model to fit to photometry",
172  dtype=str,
173  default="simple",
174  allowed={"simple": "One constant zeropoint per ccd and visit",
175  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
176  )
177  photometryVisitDegree = pexConfig.Field(
178  doc="Degree of the per-visit polynomial transform for the constrained photometry model.",
179  dtype=int,
180  default=7,
181  )
182  astrometryRefObjLoader = pexConfig.ConfigurableField(
183  target=LoadIndexedReferenceObjectsTask,
184  doc="Reference object loader for astrometric fit",
185  )
186  photometryRefObjLoader = pexConfig.ConfigurableField(
187  target=LoadIndexedReferenceObjectsTask,
188  doc="Reference object loader for photometric fit",
189  )
190  sourceSelector = sourceSelectorRegistry.makeField(
191  doc="How to select sources for cross-matching",
192  default="astrometry"
193  )
194  writeChi2ContributionFiles = pexConfig.Field(
195  dtype=bool,
196  doc="Write initial/final fit files containing the contributions to chi2.",
197  default=False
198  )
199 
200  def setDefaults(self):
201  sourceSelector = self.sourceSelector["astrometry"]
202  sourceSelector.setDefaults()
203  # don't want to lose existing flags, just add to them.
204  sourceSelector.badFlags.extend(["slot_Shape_flag"])
205  # This should be used to set the FluxField value in jointcal::JointcalControl
206  sourceSelector.sourceFluxType = 'Calib'
207 
208 
209 class JointcalTask(pipeBase.CmdLineTask):
210  """Jointly astrometrically (photometrically later) calibrate a group of images."""
211 
212  ConfigClass = JointcalConfig
213  RunnerClass = JointcalRunner
214  _DefaultName = "jointcal"
215 
216  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
217  """
218  Instantiate a JointcalTask.
219 
220  Parameters
221  ----------
222  butler : lsst.daf.persistence.Butler
223  The butler is passed to the refObjLoader constructor in case it is
224  needed. Ignored if the refObjLoader argument provides a loader directly.
225  Used to initialize the astrometry and photometry refObjLoaders.
226  profile_jointcal : bool
227  set to True to profile different stages of this jointcal run.
228  """
229  pipeBase.CmdLineTask.__init__(self, **kwargs)
230  self.profile_jointcal = profile_jointcal
231  self.makeSubtask("sourceSelector")
232  if self.config.doAstrometry:
233  self.makeSubtask('astrometryRefObjLoader', butler=butler)
234  if self.config.doPhotometry:
235  self.makeSubtask('photometryRefObjLoader', butler=butler)
236 
237  # To hold various computed metrics for use by tests
238  self.job = Job.load_metrics_package(subset='jointcal')
239 
240  # We don't need to persist config and metadata at this stage.
241  # In this way, we don't need to put a specific entry in the camera mapper policy file
242  def _getConfigName(self):
243  return None
244 
245  def _getMetadataName(self):
246  return None
247 
248  @classmethod
249  def _makeArgumentParser(cls):
250  """Create an argument parser"""
251  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
252  parser.add_argument("--profile_jointcal", default=False, action="store_true",
253  help="Profile steps of jointcal separately.")
254  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
255  ContainerClass=PerTractCcdDataIdContainer)
256  return parser
257 
258  def _build_ccdImage(self, dataRef, associations, jointcalControl):
259  """
260  Extract the necessary things from this dataRef to add a new ccdImage.
261 
262  Parameters
263  ----------
264  dataRef : lsst.daf.persistence.ButlerDataRef
265  dataRef to extract info from.
266  associations : lsst.jointcal.Associations
267  object to add the info to, to construct a new CcdImage
268  jointcalControl : jointcal.JointcalControl
269  control object for associations management
270 
271  Returns
272  ------
273  namedtuple
274  wcs : lsst.afw.image.TanWcs
275  the TAN WCS of this image, read from the calexp
276  key : namedtuple
277  a key to identify this dataRef by its visit and ccd ids
278  filter : str
279  this calexp's filter
280  """
281  if "visit" in dataRef.dataId.keys():
282  visit = dataRef.dataId["visit"]
283  else:
284  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
285 
286  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
287 
288  visitInfo = dataRef.get('calexp_visitInfo')
289  detector = dataRef.get('calexp_detector')
290  ccdId = detector.getId()
291  calib = dataRef.get('calexp_calib')
292  tanWcs = dataRef.get('calexp_wcs')
293  bbox = dataRef.get('calexp_bbox')
294  filt = dataRef.get('calexp_filter')
295  filterName = filt.getName()
296  fluxMag0 = calib.getFluxMag0()
297  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
298 
299  goodSrc = self.sourceSelector.selectSources(src)
300 
301  if len(goodSrc.sourceCat) == 0:
302  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
303  else:
304  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
305  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
306  visit, ccdId, jointcalControl)
307 
308  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
309  Key = collections.namedtuple('Key', ('visit', 'ccd'))
310  return Result(tanWcs, Key(visit, ccdId), filterName)
311 
312  @pipeBase.timeMethod
313  def run(self, dataRefs, profile_jointcal=False):
314  """
315  Jointly calibrate the astrometry and photometry across a set of images.
316 
317  Parameters
318  ----------
319  dataRefs : list of lsst.daf.persistence.ButlerDataRef
320  List of data references to the exposures to be fit.
321  profile_jointcal : bool
322  Profile the individual steps of jointcal.
323 
324  Returns
325  -------
326  pipe.base.Struct
327  struct containing:
328  * dataRefs: the provided data references that were fit (with updated WCSs)
329  * oldWcsList: the original WCS from each dataRef
330  * metrics: dictionary of internally-computed metrics for testing/validation.
331  """
332  if len(dataRefs) == 0:
333  raise ValueError('Need a non-empty list of data references!')
334 
335  exitStatus = 0 # exit status for shell
336 
337  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
338  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
339  associations = lsst.jointcal.Associations()
340 
341  visit_ccd_to_dataRef = {}
342  oldWcsList = []
343  filters = []
344  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
345  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
346  # We need the bounding-box of the focal plane for photometry visit models.
347  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
348  camera = dataRefs[0].get('camera', immediate=True)
349  self.focalPlaneBBox = camera.getFpBBox()
350  for ref in dataRefs:
351  result = self._build_ccdImage(ref, associations, jointcalControl)
352  oldWcsList.append(result.wcs)
353  visit_ccd_to_dataRef[result.key] = ref
354  filters.append(result.filter)
355  filters = collections.Counter(filters)
356 
357  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
358  commonTangentPoint = lsst.afw.coord.averageCoord(centers)
359  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition())
360  associations.setCommonTangentPoint(commonTangentPoint.getPosition())
361 
362  # Use external reference catalogs handled by LSST stack mechanism
363  # Get the bounding box overlapping all associated images
364  # ==> This is probably a bad idea to do it this way <== To be improved
365  bbox = associations.getRaDecBBox()
366  center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
367  corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
368  radius = center.angularSeparation(corner).asRadians()
369 
370  # Get astrometry_net_data path
371  anDir = lsst.utils.getPackageDir('astrometry_net_data')
372  if anDir is None:
373  raise RuntimeError("astrometry_net_data is not setup")
374 
375  # Determine a default filter associated with the catalog. See DM-9093
376  defaultFilter = filters.most_common(1)[0][0]
377  self.log.debug("Using %s band for reference flux", defaultFilter)
378 
379  # TODO: need a better way to get the tract.
380  tract = dataRefs[0].dataId['tract']
381 
382  if self.config.doAstrometry:
383  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
384  name="astrometry",
385  refObjLoader=self.astrometryRefObjLoader,
386  fit_function=self._fit_astrometry,
387  profile_jointcal=profile_jointcal,
388  tract=tract)
389  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
390  else:
391  astrometry = Astrometry(None, None, None)
392 
393  if self.config.doPhotometry:
394  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
395  name="photometry",
396  refObjLoader=self.photometryRefObjLoader,
397  fit_function=self._fit_photometry,
398  profile_jointcal=profile_jointcal,
399  tract=tract,
400  filters=filters,
401  reject_bad_fluxes=True)
402  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
403  else:
404  photometry = Photometry(None, None)
405 
406  return pipeBase.Struct(dataRefs=dataRefs,
407  oldWcsList=oldWcsList,
408  job=self.job,
409  exitStatus=exitStatus)
410 
411  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
412  name="", refObjLoader=None, filters=[], fit_function=None,
413  tract=None, profile_jointcal=False, match_cut=3.0,
414  reject_bad_fluxes=False):
415  """Load reference catalog, perform the fit, and return the result.
416 
417  Parameters
418  ----------
419  associations : lsst.jointcal.Associations
420  The star/reference star associations to fit.
421  defaultFilter : str
422  filter to load from reference catalog.
423  center : lsst.afw.coord.Coord
424  Center of field to load from reference catalog.
425  radius : lsst.afw.geom.Angle
426  On-sky radius to load from reference catalog.
427  name : str
428  Name of thing being fit: "Astrometry" or "Photometry".
429  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
430  Reference object loader to load from for fit.
431  filters : list of str, optional
432  List of filters to load from the reference catalog.
433  fit_function : function
434  function to call to perform fit (takes associations object).
435  tract : str
436  Name of tract currently being fit.
437  profile_jointcal : bool, optional
438  Separately profile the fitting step.
439  match_cut : float, optional
440  Radius in arcseconds to find cross-catalog matches to during
441  associations.associateCatalogs.
442  reject_bad_fluxes : bool, optional
443  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
444 
445  Returns
446  -------
447  Result of `fit_function()`
448  """
449  self.log.info("====== Now processing %s...", name)
450  # TODO: this should not print "trying to invert a singular transformation:"
451  # if it does that, something's not right about the WCS...
452  associations.associateCatalogs(match_cut)
453  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
454  associations.fittedStarListSize())
455 
456  skyCircle = refObjLoader.loadSkyCircle(center,
457  afwGeom.Angle(radius, afwGeom.radians),
458  defaultFilter)
459 
460  # Need memory contiguity to get reference filters as a vector.
461  if not skyCircle.refCat.isContiguous():
462  refCat = skyCircle.refCat.copy(deep=True)
463  else:
464  refCat = skyCircle.refCat
465 
466  # load the reference catalog fluxes.
467  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
468  refFluxes = {}
469  refFluxErrs = {}
470  for filt in filters:
471  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
472  refFluxes[filt] = refCat.get(filtKeys[0])
473  refFluxErrs[filt] = refCat.get(filtKeys[1])
474 
475  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
476  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
477  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
478  associations.refStarListSize())
479 
480  associations.selectFittedStars(self.config.minMeasurements)
481  self._check_star_lists(associations, name)
482  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
483  associations.refStarListSize())
484  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
485  associations.fittedStarListSize())
486  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
487  associations.nCcdImagesValidForFit())
488 
489  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
490  dataName = "{}_{}".format(tract, defaultFilter)
491  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
492  result = fit_function(associations, dataName)
493  # TODO DM-12446: turn this into a "butler save" somehow.
494  # Save reference and measurement chi2 contributions for this data
495  if self.config.writeChi2ContributionFiles:
496  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
497  result.fit.saveChi2Contributions(baseName)
498 
499  return result
500 
501  def _check_star_lists(self, associations, name):
502  # TODO: these should be len(blah), but we need this properly wrapped first.
503  if associations.nCcdImagesValidForFit() == 0:
504  raise RuntimeError('No images in the ccdImageList!')
505  if associations.fittedStarListSize() == 0:
506  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
507  if associations.refStarListSize() == 0:
508  raise RuntimeError('No stars in the {} reference star list!'.format(name))
509 
510  def _fit_photometry(self, associations, dataName=None):
511  """
512  Fit the photometric data.
513 
514  Parameters
515  ----------
516  associations : lsst.jointcal.Associations
517  The star/reference star associations to fit.
518  dataName : str
519  Name of the data being processed (e.g. "1234_HSC-Y"), for
520  identifying debugging files.
521 
522  Returns
523  -------
524  namedtuple
525  fit : lsst.jointcal.PhotometryFit
526  The photometric fitter used to perform the fit.
527  model : lsst.jointcal.PhotometryModel
528  The photometric model that was fit.
529  """
530  self.log.info("=== Starting photometric fitting...")
531 
532  # TODO: should use pex.config.RegistryField here (see DM-9195)
533  if self.config.photometryModel == "constrained":
534  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
535  self.focalPlaneBBox,
536  visitDegree=self.config.photometryVisitDegree)
537  elif self.config.photometryModel == "simple":
538  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
539 
540  fit = lsst.jointcal.PhotometryFit(associations, model)
541  chi2 = fit.computeChi2()
542  # TODO DM-12446: turn this into a "butler save" somehow.
543  # Save reference and measurement chi2 contributions for this data
544  if self.config.writeChi2ContributionFiles:
545  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
546  fit.saveChi2Contributions(baseName)
547 
548  if not np.isfinite(chi2.chi2):
549  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
550  self.log.info("Initialized: %s", str(chi2))
551  fit.minimize("Model")
552  chi2 = fit.computeChi2()
553  self.log.info(str(chi2))
554  fit.minimize("Fluxes")
555  chi2 = fit.computeChi2()
556  self.log.info(str(chi2))
557  fit.minimize("Model Fluxes")
558  chi2 = fit.computeChi2()
559  if not np.isfinite(chi2.chi2):
560  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
561  self.log.info("Fit prepared with %s", str(chi2))
562 
563  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
564 
565  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
566  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
567  return Photometry(fit, model)
568 
569  def _fit_astrometry(self, associations, dataName=None):
570  """
571  Fit the astrometric data.
572 
573  Parameters
574  ----------
575  associations : lsst.jointcal.Associations
576  The star/reference star associations to fit.
577  dataName : str
578  Name of the data being processed (e.g. "1234_HSC-Y"), for
579  identifying debugging files.
580 
581  Returns
582  -------
583  namedtuple
584  fit : lsst.jointcal.AstrometryFit
585  The astrometric fitter used to perform the fit.
586  model : lsst.jointcal.AstrometryModel
587  The astrometric model that was fit.
588  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
589  The model for the sky to tangent plane projection that was used in the fit.
590  """
591 
592  self.log.info("=== Starting astrometric fitting...")
593 
594  associations.deprojectFittedStars()
595 
596  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
597  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
598  # them so carefully?
599  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
600 
601  if self.config.astrometryModel == "constrainedPoly":
602  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
603  sky_to_tan_projection, self.config.useInputWcs, 0,
604  chipDegree=self.config.astrometryChipDegree,
605  visitDegree=self.config.astrometryVisitDegree)
606  elif self.config.astrometryModel == "simplePoly":
607  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
608  sky_to_tan_projection, self.config.useInputWcs, 0,
609  self.config.astrometrySimpleDegree)
610 
611  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
612  chi2 = fit.computeChi2()
613  # TODO DM-12446: turn this into a "butler save" somehow.
614  # Save reference and measurement chi2 contributions for this data
615  if self.config.writeChi2ContributionFiles:
616  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
617  fit.saveChi2Contributions(baseName)
618 
619  if not np.isfinite(chi2.chi2):
620  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
621  self.log.info("Initialized: %s", str(chi2))
622  fit.minimize("Distortions")
623  chi2 = fit.computeChi2()
624  self.log.info(str(chi2))
625  fit.minimize("Positions")
626  chi2 = fit.computeChi2()
627  self.log.info(str(chi2))
628  fit.minimize("Distortions Positions")
629  chi2 = fit.computeChi2()
630  self.log.info(str(chi2))
631  if not np.isfinite(chi2.chi2):
632  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
633  self.log.info("Fit prepared with %s", str(chi2))
634 
635  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
636 
637  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
638  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
639 
640  return Astrometry(fit, model, sky_to_tan_projection)
641 
642  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
643  """Run fit.minimize up to max_steps times, returning the final chi2."""
644 
645  for i in range(max_steps):
646  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
647  chi2 = fit.computeChi2()
648  if not np.isfinite(chi2.chi2):
649  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
650  self.log.info(str(chi2))
651  if r == MinimizeResult.Converged:
652  self.log.debug("fit has converged - no more outliers - redo minimixation"
653  "one more time in case we have lost accuracy in rank update")
654  # Redo minimization one more time in case we have lost accuracy in rank update
655  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
656  chi2 = fit.computeChi2()
657  self.log.info("Fit completed with: %s", str(chi2))
658  break
659  elif r == MinimizeResult.Chi2Increased:
660  self.log.warn("still some ouliers but chi2 increases - retry")
661  elif r == MinimizeResult.Failed:
662  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
663  else:
664  raise RuntimeError("Unxepected return code from minimize().")
665  else:
666  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
667 
668  return chi2
669 
670  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
671  """
672  Write the fitted astrometric results to a new 'wcs' dataRef.
673 
674  Parameters
675  ----------
676  associations : lsst.jointcal.Associations
677  The star/reference star associations to fit.
678  model : lsst.jointcal.AstrometryModel
679  The astrometric model that was fit.
680  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
681  dict of ccdImage identifiers to dataRefs that were fit
682  """
683 
684  ccdImageList = associations.getCcdImageList()
685  for ccdImage in ccdImageList:
686  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
687  ccd = ccdImage.ccdId
688  visit = ccdImage.visit
689  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
690  exp = afwImage.ExposureI(0, 0)
691  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
692  tanSip = model.produceSipWcs(ccdImage)
693  tanWcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
694  exp.setWcs(tanWcs)
695  try:
696  dataRef.put(exp, 'wcs')
697  except pexExceptions.Exception as e:
698  self.log.fatal('Failed to write updated Wcs: %s', str(e))
699  raise e
700 
701  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
702  """
703  Write the fitted photometric results to a new 'photoCalib' dataRef.
704 
705  Parameters
706  ----------
707  associations : lsst.jointcal.Associations
708  The star/reference star associations to fit.
709  model : lsst.jointcal.PhotometryModel
710  The photoometric model that was fit.
711  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
712  dict of ccdImage identifiers to dataRefs that were fit
713  """
714 
715  ccdImageList = associations.getCcdImageList()
716  for ccdImage in ccdImageList:
717  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
718  ccd = ccdImage.ccdId
719  visit = ccdImage.visit
720  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
721  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
722  photoCalib = model.toPhotoCalib(ccdImage)
723  try:
724  dataRef.put(photoCalib, 'photoCalib')
725  except pexExceptions.Exception as e:
726  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
727  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:258
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:510
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:54
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:569
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:642
def _check_star_lists(self, associations, name)
Definition: jointcal.py:501
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:701
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:20
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:36
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:414
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:670
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:313
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:216