lsst.jointcal  14.0-28-ge87de3a+2
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.pex.exceptions as pexExceptions
16 import lsst.afw.table
18 from lsst.verify import Job, Measurement
19 
20 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
21 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
22 
23 from .dataIds import PerTractCcdDataIdContainer
24 
25 import lsst.jointcal
26 from lsst.jointcal import MinimizeResult
27 
28 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
29 
30 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
31 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
32 
33 
34 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
35 def add_measurement(job, name, value):
36  meas = Measurement(job.metrics[name], value)
37  job.measurements.insert(meas)
38 
39 
40 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
41  """Subclass of TaskRunner for jointcalTask
42 
43  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
44  extracted from the command line (whereas most CmdLineTasks' run methods take
45  single dataRef, are are called repeatedly). This class transforms the processed
46  arguments generated by the ArgumentParser into the arguments expected by
47  Jointcal.run().
48 
49  See pipeBase.TaskRunner for more information.
50  """
51 
52  @staticmethod
53  def getTargetList(parsedCmd, **kwargs):
54  """
55  Return a list of tuples per tract, each containing (dataRefs, kwargs).
56 
57  Jointcal operates on lists of dataRefs simultaneously.
58  """
59  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
60  kwargs['butler'] = parsedCmd.butler
61 
62  # organize data IDs by tract
63  refListDict = {}
64  for ref in parsedCmd.id.refList:
65  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
66  # we call run() once with each tract
67  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
68  return result
69 
70  def __call__(self, args):
71  """
72  Parameters
73  ----------
74  args
75  Arguments for Task.run()
76 
77  Returns
78  -------
79  pipe.base.Struct
80  if self.doReturnResults is False:
81 
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83 
84  if self.doReturnResults is True:
85 
86  - ``result``: the result of calling jointcal.run()
87  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
88  """
89  exitStatus = 0 # exit status for shell
90 
91  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
92  dataRefList, kwargs = args
93  butler = kwargs.pop('butler')
94  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
95  result = None
96  try:
97  result = task.run(dataRefList, **kwargs)
98  exitStatus = result.exitStatus
99  job_path = butler.get('verify_job_filename')
100  result.job.write(job_path[0])
101  except Exception as e: # catch everything, sort it out later.
102  if self.doRaise:
103  raise e
104  else:
105  exitStatus = 1
106  eName = type(e).__name__
107  tract = dataRefList[0].dataId['tract']
108  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
109 
110  if self.doReturnResults:
111  return pipeBase.Struct(result=result, exitStatus=exitStatus)
112  else:
113  return pipeBase.Struct(exitStatus=exitStatus)
114 
115 
116 class JointcalConfig(pexConfig.Config):
117  """Config for JointcalTask"""
118 
119  doAstrometry = pexConfig.Field(
120  doc="Fit astrometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  doPhotometry = pexConfig.Field(
125  doc="Fit photometry and write the fitted result.",
126  dtype=bool,
127  default=True
128  )
129  coaddName = pexConfig.Field(
130  doc="Type of coadd, typically deep or goodSeeing",
131  dtype=str,
132  default="deep"
133  )
134  posError = pexConfig.Field(
135  doc="Constant term for error on position (in pixel unit)",
136  dtype=float,
137  default=0.02,
138  )
139  # TODO: DM-6885 matchCut should be an afw.geom.Angle
140  matchCut = pexConfig.Field(
141  doc="Matching radius between fitted and reference stars (arcseconds)",
142  dtype=float,
143  default=3.0,
144  )
145  minMeasurements = pexConfig.Field(
146  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
147  dtype=int,
148  default=2,
149  )
150  astrometrySimpleDegree = pexConfig.Field(
151  doc="Polynomial degree for fitting the simple astrometry model.",
152  dtype=int,
153  default=3,
154  )
155  astrometryChipDegree = pexConfig.Field(
156  doc="Degree of the per-chip transform for the constrained astrometry model.",
157  dtype=int,
158  default=2,
159  )
160  astrometryVisitDegree = pexConfig.Field(
161  doc="Degree of the per-visit transform for the constrained astrometry model.",
162  dtype=int,
163  default=3,
164  )
165  useInputWcs = pexConfig.Field(
166  doc="Use the input calexp WCSs to initialize the astrometryModel.",
167  dtype=bool,
168  default=True,
169  )
170  astrometryModel = pexConfig.ChoiceField(
171  doc="Type of model to fit to astrometry",
172  dtype=str,
173  default="simplePoly",
174  allowed={"simplePoly": "One polynomial per ccd",
175  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
176  )
177  photometryModel = pexConfig.ChoiceField(
178  doc="Type of model to fit to photometry",
179  dtype=str,
180  default="simple",
181  allowed={"simple": "One constant zeropoint per ccd and visit",
182  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
183  )
184  photometryVisitDegree = pexConfig.Field(
185  doc="Degree of the per-visit polynomial transform for the constrained photometry model.",
186  dtype=int,
187  default=7,
188  )
189  astrometryRefObjLoader = pexConfig.ConfigurableField(
190  target=LoadIndexedReferenceObjectsTask,
191  doc="Reference object loader for astrometric fit",
192  )
193  photometryRefObjLoader = pexConfig.ConfigurableField(
194  target=LoadIndexedReferenceObjectsTask,
195  doc="Reference object loader for photometric fit",
196  )
197  sourceSelector = sourceSelectorRegistry.makeField(
198  doc="How to select sources for cross-matching",
199  default="astrometry"
200  )
201  writeChi2ContributionFiles = pexConfig.Field(
202  dtype=bool,
203  doc="Write initial/final fit files containing the contributions to chi2.",
204  default=False
205  )
206 
207  def setDefaults(self):
208  sourceSelector = self.sourceSelector["astrometry"]
209  sourceSelector.setDefaults()
210  # don't want to lose existing flags, just add to them.
211  sourceSelector.badFlags.extend(["slot_Shape_flag"])
212  # This should be used to set the FluxField value in jointcal::JointcalControl
213  sourceSelector.sourceFluxType = 'Calib'
214 
215 
216 class JointcalTask(pipeBase.CmdLineTask):
217  """Jointly astrometrically and photometrically calibrate a group of images."""
218 
219  ConfigClass = JointcalConfig
220  RunnerClass = JointcalRunner
221  _DefaultName = "jointcal"
222 
223  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
224  """
225  Instantiate a JointcalTask.
226 
227  Parameters
228  ----------
229  butler : lsst.daf.persistence.Butler
230  The butler is passed to the refObjLoader constructor in case it is
231  needed. Ignored if the refObjLoader argument provides a loader directly.
232  Used to initialize the astrometry and photometry refObjLoaders.
233  profile_jointcal : bool
234  set to True to profile different stages of this jointcal run.
235  """
236  pipeBase.CmdLineTask.__init__(self, **kwargs)
237  self.profile_jointcal = profile_jointcal
238  self.makeSubtask("sourceSelector")
239  if self.config.doAstrometry:
240  self.makeSubtask('astrometryRefObjLoader', butler=butler)
241  if self.config.doPhotometry:
242  self.makeSubtask('photometryRefObjLoader', butler=butler)
243 
244  # To hold various computed metrics for use by tests
245  self.job = Job.load_metrics_package(subset='jointcal')
246 
247  # We don't need to persist config and metadata at this stage.
248  # In this way, we don't need to put a specific entry in the camera mapper policy file
249  def _getConfigName(self):
250  return None
251 
252  def _getMetadataName(self):
253  return None
254 
255  @classmethod
256  def _makeArgumentParser(cls):
257  """Create an argument parser"""
258  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
259  parser.add_argument("--profile_jointcal", default=False, action="store_true",
260  help="Profile steps of jointcal separately.")
261  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
262  ContainerClass=PerTractCcdDataIdContainer)
263  return parser
264 
265  def _build_ccdImage(self, dataRef, associations, jointcalControl):
266  """
267  Extract the necessary things from this dataRef to add a new ccdImage.
268 
269  Parameters
270  ----------
271  dataRef : lsst.daf.persistence.ButlerDataRef
272  dataRef to extract info from.
273  associations : lsst.jointcal.Associations
274  object to add the info to, to construct a new CcdImage
275  jointcalControl : jointcal.JointcalControl
276  control object for associations management
277 
278  Returns
279  ------
280  namedtuple
281  wcs : lsst.afw.geom.SkyWcs
282  the TAN WCS of this image, read from the calexp
283  key : namedtuple
284  a key to identify this dataRef by its visit and ccd ids
285  filter : str
286  this calexp's filter
287  """
288  if "visit" in dataRef.dataId.keys():
289  visit = dataRef.dataId["visit"]
290  else:
291  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
292 
293  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
294 
295  visitInfo = dataRef.get('calexp_visitInfo')
296  detector = dataRef.get('calexp_detector')
297  ccdId = detector.getId()
298  calib = dataRef.get('calexp_calib')
299  tanWcs = dataRef.get('calexp_wcs')
300  bbox = dataRef.get('calexp_bbox')
301  filt = dataRef.get('calexp_filter')
302  filterName = filt.getName()
303  fluxMag0 = calib.getFluxMag0()
304  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
305 
306  goodSrc = self.sourceSelector.selectSources(src)
307 
308  if len(goodSrc.sourceCat) == 0:
309  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
310  else:
311  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
312  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
313  visit, ccdId, jointcalControl)
314 
315  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
316  Key = collections.namedtuple('Key', ('visit', 'ccd'))
317  return Result(tanWcs, Key(visit, ccdId), filterName)
318 
319  @pipeBase.timeMethod
320  def run(self, dataRefs, profile_jointcal=False):
321  """
322  Jointly calibrate the astrometry and photometry across a set of images.
323 
324  Parameters
325  ----------
326  dataRefs : list of lsst.daf.persistence.ButlerDataRef
327  List of data references to the exposures to be fit.
328  profile_jointcal : bool
329  Profile the individual steps of jointcal.
330 
331  Returns
332  -------
333  pipe.base.Struct
334  struct containing:
335  * dataRefs: the provided data references that were fit (with updated WCSs)
336  * oldWcsList: the original WCS from each dataRef
337  * metrics: dictionary of internally-computed metrics for testing/validation.
338  """
339  if len(dataRefs) == 0:
340  raise ValueError('Need a non-empty list of data references!')
341 
342  exitStatus = 0 # exit status for shell
343 
344  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
345  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
346  associations = lsst.jointcal.Associations()
347 
348  visit_ccd_to_dataRef = {}
349  oldWcsList = []
350  filters = []
351  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
352  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
353  # We need the bounding-box of the focal plane for photometry visit models.
354  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
355  camera = dataRefs[0].get('camera', immediate=True)
356  self.focalPlaneBBox = camera.getFpBBox()
357  for ref in dataRefs:
358  result = self._build_ccdImage(ref, associations, jointcalControl)
359  oldWcsList.append(result.wcs)
360  visit_ccd_to_dataRef[result.key] = ref
361  filters.append(result.filter)
362  filters = collections.Counter(filters)
363 
364  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
365  commonTangentPoint = afwGeom.averageSpherePoint(centers)
366  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition(afwGeom.degrees))
367  associations.setCommonTangentPoint(commonTangentPoint.getPosition(afwGeom.degrees))
368 
369  # Use external reference catalogs handled by LSST stack mechanism
370  # Get the bounding box overlapping all associated images
371  # ==> This is probably a bad idea to do it this way <== To be improved
372  bbox = associations.getRaDecBBox()
373  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
374  bboxCenter = bbox.getCenter()
375  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
376  bboxMax = bbox.getMax()
377  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
378  radius = center.separation(corner).asRadians()
379 
380  # Get astrometry_net_data path
381  anDir = lsst.utils.getPackageDir('astrometry_net_data')
382  if anDir is None:
383  raise RuntimeError("astrometry_net_data is not setup")
384 
385  # Determine a default filter associated with the catalog. See DM-9093
386  defaultFilter = filters.most_common(1)[0][0]
387  self.log.debug("Using %s band for reference flux", defaultFilter)
388 
389  # TODO: need a better way to get the tract.
390  tract = dataRefs[0].dataId['tract']
391 
392  if self.config.doAstrometry:
393  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
394  name="astrometry",
395  refObjLoader=self.astrometryRefObjLoader,
396  fit_function=self._fit_astrometry,
397  profile_jointcal=profile_jointcal,
398  tract=tract)
399  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
400  else:
401  astrometry = Astrometry(None, None, None)
402 
403  if self.config.doPhotometry:
404  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
405  name="photometry",
406  refObjLoader=self.photometryRefObjLoader,
407  fit_function=self._fit_photometry,
408  profile_jointcal=profile_jointcal,
409  tract=tract,
410  filters=filters,
411  reject_bad_fluxes=True)
412  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
413  else:
414  photometry = Photometry(None, None)
415 
416  return pipeBase.Struct(dataRefs=dataRefs,
417  oldWcsList=oldWcsList,
418  job=self.job,
419  exitStatus=exitStatus)
420 
421  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
422  name="", refObjLoader=None, filters=[], fit_function=None,
423  tract=None, profile_jointcal=False, match_cut=3.0,
424  reject_bad_fluxes=False):
425  """Load reference catalog, perform the fit, and return the result.
426 
427  Parameters
428  ----------
429  associations : lsst.jointcal.Associations
430  The star/reference star associations to fit.
431  defaultFilter : str
432  filter to load from reference catalog.
433  center : lsst.afw.geom.SpherePoint
434  ICRS center of field to load from reference catalog.
435  radius : lsst.afw.geom.Angle
436  On-sky radius to load from reference catalog.
437  name : str
438  Name of thing being fit: "Astrometry" or "Photometry".
439  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
440  Reference object loader to load from for fit.
441  filters : list of str, optional
442  List of filters to load from the reference catalog.
443  fit_function : function
444  function to call to perform fit (takes associations object).
445  tract : str
446  Name of tract currently being fit.
447  profile_jointcal : bool, optional
448  Separately profile the fitting step.
449  match_cut : float, optional
450  Radius in arcseconds to find cross-catalog matches to during
451  associations.associateCatalogs.
452  reject_bad_fluxes : bool, optional
453  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
454 
455  Returns
456  -------
457  Result of `fit_function()`
458  """
459  self.log.info("====== Now processing %s...", name)
460  # TODO: this should not print "trying to invert a singular transformation:"
461  # if it does that, something's not right about the WCS...
462  associations.associateCatalogs(match_cut)
463  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
464  associations.fittedStarListSize())
465 
466  skyCircle = refObjLoader.loadSkyCircle(center,
467  afwGeom.Angle(radius, afwGeom.radians),
468  defaultFilter)
469 
470  # Need memory contiguity to get reference filters as a vector.
471  if not skyCircle.refCat.isContiguous():
472  refCat = skyCircle.refCat.copy(deep=True)
473  else:
474  refCat = skyCircle.refCat
475 
476  # load the reference catalog fluxes.
477  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
478  refFluxes = {}
479  refFluxErrs = {}
480  for filt in filters:
481  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
482  refFluxes[filt] = refCat.get(filtKeys[0])
483  refFluxErrs[filt] = refCat.get(filtKeys[1])
484 
485  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
486  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
487  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
488  associations.refStarListSize())
489 
490  associations.prepareFittedStars(self.config.minMeasurements)
491 
492  self._check_star_lists(associations, name)
493  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
494  associations.nFittedStarsWithAssociatedRefStar())
495  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
496  associations.fittedStarListSize())
497  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
498  associations.nCcdImagesValidForFit())
499 
500  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
501  dataName = "{}_{}".format(tract, defaultFilter)
502  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
503  result = fit_function(associations, dataName)
504  # TODO DM-12446: turn this into a "butler save" somehow.
505  # Save reference and measurement chi2 contributions for this data
506  if self.config.writeChi2ContributionFiles:
507  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
508  result.fit.saveChi2Contributions(baseName)
509 
510  return result
511 
512  def _check_star_lists(self, associations, name):
513  # TODO: these should be len(blah), but we need this properly wrapped first.
514  if associations.nCcdImagesValidForFit() == 0:
515  raise RuntimeError('No images in the ccdImageList!')
516  if associations.fittedStarListSize() == 0:
517  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
518  if associations.refStarListSize() == 0:
519  raise RuntimeError('No stars in the {} reference star list!'.format(name))
520 
521  def _fit_photometry(self, associations, dataName=None):
522  """
523  Fit the photometric data.
524 
525  Parameters
526  ----------
527  associations : lsst.jointcal.Associations
528  The star/reference star associations to fit.
529  dataName : str
530  Name of the data being processed (e.g. "1234_HSC-Y"), for
531  identifying debugging files.
532 
533  Returns
534  -------
535  namedtuple
536  fit : lsst.jointcal.PhotometryFit
537  The photometric fitter used to perform the fit.
538  model : lsst.jointcal.PhotometryModel
539  The photometric model that was fit.
540  """
541  self.log.info("=== Starting photometric fitting...")
542 
543  # TODO: should use pex.config.RegistryField here (see DM-9195)
544  if self.config.photometryModel == "constrained":
545  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
546  self.focalPlaneBBox,
547  visitDegree=self.config.photometryVisitDegree)
548  elif self.config.photometryModel == "simple":
549  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
550 
551  fit = lsst.jointcal.PhotometryFit(associations, model)
552  chi2 = fit.computeChi2()
553  # TODO DM-12446: turn this into a "butler save" somehow.
554  # Save reference and measurement chi2 contributions for this data
555  if self.config.writeChi2ContributionFiles:
556  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
557  fit.saveChi2Contributions(baseName)
558 
559  if not np.isfinite(chi2.chi2):
560  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
561  self.log.info("Initialized: %s", str(chi2))
562  fit.minimize("Model")
563  chi2 = fit.computeChi2()
564  self.log.info(str(chi2))
565  fit.minimize("Fluxes")
566  chi2 = fit.computeChi2()
567  self.log.info(str(chi2))
568  fit.minimize("Model Fluxes")
569  chi2 = fit.computeChi2()
570  if not np.isfinite(chi2.chi2):
571  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
572  self.log.info("Fit prepared with %s", str(chi2))
573 
574  model.freezeErrorTransform()
575  self.log.debug("Photometry error scales are frozen.")
576 
577  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
578 
579  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
580  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
581  return Photometry(fit, model)
582 
583  def _fit_astrometry(self, associations, dataName=None):
584  """
585  Fit the astrometric data.
586 
587  Parameters
588  ----------
589  associations : lsst.jointcal.Associations
590  The star/reference star associations to fit.
591  dataName : str
592  Name of the data being processed (e.g. "1234_HSC-Y"), for
593  identifying debugging files.
594 
595  Returns
596  -------
597  namedtuple
598  fit : lsst.jointcal.AstrometryFit
599  The astrometric fitter used to perform the fit.
600  model : lsst.jointcal.AstrometryModel
601  The astrometric model that was fit.
602  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
603  The model for the sky to tangent plane projection that was used in the fit.
604  """
605 
606  self.log.info("=== Starting astrometric fitting...")
607 
608  associations.deprojectFittedStars()
609 
610  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
611  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
612  # them so carefully?
613  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
614 
615  if self.config.astrometryModel == "constrainedPoly":
616  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
617  sky_to_tan_projection, self.config.useInputWcs, 0,
618  chipDegree=self.config.astrometryChipDegree,
619  visitDegree=self.config.astrometryVisitDegree)
620  elif self.config.astrometryModel == "simplePoly":
621  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
622  sky_to_tan_projection, self.config.useInputWcs, 0,
623  self.config.astrometrySimpleDegree)
624 
625  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
626  chi2 = fit.computeChi2()
627  # TODO DM-12446: turn this into a "butler save" somehow.
628  # Save reference and measurement chi2 contributions for this data
629  if self.config.writeChi2ContributionFiles:
630  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
631  fit.saveChi2Contributions(baseName)
632 
633  if not np.isfinite(chi2.chi2):
634  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
635  self.log.info("Initialized: %s", str(chi2))
636  fit.minimize("Distortions")
637  chi2 = fit.computeChi2()
638  self.log.info(str(chi2))
639  fit.minimize("Positions")
640  chi2 = fit.computeChi2()
641  self.log.info(str(chi2))
642  fit.minimize("Distortions Positions")
643  chi2 = fit.computeChi2()
644  self.log.info(str(chi2))
645  if not np.isfinite(chi2.chi2):
646  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
647  self.log.info("Fit prepared with %s", str(chi2))
648 
649  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
650 
651  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
652  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
653 
654  return Astrometry(fit, model, sky_to_tan_projection)
655 
656  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
657  """Run fit.minimize up to max_steps times, returning the final chi2."""
658 
659  for i in range(max_steps):
660  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
661  chi2 = fit.computeChi2()
662  if not np.isfinite(chi2.chi2):
663  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
664  self.log.info(str(chi2))
665  if r == MinimizeResult.Converged:
666  self.log.debug("fit has converged - no more outliers - redo minimixation"
667  "one more time in case we have lost accuracy in rank update")
668  # Redo minimization one more time in case we have lost accuracy in rank update
669  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
670  chi2 = fit.computeChi2()
671  self.log.info("Fit completed with: %s", str(chi2))
672  break
673  elif r == MinimizeResult.Chi2Increased:
674  self.log.warn("still some ouliers but chi2 increases - retry")
675  elif r == MinimizeResult.Failed:
676  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
677  else:
678  raise RuntimeError("Unxepected return code from minimize().")
679  else:
680  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
681 
682  return chi2
683 
684  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
685  """
686  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
687 
688  Parameters
689  ----------
690  associations : lsst.jointcal.Associations
691  The star/reference star associations to fit.
692  model : lsst.jointcal.AstrometryModel
693  The astrometric model that was fit.
694  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
695  dict of ccdImage identifiers to dataRefs that were fit
696  """
697 
698  ccdImageList = associations.getCcdImageList()
699  for ccdImage in ccdImageList:
700  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
701  ccd = ccdImage.ccdId
702  visit = ccdImage.visit
703  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
704  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
705  tanSip = model.produceSipWcs(ccdImage)
706  wcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
707  try:
708  dataRef.put(wcs, 'jointcal_wcs')
709  except pexExceptions.Exception as e:
710  self.log.fatal('Failed to write updated Wcs: %s', str(e))
711  raise e
712 
713  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
714  """
715  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
716 
717  Parameters
718  ----------
719  associations : lsst.jointcal.Associations
720  The star/reference star associations to fit.
721  model : lsst.jointcal.PhotometryModel
722  The photoometric model that was fit.
723  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
724  dict of ccdImage identifiers to dataRefs that were fit
725  """
726 
727  ccdImageList = associations.getCcdImageList()
728  for ccdImage in ccdImageList:
729  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
730  ccd = ccdImage.ccdId
731  visit = ccdImage.visit
732  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
733  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
734  photoCalib = model.toPhotoCalib(ccdImage)
735  try:
736  dataRef.put(photoCalib, 'jointcal_photoCalib')
737  except pexExceptions.Exception as e:
738  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
739  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:265
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:521
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:53
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:583
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:656
def _check_star_lists(self, associations, name)
Definition: jointcal.py:512
std::shared_ptr< lsst::afw::geom::SkyWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:713
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:35
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:424
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:684
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:320
Photometric response model which has a single photometric factor per CcdImage.
SpherePoint averageSpherePoint(std::vector< SpherePoint > const &coords)
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:223