lsst.jointcal  15.0-7-gab4c137+2
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.pex.exceptions as pexExceptions
16 import lsst.afw.table
18 from lsst.verify import Job, Measurement
19 
20 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
21 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
22 
23 from .dataIds import PerTractCcdDataIdContainer
24 
25 import lsst.jointcal
26 from lsst.jointcal import MinimizeResult
27 
28 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
29 
30 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
31 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
32 
33 
34 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
35 def add_measurement(job, name, value):
36  meas = Measurement(job.metrics[name], value)
37  job.measurements.insert(meas)
38 
39 
40 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
41  """Subclass of TaskRunner for jointcalTask
42 
43  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
44  extracted from the command line (whereas most CmdLineTasks' run methods take
45  single dataRef, are are called repeatedly). This class transforms the processed
46  arguments generated by the ArgumentParser into the arguments expected by
47  Jointcal.run().
48 
49  See pipeBase.TaskRunner for more information.
50  """
51 
52  @staticmethod
53  def getTargetList(parsedCmd, **kwargs):
54  """
55  Return a list of tuples per tract, each containing (dataRefs, kwargs).
56 
57  Jointcal operates on lists of dataRefs simultaneously.
58  """
59  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
60  kwargs['butler'] = parsedCmd.butler
61 
62  # organize data IDs by tract
63  refListDict = {}
64  for ref in parsedCmd.id.refList:
65  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
66  # we call run() once with each tract
67  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
68  return result
69 
70  def __call__(self, args):
71  """
72  Parameters
73  ----------
74  args
75  Arguments for Task.run()
76 
77  Returns
78  -------
79  pipe.base.Struct
80  if self.doReturnResults is False:
81 
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83 
84  if self.doReturnResults is True:
85 
86  - ``result``: the result of calling jointcal.run()
87  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
88  """
89  exitStatus = 0 # exit status for shell
90 
91  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
92  dataRefList, kwargs = args
93  butler = kwargs.pop('butler')
94  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
95  result = None
96  try:
97  result = task.run(dataRefList, **kwargs)
98  exitStatus = result.exitStatus
99  job_path = butler.get('verify_job_filename')
100  result.job.write(job_path[0])
101  except Exception as e: # catch everything, sort it out later.
102  if self.doRaise:
103  raise e
104  else:
105  exitStatus = 1
106  eName = type(e).__name__
107  tract = dataRefList[0].dataId['tract']
108  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
109 
110  if self.doReturnResults:
111  return pipeBase.Struct(result=result, exitStatus=exitStatus)
112  else:
113  return pipeBase.Struct(exitStatus=exitStatus)
114 
115 
116 class JointcalConfig(pexConfig.Config):
117  """Config for JointcalTask"""
118 
119  doAstrometry = pexConfig.Field(
120  doc="Fit astrometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  doPhotometry = pexConfig.Field(
125  doc="Fit photometry and write the fitted result.",
126  dtype=bool,
127  default=True
128  )
129  coaddName = pexConfig.Field(
130  doc="Type of coadd, typically deep or goodSeeing",
131  dtype=str,
132  default="deep"
133  )
134  posError = pexConfig.Field(
135  doc="Constant term for error on position (in pixel unit)",
136  dtype=float,
137  default=0.02,
138  )
139  # TODO: DM-6885 matchCut should be an afw.geom.Angle
140  matchCut = pexConfig.Field(
141  doc="Matching radius between fitted and reference stars (arcseconds)",
142  dtype=float,
143  default=3.0,
144  )
145  minMeasurements = pexConfig.Field(
146  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
147  dtype=int,
148  default=2,
149  )
150  astrometrySimpleOrder = pexConfig.Field(
151  doc="Polynomial order for fitting the simple astrometry model.",
152  dtype=int,
153  default=3,
154  )
155  astrometryChipOrder = pexConfig.Field(
156  doc="Order of the per-chip transform for the constrained astrometry model.",
157  dtype=int,
158  default=1,
159  )
160  astrometryVisitOrder = pexConfig.Field(
161  doc="Order of the per-visit transform for the constrained astrometry model.",
162  dtype=int,
163  default=5,
164  )
165  useInputWcs = pexConfig.Field(
166  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
167  dtype=bool,
168  default=True,
169  )
170  astrometryModel = pexConfig.ChoiceField(
171  doc="Type of model to fit to astrometry",
172  dtype=str,
173  default="simple",
174  allowed={"simple": "One polynomial per ccd",
175  "constrained": "One polynomial per ccd, and one polynomial per visit"}
176  )
177  photometryModel = pexConfig.ChoiceField(
178  doc="Type of model to fit to photometry",
179  dtype=str,
180  default="simple",
181  allowed={"simple": "One constant zeropoint per ccd and visit",
182  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
183  )
184  photometryVisitOrder = pexConfig.Field(
185  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
186  dtype=int,
187  default=7,
188  )
189  astrometryRefObjLoader = pexConfig.ConfigurableField(
190  target=LoadIndexedReferenceObjectsTask,
191  doc="Reference object loader for astrometric fit",
192  )
193  photometryRefObjLoader = pexConfig.ConfigurableField(
194  target=LoadIndexedReferenceObjectsTask,
195  doc="Reference object loader for photometric fit",
196  )
197  sourceSelector = sourceSelectorRegistry.makeField(
198  doc="How to select sources for cross-matching",
199  default="astrometry"
200  )
201  writeChi2ContributionFiles = pexConfig.Field(
202  dtype=bool,
203  doc="Write initial/final fit files containing the contributions to chi2.",
204  default=False
205  )
206 
207  def setDefaults(self):
208  sourceSelector = self.sourceSelector["astrometry"]
209  sourceSelector.setDefaults()
210  # don't want to lose existing flags, just add to them.
211  sourceSelector.badFlags.extend(["slot_Shape_flag"])
212  # This should be used to set the FluxField value in jointcal::JointcalControl
213  sourceSelector.sourceFluxType = 'Calib'
214 
215 
216 class JointcalTask(pipeBase.CmdLineTask):
217  """Jointly astrometrically and photometrically calibrate a group of images."""
218 
219  ConfigClass = JointcalConfig
220  RunnerClass = JointcalRunner
221  _DefaultName = "jointcal"
222 
223  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
224  """
225  Instantiate a JointcalTask.
226 
227  Parameters
228  ----------
229  butler : lsst.daf.persistence.Butler
230  The butler is passed to the refObjLoader constructor in case it is
231  needed. Ignored if the refObjLoader argument provides a loader directly.
232  Used to initialize the astrometry and photometry refObjLoaders.
233  profile_jointcal : bool
234  set to True to profile different stages of this jointcal run.
235  """
236  pipeBase.CmdLineTask.__init__(self, **kwargs)
237  self.profile_jointcal = profile_jointcal
238  self.makeSubtask("sourceSelector")
239  if self.config.doAstrometry:
240  self.makeSubtask('astrometryRefObjLoader', butler=butler)
241  if self.config.doPhotometry:
242  self.makeSubtask('photometryRefObjLoader', butler=butler)
243 
244  # To hold various computed metrics for use by tests
245  self.job = Job.load_metrics_package(subset='jointcal')
246 
247  # We don't need to persist config and metadata at this stage.
248  # In this way, we don't need to put a specific entry in the camera mapper policy file
249  def _getConfigName(self):
250  return None
251 
252  def _getMetadataName(self):
253  return None
254 
255  @classmethod
256  def _makeArgumentParser(cls):
257  """Create an argument parser"""
258  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
259  parser.add_argument("--profile_jointcal", default=False, action="store_true",
260  help="Profile steps of jointcal separately.")
261  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
262  ContainerClass=PerTractCcdDataIdContainer)
263  return parser
264 
265  def _build_ccdImage(self, dataRef, associations, jointcalControl):
266  """
267  Extract the necessary things from this dataRef to add a new ccdImage.
268 
269  Parameters
270  ----------
271  dataRef : lsst.daf.persistence.ButlerDataRef
272  dataRef to extract info from.
273  associations : lsst.jointcal.Associations
274  object to add the info to, to construct a new CcdImage
275  jointcalControl : jointcal.JointcalControl
276  control object for associations management
277 
278  Returns
279  ------
280  namedtuple
281  wcs : lsst.afw.geom.SkyWcs
282  the TAN WCS of this image, read from the calexp
283  key : namedtuple
284  a key to identify this dataRef by its visit and ccd ids
285  filter : str
286  this calexp's filter
287  """
288  if "visit" in dataRef.dataId.keys():
289  visit = dataRef.dataId["visit"]
290  else:
291  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
292 
293  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
294 
295  visitInfo = dataRef.get('calexp_visitInfo')
296  detector = dataRef.get('calexp_detector')
297  ccdId = detector.getId()
298  calib = dataRef.get('calexp_calib')
299  tanWcs = dataRef.get('calexp_wcs')
300  bbox = dataRef.get('calexp_bbox')
301  filt = dataRef.get('calexp_filter')
302  filterName = filt.getName()
303  fluxMag0 = calib.getFluxMag0()
304  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
305 
306  goodSrc = self.sourceSelector.selectSources(src)
307 
308  if len(goodSrc.sourceCat) == 0:
309  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
310  else:
311  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
312  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
313  visit, ccdId, jointcalControl)
314 
315  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
316  Key = collections.namedtuple('Key', ('visit', 'ccd'))
317  return Result(tanWcs, Key(visit, ccdId), filterName)
318 
319  @pipeBase.timeMethod
320  def run(self, dataRefs, profile_jointcal=False):
321  """
322  Jointly calibrate the astrometry and photometry across a set of images.
323 
324  Parameters
325  ----------
326  dataRefs : list of lsst.daf.persistence.ButlerDataRef
327  List of data references to the exposures to be fit.
328  profile_jointcal : bool
329  Profile the individual steps of jointcal.
330 
331  Returns
332  -------
333  pipe.base.Struct
334  struct containing:
335  * dataRefs: the provided data references that were fit (with updated WCSs)
336  * oldWcsList: the original WCS from each dataRef
337  * metrics: dictionary of internally-computed metrics for testing/validation.
338  """
339  if len(dataRefs) == 0:
340  raise ValueError('Need a non-empty list of data references!')
341 
342  exitStatus = 0 # exit status for shell
343 
344  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
345  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
346  associations = lsst.jointcal.Associations()
347 
348  visit_ccd_to_dataRef = {}
349  oldWcsList = []
350  filters = []
351  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
352  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
353  # We need the bounding-box of the focal plane for photometry visit models.
354  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
355  camera = dataRefs[0].get('camera', immediate=True)
356  self.focalPlaneBBox = camera.getFpBBox()
357  for ref in dataRefs:
358  result = self._build_ccdImage(ref, associations, jointcalControl)
359  oldWcsList.append(result.wcs)
360  visit_ccd_to_dataRef[result.key] = ref
361  filters.append(result.filter)
362  filters = collections.Counter(filters)
363 
364  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
365  commonTangentPoint = afwGeom.averageSpherePoint(centers)
366  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition(afwGeom.degrees))
367  associations.setCommonTangentPoint(commonTangentPoint.getPosition(afwGeom.degrees))
368 
369  # Use external reference catalogs handled by LSST stack mechanism
370  # Get the bounding box overlapping all associated images
371  # ==> This is probably a bad idea to do it this way <== To be improved
372  bbox = associations.getRaDecBBox()
373  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
374  bboxCenter = bbox.getCenter()
375  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
376  bboxMax = bbox.getMax()
377  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
378  radius = center.separation(corner).asRadians()
379 
380  # Get astrometry_net_data path
381  anDir = lsst.utils.getPackageDir('astrometry_net_data')
382  if anDir is None:
383  raise RuntimeError("astrometry_net_data is not setup")
384 
385  # Determine a default filter associated with the catalog. See DM-9093
386  defaultFilter = filters.most_common(1)[0][0]
387  self.log.debug("Using %s band for reference flux", defaultFilter)
388 
389  # TODO: need a better way to get the tract.
390  tract = dataRefs[0].dataId['tract']
391 
392  if self.config.doAstrometry:
393  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
394  name="astrometry",
395  refObjLoader=self.astrometryRefObjLoader,
396  fit_function=self._fit_astrometry,
397  profile_jointcal=profile_jointcal,
398  tract=tract)
399  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
400  else:
401  astrometry = Astrometry(None, None, None)
402 
403  if self.config.doPhotometry:
404  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
405  name="photometry",
406  refObjLoader=self.photometryRefObjLoader,
407  fit_function=self._fit_photometry,
408  profile_jointcal=profile_jointcal,
409  tract=tract,
410  filters=filters,
411  reject_bad_fluxes=True)
412  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
413  else:
414  photometry = Photometry(None, None)
415 
416  return pipeBase.Struct(dataRefs=dataRefs,
417  oldWcsList=oldWcsList,
418  job=self.job,
419  exitStatus=exitStatus)
420 
421  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
422  name="", refObjLoader=None, filters=[], fit_function=None,
423  tract=None, profile_jointcal=False, match_cut=3.0,
424  reject_bad_fluxes=False):
425  """Load reference catalog, perform the fit, and return the result.
426 
427  Parameters
428  ----------
429  associations : lsst.jointcal.Associations
430  The star/reference star associations to fit.
431  defaultFilter : str
432  filter to load from reference catalog.
433  center : lsst.afw.geom.SpherePoint
434  ICRS center of field to load from reference catalog.
435  radius : lsst.afw.geom.Angle
436  On-sky radius to load from reference catalog.
437  name : str
438  Name of thing being fit: "Astrometry" or "Photometry".
439  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
440  Reference object loader to load from for fit.
441  filters : list of str, optional
442  List of filters to load from the reference catalog.
443  fit_function : function
444  function to call to perform fit (takes associations object).
445  tract : str
446  Name of tract currently being fit.
447  profile_jointcal : bool, optional
448  Separately profile the fitting step.
449  match_cut : float, optional
450  Radius in arcseconds to find cross-catalog matches to during
451  associations.associateCatalogs.
452  reject_bad_fluxes : bool, optional
453  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
454 
455  Returns
456  -------
457  Result of `fit_function()`
458  """
459  self.log.info("====== Now processing %s...", name)
460  # TODO: this should not print "trying to invert a singular transformation:"
461  # if it does that, something's not right about the WCS...
462  associations.associateCatalogs(match_cut)
463  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
464  associations.fittedStarListSize())
465 
466  skyCircle = refObjLoader.loadSkyCircle(center,
467  afwGeom.Angle(radius, afwGeom.radians),
468  defaultFilter)
469 
470  # Need memory contiguity to get reference filters as a vector.
471  if not skyCircle.refCat.isContiguous():
472  refCat = skyCircle.refCat.copy(deep=True)
473  else:
474  refCat = skyCircle.refCat
475 
476  # load the reference catalog fluxes.
477  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
478  refFluxes = {}
479  refFluxErrs = {}
480  for filt in filters:
481  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
482  refFluxes[filt] = refCat.get(filtKeys[0])
483  refFluxErrs[filt] = refCat.get(filtKeys[1])
484 
485  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
486  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
487  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
488  associations.refStarListSize())
489 
490  associations.prepareFittedStars(self.config.minMeasurements)
491 
492  self._check_star_lists(associations, name)
493  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
494  associations.nFittedStarsWithAssociatedRefStar())
495  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
496  associations.fittedStarListSize())
497  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
498  associations.nCcdImagesValidForFit())
499 
500  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
501  dataName = "{}_{}".format(tract, defaultFilter)
502  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
503  result = fit_function(associations, dataName)
504  # TODO DM-12446: turn this into a "butler save" somehow.
505  # Save reference and measurement chi2 contributions for this data
506  if self.config.writeChi2ContributionFiles:
507  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
508  result.fit.saveChi2Contributions(baseName)
509 
510  return result
511 
512  def _check_star_lists(self, associations, name):
513  # TODO: these should be len(blah), but we need this properly wrapped first.
514  if associations.nCcdImagesValidForFit() == 0:
515  raise RuntimeError('No images in the ccdImageList!')
516  if associations.fittedStarListSize() == 0:
517  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
518  if associations.refStarListSize() == 0:
519  raise RuntimeError('No stars in the {} reference star list!'.format(name))
520 
521  def _fit_photometry(self, associations, dataName=None):
522  """
523  Fit the photometric data.
524 
525  Parameters
526  ----------
527  associations : lsst.jointcal.Associations
528  The star/reference star associations to fit.
529  dataName : str
530  Name of the data being processed (e.g. "1234_HSC-Y"), for
531  identifying debugging files.
532 
533  Returns
534  -------
535  namedtuple
536  fit : lsst.jointcal.PhotometryFit
537  The photometric fitter used to perform the fit.
538  model : lsst.jointcal.PhotometryModel
539  The photometric model that was fit.
540  """
541  self.log.info("=== Starting photometric fitting...")
542 
543  # TODO: should use pex.config.RegistryField here (see DM-9195)
544  if self.config.photometryModel == "constrained":
545  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
546  self.focalPlaneBBox,
547  visitOrder=self.config.photometryVisitOrder)
548  elif self.config.photometryModel == "simple":
549  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
550 
551  fit = lsst.jointcal.PhotometryFit(associations, model)
552  chi2 = fit.computeChi2()
553  # TODO DM-12446: turn this into a "butler save" somehow.
554  # Save reference and measurement chi2 contributions for this data
555  if self.config.writeChi2ContributionFiles:
556  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
557  fit.saveChi2Contributions(baseName)
558 
559  if not np.isfinite(chi2.chi2):
560  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
561  self.log.info("Initialized: %s", str(chi2))
562  # The constrained model needs the visit transfo fit first; the chip
563  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
564  if self.config.photometryModel == "constrained":
565  # TODO: (related to DM-8046): implement Visit/Chip choice
566  fit.minimize("ModelVisit")
567  chi2 = fit.computeChi2()
568  self.log.info(str(chi2))
569  fit.minimize("Model")
570  chi2 = fit.computeChi2()
571  self.log.info(str(chi2))
572  fit.minimize("Fluxes")
573  chi2 = fit.computeChi2()
574  self.log.info(str(chi2))
575  fit.minimize("Model Fluxes")
576  chi2 = fit.computeChi2()
577  if not np.isfinite(chi2.chi2):
578  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
579  self.log.info("Fit prepared with %s", str(chi2))
580 
581  model.freezeErrorTransform()
582  self.log.debug("Photometry error scales are frozen.")
583 
584  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
585 
586  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
587  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
588  return Photometry(fit, model)
589 
590  def _fit_astrometry(self, associations, dataName=None):
591  """
592  Fit the astrometric data.
593 
594  Parameters
595  ----------
596  associations : lsst.jointcal.Associations
597  The star/reference star associations to fit.
598  dataName : str
599  Name of the data being processed (e.g. "1234_HSC-Y"), for
600  identifying debugging files.
601 
602  Returns
603  -------
604  namedtuple
605  fit : lsst.jointcal.AstrometryFit
606  The astrometric fitter used to perform the fit.
607  model : lsst.jointcal.AstrometryModel
608  The astrometric model that was fit.
609  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
610  The model for the sky to tangent plane projection that was used in the fit.
611  """
612 
613  self.log.info("=== Starting astrometric fitting...")
614 
615  associations.deprojectFittedStars()
616 
617  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
618  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
619  # them so carefully?
620  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
621 
622  if self.config.astrometryModel == "constrained":
623  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
624  sky_to_tan_projection,
625  chipOrder=self.config.astrometryChipOrder,
626  visitOrder=self.config.astrometryVisitOrder)
627  elif self.config.astrometryModel == "simple":
628  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
629  sky_to_tan_projection,
630  self.config.useInputWcs,
631  nNotFit=0,
632  order=self.config.astrometrySimpleOrder)
633 
634  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
635  chi2 = fit.computeChi2()
636  # TODO DM-12446: turn this into a "butler save" somehow.
637  # Save reference and measurement chi2 contributions for this data
638  if self.config.writeChi2ContributionFiles:
639  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
640  fit.saveChi2Contributions(baseName)
641 
642  if not np.isfinite(chi2.chi2):
643  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
644  self.log.info("Initialized: %s", str(chi2))
645  # The constrained model needs the visit transfo fit first; the chip
646  # transfo is initialized from the detector's cameraGeom, so it's close.
647  if self.config.astrometryModel == "constrained":
648  fit.minimize("DistortionsVisit")
649  chi2 = fit.computeChi2()
650  self.log.info(str(chi2))
651  fit.minimize("Distortions")
652  chi2 = fit.computeChi2()
653  self.log.info(str(chi2))
654  fit.minimize("Positions")
655  chi2 = fit.computeChi2()
656  self.log.info(str(chi2))
657  fit.minimize("Distortions Positions")
658  chi2 = fit.computeChi2()
659  self.log.info(str(chi2))
660  if not np.isfinite(chi2.chi2):
661  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
662  self.log.info("Fit prepared with %s", str(chi2))
663 
664  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
665 
666  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
667  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
668 
669  return Astrometry(fit, model, sky_to_tan_projection)
670 
671  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
672  """Run fit.minimize up to max_steps times, returning the final chi2."""
673 
674  for i in range(max_steps):
675  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
676  chi2 = fit.computeChi2()
677  if not np.isfinite(chi2.chi2):
678  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
679  self.log.info(str(chi2))
680  if r == MinimizeResult.Converged:
681  self.log.debug("fit has converged - no more outliers - redo minimization "
682  "one more time in case we have lost accuracy in rank update.")
683  # Redo minimization one more time in case we have lost accuracy in rank update
684  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
685  chi2 = fit.computeChi2()
686  self.log.info("Fit completed with: %s", str(chi2))
687  break
688  elif r == MinimizeResult.Chi2Increased:
689  self.log.warn("still some ouliers but chi2 increases - retry")
690  elif r == MinimizeResult.Failed:
691  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
692  else:
693  raise RuntimeError("Unxepected return code from minimize().")
694  else:
695  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
696 
697  return chi2
698 
699  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
700  """
701  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
702 
703  Parameters
704  ----------
705  associations : lsst.jointcal.Associations
706  The star/reference star associations to fit.
707  model : lsst.jointcal.AstrometryModel
708  The astrometric model that was fit.
709  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
710  dict of ccdImage identifiers to dataRefs that were fit
711  """
712 
713  ccdImageList = associations.getCcdImageList()
714  for ccdImage in ccdImageList:
715  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
716  ccd = ccdImage.ccdId
717  visit = ccdImage.visit
718  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
719  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
720  skyWcs = model.makeSkyWcs(ccdImage)
721  try:
722  dataRef.put(skyWcs, 'jointcal_wcs')
723  except pexExceptions.Exception as e:
724  self.log.fatal('Failed to write updated Wcs: %s', str(e))
725  raise e
726 
727  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
728  """
729  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
730 
731  Parameters
732  ----------
733  associations : lsst.jointcal.Associations
734  The star/reference star associations to fit.
735  model : lsst.jointcal.PhotometryModel
736  The photoometric model that was fit.
737  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
738  dict of ccdImage identifiers to dataRefs that were fit
739  """
740 
741  ccdImageList = associations.getCcdImageList()
742  for ccdImage in ccdImageList:
743  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
744  ccd = ccdImage.ccdId
745  visit = ccdImage.visit
746  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
747  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
748  photoCalib = model.toPhotoCalib(ccdImage)
749  try:
750  dataRef.put(photoCalib, 'jointcal_photoCalib')
751  except pexExceptions.Exception as e:
752  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
753  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:265
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:521
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:53
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:590
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:671
def _check_star_lists(self, associations, name)
Definition: jointcal.py:512
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:727
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:35
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:424
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:699
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:320
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:223