lsst.jointcal  14.0-27-ga47eb89+2
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 
3 from __future__ import division, absolute_import, print_function
4 from builtins import str
5 from builtins import range
6 
7 import collections
8 import numpy as np
9 
10 import lsst.utils
11 import lsst.pex.config as pexConfig
12 import lsst.pipe.base as pipeBase
13 import lsst.afw.image as afwImage
14 import lsst.afw.geom as afwGeom
15 import lsst.afw.coord as afwCoord
16 import lsst.pex.exceptions as pexExceptions
17 import lsst.afw.table
19 from lsst.verify import Job, Measurement
20 
21 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
22 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
23 
24 from .dataIds import PerTractCcdDataIdContainer
25 
26 import lsst.jointcal
27 from lsst.jointcal import MinimizeResult
28 
29 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
30 
31 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
32 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
33 
34 
35 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
36 def add_measurement(job, name, value):
37  meas = Measurement(job.metrics[name], value)
38  job.measurements.insert(meas)
39 
40 
41 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
42  """Subclass of TaskRunner for jointcalTask
43 
44  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
45  extracted from the command line (whereas most CmdLineTasks' run methods take
46  single dataRef, are are called repeatedly). This class transforms the processed
47  arguments generated by the ArgumentParser into the arguments expected by
48  Jointcal.run().
49 
50  See pipeBase.TaskRunner for more information.
51  """
52 
53  @staticmethod
54  def getTargetList(parsedCmd, **kwargs):
55  """
56  Return a list of tuples per tract, each containing (dataRefs, kwargs).
57 
58  Jointcal operates on lists of dataRefs simultaneously.
59  """
60  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
61  kwargs['butler'] = parsedCmd.butler
62 
63  # organize data IDs by tract
64  refListDict = {}
65  for ref in parsedCmd.id.refList:
66  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
67  # we call run() once with each tract
68  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
69  return result
70 
71  def __call__(self, args):
72  """
73  Parameters
74  ----------
75  args
76  Arguments for Task.run()
77 
78  Returns
79  -------
80  pipe.base.Struct
81  if self.doReturnResults is False:
82 
83  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
84 
85  if self.doReturnResults is True:
86 
87  - ``result``: the result of calling jointcal.run()
88  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
89  """
90  exitStatus = 0 # exit status for shell
91 
92  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
93  dataRefList, kwargs = args
94  butler = kwargs.pop('butler')
95  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
96  result = None
97  try:
98  result = task.run(dataRefList, **kwargs)
99  exitStatus = result.exitStatus
100  job_path = butler.get('verify_job_filename')
101  result.job.write(job_path[0])
102  except Exception as e: # catch everything, sort it out later.
103  if self.doRaise:
104  raise e
105  else:
106  exitStatus = 1
107  eName = type(e).__name__
108  tract = dataRefList[0].dataId['tract']
109  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
110 
111  if self.doReturnResults:
112  return pipeBase.Struct(result=result, exitStatus=exitStatus)
113  else:
114  return pipeBase.Struct(exitStatus=exitStatus)
115 
116 
117 class JointcalConfig(pexConfig.Config):
118  """Config for JointcalTask"""
119 
120  doAstrometry = pexConfig.Field(
121  doc="Fit astrometry and write the fitted result.",
122  dtype=bool,
123  default=True
124  )
125  doPhotometry = pexConfig.Field(
126  doc="Fit photometry and write the fitted result.",
127  dtype=bool,
128  default=True
129  )
130  coaddName = pexConfig.Field(
131  doc="Type of coadd, typically deep or goodSeeing",
132  dtype=str,
133  default="deep"
134  )
135  posError = pexConfig.Field(
136  doc="Constant term for error on position (in pixel unit)",
137  dtype=float,
138  default=0.02,
139  )
140  # TODO: DM-6885 matchCut should be an afw.geom.Angle
141  matchCut = pexConfig.Field(
142  doc="Matching radius between fitted and reference stars (arcseconds)",
143  dtype=float,
144  default=3.0,
145  )
146  minMeasurements = pexConfig.Field(
147  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
148  dtype=int,
149  default=2,
150  )
151  astrometrySimpleDegree = pexConfig.Field(
152  doc="Polynomial degree for fitting the simple astrometry model.",
153  dtype=int,
154  default=3,
155  )
156  astrometryChipDegree = pexConfig.Field(
157  doc="Degree of the per-chip transform for the constrained astrometry model.",
158  dtype=int,
159  default=2,
160  )
161  astrometryVisitDegree = pexConfig.Field(
162  doc="Degree of the per-visit transform for the constrained astrometry model.",
163  dtype=int,
164  default=3,
165  )
166  useInputWcs = pexConfig.Field(
167  doc="Use the input calexp WCSs to initialize the astrometryModel.",
168  dtype=bool,
169  default=True,
170  )
171  astrometryModel = pexConfig.ChoiceField(
172  doc="Type of model to fit to astrometry",
173  dtype=str,
174  default="simplePoly",
175  allowed={"simplePoly": "One polynomial per ccd",
176  "constrainedPoly": "One polynomial per ccd, and one polynomial per visit"}
177  )
178  photometryModel = pexConfig.ChoiceField(
179  doc="Type of model to fit to photometry",
180  dtype=str,
181  default="simple",
182  allowed={"simple": "One constant zeropoint per ccd and visit",
183  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
184  )
185  photometryVisitDegree = pexConfig.Field(
186  doc="Degree of the per-visit polynomial transform for the constrained photometry model.",
187  dtype=int,
188  default=7,
189  )
190  astrometryRefObjLoader = pexConfig.ConfigurableField(
191  target=LoadIndexedReferenceObjectsTask,
192  doc="Reference object loader for astrometric fit",
193  )
194  photometryRefObjLoader = pexConfig.ConfigurableField(
195  target=LoadIndexedReferenceObjectsTask,
196  doc="Reference object loader for photometric fit",
197  )
198  sourceSelector = sourceSelectorRegistry.makeField(
199  doc="How to select sources for cross-matching",
200  default="astrometry"
201  )
202  writeChi2ContributionFiles = pexConfig.Field(
203  dtype=bool,
204  doc="Write initial/final fit files containing the contributions to chi2.",
205  default=False
206  )
207 
208  def setDefaults(self):
209  sourceSelector = self.sourceSelector["astrometry"]
210  sourceSelector.setDefaults()
211  # don't want to lose existing flags, just add to them.
212  sourceSelector.badFlags.extend(["slot_Shape_flag"])
213  # This should be used to set the FluxField value in jointcal::JointcalControl
214  sourceSelector.sourceFluxType = 'Calib'
215 
216 
217 class JointcalTask(pipeBase.CmdLineTask):
218  """Jointly astrometrically and photometrically calibrate a group of images."""
219 
220  ConfigClass = JointcalConfig
221  RunnerClass = JointcalRunner
222  _DefaultName = "jointcal"
223 
224  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
225  """
226  Instantiate a JointcalTask.
227 
228  Parameters
229  ----------
230  butler : lsst.daf.persistence.Butler
231  The butler is passed to the refObjLoader constructor in case it is
232  needed. Ignored if the refObjLoader argument provides a loader directly.
233  Used to initialize the astrometry and photometry refObjLoaders.
234  profile_jointcal : bool
235  set to True to profile different stages of this jointcal run.
236  """
237  pipeBase.CmdLineTask.__init__(self, **kwargs)
238  self.profile_jointcal = profile_jointcal
239  self.makeSubtask("sourceSelector")
240  if self.config.doAstrometry:
241  self.makeSubtask('astrometryRefObjLoader', butler=butler)
242  if self.config.doPhotometry:
243  self.makeSubtask('photometryRefObjLoader', butler=butler)
244 
245  # To hold various computed metrics for use by tests
246  self.job = Job.load_metrics_package(subset='jointcal')
247 
248  # We don't need to persist config and metadata at this stage.
249  # In this way, we don't need to put a specific entry in the camera mapper policy file
250  def _getConfigName(self):
251  return None
252 
253  def _getMetadataName(self):
254  return None
255 
256  @classmethod
257  def _makeArgumentParser(cls):
258  """Create an argument parser"""
259  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
260  parser.add_argument("--profile_jointcal", default=False, action="store_true",
261  help="Profile steps of jointcal separately.")
262  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
263  ContainerClass=PerTractCcdDataIdContainer)
264  return parser
265 
266  def _build_ccdImage(self, dataRef, associations, jointcalControl):
267  """
268  Extract the necessary things from this dataRef to add a new ccdImage.
269 
270  Parameters
271  ----------
272  dataRef : lsst.daf.persistence.ButlerDataRef
273  dataRef to extract info from.
274  associations : lsst.jointcal.Associations
275  object to add the info to, to construct a new CcdImage
276  jointcalControl : jointcal.JointcalControl
277  control object for associations management
278 
279  Returns
280  ------
281  namedtuple
282  wcs : lsst.afw.geom.SkyWcs
283  the TAN WCS of this image, read from the calexp
284  key : namedtuple
285  a key to identify this dataRef by its visit and ccd ids
286  filter : str
287  this calexp's filter
288  """
289  if "visit" in dataRef.dataId.keys():
290  visit = dataRef.dataId["visit"]
291  else:
292  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
293 
294  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
295 
296  visitInfo = dataRef.get('calexp_visitInfo')
297  detector = dataRef.get('calexp_detector')
298  ccdId = detector.getId()
299  calib = dataRef.get('calexp_calib')
300  tanWcs = dataRef.get('calexp_wcs')
301  bbox = dataRef.get('calexp_bbox')
302  filt = dataRef.get('calexp_filter')
303  filterName = filt.getName()
304  fluxMag0 = calib.getFluxMag0()
305  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
306 
307  goodSrc = self.sourceSelector.selectSources(src)
308 
309  if len(goodSrc.sourceCat) == 0:
310  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
311  else:
312  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
313  associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
314  visit, ccdId, jointcalControl)
315 
316  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
317  Key = collections.namedtuple('Key', ('visit', 'ccd'))
318  return Result(tanWcs, Key(visit, ccdId), filterName)
319 
320  @pipeBase.timeMethod
321  def run(self, dataRefs, profile_jointcal=False):
322  """
323  Jointly calibrate the astrometry and photometry across a set of images.
324 
325  Parameters
326  ----------
327  dataRefs : list of lsst.daf.persistence.ButlerDataRef
328  List of data references to the exposures to be fit.
329  profile_jointcal : bool
330  Profile the individual steps of jointcal.
331 
332  Returns
333  -------
334  pipe.base.Struct
335  struct containing:
336  * dataRefs: the provided data references that were fit (with updated WCSs)
337  * oldWcsList: the original WCS from each dataRef
338  * metrics: dictionary of internally-computed metrics for testing/validation.
339  """
340  if len(dataRefs) == 0:
341  raise ValueError('Need a non-empty list of data references!')
342 
343  exitStatus = 0 # exit status for shell
344 
345  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
346  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
347  associations = lsst.jointcal.Associations()
348 
349  visit_ccd_to_dataRef = {}
350  oldWcsList = []
351  filters = []
352  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
353  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
354  # We need the bounding-box of the focal plane for photometry visit models.
355  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
356  camera = dataRefs[0].get('camera', immediate=True)
357  self.focalPlaneBBox = camera.getFpBBox()
358  for ref in dataRefs:
359  result = self._build_ccdImage(ref, associations, jointcalControl)
360  oldWcsList.append(result.wcs)
361  visit_ccd_to_dataRef[result.key] = ref
362  filters.append(result.filter)
363  filters = collections.Counter(filters)
364 
365  centers = [ccdImage.getBoresightRaDec() for ccdImage in associations.getCcdImageList()]
366  commonTangentPoint = lsst.afw.coord.averageCoord(centers)
367  self.log.debug("Using common tangent point: %s", commonTangentPoint.getPosition())
368  associations.setCommonTangentPoint(commonTangentPoint.getPosition())
369 
370  # Use external reference catalogs handled by LSST stack mechanism
371  # Get the bounding box overlapping all associated images
372  # ==> This is probably a bad idea to do it this way <== To be improved
373  bbox = associations.getRaDecBBox()
374  center = afwCoord.IcrsCoord(bbox.getCenter(), afwGeom.degrees)
375  corner = afwCoord.IcrsCoord(bbox.getMax(), afwGeom.degrees)
376  radius = center.angularSeparation(corner).asRadians()
377 
378  # Get astrometry_net_data path
379  anDir = lsst.utils.getPackageDir('astrometry_net_data')
380  if anDir is None:
381  raise RuntimeError("astrometry_net_data is not setup")
382 
383  # Determine a default filter associated with the catalog. See DM-9093
384  defaultFilter = filters.most_common(1)[0][0]
385  self.log.debug("Using %s band for reference flux", defaultFilter)
386 
387  # TODO: need a better way to get the tract.
388  tract = dataRefs[0].dataId['tract']
389 
390  if self.config.doAstrometry:
391  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
392  name="astrometry",
393  refObjLoader=self.astrometryRefObjLoader,
394  fit_function=self._fit_astrometry,
395  profile_jointcal=profile_jointcal,
396  tract=tract)
397  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
398  else:
399  astrometry = Astrometry(None, None, None)
400 
401  if self.config.doPhotometry:
402  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
403  name="photometry",
404  refObjLoader=self.photometryRefObjLoader,
405  fit_function=self._fit_photometry,
406  profile_jointcal=profile_jointcal,
407  tract=tract,
408  filters=filters,
409  reject_bad_fluxes=True)
410  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
411  else:
412  photometry = Photometry(None, None)
413 
414  return pipeBase.Struct(dataRefs=dataRefs,
415  oldWcsList=oldWcsList,
416  job=self.job,
417  exitStatus=exitStatus)
418 
419  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
420  name="", refObjLoader=None, filters=[], fit_function=None,
421  tract=None, profile_jointcal=False, match_cut=3.0,
422  reject_bad_fluxes=False):
423  """Load reference catalog, perform the fit, and return the result.
424 
425  Parameters
426  ----------
427  associations : lsst.jointcal.Associations
428  The star/reference star associations to fit.
429  defaultFilter : str
430  filter to load from reference catalog.
431  center : lsst.afw.coord.IcrsCoord
432  Center of field to load from reference catalog.
433  radius : lsst.afw.geom.Angle
434  On-sky radius to load from reference catalog.
435  name : str
436  Name of thing being fit: "Astrometry" or "Photometry".
437  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
438  Reference object loader to load from for fit.
439  filters : list of str, optional
440  List of filters to load from the reference catalog.
441  fit_function : function
442  function to call to perform fit (takes associations object).
443  tract : str
444  Name of tract currently being fit.
445  profile_jointcal : bool, optional
446  Separately profile the fitting step.
447  match_cut : float, optional
448  Radius in arcseconds to find cross-catalog matches to during
449  associations.associateCatalogs.
450  reject_bad_fluxes : bool, optional
451  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
452 
453  Returns
454  -------
455  Result of `fit_function()`
456  """
457  self.log.info("====== Now processing %s...", name)
458  # TODO: this should not print "trying to invert a singular transformation:"
459  # if it does that, something's not right about the WCS...
460  associations.associateCatalogs(match_cut)
461  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
462  associations.fittedStarListSize())
463 
464  skyCircle = refObjLoader.loadSkyCircle(center,
465  afwGeom.Angle(radius, afwGeom.radians),
466  defaultFilter)
467 
468  # Need memory contiguity to get reference filters as a vector.
469  if not skyCircle.refCat.isContiguous():
470  refCat = skyCircle.refCat.copy(deep=True)
471  else:
472  refCat = skyCircle.refCat
473 
474  # load the reference catalog fluxes.
475  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
476  refFluxes = {}
477  refFluxErrs = {}
478  for filt in filters:
479  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
480  refFluxes[filt] = refCat.get(filtKeys[0])
481  refFluxErrs[filt] = refCat.get(filtKeys[1])
482 
483  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
484  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
485  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
486  associations.refStarListSize())
487 
488  associations.prepareFittedStars(self.config.minMeasurements)
489 
490  self._check_star_lists(associations, name)
491  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
492  associations.nFittedStarsWithAssociatedRefStar())
493  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
494  associations.fittedStarListSize())
495  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
496  associations.nCcdImagesValidForFit())
497 
498  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
499  dataName = "{}_{}".format(tract, defaultFilter)
500  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
501  result = fit_function(associations, dataName)
502  # TODO DM-12446: turn this into a "butler save" somehow.
503  # Save reference and measurement chi2 contributions for this data
504  if self.config.writeChi2ContributionFiles:
505  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
506  result.fit.saveChi2Contributions(baseName)
507 
508  return result
509 
510  def _check_star_lists(self, associations, name):
511  # TODO: these should be len(blah), but we need this properly wrapped first.
512  if associations.nCcdImagesValidForFit() == 0:
513  raise RuntimeError('No images in the ccdImageList!')
514  if associations.fittedStarListSize() == 0:
515  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
516  if associations.refStarListSize() == 0:
517  raise RuntimeError('No stars in the {} reference star list!'.format(name))
518 
519  def _fit_photometry(self, associations, dataName=None):
520  """
521  Fit the photometric data.
522 
523  Parameters
524  ----------
525  associations : lsst.jointcal.Associations
526  The star/reference star associations to fit.
527  dataName : str
528  Name of the data being processed (e.g. "1234_HSC-Y"), for
529  identifying debugging files.
530 
531  Returns
532  -------
533  namedtuple
534  fit : lsst.jointcal.PhotometryFit
535  The photometric fitter used to perform the fit.
536  model : lsst.jointcal.PhotometryModel
537  The photometric model that was fit.
538  """
539  self.log.info("=== Starting photometric fitting...")
540 
541  # TODO: should use pex.config.RegistryField here (see DM-9195)
542  if self.config.photometryModel == "constrained":
543  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
544  self.focalPlaneBBox,
545  visitDegree=self.config.photometryVisitDegree)
546  elif self.config.photometryModel == "simple":
547  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
548 
549  fit = lsst.jointcal.PhotometryFit(associations, model)
550  chi2 = fit.computeChi2()
551  # TODO DM-12446: turn this into a "butler save" somehow.
552  # Save reference and measurement chi2 contributions for this data
553  if self.config.writeChi2ContributionFiles:
554  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
555  fit.saveChi2Contributions(baseName)
556 
557  if not np.isfinite(chi2.chi2):
558  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
559  self.log.info("Initialized: %s", str(chi2))
560  fit.minimize("Model")
561  chi2 = fit.computeChi2()
562  self.log.info(str(chi2))
563  fit.minimize("Fluxes")
564  chi2 = fit.computeChi2()
565  self.log.info(str(chi2))
566  fit.minimize("Model Fluxes")
567  chi2 = fit.computeChi2()
568  if not np.isfinite(chi2.chi2):
569  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
570  self.log.info("Fit prepared with %s", str(chi2))
571 
572  model.freezeErrorTransform()
573  self.log.debug("Photometry error scales are frozen.")
574 
575  chi2 = self._iterate_fit(fit, model, 20, "photometry", "Model Fluxes")
576 
577  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
578  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
579  return Photometry(fit, model)
580 
581  def _fit_astrometry(self, associations, dataName=None):
582  """
583  Fit the astrometric data.
584 
585  Parameters
586  ----------
587  associations : lsst.jointcal.Associations
588  The star/reference star associations to fit.
589  dataName : str
590  Name of the data being processed (e.g. "1234_HSC-Y"), for
591  identifying debugging files.
592 
593  Returns
594  -------
595  namedtuple
596  fit : lsst.jointcal.AstrometryFit
597  The astrometric fitter used to perform the fit.
598  model : lsst.jointcal.AstrometryModel
599  The astrometric model that was fit.
600  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
601  The model for the sky to tangent plane projection that was used in the fit.
602  """
603 
604  self.log.info("=== Starting astrometric fitting...")
605 
606  associations.deprojectFittedStars()
607 
608  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
609  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
610  # them so carefully?
611  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
612 
613  if self.config.astrometryModel == "constrainedPoly":
614  model = lsst.jointcal.ConstrainedPolyModel(associations.getCcdImageList(),
615  sky_to_tan_projection, self.config.useInputWcs, 0,
616  chipDegree=self.config.astrometryChipDegree,
617  visitDegree=self.config.astrometryVisitDegree)
618  elif self.config.astrometryModel == "simplePoly":
619  model = lsst.jointcal.SimplePolyModel(associations.getCcdImageList(),
620  sky_to_tan_projection, self.config.useInputWcs, 0,
621  self.config.astrometrySimpleDegree)
622 
623  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
624  chi2 = fit.computeChi2()
625  # TODO DM-12446: turn this into a "butler save" somehow.
626  # Save reference and measurement chi2 contributions for this data
627  if self.config.writeChi2ContributionFiles:
628  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
629  fit.saveChi2Contributions(baseName)
630 
631  if not np.isfinite(chi2.chi2):
632  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
633  self.log.info("Initialized: %s", str(chi2))
634  fit.minimize("Distortions")
635  chi2 = fit.computeChi2()
636  self.log.info(str(chi2))
637  fit.minimize("Positions")
638  chi2 = fit.computeChi2()
639  self.log.info(str(chi2))
640  fit.minimize("Distortions Positions")
641  chi2 = fit.computeChi2()
642  self.log.info(str(chi2))
643  if not np.isfinite(chi2.chi2):
644  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
645  self.log.info("Fit prepared with %s", str(chi2))
646 
647  chi2 = self._iterate_fit(fit, model, 20, "astrometry", "Distortions Positions")
648 
649  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
650  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
651 
652  return Astrometry(fit, model, sky_to_tan_projection)
653 
654  def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
655  """Run fit.minimize up to max_steps times, returning the final chi2."""
656 
657  for i in range(max_steps):
658  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
659  chi2 = fit.computeChi2()
660  if not np.isfinite(chi2.chi2):
661  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
662  self.log.info(str(chi2))
663  if r == MinimizeResult.Converged:
664  self.log.debug("fit has converged - no more outliers - redo minimixation"
665  "one more time in case we have lost accuracy in rank update")
666  # Redo minimization one more time in case we have lost accuracy in rank update
667  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
668  chi2 = fit.computeChi2()
669  self.log.info("Fit completed with: %s", str(chi2))
670  break
671  elif r == MinimizeResult.Chi2Increased:
672  self.log.warn("still some ouliers but chi2 increases - retry")
673  elif r == MinimizeResult.Failed:
674  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
675  else:
676  raise RuntimeError("Unxepected return code from minimize().")
677  else:
678  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
679 
680  return chi2
681 
682  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
683  """
684  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
685 
686  Parameters
687  ----------
688  associations : lsst.jointcal.Associations
689  The star/reference star associations to fit.
690  model : lsst.jointcal.AstrometryModel
691  The astrometric model that was fit.
692  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
693  dict of ccdImage identifiers to dataRefs that were fit
694  """
695 
696  ccdImageList = associations.getCcdImageList()
697  for ccdImage in ccdImageList:
698  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
699  ccd = ccdImage.ccdId
700  visit = ccdImage.visit
701  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
702  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
703  tanSip = model.produceSipWcs(ccdImage)
704  wcs = lsst.jointcal.gtransfoToTanWcs(tanSip, ccdImage.imageFrame, False)
705  try:
706  dataRef.put(wcs, 'jointcal_wcs')
707  except pexExceptions.Exception as e:
708  self.log.fatal('Failed to write updated Wcs: %s', str(e))
709  raise e
710 
711  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
712  """
713  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
714 
715  Parameters
716  ----------
717  associations : lsst.jointcal.Associations
718  The star/reference star associations to fit.
719  model : lsst.jointcal.PhotometryModel
720  The photoometric model that was fit.
721  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
722  dict of ccdImage identifiers to dataRefs that were fit
723  """
724 
725  ccdImageList = associations.getCcdImageList()
726  for ccdImage in ccdImageList:
727  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
728  ccd = ccdImage.ccdId
729  visit = ccdImage.visit
730  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
731  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
732  photoCalib = model.toPhotoCalib(ccdImage)
733  try:
734  dataRef.put(photoCalib, 'jointcal_photoCalib')
735  except pexExceptions.Exception as e:
736  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
737  raise e
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:266
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:519
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:54
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:581
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:654
def _check_star_lists(self, associations, name)
Definition: jointcal.py:510
std::shared_ptr< lsst::afw::geom::SkyWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:28
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:711
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:36
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:422
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:682
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:321
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:224