lsst.jointcal  16.0-17-g0bdc215+2
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.runDataRef().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call runDataRef() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.runDataRef()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.runDataRef()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.runDataRef(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  allowLineSearch = pexConfig.Field(
156  doc="Allow a line search during minimization, if it is reasonable for the model"
157  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
158  dtype=bool,
159  default=False
160  )
161  astrometrySimpleOrder = pexConfig.Field(
162  doc="Polynomial order for fitting the simple astrometry model.",
163  dtype=int,
164  default=3,
165  )
166  astrometryChipOrder = pexConfig.Field(
167  doc="Order of the per-chip transform for the constrained astrometry model.",
168  dtype=int,
169  default=1,
170  )
171  astrometryVisitOrder = pexConfig.Field(
172  doc="Order of the per-visit transform for the constrained astrometry model.",
173  dtype=int,
174  default=5,
175  )
176  useInputWcs = pexConfig.Field(
177  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
178  dtype=bool,
179  default=True,
180  )
181  astrometryModel = pexConfig.ChoiceField(
182  doc="Type of model to fit to astrometry",
183  dtype=str,
184  default="constrained",
185  allowed={"simple": "One polynomial per ccd",
186  "constrained": "One polynomial per ccd, and one polynomial per visit"}
187  )
188  photometryModel = pexConfig.ChoiceField(
189  doc="Type of model to fit to photometry",
190  dtype=str,
191  default="constrainedMagnitude",
192  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
193  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
194  " fitting in flux space.",
195  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
196  " fitting in magnitude space.",
197  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
198  " fitting in magnitude space.",
199  }
200  )
201  photometryVisitOrder = pexConfig.Field(
202  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
203  dtype=int,
204  default=7,
205  )
206  photometryDoRankUpdate = pexConfig.Field(
207  doc="Do the rank update step during minimization. "
208  "Skipping this can help deal with models that are too non-linear.",
209  dtype=bool,
210  default=True,
211  )
212  astrometryDoRankUpdate = pexConfig.Field(
213  doc="Do the rank update step during minimization (should not change the astrometry fit). "
214  "Skipping this can help deal with models that are too non-linear.",
215  dtype=bool,
216  default=True,
217  )
218  outlierRejectSigma = pexConfig.Field(
219  doc="How many sigma to reject outliers at during minimization.",
220  dtype=float,
221  default=5.0,
222  )
223  maxPhotometrySteps = pexConfig.Field(
224  doc="Maximum number of minimize iterations to take when fitting photometry.",
225  dtype=int,
226  default=20,
227  )
228  maxAstrometrySteps = pexConfig.Field(
229  doc="Maximum number of minimize iterations to take when fitting photometry.",
230  dtype=int,
231  default=20,
232  )
233  astrometryRefObjLoader = pexConfig.ConfigurableField(
234  target=LoadIndexedReferenceObjectsTask,
235  doc="Reference object loader for astrometric fit",
236  )
237  photometryRefObjLoader = pexConfig.ConfigurableField(
238  target=LoadIndexedReferenceObjectsTask,
239  doc="Reference object loader for photometric fit",
240  )
241  sourceSelector = sourceSelectorRegistry.makeField(
242  doc="How to select sources for cross-matching",
243  default="astrometry"
244  )
245  writeInitMatrix = pexConfig.Field(
246  dtype=bool,
247  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
248  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
249  "Note that these files are the dense versions of the matrix, and so may be very large.",
250  default=False
251  )
252  writeChi2ContributionFiles = pexConfig.Field(
253  dtype=bool,
254  doc="Write initial/final fit files containing the contributions to chi2.",
255  default=False
256  )
257  sourceFluxType = pexConfig.Field(
258  dtype=str,
259  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
260  default='Calib'
261  )
262 
263  def setDefaults(self):
264  sourceSelector = self.sourceSelector["astrometry"]
265  sourceSelector.setDefaults()
266  # don't want to lose existing flags, just add to them.
267  sourceSelector.badFlags.extend(["slot_Shape_flag"])
268  # This should be used to set the FluxField value in jointcal::JointcalControl
269  sourceSelector.sourceFluxType = self.sourceFluxType
270 
271 
272 class JointcalTask(pipeBase.CmdLineTask):
273  """Jointly astrometrically and photometrically calibrate a group of images."""
274 
275  ConfigClass = JointcalConfig
276  RunnerClass = JointcalRunner
277  _DefaultName = "jointcal"
278 
279  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
280  """
281  Instantiate a JointcalTask.
282 
283  Parameters
284  ----------
285  butler : lsst.daf.persistence.Butler
286  The butler is passed to the refObjLoader constructor in case it is
287  needed. Ignored if the refObjLoader argument provides a loader directly.
288  Used to initialize the astrometry and photometry refObjLoaders.
289  profile_jointcal : bool
290  set to True to profile different stages of this jointcal run.
291  """
292  pipeBase.CmdLineTask.__init__(self, **kwargs)
293  self.profile_jointcal = profile_jointcal
294  self.makeSubtask("sourceSelector")
295  if self.config.doAstrometry:
296  self.makeSubtask('astrometryRefObjLoader', butler=butler)
297  if self.config.doPhotometry:
298  self.makeSubtask('photometryRefObjLoader', butler=butler)
299 
300  # To hold various computed metrics for use by tests
301  self.job = Job.load_metrics_package(subset='jointcal')
302 
303  # We don't need to persist config and metadata at this stage.
304  # In this way, we don't need to put a specific entry in the camera mapper policy file
305  def _getConfigName(self):
306  return None
307 
308  def _getMetadataName(self):
309  return None
310 
311  @classmethod
312  def _makeArgumentParser(cls):
313  """Create an argument parser"""
314  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
315  parser.add_argument("--profile_jointcal", default=False, action="store_true",
316  help="Profile steps of jointcal separately.")
317  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
318  ContainerClass=PerTractCcdDataIdContainer)
319  return parser
320 
321  def _build_ccdImage(self, dataRef, associations, jointcalControl):
322  """
323  Extract the necessary things from this dataRef to add a new ccdImage.
324 
325  Parameters
326  ----------
327  dataRef : lsst.daf.persistence.ButlerDataRef
328  dataRef to extract info from.
329  associations : lsst.jointcal.Associations
330  object to add the info to, to construct a new CcdImage
331  jointcalControl : jointcal.JointcalControl
332  control object for associations management
333 
334  Returns
335  ------
336  namedtuple
337  wcs : lsst.afw.geom.SkyWcs
338  the TAN WCS of this image, read from the calexp
339  key : namedtuple
340  a key to identify this dataRef by its visit and ccd ids
341  filter : str
342  this calexp's filter
343  """
344  if "visit" in dataRef.dataId.keys():
345  visit = dataRef.dataId["visit"]
346  else:
347  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
348 
349  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
350 
351  visitInfo = dataRef.get('calexp_visitInfo')
352  detector = dataRef.get('calexp_detector')
353  ccdId = detector.getId()
354  calib = dataRef.get('calexp_calib')
355  tanWcs = dataRef.get('calexp_wcs')
356  bbox = dataRef.get('calexp_bbox')
357  filt = dataRef.get('calexp_filter')
358  filterName = filt.getName()
359  fluxMag0 = calib.getFluxMag0()
360  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
361 
362  goodSrc = self.sourceSelector.run(src)
363 
364  if len(goodSrc.sourceCat) == 0:
365  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
366  else:
367  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
368  associations.createCcdImage(goodSrc.sourceCat,
369  tanWcs,
370  visitInfo,
371  bbox,
372  filterName,
373  photoCalib,
374  detector,
375  visit,
376  ccdId,
377  jointcalControl)
378 
379  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
380  Key = collections.namedtuple('Key', ('visit', 'ccd'))
381  return Result(tanWcs, Key(visit, ccdId), filterName)
382 
383  @pipeBase.timeMethod
384  def runDataRef(self, dataRefs, profile_jointcal=False):
385  """
386  Jointly calibrate the astrometry and photometry across a set of images.
387 
388  Parameters
389  ----------
390  dataRefs : list of lsst.daf.persistence.ButlerDataRef
391  List of data references to the exposures to be fit.
392  profile_jointcal : bool
393  Profile the individual steps of jointcal.
394 
395  Returns
396  -------
397  pipe.base.Struct
398  struct containing:
399  * dataRefs: the provided data references that were fit (with updated WCSs)
400  * oldWcsList: the original WCS from each dataRef
401  * metrics: dictionary of internally-computed metrics for testing/validation.
402  """
403  if len(dataRefs) == 0:
404  raise ValueError('Need a non-empty list of data references!')
405 
406  exitStatus = 0 # exit status for shell
407 
408  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
409  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
410  associations = lsst.jointcal.Associations()
411 
412  visit_ccd_to_dataRef = {}
413  oldWcsList = []
414  filters = []
415  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
416  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
417  # We need the bounding-box of the focal plane for photometry visit models.
418  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
419  camera = dataRefs[0].get('camera', immediate=True)
420  self.focalPlaneBBox = camera.getFpBBox()
421  for ref in dataRefs:
422  result = self._build_ccdImage(ref, associations, jointcalControl)
423  oldWcsList.append(result.wcs)
424  visit_ccd_to_dataRef[result.key] = ref
425  filters.append(result.filter)
426  filters = collections.Counter(filters)
427 
428  associations.computeCommonTangentPoint()
429 
430  # Use external reference catalogs handled by LSST stack mechanism
431  # Get the bounding box overlapping all associated images
432  # ==> This is probably a bad idea to do it this way <== To be improved
433  bbox = associations.getRaDecBBox()
434  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
435  bboxCenter = bbox.getCenter()
436  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
437  bboxMax = bbox.getMax()
438  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
439  radius = center.separation(corner).asRadians()
440 
441  # Get astrometry_net_data path
442  anDir = lsst.utils.getPackageDir('astrometry_net_data')
443  if anDir is None:
444  raise RuntimeError("astrometry_net_data is not setup")
445 
446  # Determine a default filter associated with the catalog. See DM-9093
447  defaultFilter = filters.most_common(1)[0][0]
448  self.log.debug("Using %s band for reference flux", defaultFilter)
449 
450  # TODO: need a better way to get the tract.
451  tract = dataRefs[0].dataId['tract']
452 
453  if self.config.doAstrometry:
454  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
455  name="astrometry",
456  refObjLoader=self.astrometryRefObjLoader,
457  fit_function=self._fit_astrometry,
458  profile_jointcal=profile_jointcal,
459  tract=tract)
460  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
461  else:
462  astrometry = Astrometry(None, None, None)
463 
464  if self.config.doPhotometry:
465  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
466  name="photometry",
467  refObjLoader=self.photometryRefObjLoader,
468  fit_function=self._fit_photometry,
469  profile_jointcal=profile_jointcal,
470  tract=tract,
471  filters=filters,
472  reject_bad_fluxes=True)
473  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
474  else:
475  photometry = Photometry(None, None)
476 
477  return pipeBase.Struct(dataRefs=dataRefs,
478  oldWcsList=oldWcsList,
479  job=self.job,
480  exitStatus=exitStatus)
481 
482  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
483  name="", refObjLoader=None, filters=[], fit_function=None,
484  tract=None, profile_jointcal=False, match_cut=3.0,
485  reject_bad_fluxes=False):
486  """Load reference catalog, perform the fit, and return the result.
487 
488  Parameters
489  ----------
490  associations : lsst.jointcal.Associations
491  The star/reference star associations to fit.
492  defaultFilter : str
493  filter to load from reference catalog.
494  center : lsst.afw.geom.SpherePoint
495  ICRS center of field to load from reference catalog.
496  radius : lsst.afw.geom.Angle
497  On-sky radius to load from reference catalog.
498  name : str
499  Name of thing being fit: "Astrometry" or "Photometry".
500  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
501  Reference object loader to load from for fit.
502  filters : list of str, optional
503  List of filters to load from the reference catalog.
504  fit_function : function
505  function to call to perform fit (takes associations object).
506  tract : str
507  Name of tract currently being fit.
508  profile_jointcal : bool, optional
509  Separately profile the fitting step.
510  match_cut : float, optional
511  Radius in arcseconds to find cross-catalog matches to during
512  associations.associateCatalogs.
513  reject_bad_fluxes : bool, optional
514  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
515 
516  Returns
517  -------
518  Result of `fit_function()`
519  """
520  self.log.info("====== Now processing %s...", name)
521  # TODO: this should not print "trying to invert a singular transformation:"
522  # if it does that, something's not right about the WCS...
523  associations.associateCatalogs(match_cut)
524  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
525  associations.fittedStarListSize())
526 
527  skyCircle = refObjLoader.loadSkyCircle(center,
528  afwGeom.Angle(radius, afwGeom.radians),
529  defaultFilter)
530 
531  # Need memory contiguity to get reference filters as a vector.
532  if not skyCircle.refCat.isContiguous():
533  refCat = skyCircle.refCat.copy(deep=True)
534  else:
535  refCat = skyCircle.refCat
536 
537  # load the reference catalog fluxes.
538  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
539  refFluxes = {}
540  refFluxErrs = {}
541  for filt in filters:
542  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
543  refFluxes[filt] = refCat.get(filtKeys[0])
544  refFluxErrs[filt] = refCat.get(filtKeys[1])
545 
546  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
547  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
548  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
549  associations.refStarListSize())
550 
551  associations.prepareFittedStars(self.config.minMeasurements)
552 
553  self._check_star_lists(associations, name)
554  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
555  associations.nFittedStarsWithAssociatedRefStar())
556  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
557  associations.fittedStarListSize())
558  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
559  associations.nCcdImagesValidForFit())
560 
561  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
562  dataName = "{}_{}".format(tract, defaultFilter)
563  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
564  result = fit_function(associations, dataName)
565  # TODO DM-12446: turn this into a "butler save" somehow.
566  # Save reference and measurement chi2 contributions for this data
567  if self.config.writeChi2ContributionFiles:
568  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
569  result.fit.saveChi2Contributions(baseName)
570 
571  return result
572 
573  def _check_star_lists(self, associations, name):
574  # TODO: these should be len(blah), but we need this properly wrapped first.
575  if associations.nCcdImagesValidForFit() == 0:
576  raise RuntimeError('No images in the ccdImageList!')
577  if associations.fittedStarListSize() == 0:
578  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
579  if associations.refStarListSize() == 0:
580  raise RuntimeError('No stars in the {} reference star list!'.format(name))
581 
582  def _fit_photometry(self, associations, dataName=None):
583  """
584  Fit the photometric data.
585 
586  Parameters
587  ----------
588  associations : lsst.jointcal.Associations
589  The star/reference star associations to fit.
590  dataName : str
591  Name of the data being processed (e.g. "1234_HSC-Y"), for
592  identifying debugging files.
593 
594  Returns
595  -------
596  namedtuple
597  fit : lsst.jointcal.PhotometryFit
598  The photometric fitter used to perform the fit.
599  model : lsst.jointcal.PhotometryModel
600  The photometric model that was fit.
601  """
602  self.log.info("=== Starting photometric fitting...")
603 
604  # TODO: should use pex.config.RegistryField here (see DM-9195)
605  if self.config.photometryModel == "constrainedFlux":
606  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
607  self.focalPlaneBBox,
608  visitOrder=self.config.photometryVisitOrder)
609  # potentially nonlinear problem, so we may need a line search to converge.
610  doLineSearch = self.config.allowLineSearch
611  elif self.config.photometryModel == "constrainedMagnitude":
612  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
613  self.focalPlaneBBox,
614  visitOrder=self.config.photometryVisitOrder)
615  # potentially nonlinear problem, so we may need a line search to converge.
616  doLineSearch = self.config.allowLineSearch
617  elif self.config.photometryModel == "simpleFlux":
618  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList())
619  doLineSearch = False # purely linear in model parameters, so no line search needed
620  elif self.config.photometryModel == "simpleMagnitude":
621  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList())
622  doLineSearch = False # purely linear in model parameters, so no line search needed
623 
624  fit = lsst.jointcal.PhotometryFit(associations, model)
625  chi2 = fit.computeChi2()
626  # TODO DM-12446: turn this into a "butler save" somehow.
627  # Save reference and measurement chi2 contributions for this data
628  if self.config.writeChi2ContributionFiles:
629  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
630  fit.saveChi2Contributions(baseName)
631 
632  if not np.isfinite(chi2.chi2):
633  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
634  self.log.info("Initialized: %s", str(chi2))
635  # The constrained model needs the visit transfo fit first; the chip
636  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
637  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
638  if self.config.photometryModel.startswith("constrained"):
639  # no line search: should be purely (or nearly) linear,
640  # and we want a large step size to initialize with.
641  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
642  chi2 = fit.computeChi2()
643  self.log.info(str(chi2))
644  dumpMatrixFile = "" # so we don't redo the output on the next step
645  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
646  chi2 = fit.computeChi2()
647  self.log.info(str(chi2))
648  fit.minimize("Fluxes") # no line search: always purely linear.
649  chi2 = fit.computeChi2()
650  self.log.info(str(chi2))
651  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
652  chi2 = fit.computeChi2()
653  if not np.isfinite(chi2.chi2):
654  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
655  self.log.info("Fit prepared with %s", str(chi2))
656 
657  model.freezeErrorTransform()
658  self.log.debug("Photometry error scales are frozen.")
659 
660  chi2 = self._iterate_fit(associations,
661  fit,
662  self.config.maxPhotometrySteps,
663  "photometry",
664  "Model Fluxes",
665  doRankUpdate=self.config.photometryDoRankUpdate,
666  doLineSearch=doLineSearch,
667  dataName=dataName)
668 
669  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
670  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
671  return Photometry(fit, model)
672 
673  def _fit_astrometry(self, associations, dataName=None):
674  """
675  Fit the astrometric data.
676 
677  Parameters
678  ----------
679  associations : lsst.jointcal.Associations
680  The star/reference star associations to fit.
681  dataName : str
682  Name of the data being processed (e.g. "1234_HSC-Y"), for
683  identifying debugging files.
684 
685  Returns
686  -------
687  namedtuple
688  fit : lsst.jointcal.AstrometryFit
689  The astrometric fitter used to perform the fit.
690  model : lsst.jointcal.AstrometryModel
691  The astrometric model that was fit.
692  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
693  The model for the sky to tangent plane projection that was used in the fit.
694  """
695 
696  self.log.info("=== Starting astrometric fitting...")
697 
698  associations.deprojectFittedStars()
699 
700  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
701  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
702  # them so carefully?
703  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
704 
705  if self.config.astrometryModel == "constrained":
706  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
707  sky_to_tan_projection,
708  chipOrder=self.config.astrometryChipOrder,
709  visitOrder=self.config.astrometryVisitOrder)
710  elif self.config.astrometryModel == "simple":
711  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
712  sky_to_tan_projection,
713  self.config.useInputWcs,
714  nNotFit=0,
715  order=self.config.astrometrySimpleOrder)
716 
717  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
718  chi2 = fit.computeChi2()
719  # TODO DM-12446: turn this into a "butler save" somehow.
720  # Save reference and measurement chi2 contributions for this data
721  if self.config.writeChi2ContributionFiles:
722  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
723  fit.saveChi2Contributions(baseName)
724 
725  if not np.isfinite(chi2.chi2):
726  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
727  self.log.info("Initialized: %s", str(chi2))
728  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
729  # The constrained model needs the visit transfo fit first; the chip
730  # transfo is initialized from the detector's cameraGeom, so it's close.
731  if self.config.astrometryModel == "constrained":
732  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
733  chi2 = fit.computeChi2()
734  self.log.info(str(chi2))
735  dumpMatrixFile = "" # so we don't redo the output on the next step
736  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
737  chi2 = fit.computeChi2()
738  self.log.info(str(chi2))
739  fit.minimize("Positions")
740  chi2 = fit.computeChi2()
741  self.log.info(str(chi2))
742  fit.minimize("Distortions Positions")
743  chi2 = fit.computeChi2()
744  self.log.info(str(chi2))
745  if not np.isfinite(chi2.chi2):
746  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
747  self.log.info("Fit prepared with %s", str(chi2))
748 
749  chi2 = self._iterate_fit(associations,
750  fit,
751  self.config.maxAstrometrySteps,
752  "astrometry",
753  "Distortions Positions",
754  doRankUpdate=self.config.astrometryDoRankUpdate,
755  dataName=dataName)
756 
757  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
758  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
759 
760  return Astrometry(fit, model, sky_to_tan_projection)
761 
762  def _check_stars(self, associations):
763  """Count measured and reference stars per ccd and warn/log them."""
764  for ccdImage in associations.getCcdImageList():
765  nMeasuredStars, nRefStars = ccdImage.countStars()
766  self.log.debug("ccdImage %s has %s measured and %s reference stars",
767  ccdImage.getName(), nMeasuredStars, nRefStars)
768  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
769  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
770  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
771  if nRefStars < self.config.minRefStarsPerCcd:
772  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
773  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
774 
775  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
776  dataName="",
777  doRankUpdate=True,
778  doLineSearch=False):
779  """Run fitter.minimize up to max_steps times, returning the final chi2.
780 
781  Parameters
782  ----------
783  associations : `lsst.jointcal.Associations`
784  The star/reference star associations to fit.
785  fitter : `lsst.jointcal.FitterBase`
786  The fitter to use for minimization.
787  max_steps : `int`
788  Maximum number of steps to run outlier rejection before declaring
789  convergence failure.
790  name : {'photometry' or 'astrometry'}
791  What type of data are we fitting (for logs and debugging files).
792  whatToFit : `str`
793  Passed to ``fitter.minimize()`` to define the parameters to fit.
794  dataName : str, optional
795  Descriptive name for this dataset (e.g. tract and filter),
796  for debugging.
797  doRankUpdate : bool, optional
798  Do an Eigen rank update during minimization, or recompute the full
799  matrix and gradient?
800  doLineSearch : bool, optional
801  Do a line search for the optimum step during minimization?
802 
803  Returns
804  -------
805  chi2: `lsst.jointcal.Chi2Statistic`
806  The final chi2 after the fit converges, or is forced to end.
807 
808  Raises
809  ------
810  FloatingPointError
811  Raised if the fitter fails with a non-finite value.
812  RuntimeError
813  Raised if the fitter fails for some other reason;
814  log messages will provide further details.
815  """
816  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
817  for i in range(max_steps):
818  result = fitter.minimize(whatToFit,
819  self.config.outlierRejectSigma,
820  doRankUpdate=doRankUpdate,
821  doLineSearch=doLineSearch,
822  dumpMatrixFile=dumpMatrixFile)
823  dumpMatrixFile = "" # clear it so we don't write the matrix again.
824  chi2 = fitter.computeChi2()
825  self._check_stars(associations)
826  self.log.info(str(chi2))
827  if result == MinimizeResult.Converged:
828  if doRankUpdate:
829  self.log.debug("fit has converged - no more outliers - redo minimization "
830  "one more time in case we have lost accuracy in rank update.")
831  # Redo minimization one more time in case we have lost accuracy in rank update
832  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
833  chi2 = fitter.computeChi2()
834  self.log.info("Fit completed with: %s", str(chi2))
835 
836  # log a message for a large final chi2, TODO: DM-15247 for something better
837  if chi2.chi2/chi2.ndof >= 4.0:
838  self.log.error("Potentially bad fit: High chi-squared/ndof.")
839 
840  break
841  elif result == MinimizeResult.Chi2Increased:
842  self.log.warn("still some outliers but chi2 increases - retry")
843  elif result == MinimizeResult.NonFinite:
844  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
845  # TODO DM-12446: turn this into a "butler save" somehow.
846  fitter.saveChi2Contributions(filename)
847  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
848  raise FloatingPointError(msg.format(filename))
849  elif result == MinimizeResult.Failed:
850  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
851  else:
852  raise RuntimeError("Unxepected return code from minimize().")
853  else:
854  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
855 
856  return chi2
857 
858  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
859  """
860  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
861 
862  Parameters
863  ----------
864  associations : lsst.jointcal.Associations
865  The star/reference star associations to fit.
866  model : lsst.jointcal.AstrometryModel
867  The astrometric model that was fit.
868  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
869  dict of ccdImage identifiers to dataRefs that were fit
870  """
871 
872  ccdImageList = associations.getCcdImageList()
873  for ccdImage in ccdImageList:
874  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
875  ccd = ccdImage.ccdId
876  visit = ccdImage.visit
877  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
878  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
879  skyWcs = model.makeSkyWcs(ccdImage)
880  try:
881  dataRef.put(skyWcs, 'jointcal_wcs')
882  except pexExceptions.Exception as e:
883  self.log.fatal('Failed to write updated Wcs: %s', str(e))
884  raise e
885 
886  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
887  """
888  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
889 
890  Parameters
891  ----------
892  associations : lsst.jointcal.Associations
893  The star/reference star associations to fit.
894  model : lsst.jointcal.PhotometryModel
895  The photoometric model that was fit.
896  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
897  dict of ccdImage identifiers to dataRefs that were fit
898  """
899 
900  ccdImageList = associations.getCcdImageList()
901  for ccdImage in ccdImageList:
902  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
903  ccd = ccdImage.ccdId
904  visit = ccdImage.visit
905  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
906  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
907  photoCalib = model.toPhotoCalib(ccdImage)
908  try:
909  dataRef.put(photoCalib, 'jointcal_photoCalib')
910  except pexExceptions.Exception as e:
911  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
912  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:384
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:321
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:582
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:673
def _check_star_lists(self, associations, name)
Definition: jointcal.py:573
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:778
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:886
def _check_stars(self, associations)
Definition: jointcal.py:762
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:485
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:858
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:279