lsst.jointcal  16.0-18-gdf247dd+7
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.runDataRef().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call runDataRef() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.runDataRef()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.runDataRef()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.runDataRef(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  positionErrorPedestal = pexConfig.Field(
130  doc="Systematic term to apply to the measured position error (pixels)",
131  dtype=float,
132  default=0.02,
133  )
134  photometryErrorPedestal = pexConfig.Field(
135  doc="Systematic term to apply to the measured error on flux or magnitude as a "
136  "fraction of source flux or magnitude delta (e.g. 0.05 is 5\% of flux or +50 millimag).",
137  dtype=float,
138  default=0.0,
139  )
140  # TODO: DM-6885 matchCut should be an afw.geom.Angle
141  matchCut = pexConfig.Field(
142  doc="Matching radius between fitted and reference stars (arcseconds)",
143  dtype=float,
144  default=3.0,
145  )
146  minMeasurements = pexConfig.Field(
147  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
148  dtype=int,
149  default=2,
150  )
151  minMeasuredStarsPerCcd = pexConfig.Field(
152  doc="Minimum number of measuredStars per ccdImage before printing warnings",
153  dtype=int,
154  default=100,
155  )
156  minRefStarsPerCcd = pexConfig.Field(
157  doc="Minimum number of measuredStars per ccdImage before printing warnings",
158  dtype=int,
159  default=30,
160  )
161  allowLineSearch = pexConfig.Field(
162  doc="Allow a line search during minimization, if it is reasonable for the model"
163  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
164  dtype=bool,
165  default=False
166  )
167  astrometrySimpleOrder = pexConfig.Field(
168  doc="Polynomial order for fitting the simple astrometry model.",
169  dtype=int,
170  default=3,
171  )
172  astrometryChipOrder = pexConfig.Field(
173  doc="Order of the per-chip transform for the constrained astrometry model.",
174  dtype=int,
175  default=1,
176  )
177  astrometryVisitOrder = pexConfig.Field(
178  doc="Order of the per-visit transform for the constrained astrometry model.",
179  dtype=int,
180  default=5,
181  )
182  useInputWcs = pexConfig.Field(
183  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
184  dtype=bool,
185  default=True,
186  )
187  astrometryModel = pexConfig.ChoiceField(
188  doc="Type of model to fit to astrometry",
189  dtype=str,
190  default="constrained",
191  allowed={"simple": "One polynomial per ccd",
192  "constrained": "One polynomial per ccd, and one polynomial per visit"}
193  )
194  photometryModel = pexConfig.ChoiceField(
195  doc="Type of model to fit to photometry",
196  dtype=str,
197  default="constrainedMagnitude",
198  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
199  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
200  " fitting in flux space.",
201  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
202  " fitting in magnitude space.",
203  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
204  " fitting in magnitude space.",
205  }
206  )
207  photometryVisitOrder = pexConfig.Field(
208  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
209  dtype=int,
210  default=7,
211  )
212  photometryDoRankUpdate = pexConfig.Field(
213  doc="Do the rank update step during minimization. "
214  "Skipping this can help deal with models that are too non-linear.",
215  dtype=bool,
216  default=True,
217  )
218  astrometryDoRankUpdate = pexConfig.Field(
219  doc="Do the rank update step during minimization (should not change the astrometry fit). "
220  "Skipping this can help deal with models that are too non-linear.",
221  dtype=bool,
222  default=True,
223  )
224  outlierRejectSigma = pexConfig.Field(
225  doc="How many sigma to reject outliers at during minimization.",
226  dtype=float,
227  default=5.0,
228  )
229  maxPhotometrySteps = pexConfig.Field(
230  doc="Maximum number of minimize iterations to take when fitting photometry.",
231  dtype=int,
232  default=20,
233  )
234  maxAstrometrySteps = pexConfig.Field(
235  doc="Maximum number of minimize iterations to take when fitting photometry.",
236  dtype=int,
237  default=20,
238  )
239  astrometryRefObjLoader = pexConfig.ConfigurableField(
240  target=LoadIndexedReferenceObjectsTask,
241  doc="Reference object loader for astrometric fit",
242  )
243  photometryRefObjLoader = pexConfig.ConfigurableField(
244  target=LoadIndexedReferenceObjectsTask,
245  doc="Reference object loader for photometric fit",
246  )
247  sourceSelector = sourceSelectorRegistry.makeField(
248  doc="How to select sources for cross-matching",
249  default="astrometry"
250  )
251  writeInitMatrix = pexConfig.Field(
252  dtype=bool,
253  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
254  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
255  "Note that these files are the dense versions of the matrix, and so may be very large.",
256  default=False
257  )
258  writeChi2ContributionFiles = pexConfig.Field(
259  dtype=bool,
260  doc="Write initial/final fit files containing the contributions to chi2.",
261  default=False
262  )
263  sourceFluxType = pexConfig.Field(
264  dtype=str,
265  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
266  default='Calib'
267  )
268 
269  def setDefaults(self):
270  sourceSelector = self.sourceSelector["astrometry"]
271  sourceSelector.setDefaults()
272  # don't want to lose existing flags, just add to them.
273  sourceSelector.badFlags.extend(["slot_Shape_flag"])
274  # This should be used to set the FluxField value in jointcal::JointcalControl
275  sourceSelector.sourceFluxType = self.sourceFluxType
276 
277 
278 class JointcalTask(pipeBase.CmdLineTask):
279  """Jointly astrometrically and photometrically calibrate a group of images."""
280 
281  ConfigClass = JointcalConfig
282  RunnerClass = JointcalRunner
283  _DefaultName = "jointcal"
284 
285  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
286  """
287  Instantiate a JointcalTask.
288 
289  Parameters
290  ----------
291  butler : lsst.daf.persistence.Butler
292  The butler is passed to the refObjLoader constructor in case it is
293  needed. Ignored if the refObjLoader argument provides a loader directly.
294  Used to initialize the astrometry and photometry refObjLoaders.
295  profile_jointcal : bool
296  set to True to profile different stages of this jointcal run.
297  """
298  pipeBase.CmdLineTask.__init__(self, **kwargs)
299  self.profile_jointcal = profile_jointcal
300  self.makeSubtask("sourceSelector")
301  if self.config.doAstrometry:
302  self.makeSubtask('astrometryRefObjLoader', butler=butler)
303  if self.config.doPhotometry:
304  self.makeSubtask('photometryRefObjLoader', butler=butler)
305 
306  # To hold various computed metrics for use by tests
307  self.job = Job.load_metrics_package(subset='jointcal')
308 
309  # We don't need to persist config and metadata at this stage.
310  # In this way, we don't need to put a specific entry in the camera mapper policy file
311  def _getConfigName(self):
312  return None
313 
314  def _getMetadataName(self):
315  return None
316 
317  @classmethod
318  def _makeArgumentParser(cls):
319  """Create an argument parser"""
320  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
321  parser.add_argument("--profile_jointcal", default=False, action="store_true",
322  help="Profile steps of jointcal separately.")
323  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
324  ContainerClass=PerTractCcdDataIdContainer)
325  return parser
326 
327  def _build_ccdImage(self, dataRef, associations, jointcalControl):
328  """
329  Extract the necessary things from this dataRef to add a new ccdImage.
330 
331  Parameters
332  ----------
333  dataRef : lsst.daf.persistence.ButlerDataRef
334  dataRef to extract info from.
335  associations : lsst.jointcal.Associations
336  object to add the info to, to construct a new CcdImage
337  jointcalControl : jointcal.JointcalControl
338  control object for associations management
339 
340  Returns
341  ------
342  namedtuple
343  wcs : lsst.afw.geom.SkyWcs
344  the TAN WCS of this image, read from the calexp
345  key : namedtuple
346  a key to identify this dataRef by its visit and ccd ids
347  filter : str
348  this calexp's filter
349  """
350  if "visit" in dataRef.dataId.keys():
351  visit = dataRef.dataId["visit"]
352  else:
353  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
354 
355  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
356 
357  visitInfo = dataRef.get('calexp_visitInfo')
358  detector = dataRef.get('calexp_detector')
359  ccdId = detector.getId()
360  calib = dataRef.get('calexp_calib')
361  tanWcs = dataRef.get('calexp_wcs')
362  bbox = dataRef.get('calexp_bbox')
363  filt = dataRef.get('calexp_filter')
364  filterName = filt.getName()
365  fluxMag0 = calib.getFluxMag0()
366  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
367 
368  goodSrc = self.sourceSelector.run(src)
369 
370  if len(goodSrc.sourceCat) == 0:
371  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
372  else:
373  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
374  associations.createCcdImage(goodSrc.sourceCat,
375  tanWcs,
376  visitInfo,
377  bbox,
378  filterName,
379  photoCalib,
380  detector,
381  visit,
382  ccdId,
383  jointcalControl)
384 
385  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
386  Key = collections.namedtuple('Key', ('visit', 'ccd'))
387  return Result(tanWcs, Key(visit, ccdId), filterName)
388 
389  @pipeBase.timeMethod
390  def runDataRef(self, dataRefs, profile_jointcal=False):
391  """
392  Jointly calibrate the astrometry and photometry across a set of images.
393 
394  Parameters
395  ----------
396  dataRefs : list of lsst.daf.persistence.ButlerDataRef
397  List of data references to the exposures to be fit.
398  profile_jointcal : bool
399  Profile the individual steps of jointcal.
400 
401  Returns
402  -------
403  pipe.base.Struct
404  struct containing:
405  * dataRefs: the provided data references that were fit (with updated WCSs)
406  * oldWcsList: the original WCS from each dataRef
407  * metrics: dictionary of internally-computed metrics for testing/validation.
408  """
409  if len(dataRefs) == 0:
410  raise ValueError('Need a non-empty list of data references!')
411 
412  exitStatus = 0 # exit status for shell
413 
414  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
415  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
416  associations = lsst.jointcal.Associations()
417 
418  visit_ccd_to_dataRef = {}
419  oldWcsList = []
420  filters = []
421  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
422  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
423  # We need the bounding-box of the focal plane for photometry visit models.
424  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
425  camera = dataRefs[0].get('camera', immediate=True)
426  self.focalPlaneBBox = camera.getFpBBox()
427  for ref in dataRefs:
428  result = self._build_ccdImage(ref, associations, jointcalControl)
429  oldWcsList.append(result.wcs)
430  visit_ccd_to_dataRef[result.key] = ref
431  filters.append(result.filter)
432  filters = collections.Counter(filters)
433 
434  associations.computeCommonTangentPoint()
435 
436  # Use external reference catalogs handled by LSST stack mechanism
437  # Get the bounding box overlapping all associated images
438  # ==> This is probably a bad idea to do it this way <== To be improved
439  bbox = associations.getRaDecBBox()
440  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
441  bboxCenter = bbox.getCenter()
442  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
443  bboxMax = bbox.getMax()
444  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
445  radius = center.separation(corner).asRadians()
446 
447  # Get astrometry_net_data path
448  anDir = lsst.utils.getPackageDir('astrometry_net_data')
449  if anDir is None:
450  raise RuntimeError("astrometry_net_data is not setup")
451 
452  # Determine a default filter associated with the catalog. See DM-9093
453  defaultFilter = filters.most_common(1)[0][0]
454  self.log.debug("Using %s band for reference flux", defaultFilter)
455 
456  # TODO: need a better way to get the tract.
457  tract = dataRefs[0].dataId['tract']
458 
459  if self.config.doAstrometry:
460  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
461  name="astrometry",
462  refObjLoader=self.astrometryRefObjLoader,
463  fit_function=self._fit_astrometry,
464  profile_jointcal=profile_jointcal,
465  tract=tract)
466  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
467  else:
468  astrometry = Astrometry(None, None, None)
469 
470  if self.config.doPhotometry:
471  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
472  name="photometry",
473  refObjLoader=self.photometryRefObjLoader,
474  fit_function=self._fit_photometry,
475  profile_jointcal=profile_jointcal,
476  tract=tract,
477  filters=filters,
478  reject_bad_fluxes=True)
479  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
480  else:
481  photometry = Photometry(None, None)
482 
483  return pipeBase.Struct(dataRefs=dataRefs,
484  oldWcsList=oldWcsList,
485  job=self.job,
486  exitStatus=exitStatus)
487 
488  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
489  name="", refObjLoader=None, filters=[], fit_function=None,
490  tract=None, profile_jointcal=False, match_cut=3.0,
491  reject_bad_fluxes=False):
492  """Load reference catalog, perform the fit, and return the result.
493 
494  Parameters
495  ----------
496  associations : lsst.jointcal.Associations
497  The star/reference star associations to fit.
498  defaultFilter : str
499  filter to load from reference catalog.
500  center : lsst.afw.geom.SpherePoint
501  ICRS center of field to load from reference catalog.
502  radius : lsst.afw.geom.Angle
503  On-sky radius to load from reference catalog.
504  name : str
505  Name of thing being fit: "Astrometry" or "Photometry".
506  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
507  Reference object loader to load from for fit.
508  filters : list of str, optional
509  List of filters to load from the reference catalog.
510  fit_function : function
511  function to call to perform fit (takes associations object).
512  tract : str
513  Name of tract currently being fit.
514  profile_jointcal : bool, optional
515  Separately profile the fitting step.
516  match_cut : float, optional
517  Radius in arcseconds to find cross-catalog matches to during
518  associations.associateCatalogs.
519  reject_bad_fluxes : bool, optional
520  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
521 
522  Returns
523  -------
524  Result of `fit_function()`
525  """
526  self.log.info("====== Now processing %s...", name)
527  # TODO: this should not print "trying to invert a singular transformation:"
528  # if it does that, something's not right about the WCS...
529  associations.associateCatalogs(match_cut)
530  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
531  associations.fittedStarListSize())
532 
533  skyCircle = refObjLoader.loadSkyCircle(center,
534  afwGeom.Angle(radius, afwGeom.radians),
535  defaultFilter)
536 
537  # Need memory contiguity to get reference filters as a vector.
538  if not skyCircle.refCat.isContiguous():
539  refCat = skyCircle.refCat.copy(deep=True)
540  else:
541  refCat = skyCircle.refCat
542 
543  # load the reference catalog fluxes.
544  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
545  refFluxes = {}
546  refFluxErrs = {}
547  for filt in filters:
548  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
549  refFluxes[filt] = refCat.get(filtKeys[0])
550  refFluxErrs[filt] = refCat.get(filtKeys[1])
551 
552  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
553  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
554  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
555  associations.refStarListSize())
556 
557  associations.prepareFittedStars(self.config.minMeasurements)
558 
559  self._check_star_lists(associations, name)
560  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
561  associations.nFittedStarsWithAssociatedRefStar())
562  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
563  associations.fittedStarListSize())
564  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
565  associations.nCcdImagesValidForFit())
566 
567  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
568  dataName = "{}_{}".format(tract, defaultFilter)
569  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
570  result = fit_function(associations, dataName)
571  # TODO DM-12446: turn this into a "butler save" somehow.
572  # Save reference and measurement chi2 contributions for this data
573  if self.config.writeChi2ContributionFiles:
574  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
575  result.fit.saveChi2Contributions(baseName)
576 
577  return result
578 
579  def _check_star_lists(self, associations, name):
580  # TODO: these should be len(blah), but we need this properly wrapped first.
581  if associations.nCcdImagesValidForFit() == 0:
582  raise RuntimeError('No images in the ccdImageList!')
583  if associations.fittedStarListSize() == 0:
584  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
585  if associations.refStarListSize() == 0:
586  raise RuntimeError('No stars in the {} reference star list!'.format(name))
587 
588  def _fit_photometry(self, associations, dataName=None):
589  """
590  Fit the photometric data.
591 
592  Parameters
593  ----------
594  associations : lsst.jointcal.Associations
595  The star/reference star associations to fit.
596  dataName : str
597  Name of the data being processed (e.g. "1234_HSC-Y"), for
598  identifying debugging files.
599 
600  Returns
601  -------
602  namedtuple
603  fit : lsst.jointcal.PhotometryFit
604  The photometric fitter used to perform the fit.
605  model : lsst.jointcal.PhotometryModel
606  The photometric model that was fit.
607  """
608  self.log.info("=== Starting photometric fitting...")
609 
610  # TODO: should use pex.config.RegistryField here (see DM-9195)
611  if self.config.photometryModel == "constrainedFlux":
612  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
613  self.focalPlaneBBox,
614  visitOrder=self.config.photometryVisitOrder,
615  errorPedestal=self.config.photometryErrorPedestal)
616  # potentially nonlinear problem, so we may need a line search to converge.
617  doLineSearch = self.config.allowLineSearch
618  elif self.config.photometryModel == "constrainedMagnitude":
619  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
620  self.focalPlaneBBox,
621  visitOrder=self.config.photometryVisitOrder,
622  errorPedestal=self.config.photometryErrorPedestal)
623  # potentially nonlinear problem, so we may need a line search to converge.
624  doLineSearch = self.config.allowLineSearch
625  elif self.config.photometryModel == "simpleFlux":
626  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
627  errorPedestal=self.config.photometryErrorPedestal)
628  doLineSearch = False # purely linear in model parameters, so no line search needed
629  elif self.config.photometryModel == "simpleMagnitude":
630  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
631  errorPedestal=self.config.photometryErrorPedestal)
632  doLineSearch = False # purely linear in model parameters, so no line search needed
633 
634  fit = lsst.jointcal.PhotometryFit(associations, model)
635  chi2 = fit.computeChi2()
636  # TODO DM-12446: turn this into a "butler save" somehow.
637  # Save reference and measurement chi2 contributions for this data
638  if self.config.writeChi2ContributionFiles:
639  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
640  fit.saveChi2Contributions(baseName)
641 
642  if not np.isfinite(chi2.chi2):
643  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
644  self.log.info("Initialized: %s", str(chi2))
645  # The constrained model needs the visit transfo fit first; the chip
646  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
647  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
648  if self.config.photometryModel.startswith("constrained"):
649  # no line search: should be purely (or nearly) linear,
650  # and we want a large step size to initialize with.
651  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
652  chi2 = fit.computeChi2()
653  self.log.info(str(chi2))
654  dumpMatrixFile = "" # so we don't redo the output on the next step
655  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
656  chi2 = fit.computeChi2()
657  self.log.info(str(chi2))
658  fit.minimize("Fluxes") # no line search: always purely linear.
659  chi2 = fit.computeChi2()
660  self.log.info(str(chi2))
661  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
662  chi2 = fit.computeChi2()
663  if not np.isfinite(chi2.chi2):
664  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
665  self.log.info("Fit prepared with %s", str(chi2))
666 
667  model.freezeErrorTransform()
668  self.log.debug("Photometry error scales are frozen.")
669 
670  chi2 = self._iterate_fit(associations,
671  fit,
672  self.config.maxPhotometrySteps,
673  "photometry",
674  "Model Fluxes",
675  doRankUpdate=self.config.photometryDoRankUpdate,
676  doLineSearch=doLineSearch,
677  dataName=dataName)
678 
679  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
680  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
681  return Photometry(fit, model)
682 
683  def _fit_astrometry(self, associations, dataName=None):
684  """
685  Fit the astrometric data.
686 
687  Parameters
688  ----------
689  associations : lsst.jointcal.Associations
690  The star/reference star associations to fit.
691  dataName : str
692  Name of the data being processed (e.g. "1234_HSC-Y"), for
693  identifying debugging files.
694 
695  Returns
696  -------
697  namedtuple
698  fit : lsst.jointcal.AstrometryFit
699  The astrometric fitter used to perform the fit.
700  model : lsst.jointcal.AstrometryModel
701  The astrometric model that was fit.
702  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
703  The model for the sky to tangent plane projection that was used in the fit.
704  """
705 
706  self.log.info("=== Starting astrometric fitting...")
707 
708  associations.deprojectFittedStars()
709 
710  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
711  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
712  # them so carefully?
713  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
714 
715  if self.config.astrometryModel == "constrained":
716  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
717  sky_to_tan_projection,
718  chipOrder=self.config.astrometryChipOrder,
719  visitOrder=self.config.astrometryVisitOrder)
720  elif self.config.astrometryModel == "simple":
721  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
722  sky_to_tan_projection,
723  self.config.useInputWcs,
724  nNotFit=0,
725  order=self.config.astrometrySimpleOrder)
726 
727  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
728  chi2 = fit.computeChi2()
729  # TODO DM-12446: turn this into a "butler save" somehow.
730  # Save reference and measurement chi2 contributions for this data
731  if self.config.writeChi2ContributionFiles:
732  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
733  fit.saveChi2Contributions(baseName)
734 
735  if not np.isfinite(chi2.chi2):
736  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
737  self.log.info("Initialized: %s", str(chi2))
738  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
739  # The constrained model needs the visit transfo fit first; the chip
740  # transfo is initialized from the detector's cameraGeom, so it's close.
741  if self.config.astrometryModel == "constrained":
742  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
743  chi2 = fit.computeChi2()
744  self.log.info(str(chi2))
745  dumpMatrixFile = "" # so we don't redo the output on the next step
746  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
747  chi2 = fit.computeChi2()
748  self.log.info(str(chi2))
749  fit.minimize("Positions")
750  chi2 = fit.computeChi2()
751  self.log.info(str(chi2))
752  fit.minimize("Distortions Positions")
753  chi2 = fit.computeChi2()
754  self.log.info(str(chi2))
755  if not np.isfinite(chi2.chi2):
756  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
757  self.log.info("Fit prepared with %s", str(chi2))
758 
759  chi2 = self._iterate_fit(associations,
760  fit,
761  self.config.maxAstrometrySteps,
762  "astrometry",
763  "Distortions Positions",
764  doRankUpdate=self.config.astrometryDoRankUpdate,
765  dataName=dataName)
766 
767  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
768  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
769 
770  return Astrometry(fit, model, sky_to_tan_projection)
771 
772  def _check_stars(self, associations):
773  """Count measured and reference stars per ccd and warn/log them."""
774  for ccdImage in associations.getCcdImageList():
775  nMeasuredStars, nRefStars = ccdImage.countStars()
776  self.log.debug("ccdImage %s has %s measured and %s reference stars",
777  ccdImage.getName(), nMeasuredStars, nRefStars)
778  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
779  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
780  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
781  if nRefStars < self.config.minRefStarsPerCcd:
782  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
783  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
784 
785  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
786  dataName="",
787  doRankUpdate=True,
788  doLineSearch=False):
789  """Run fitter.minimize up to max_steps times, returning the final chi2.
790 
791  Parameters
792  ----------
793  associations : `lsst.jointcal.Associations`
794  The star/reference star associations to fit.
795  fitter : `lsst.jointcal.FitterBase`
796  The fitter to use for minimization.
797  max_steps : `int`
798  Maximum number of steps to run outlier rejection before declaring
799  convergence failure.
800  name : {'photometry' or 'astrometry'}
801  What type of data are we fitting (for logs and debugging files).
802  whatToFit : `str`
803  Passed to ``fitter.minimize()`` to define the parameters to fit.
804  dataName : str, optional
805  Descriptive name for this dataset (e.g. tract and filter),
806  for debugging.
807  doRankUpdate : bool, optional
808  Do an Eigen rank update during minimization, or recompute the full
809  matrix and gradient?
810  doLineSearch : bool, optional
811  Do a line search for the optimum step during minimization?
812 
813  Returns
814  -------
815  chi2: `lsst.jointcal.Chi2Statistic`
816  The final chi2 after the fit converges, or is forced to end.
817 
818  Raises
819  ------
820  FloatingPointError
821  Raised if the fitter fails with a non-finite value.
822  RuntimeError
823  Raised if the fitter fails for some other reason;
824  log messages will provide further details.
825  """
826  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
827  for i in range(max_steps):
828  result = fitter.minimize(whatToFit,
829  self.config.outlierRejectSigma,
830  doRankUpdate=doRankUpdate,
831  doLineSearch=doLineSearch,
832  dumpMatrixFile=dumpMatrixFile)
833  dumpMatrixFile = "" # clear it so we don't write the matrix again.
834  chi2 = fitter.computeChi2()
835  self._check_stars(associations)
836  self.log.info(str(chi2))
837  if result == MinimizeResult.Converged:
838  if doRankUpdate:
839  self.log.debug("fit has converged - no more outliers - redo minimization "
840  "one more time in case we have lost accuracy in rank update.")
841  # Redo minimization one more time in case we have lost accuracy in rank update
842  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
843  chi2 = fitter.computeChi2()
844  self.log.info("Fit completed with: %s", str(chi2))
845 
846  # log a message for a large final chi2, TODO: DM-15247 for something better
847  if chi2.chi2/chi2.ndof >= 4.0:
848  self.log.error("Potentially bad fit: High chi-squared/ndof.")
849 
850  break
851  elif result == MinimizeResult.Chi2Increased:
852  self.log.warn("still some outliers but chi2 increases - retry")
853  elif result == MinimizeResult.NonFinite:
854  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
855  # TODO DM-12446: turn this into a "butler save" somehow.
856  fitter.saveChi2Contributions(filename)
857  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
858  raise FloatingPointError(msg.format(filename))
859  elif result == MinimizeResult.Failed:
860  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
861  else:
862  raise RuntimeError("Unxepected return code from minimize().")
863  else:
864  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
865 
866  return chi2
867 
868  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
869  """
870  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
871 
872  Parameters
873  ----------
874  associations : lsst.jointcal.Associations
875  The star/reference star associations to fit.
876  model : lsst.jointcal.AstrometryModel
877  The astrometric model that was fit.
878  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
879  dict of ccdImage identifiers to dataRefs that were fit
880  """
881 
882  ccdImageList = associations.getCcdImageList()
883  for ccdImage in ccdImageList:
884  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
885  ccd = ccdImage.ccdId
886  visit = ccdImage.visit
887  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
888  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
889  skyWcs = model.makeSkyWcs(ccdImage)
890  try:
891  dataRef.put(skyWcs, 'jointcal_wcs')
892  except pexExceptions.Exception as e:
893  self.log.fatal('Failed to write updated Wcs: %s', str(e))
894  raise e
895 
896  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
897  """
898  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
899 
900  Parameters
901  ----------
902  associations : lsst.jointcal.Associations
903  The star/reference star associations to fit.
904  model : lsst.jointcal.PhotometryModel
905  The photoometric model that was fit.
906  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
907  dict of ccdImage identifiers to dataRefs that were fit
908  """
909 
910  ccdImageList = associations.getCcdImageList()
911  for ccdImage in ccdImageList:
912  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
913  ccd = ccdImage.ccdId
914  visit = ccdImage.visit
915  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
916  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
917  photoCalib = model.toPhotoCalib(ccdImage)
918  try:
919  dataRef.put(photoCalib, 'jointcal_photoCalib')
920  except pexExceptions.Exception as e:
921  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
922  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:390
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:327
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:588
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:683
def _check_star_lists(self, associations, name)
Definition: jointcal.py:579
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:788
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:896
def _check_stars(self, associations)
Definition: jointcal.py:772
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:491
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:868
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:285