lsst.jointcal  16.0-23-gcb65559+3
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 
25 import lsst.utils
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.afw.image as afwImage
29 import lsst.afw.geom as afwGeom
30 import lsst.pex.exceptions as pexExceptions
31 import lsst.afw.table
33 from lsst.verify import Job, Measurement
34 
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
37 
38 from .dataIds import PerTractCcdDataIdContainer
39 
40 import lsst.jointcal
41 from lsst.jointcal import MinimizeResult
42 
43 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
44 
45 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
46 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
47 
48 
49 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
50 def add_measurement(job, name, value):
51  meas = Measurement(job.metrics[name], value)
52  job.measurements.insert(meas)
53 
54 
55 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
56  """Subclass of TaskRunner for jointcalTask
57 
58  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
59  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
60  single dataRef, are are called repeatedly). This class transforms the processed
61  arguments generated by the ArgumentParser into the arguments expected by
62  Jointcal.runDataRef().
63 
64  See pipeBase.TaskRunner for more information.
65  """
66 
67  @staticmethod
68  def getTargetList(parsedCmd, **kwargs):
69  """
70  Return a list of tuples per tract, each containing (dataRefs, kwargs).
71 
72  Jointcal operates on lists of dataRefs simultaneously.
73  """
74  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
75  kwargs['butler'] = parsedCmd.butler
76 
77  # organize data IDs by tract
78  refListDict = {}
79  for ref in parsedCmd.id.refList:
80  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
81  # we call runDataRef() once with each tract
82  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
83  return result
84 
85  def __call__(self, args):
86  """
87  Parameters
88  ----------
89  args
90  Arguments for Task.runDataRef()
91 
92  Returns
93  -------
94  pipe.base.Struct
95  if self.doReturnResults is False:
96 
97  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
98 
99  if self.doReturnResults is True:
100 
101  - ``result``: the result of calling jointcal.runDataRef()
102  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
103  """
104  exitStatus = 0 # exit status for shell
105 
106  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
107  dataRefList, kwargs = args
108  butler = kwargs.pop('butler')
109  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
110  result = None
111  try:
112  result = task.runDataRef(dataRefList, **kwargs)
113  exitStatus = result.exitStatus
114  job_path = butler.get('verify_job_filename')
115  result.job.write(job_path[0])
116  except Exception as e: # catch everything, sort it out later.
117  if self.doRaise:
118  raise e
119  else:
120  exitStatus = 1
121  eName = type(e).__name__
122  tract = dataRefList[0].dataId['tract']
123  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
124 
125  if self.doReturnResults:
126  return pipeBase.Struct(result=result, exitStatus=exitStatus)
127  else:
128  return pipeBase.Struct(exitStatus=exitStatus)
129 
130 
131 class JointcalConfig(pexConfig.Config):
132  """Config for JointcalTask"""
133 
134  doAstrometry = pexConfig.Field(
135  doc="Fit astrometry and write the fitted result.",
136  dtype=bool,
137  default=True
138  )
139  doPhotometry = pexConfig.Field(
140  doc="Fit photometry and write the fitted result.",
141  dtype=bool,
142  default=True
143  )
144  coaddName = pexConfig.Field(
145  doc="Type of coadd, typically deep or goodSeeing",
146  dtype=str,
147  default="deep"
148  )
149  positionErrorPedestal = pexConfig.Field(
150  doc="Systematic term to apply to the measured position error (pixels)",
151  dtype=float,
152  default=0.02,
153  )
154  photometryErrorPedestal = pexConfig.Field(
155  doc="Systematic term to apply to the measured error on flux or magnitude as a "
156  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
157  dtype=float,
158  default=0.0,
159  )
160  # TODO: DM-6885 matchCut should be an afw.geom.Angle
161  matchCut = pexConfig.Field(
162  doc="Matching radius between fitted and reference stars (arcseconds)",
163  dtype=float,
164  default=3.0,
165  )
166  minMeasurements = pexConfig.Field(
167  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
168  dtype=int,
169  default=2,
170  )
171  minMeasuredStarsPerCcd = pexConfig.Field(
172  doc="Minimum number of measuredStars per ccdImage before printing warnings",
173  dtype=int,
174  default=100,
175  )
176  minRefStarsPerCcd = pexConfig.Field(
177  doc="Minimum number of measuredStars per ccdImage before printing warnings",
178  dtype=int,
179  default=30,
180  )
181  allowLineSearch = pexConfig.Field(
182  doc="Allow a line search during minimization, if it is reasonable for the model"
183  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
184  dtype=bool,
185  default=False
186  )
187  astrometrySimpleOrder = pexConfig.Field(
188  doc="Polynomial order for fitting the simple astrometry model.",
189  dtype=int,
190  default=3,
191  )
192  astrometryChipOrder = pexConfig.Field(
193  doc="Order of the per-chip transform for the constrained astrometry model.",
194  dtype=int,
195  default=1,
196  )
197  astrometryVisitOrder = pexConfig.Field(
198  doc="Order of the per-visit transform for the constrained astrometry model.",
199  dtype=int,
200  default=5,
201  )
202  useInputWcs = pexConfig.Field(
203  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
204  dtype=bool,
205  default=True,
206  )
207  astrometryModel = pexConfig.ChoiceField(
208  doc="Type of model to fit to astrometry",
209  dtype=str,
210  default="constrained",
211  allowed={"simple": "One polynomial per ccd",
212  "constrained": "One polynomial per ccd, and one polynomial per visit"}
213  )
214  photometryModel = pexConfig.ChoiceField(
215  doc="Type of model to fit to photometry",
216  dtype=str,
217  default="constrainedMagnitude",
218  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
219  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
220  " fitting in flux space.",
221  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
222  " fitting in magnitude space.",
223  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
224  " fitting in magnitude space.",
225  }
226  )
227  photometryVisitOrder = pexConfig.Field(
228  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
229  dtype=int,
230  default=7,
231  )
232  photometryDoRankUpdate = pexConfig.Field(
233  doc="Do the rank update step during minimization. "
234  "Skipping this can help deal with models that are too non-linear.",
235  dtype=bool,
236  default=True,
237  )
238  astrometryDoRankUpdate = pexConfig.Field(
239  doc="Do the rank update step during minimization (should not change the astrometry fit). "
240  "Skipping this can help deal with models that are too non-linear.",
241  dtype=bool,
242  default=True,
243  )
244  outlierRejectSigma = pexConfig.Field(
245  doc="How many sigma to reject outliers at during minimization.",
246  dtype=float,
247  default=5.0,
248  )
249  maxPhotometrySteps = pexConfig.Field(
250  doc="Maximum number of minimize iterations to take when fitting photometry.",
251  dtype=int,
252  default=20,
253  )
254  maxAstrometrySteps = pexConfig.Field(
255  doc="Maximum number of minimize iterations to take when fitting photometry.",
256  dtype=int,
257  default=20,
258  )
259  astrometryRefObjLoader = pexConfig.ConfigurableField(
260  target=LoadIndexedReferenceObjectsTask,
261  doc="Reference object loader for astrometric fit",
262  )
263  photometryRefObjLoader = pexConfig.ConfigurableField(
264  target=LoadIndexedReferenceObjectsTask,
265  doc="Reference object loader for photometric fit",
266  )
267  sourceSelector = sourceSelectorRegistry.makeField(
268  doc="How to select sources for cross-matching",
269  default="astrometry"
270  )
271  writeInitMatrix = pexConfig.Field(
272  dtype=bool,
273  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
274  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
275  "Note that these files are the dense versions of the matrix, and so may be very large.",
276  default=False
277  )
278  writeChi2ContributionFiles = pexConfig.Field(
279  dtype=bool,
280  doc="Write initial/final fit files containing the contributions to chi2.",
281  default=False
282  )
283  sourceFluxType = pexConfig.Field(
284  dtype=str,
285  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
286  default='Calib'
287  )
288 
289  def setDefaults(self):
290  sourceSelector = self.sourceSelector["astrometry"]
291  sourceSelector.setDefaults()
292  # don't want to lose existing flags, just add to them.
293  sourceSelector.badFlags.extend(["slot_Shape_flag"])
294  # This should be used to set the FluxField value in jointcal::JointcalControl
295  sourceSelector.sourceFluxType = self.sourceFluxType
296 
297 
298 class JointcalTask(pipeBase.CmdLineTask):
299  """Jointly astrometrically and photometrically calibrate a group of images."""
300 
301  ConfigClass = JointcalConfig
302  RunnerClass = JointcalRunner
303  _DefaultName = "jointcal"
304 
305  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
306  """
307  Instantiate a JointcalTask.
308 
309  Parameters
310  ----------
311  butler : lsst.daf.persistence.Butler
312  The butler is passed to the refObjLoader constructor in case it is
313  needed. Ignored if the refObjLoader argument provides a loader directly.
314  Used to initialize the astrometry and photometry refObjLoaders.
315  profile_jointcal : bool
316  set to True to profile different stages of this jointcal run.
317  """
318  pipeBase.CmdLineTask.__init__(self, **kwargs)
319  self.profile_jointcal = profile_jointcal
320  self.makeSubtask("sourceSelector")
321  if self.config.doAstrometry:
322  self.makeSubtask('astrometryRefObjLoader', butler=butler)
323  if self.config.doPhotometry:
324  self.makeSubtask('photometryRefObjLoader', butler=butler)
325 
326  # To hold various computed metrics for use by tests
327  self.job = Job.load_metrics_package(subset='jointcal')
328 
329  # We don't need to persist config and metadata at this stage.
330  # In this way, we don't need to put a specific entry in the camera mapper policy file
331  def _getConfigName(self):
332  return None
333 
334  def _getMetadataName(self):
335  return None
336 
337  @classmethod
338  def _makeArgumentParser(cls):
339  """Create an argument parser"""
340  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
341  parser.add_argument("--profile_jointcal", default=False, action="store_true",
342  help="Profile steps of jointcal separately.")
343  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
344  ContainerClass=PerTractCcdDataIdContainer)
345  return parser
346 
347  def _build_ccdImage(self, dataRef, associations, jointcalControl):
348  """
349  Extract the necessary things from this dataRef to add a new ccdImage.
350 
351  Parameters
352  ----------
353  dataRef : lsst.daf.persistence.ButlerDataRef
354  dataRef to extract info from.
355  associations : lsst.jointcal.Associations
356  object to add the info to, to construct a new CcdImage
357  jointcalControl : jointcal.JointcalControl
358  control object for associations management
359 
360  Returns
361  ------
362  namedtuple
363  wcs : lsst.afw.geom.SkyWcs
364  the TAN WCS of this image, read from the calexp
365  key : namedtuple
366  a key to identify this dataRef by its visit and ccd ids
367  filter : str
368  this calexp's filter
369  """
370  if "visit" in dataRef.dataId.keys():
371  visit = dataRef.dataId["visit"]
372  else:
373  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
374 
375  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
376 
377  visitInfo = dataRef.get('calexp_visitInfo')
378  detector = dataRef.get('calexp_detector')
379  ccdId = detector.getId()
380  calib = dataRef.get('calexp_calib')
381  tanWcs = dataRef.get('calexp_wcs')
382  bbox = dataRef.get('calexp_bbox')
383  filt = dataRef.get('calexp_filter')
384  filterName = filt.getName()
385  fluxMag0 = calib.getFluxMag0()
386  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
387 
388  goodSrc = self.sourceSelector.run(src)
389 
390  if len(goodSrc.sourceCat) == 0:
391  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
392  else:
393  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
394  associations.createCcdImage(goodSrc.sourceCat,
395  tanWcs,
396  visitInfo,
397  bbox,
398  filterName,
399  photoCalib,
400  detector,
401  visit,
402  ccdId,
403  jointcalControl)
404 
405  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
406  Key = collections.namedtuple('Key', ('visit', 'ccd'))
407  return Result(tanWcs, Key(visit, ccdId), filterName)
408 
409  @pipeBase.timeMethod
410  def runDataRef(self, dataRefs, profile_jointcal=False):
411  """
412  Jointly calibrate the astrometry and photometry across a set of images.
413 
414  Parameters
415  ----------
416  dataRefs : list of lsst.daf.persistence.ButlerDataRef
417  List of data references to the exposures to be fit.
418  profile_jointcal : bool
419  Profile the individual steps of jointcal.
420 
421  Returns
422  -------
423  pipe.base.Struct
424  struct containing:
425  * dataRefs: the provided data references that were fit (with updated WCSs)
426  * oldWcsList: the original WCS from each dataRef
427  * metrics: dictionary of internally-computed metrics for testing/validation.
428  """
429  if len(dataRefs) == 0:
430  raise ValueError('Need a non-empty list of data references!')
431 
432  exitStatus = 0 # exit status for shell
433 
434  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
435  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
436  associations = lsst.jointcal.Associations()
437 
438  visit_ccd_to_dataRef = {}
439  oldWcsList = []
440  filters = []
441  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
442  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
443  # We need the bounding-box of the focal plane for photometry visit models.
444  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
445  camera = dataRefs[0].get('camera', immediate=True)
446  self.focalPlaneBBox = camera.getFpBBox()
447  for ref in dataRefs:
448  result = self._build_ccdImage(ref, associations, jointcalControl)
449  oldWcsList.append(result.wcs)
450  visit_ccd_to_dataRef[result.key] = ref
451  filters.append(result.filter)
452  filters = collections.Counter(filters)
453 
454  associations.computeCommonTangentPoint()
455 
456  # Use external reference catalogs handled by LSST stack mechanism
457  # Get the bounding box overlapping all associated images
458  # ==> This is probably a bad idea to do it this way <== To be improved
459  bbox = associations.getRaDecBBox()
460  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
461  bboxCenter = bbox.getCenter()
462  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
463  bboxMax = bbox.getMax()
464  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
465  radius = center.separation(corner).asRadians()
466 
467  # Get astrometry_net_data path
468  anDir = lsst.utils.getPackageDir('astrometry_net_data')
469  if anDir is None:
470  raise RuntimeError("astrometry_net_data is not setup")
471 
472  # Determine a default filter associated with the catalog. See DM-9093
473  defaultFilter = filters.most_common(1)[0][0]
474  self.log.debug("Using %s band for reference flux", defaultFilter)
475 
476  # TODO: need a better way to get the tract.
477  tract = dataRefs[0].dataId['tract']
478 
479  if self.config.doAstrometry:
480  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
481  name="astrometry",
482  refObjLoader=self.astrometryRefObjLoader,
483  fit_function=self._fit_astrometry,
484  profile_jointcal=profile_jointcal,
485  tract=tract)
486  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
487  else:
488  astrometry = Astrometry(None, None, None)
489 
490  if self.config.doPhotometry:
491  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
492  name="photometry",
493  refObjLoader=self.photometryRefObjLoader,
494  fit_function=self._fit_photometry,
495  profile_jointcal=profile_jointcal,
496  tract=tract,
497  filters=filters,
498  reject_bad_fluxes=True)
499  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
500  else:
501  photometry = Photometry(None, None)
502 
503  return pipeBase.Struct(dataRefs=dataRefs,
504  oldWcsList=oldWcsList,
505  job=self.job,
506  exitStatus=exitStatus)
507 
508  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
509  name="", refObjLoader=None, filters=[], fit_function=None,
510  tract=None, profile_jointcal=False, match_cut=3.0,
511  reject_bad_fluxes=False):
512  """Load reference catalog, perform the fit, and return the result.
513 
514  Parameters
515  ----------
516  associations : lsst.jointcal.Associations
517  The star/reference star associations to fit.
518  defaultFilter : str
519  filter to load from reference catalog.
520  center : lsst.afw.geom.SpherePoint
521  ICRS center of field to load from reference catalog.
522  radius : lsst.afw.geom.Angle
523  On-sky radius to load from reference catalog.
524  name : str
525  Name of thing being fit: "Astrometry" or "Photometry".
526  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
527  Reference object loader to load from for fit.
528  filters : list of str, optional
529  List of filters to load from the reference catalog.
530  fit_function : function
531  function to call to perform fit (takes associations object).
532  tract : str
533  Name of tract currently being fit.
534  profile_jointcal : bool, optional
535  Separately profile the fitting step.
536  match_cut : float, optional
537  Radius in arcseconds to find cross-catalog matches to during
538  associations.associateCatalogs.
539  reject_bad_fluxes : bool, optional
540  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
541 
542  Returns
543  -------
544  Result of `fit_function()`
545  """
546  self.log.info("====== Now processing %s...", name)
547  # TODO: this should not print "trying to invert a singular transformation:"
548  # if it does that, something's not right about the WCS...
549  associations.associateCatalogs(match_cut)
550  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
551  associations.fittedStarListSize())
552 
553  skyCircle = refObjLoader.loadSkyCircle(center,
554  afwGeom.Angle(radius, afwGeom.radians),
555  defaultFilter)
556 
557  # Need memory contiguity to get reference filters as a vector.
558  if not skyCircle.refCat.isContiguous():
559  refCat = skyCircle.refCat.copy(deep=True)
560  else:
561  refCat = skyCircle.refCat
562 
563  # load the reference catalog fluxes.
564  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
565  refFluxes = {}
566  refFluxErrs = {}
567  for filt in filters:
568  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
569  refFluxes[filt] = refCat.get(filtKeys[0])
570  refFluxErrs[filt] = refCat.get(filtKeys[1])
571 
572  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
573  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
574  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
575  associations.refStarListSize())
576 
577  associations.prepareFittedStars(self.config.minMeasurements)
578 
579  self._check_star_lists(associations, name)
580  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
581  associations.nFittedStarsWithAssociatedRefStar())
582  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
583  associations.fittedStarListSize())
584  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
585  associations.nCcdImagesValidForFit())
586 
587  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
588  dataName = "{}_{}".format(tract, defaultFilter)
589  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
590  result = fit_function(associations, dataName)
591  # TODO DM-12446: turn this into a "butler save" somehow.
592  # Save reference and measurement chi2 contributions for this data
593  if self.config.writeChi2ContributionFiles:
594  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
595  result.fit.saveChi2Contributions(baseName)
596 
597  return result
598 
599  def _check_star_lists(self, associations, name):
600  # TODO: these should be len(blah), but we need this properly wrapped first.
601  if associations.nCcdImagesValidForFit() == 0:
602  raise RuntimeError('No images in the ccdImageList!')
603  if associations.fittedStarListSize() == 0:
604  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
605  if associations.refStarListSize() == 0:
606  raise RuntimeError('No stars in the {} reference star list!'.format(name))
607 
608  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model"):
609  """Compute chi2, log it, validate the model, and return chi2."""
610  chi2 = fit.computeChi2()
611  self.log.info("%s %s", chi2Label, chi2)
612  self._check_stars(associations)
613  if not np.isfinite(chi2.chi2):
614  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
615  if not model.validate(associations.getCcdImageList()):
616  raise ValueError("Model is not valid: check log messages for warnings.")
617  return chi2
618 
619  def _fit_photometry(self, associations, dataName=None):
620  """
621  Fit the photometric data.
622 
623  Parameters
624  ----------
625  associations : lsst.jointcal.Associations
626  The star/reference star associations to fit.
627  dataName : str
628  Name of the data being processed (e.g. "1234_HSC-Y"), for
629  identifying debugging files.
630 
631  Returns
632  -------
633  namedtuple
634  fit : lsst.jointcal.PhotometryFit
635  The photometric fitter used to perform the fit.
636  model : lsst.jointcal.PhotometryModel
637  The photometric model that was fit.
638  """
639  self.log.info("=== Starting photometric fitting...")
640 
641  # TODO: should use pex.config.RegistryField here (see DM-9195)
642  if self.config.photometryModel == "constrainedFlux":
643  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
644  self.focalPlaneBBox,
645  visitOrder=self.config.photometryVisitOrder,
646  errorPedestal=self.config.photometryErrorPedestal)
647  # potentially nonlinear problem, so we may need a line search to converge.
648  doLineSearch = self.config.allowLineSearch
649  elif self.config.photometryModel == "constrainedMagnitude":
650  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
651  self.focalPlaneBBox,
652  visitOrder=self.config.photometryVisitOrder,
653  errorPedestal=self.config.photometryErrorPedestal)
654  # potentially nonlinear problem, so we may need a line search to converge.
655  doLineSearch = self.config.allowLineSearch
656  elif self.config.photometryModel == "simpleFlux":
657  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
658  errorPedestal=self.config.photometryErrorPedestal)
659  doLineSearch = False # purely linear in model parameters, so no line search needed
660  elif self.config.photometryModel == "simpleMagnitude":
661  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
662  errorPedestal=self.config.photometryErrorPedestal)
663  doLineSearch = False # purely linear in model parameters, so no line search needed
664 
665  fit = lsst.jointcal.PhotometryFit(associations, model)
666  self._logChi2AndValidate(associations, fit, model, "Initialized")
667 
668  # TODO DM-12446: turn this into a "butler save" somehow.
669  # Save reference and measurement chi2 contributions for this data
670  if self.config.writeChi2ContributionFiles:
671  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
672  fit.saveChi2Contributions(baseName)
673 
674  # The constrained model needs the visit transfo fit first; the chip
675  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
676  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
677  if self.config.photometryModel.startswith("constrained"):
678  # no line search: should be purely (or nearly) linear,
679  # and we want a large step size to initialize with.
680  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
681  self._logChi2AndValidate(associations, fit, model)
682  dumpMatrixFile = "" # so we don't redo the output on the next step
683 
684  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
685  self._logChi2AndValidate(associations, fit, model)
686 
687  fit.minimize("Fluxes") # no line search: always purely linear.
688  self._logChi2AndValidate(associations, fit, model)
689 
690  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
691  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
692 
693  model.freezeErrorTransform()
694  self.log.debug("Photometry error scales are frozen.")
695 
696  chi2 = self._iterate_fit(associations,
697  fit,
698  self.config.maxPhotometrySteps,
699  "photometry",
700  "Model Fluxes",
701  doRankUpdate=self.config.photometryDoRankUpdate,
702  doLineSearch=doLineSearch,
703  dataName=dataName)
704 
705  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
706  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
707  return Photometry(fit, model)
708 
709  def _fit_astrometry(self, associations, dataName=None):
710  """
711  Fit the astrometric data.
712 
713  Parameters
714  ----------
715  associations : lsst.jointcal.Associations
716  The star/reference star associations to fit.
717  dataName : str
718  Name of the data being processed (e.g. "1234_HSC-Y"), for
719  identifying debugging files.
720 
721  Returns
722  -------
723  namedtuple
724  fit : lsst.jointcal.AstrometryFit
725  The astrometric fitter used to perform the fit.
726  model : lsst.jointcal.AstrometryModel
727  The astrometric model that was fit.
728  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
729  The model for the sky to tangent plane projection that was used in the fit.
730  """
731 
732  self.log.info("=== Starting astrometric fitting...")
733 
734  associations.deprojectFittedStars()
735 
736  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
737  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
738  # them so carefully?
739  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
740 
741  if self.config.astrometryModel == "constrained":
742  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
743  sky_to_tan_projection,
744  chipOrder=self.config.astrometryChipOrder,
745  visitOrder=self.config.astrometryVisitOrder)
746  elif self.config.astrometryModel == "simple":
747  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
748  sky_to_tan_projection,
749  self.config.useInputWcs,
750  nNotFit=0,
751  order=self.config.astrometrySimpleOrder)
752 
753  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
754  self._logChi2AndValidate(associations, fit, model, "Initial")
755 
756  # TODO DM-12446: turn this into a "butler save" somehow.
757  # Save reference and measurement chi2 contributions for this data
758  if self.config.writeChi2ContributionFiles:
759  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
760  fit.saveChi2Contributions(baseName)
761 
762  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
763  # The constrained model needs the visit transfo fit first; the chip
764  # transfo is initialized from the detector's cameraGeom, so it's close.
765  if self.config.astrometryModel == "constrained":
766  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
767  self._logChi2AndValidate(associations, fit, model)
768  dumpMatrixFile = "" # so we don't redo the output on the next step
769 
770  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
771  self._logChi2AndValidate(associations, fit, model)
772 
773  fit.minimize("Positions")
774  self._logChi2AndValidate(associations, fit, model)
775 
776  fit.minimize("Distortions Positions")
777  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
778 
779  chi2 = self._iterate_fit(associations,
780  fit,
781  self.config.maxAstrometrySteps,
782  "astrometry",
783  "Distortions Positions",
784  doRankUpdate=self.config.astrometryDoRankUpdate,
785  dataName=dataName)
786 
787  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
788  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
789 
790  return Astrometry(fit, model, sky_to_tan_projection)
791 
792  def _check_stars(self, associations):
793  """Count measured and reference stars per ccd and warn/log them."""
794  for ccdImage in associations.getCcdImageList():
795  nMeasuredStars, nRefStars = ccdImage.countStars()
796  self.log.debug("ccdImage %s has %s measured and %s reference stars",
797  ccdImage.getName(), nMeasuredStars, nRefStars)
798  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
799  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
800  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
801  if nRefStars < self.config.minRefStarsPerCcd:
802  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
803  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
804 
805  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
806  dataName="",
807  doRankUpdate=True,
808  doLineSearch=False):
809  """Run fitter.minimize up to max_steps times, returning the final chi2.
810 
811  Parameters
812  ----------
813  associations : `lsst.jointcal.Associations`
814  The star/reference star associations to fit.
815  fitter : `lsst.jointcal.FitterBase`
816  The fitter to use for minimization.
817  max_steps : `int`
818  Maximum number of steps to run outlier rejection before declaring
819  convergence failure.
820  name : {'photometry' or 'astrometry'}
821  What type of data are we fitting (for logs and debugging files).
822  whatToFit : `str`
823  Passed to ``fitter.minimize()`` to define the parameters to fit.
824  dataName : str, optional
825  Descriptive name for this dataset (e.g. tract and filter),
826  for debugging.
827  doRankUpdate : bool, optional
828  Do an Eigen rank update during minimization, or recompute the full
829  matrix and gradient?
830  doLineSearch : bool, optional
831  Do a line search for the optimum step during minimization?
832 
833  Returns
834  -------
835  chi2: `lsst.jointcal.Chi2Statistic`
836  The final chi2 after the fit converges, or is forced to end.
837 
838  Raises
839  ------
840  FloatingPointError
841  Raised if the fitter fails with a non-finite value.
842  RuntimeError
843  Raised if the fitter fails for some other reason;
844  log messages will provide further details.
845  """
846  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
847  for i in range(max_steps):
848  result = fitter.minimize(whatToFit,
849  self.config.outlierRejectSigma,
850  doRankUpdate=doRankUpdate,
851  doLineSearch=doLineSearch,
852  dumpMatrixFile=dumpMatrixFile)
853  dumpMatrixFile = "" # clear it so we don't write the matrix again.
854  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel())
855 
856  if result == MinimizeResult.Converged:
857  if doRankUpdate:
858  self.log.debug("fit has converged - no more outliers - redo minimization "
859  "one more time in case we have lost accuracy in rank update.")
860  # Redo minimization one more time in case we have lost accuracy in rank update
861  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
862  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
863 
864  # log a message for a large final chi2, TODO: DM-15247 for something better
865  if chi2.chi2/chi2.ndof >= 4.0:
866  self.log.error("Potentially bad fit: High chi-squared/ndof.")
867 
868  break
869  elif result == MinimizeResult.Chi2Increased:
870  self.log.warn("still some outliers but chi2 increases - retry")
871  elif result == MinimizeResult.NonFinite:
872  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
873  # TODO DM-12446: turn this into a "butler save" somehow.
874  fitter.saveChi2Contributions(filename)
875  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
876  raise FloatingPointError(msg.format(filename))
877  elif result == MinimizeResult.Failed:
878  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
879  else:
880  raise RuntimeError("Unxepected return code from minimize().")
881  else:
882  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
883 
884  return chi2
885 
886  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
887  """
888  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
889 
890  Parameters
891  ----------
892  associations : lsst.jointcal.Associations
893  The star/reference star associations to fit.
894  model : lsst.jointcal.AstrometryModel
895  The astrometric model that was fit.
896  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
897  dict of ccdImage identifiers to dataRefs that were fit
898  """
899 
900  ccdImageList = associations.getCcdImageList()
901  for ccdImage in ccdImageList:
902  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
903  ccd = ccdImage.ccdId
904  visit = ccdImage.visit
905  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
906  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
907  skyWcs = model.makeSkyWcs(ccdImage)
908  try:
909  dataRef.put(skyWcs, 'jointcal_wcs')
910  except pexExceptions.Exception as e:
911  self.log.fatal('Failed to write updated Wcs: %s', str(e))
912  raise e
913 
914  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
915  """
916  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
917 
918  Parameters
919  ----------
920  associations : lsst.jointcal.Associations
921  The star/reference star associations to fit.
922  model : lsst.jointcal.PhotometryModel
923  The photoometric model that was fit.
924  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
925  dict of ccdImage identifiers to dataRefs that were fit
926  """
927 
928  ccdImageList = associations.getCcdImageList()
929  for ccdImage in ccdImageList:
930  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
931  ccd = ccdImage.ccdId
932  visit = ccdImage.visit
933  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
934  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
935  photoCalib = model.toPhotoCalib(ccdImage)
936  try:
937  dataRef.put(photoCalib, 'jointcal_photoCalib')
938  except pexExceptions.Exception as e:
939  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
940  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:410
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:347
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:619
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:68
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:709
def _check_star_lists(self, associations, name)
Definition: jointcal.py:599
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model")
Definition: jointcal.py:608
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:808
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:914
def _check_stars(self, associations)
Definition: jointcal.py:792
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:50
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:511
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:886
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:305