lsst.jointcal  16.0-28-gfc9ea6c+6
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 
25 import lsst.utils
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.afw.image as afwImage
29 import lsst.afw.geom as afwGeom
30 import lsst.pex.exceptions as pexExceptions
31 import lsst.afw.table
33 from lsst.verify import Job, Measurement
34 
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
37 
38 from .dataIds import PerTractCcdDataIdContainer
39 
40 import lsst.jointcal
41 from lsst.jointcal import MinimizeResult
42 
43 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
44 
45 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
46 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
47 
48 
49 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
50 def add_measurement(job, name, value):
51  meas = Measurement(job.metrics[name], value)
52  job.measurements.insert(meas)
53 
54 
55 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
56  """Subclass of TaskRunner for jointcalTask
57 
58  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
59  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
60  single dataRef, are are called repeatedly). This class transforms the processed
61  arguments generated by the ArgumentParser into the arguments expected by
62  Jointcal.runDataRef().
63 
64  See pipeBase.TaskRunner for more information.
65  """
66 
67  @staticmethod
68  def getTargetList(parsedCmd, **kwargs):
69  """
70  Return a list of tuples per tract, each containing (dataRefs, kwargs).
71 
72  Jointcal operates on lists of dataRefs simultaneously.
73  """
74  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
75  kwargs['butler'] = parsedCmd.butler
76 
77  # organize data IDs by tract
78  refListDict = {}
79  for ref in parsedCmd.id.refList:
80  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
81  # we call runDataRef() once with each tract
82  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
83  return result
84 
85  def __call__(self, args):
86  """
87  Parameters
88  ----------
89  args
90  Arguments for Task.runDataRef()
91 
92  Returns
93  -------
94  pipe.base.Struct
95  if self.doReturnResults is False:
96 
97  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
98 
99  if self.doReturnResults is True:
100 
101  - ``result``: the result of calling jointcal.runDataRef()
102  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
103  """
104  exitStatus = 0 # exit status for shell
105 
106  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
107  dataRefList, kwargs = args
108  butler = kwargs.pop('butler')
109  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
110  result = None
111  try:
112  result = task.runDataRef(dataRefList, **kwargs)
113  exitStatus = result.exitStatus
114  job_path = butler.get('verify_job_filename')
115  result.job.write(job_path[0])
116  except Exception as e: # catch everything, sort it out later.
117  if self.doRaise:
118  raise e
119  else:
120  exitStatus = 1
121  eName = type(e).__name__
122  tract = dataRefList[0].dataId['tract']
123  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
124 
125  if self.doReturnResults:
126  return pipeBase.Struct(result=result, exitStatus=exitStatus)
127  else:
128  return pipeBase.Struct(exitStatus=exitStatus)
129 
130 
131 class JointcalConfig(pexConfig.Config):
132  """Config for JointcalTask"""
133 
134  doAstrometry = pexConfig.Field(
135  doc="Fit astrometry and write the fitted result.",
136  dtype=bool,
137  default=True
138  )
139  doPhotometry = pexConfig.Field(
140  doc="Fit photometry and write the fitted result.",
141  dtype=bool,
142  default=True
143  )
144  coaddName = pexConfig.Field(
145  doc="Type of coadd, typically deep or goodSeeing",
146  dtype=str,
147  default="deep"
148  )
149  positionErrorPedestal = pexConfig.Field(
150  doc="Systematic term to apply to the measured position error (pixels)",
151  dtype=float,
152  default=0.02,
153  )
154  photometryErrorPedestal = pexConfig.Field(
155  doc="Systematic term to apply to the measured error on flux or magnitude as a "
156  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
157  dtype=float,
158  default=0.0,
159  )
160  # TODO: DM-6885 matchCut should be an afw.geom.Angle
161  matchCut = pexConfig.Field(
162  doc="Matching radius between fitted and reference stars (arcseconds)",
163  dtype=float,
164  default=3.0,
165  )
166  minMeasurements = pexConfig.Field(
167  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
168  dtype=int,
169  default=2,
170  )
171  minMeasuredStarsPerCcd = pexConfig.Field(
172  doc="Minimum number of measuredStars per ccdImage before printing warnings",
173  dtype=int,
174  default=100,
175  )
176  minRefStarsPerCcd = pexConfig.Field(
177  doc="Minimum number of measuredStars per ccdImage before printing warnings",
178  dtype=int,
179  default=30,
180  )
181  allowLineSearch = pexConfig.Field(
182  doc="Allow a line search during minimization, if it is reasonable for the model"
183  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
184  dtype=bool,
185  default=False
186  )
187  astrometrySimpleOrder = pexConfig.Field(
188  doc="Polynomial order for fitting the simple astrometry model.",
189  dtype=int,
190  default=3,
191  )
192  astrometryChipOrder = pexConfig.Field(
193  doc="Order of the per-chip transform for the constrained astrometry model.",
194  dtype=int,
195  default=1,
196  )
197  astrometryVisitOrder = pexConfig.Field(
198  doc="Order of the per-visit transform for the constrained astrometry model.",
199  dtype=int,
200  default=5,
201  )
202  useInputWcs = pexConfig.Field(
203  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
204  dtype=bool,
205  default=True,
206  )
207  astrometryModel = pexConfig.ChoiceField(
208  doc="Type of model to fit to astrometry",
209  dtype=str,
210  default="constrained",
211  allowed={"simple": "One polynomial per ccd",
212  "constrained": "One polynomial per ccd, and one polynomial per visit"}
213  )
214  photometryModel = pexConfig.ChoiceField(
215  doc="Type of model to fit to photometry",
216  dtype=str,
217  default="constrainedMagnitude",
218  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
219  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
220  " fitting in flux space.",
221  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
222  " fitting in magnitude space.",
223  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
224  " fitting in magnitude space.",
225  }
226  )
227  photometryVisitOrder = pexConfig.Field(
228  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
229  dtype=int,
230  default=7,
231  )
232  photometryDoRankUpdate = pexConfig.Field(
233  doc="Do the rank update step during minimization. "
234  "Skipping this can help deal with models that are too non-linear.",
235  dtype=bool,
236  default=True,
237  )
238  astrometryDoRankUpdate = pexConfig.Field(
239  doc="Do the rank update step during minimization (should not change the astrometry fit). "
240  "Skipping this can help deal with models that are too non-linear.",
241  dtype=bool,
242  default=True,
243  )
244  outlierRejectSigma = pexConfig.Field(
245  doc="How many sigma to reject outliers at during minimization.",
246  dtype=float,
247  default=5.0,
248  )
249  maxPhotometrySteps = pexConfig.Field(
250  doc="Maximum number of minimize iterations to take when fitting photometry.",
251  dtype=int,
252  default=20,
253  )
254  maxAstrometrySteps = pexConfig.Field(
255  doc="Maximum number of minimize iterations to take when fitting photometry.",
256  dtype=int,
257  default=20,
258  )
259  astrometryRefObjLoader = pexConfig.ConfigurableField(
260  target=LoadIndexedReferenceObjectsTask,
261  doc="Reference object loader for astrometric fit",
262  )
263  photometryRefObjLoader = pexConfig.ConfigurableField(
264  target=LoadIndexedReferenceObjectsTask,
265  doc="Reference object loader for photometric fit",
266  )
267  sourceSelector = sourceSelectorRegistry.makeField(
268  doc="How to select sources for cross-matching",
269  default="astrometry"
270  )
271  writeInitMatrix = pexConfig.Field(
272  dtype=bool,
273  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
274  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
275  "Note that these files are the dense versions of the matrix, and so may be very large.",
276  default=False
277  )
278  writeChi2ContributionFiles = pexConfig.Field(
279  dtype=bool,
280  doc="Write initial/final fit files containing the contributions to chi2.",
281  default=False
282  )
283  sourceFluxType = pexConfig.Field(
284  dtype=str,
285  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
286  default='Calib'
287  )
288 
289  def setDefaults(self):
290  sourceSelector = self.sourceSelector["astrometry"]
291  sourceSelector.setDefaults()
292  # don't want to lose existing flags, just add to them.
293  sourceSelector.badFlags.extend(["slot_Shape_flag"])
294  # This should be used to set the FluxField value in jointcal::JointcalControl
295  sourceSelector.sourceFluxType = self.sourceFluxType
296 
297 
298 class JointcalTask(pipeBase.CmdLineTask):
299  """Jointly astrometrically and photometrically calibrate a group of images."""
300 
301  ConfigClass = JointcalConfig
302  RunnerClass = JointcalRunner
303  _DefaultName = "jointcal"
304 
305  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
306  """
307  Instantiate a JointcalTask.
308 
309  Parameters
310  ----------
311  butler : lsst.daf.persistence.Butler
312  The butler is passed to the refObjLoader constructor in case it is
313  needed. Ignored if the refObjLoader argument provides a loader directly.
314  Used to initialize the astrometry and photometry refObjLoaders.
315  profile_jointcal : bool
316  set to True to profile different stages of this jointcal run.
317  """
318  pipeBase.CmdLineTask.__init__(self, **kwargs)
319  self.profile_jointcal = profile_jointcal
320  self.makeSubtask("sourceSelector")
321  if self.config.doAstrometry:
322  self.makeSubtask('astrometryRefObjLoader', butler=butler)
323  if self.config.doPhotometry:
324  self.makeSubtask('photometryRefObjLoader', butler=butler)
325 
326  # To hold various computed metrics for use by tests
327  self.job = Job.load_metrics_package(subset='jointcal')
328 
329  # We don't need to persist config and metadata at this stage.
330  # In this way, we don't need to put a specific entry in the camera mapper policy file
331  def _getConfigName(self):
332  return None
333 
334  def _getMetadataName(self):
335  return None
336 
337  @classmethod
338  def _makeArgumentParser(cls):
339  """Create an argument parser"""
340  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
341  parser.add_argument("--profile_jointcal", default=False, action="store_true",
342  help="Profile steps of jointcal separately.")
343  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
344  ContainerClass=PerTractCcdDataIdContainer)
345  return parser
346 
347  def _build_ccdImage(self, dataRef, associations, jointcalControl):
348  """
349  Extract the necessary things from this dataRef to add a new ccdImage.
350 
351  Parameters
352  ----------
353  dataRef : lsst.daf.persistence.ButlerDataRef
354  dataRef to extract info from.
355  associations : lsst.jointcal.Associations
356  object to add the info to, to construct a new CcdImage
357  jointcalControl : jointcal.JointcalControl
358  control object for associations management
359 
360  Returns
361  ------
362  namedtuple
363  wcs : lsst.afw.geom.SkyWcs
364  the TAN WCS of this image, read from the calexp
365  key : namedtuple
366  a key to identify this dataRef by its visit and ccd ids
367  filter : str
368  this calexp's filter
369  """
370  if "visit" in dataRef.dataId.keys():
371  visit = dataRef.dataId["visit"]
372  else:
373  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
374 
375  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
376 
377  visitInfo = dataRef.get('calexp_visitInfo')
378  detector = dataRef.get('calexp_detector')
379  ccdId = detector.getId()
380  calib = dataRef.get('calexp_calib')
381  tanWcs = dataRef.get('calexp_wcs')
382  bbox = dataRef.get('calexp_bbox')
383  filt = dataRef.get('calexp_filter')
384  filterName = filt.getName()
385  fluxMag0 = calib.getFluxMag0()
386  # TODO: need to scale these until DM-10153 is completed and PhotoCalib has replaced Calib entirely
387  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
388  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
389  referenceFlux*fluxMag0[1]/fluxMag0[0]**2, bbox)
390 
391  goodSrc = self.sourceSelector.run(src)
392 
393  if len(goodSrc.sourceCat) == 0:
394  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
395  else:
396  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
397  associations.createCcdImage(goodSrc.sourceCat,
398  tanWcs,
399  visitInfo,
400  bbox,
401  filterName,
402  photoCalib,
403  detector,
404  visit,
405  ccdId,
406  jointcalControl)
407 
408  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
409  Key = collections.namedtuple('Key', ('visit', 'ccd'))
410  return Result(tanWcs, Key(visit, ccdId), filterName)
411 
412  @pipeBase.timeMethod
413  def runDataRef(self, dataRefs, profile_jointcal=False):
414  """
415  Jointly calibrate the astrometry and photometry across a set of images.
416 
417  Parameters
418  ----------
419  dataRefs : list of lsst.daf.persistence.ButlerDataRef
420  List of data references to the exposures to be fit.
421  profile_jointcal : bool
422  Profile the individual steps of jointcal.
423 
424  Returns
425  -------
426  pipe.base.Struct
427  struct containing:
428  * dataRefs: the provided data references that were fit (with updated WCSs)
429  * oldWcsList: the original WCS from each dataRef
430  * metrics: dictionary of internally-computed metrics for testing/validation.
431  """
432  if len(dataRefs) == 0:
433  raise ValueError('Need a non-empty list of data references!')
434 
435  exitStatus = 0 # exit status for shell
436 
437  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
438  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
439  associations = lsst.jointcal.Associations()
440 
441  visit_ccd_to_dataRef = {}
442  oldWcsList = []
443  filters = []
444  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
445  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
446  # We need the bounding-box of the focal plane for photometry visit models.
447  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
448  camera = dataRefs[0].get('camera', immediate=True)
449  self.focalPlaneBBox = camera.getFpBBox()
450  for ref in dataRefs:
451  result = self._build_ccdImage(ref, associations, jointcalControl)
452  oldWcsList.append(result.wcs)
453  visit_ccd_to_dataRef[result.key] = ref
454  filters.append(result.filter)
455  filters = collections.Counter(filters)
456 
457  associations.computeCommonTangentPoint()
458 
459  # Use external reference catalogs handled by LSST stack mechanism
460  # Get the bounding box overlapping all associated images
461  # ==> This is probably a bad idea to do it this way <== To be improved
462  bbox = associations.getRaDecBBox()
463  bboxCenter = bbox.getCenter()
464  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
465  bboxMax = bbox.getMax()
466  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
467  radius = center.separation(corner).asRadians()
468 
469  # Get astrometry_net_data path
470  anDir = lsst.utils.getPackageDir('astrometry_net_data')
471  if anDir is None:
472  raise RuntimeError("astrometry_net_data is not setup")
473 
474  # Determine a default filter associated with the catalog. See DM-9093
475  defaultFilter = filters.most_common(1)[0][0]
476  self.log.debug("Using %s band for reference flux", defaultFilter)
477 
478  # TODO: need a better way to get the tract.
479  tract = dataRefs[0].dataId['tract']
480 
481  if self.config.doAstrometry:
482  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
483  name="astrometry",
484  refObjLoader=self.astrometryRefObjLoader,
485  fit_function=self._fit_astrometry,
486  profile_jointcal=profile_jointcal,
487  tract=tract)
488  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
489  else:
490  astrometry = Astrometry(None, None, None)
491 
492  if self.config.doPhotometry:
493  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
494  name="photometry",
495  refObjLoader=self.photometryRefObjLoader,
496  fit_function=self._fit_photometry,
497  profile_jointcal=profile_jointcal,
498  tract=tract,
499  filters=filters,
500  reject_bad_fluxes=True)
501  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
502  else:
503  photometry = Photometry(None, None)
504 
505  return pipeBase.Struct(dataRefs=dataRefs,
506  oldWcsList=oldWcsList,
507  job=self.job,
508  exitStatus=exitStatus)
509 
510  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
511  name="", refObjLoader=None, filters=[], fit_function=None,
512  tract=None, profile_jointcal=False, match_cut=3.0,
513  reject_bad_fluxes=False):
514  """Load reference catalog, perform the fit, and return the result.
515 
516  Parameters
517  ----------
518  associations : lsst.jointcal.Associations
519  The star/reference star associations to fit.
520  defaultFilter : str
521  filter to load from reference catalog.
522  center : lsst.afw.geom.SpherePoint
523  ICRS center of field to load from reference catalog.
524  radius : lsst.afw.geom.Angle
525  On-sky radius to load from reference catalog.
526  name : str
527  Name of thing being fit: "Astrometry" or "Photometry".
528  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
529  Reference object loader to load from for fit.
530  filters : list of str, optional
531  List of filters to load from the reference catalog.
532  fit_function : function
533  function to call to perform fit (takes associations object).
534  tract : str
535  Name of tract currently being fit.
536  profile_jointcal : bool, optional
537  Separately profile the fitting step.
538  match_cut : float, optional
539  Radius in arcseconds to find cross-catalog matches to during
540  associations.associateCatalogs.
541  reject_bad_fluxes : bool, optional
542  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
543 
544  Returns
545  -------
546  Result of `fit_function()`
547  """
548  self.log.info("====== Now processing %s...", name)
549  # TODO: this should not print "trying to invert a singular transformation:"
550  # if it does that, something's not right about the WCS...
551  associations.associateCatalogs(match_cut)
552  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
553  associations.fittedStarListSize())
554 
555  skyCircle = refObjLoader.loadSkyCircle(center,
556  afwGeom.Angle(radius, afwGeom.radians),
557  defaultFilter)
558 
559  # Need memory contiguity to get reference filters as a vector.
560  if not skyCircle.refCat.isContiguous():
561  refCat = skyCircle.refCat.copy(deep=True)
562  else:
563  refCat = skyCircle.refCat
564 
565  # load the reference catalog fluxes.
566  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
567  refFluxes = {}
568  refFluxErrs = {}
569  for filt in filters:
570  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
571  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
572  refFluxes[filt] = 1e9*refCat.get(filtKeys[0])
573  refFluxErrs[filt] = 1e9*refCat.get(filtKeys[1])
574 
575  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
576  refCat[skyCircle.fluxField] *= 1e9
577  try:
578  refCat[skyCircle.fluxField+'Err'] *= 1e9
579  except KeyError:
580  # not all existing refcats have an error field.
581  pass
582 
583  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
584  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
585  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
586  associations.refStarListSize())
587 
588  associations.prepareFittedStars(self.config.minMeasurements)
589 
590  self._check_star_lists(associations, name)
591  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
592  associations.nFittedStarsWithAssociatedRefStar())
593  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
594  associations.fittedStarListSize())
595  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
596  associations.nCcdImagesValidForFit())
597 
598  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
599  dataName = "{}_{}".format(tract, defaultFilter)
600  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
601  result = fit_function(associations, dataName)
602  # TODO DM-12446: turn this into a "butler save" somehow.
603  # Save reference and measurement chi2 contributions for this data
604  if self.config.writeChi2ContributionFiles:
605  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
606  result.fit.saveChi2Contributions(baseName)
607 
608  return result
609 
610  def _check_star_lists(self, associations, name):
611  # TODO: these should be len(blah), but we need this properly wrapped first.
612  if associations.nCcdImagesValidForFit() == 0:
613  raise RuntimeError('No images in the ccdImageList!')
614  if associations.fittedStarListSize() == 0:
615  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
616  if associations.refStarListSize() == 0:
617  raise RuntimeError('No stars in the {} reference star list!'.format(name))
618 
619  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model"):
620  """Compute chi2, log it, validate the model, and return chi2."""
621  chi2 = fit.computeChi2()
622  self.log.info("%s %s", chi2Label, chi2)
623  self._check_stars(associations)
624  if not np.isfinite(chi2.chi2):
625  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
626  if not model.validate(associations.getCcdImageList()):
627  raise ValueError("Model is not valid: check log messages for warnings.")
628  return chi2
629 
630  def _fit_photometry(self, associations, dataName=None):
631  """
632  Fit the photometric data.
633 
634  Parameters
635  ----------
636  associations : lsst.jointcal.Associations
637  The star/reference star associations to fit.
638  dataName : str
639  Name of the data being processed (e.g. "1234_HSC-Y"), for
640  identifying debugging files.
641 
642  Returns
643  -------
644  namedtuple
645  fit : lsst.jointcal.PhotometryFit
646  The photometric fitter used to perform the fit.
647  model : lsst.jointcal.PhotometryModel
648  The photometric model that was fit.
649  """
650  self.log.info("=== Starting photometric fitting...")
651 
652  # TODO: should use pex.config.RegistryField here (see DM-9195)
653  if self.config.photometryModel == "constrainedFlux":
654  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
655  self.focalPlaneBBox,
656  visitOrder=self.config.photometryVisitOrder,
657  errorPedestal=self.config.photometryErrorPedestal)
658  # potentially nonlinear problem, so we may need a line search to converge.
659  doLineSearch = self.config.allowLineSearch
660  elif self.config.photometryModel == "constrainedMagnitude":
661  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
662  self.focalPlaneBBox,
663  visitOrder=self.config.photometryVisitOrder,
664  errorPedestal=self.config.photometryErrorPedestal)
665  # potentially nonlinear problem, so we may need a line search to converge.
666  doLineSearch = self.config.allowLineSearch
667  elif self.config.photometryModel == "simpleFlux":
668  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
669  errorPedestal=self.config.photometryErrorPedestal)
670  doLineSearch = False # purely linear in model parameters, so no line search needed
671  elif self.config.photometryModel == "simpleMagnitude":
672  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
673  errorPedestal=self.config.photometryErrorPedestal)
674  doLineSearch = False # purely linear in model parameters, so no line search needed
675 
676  fit = lsst.jointcal.PhotometryFit(associations, model)
677  self._logChi2AndValidate(associations, fit, model, "Initialized")
678 
679  # TODO DM-12446: turn this into a "butler save" somehow.
680  # Save reference and measurement chi2 contributions for this data
681  if self.config.writeChi2ContributionFiles:
682  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
683  fit.saveChi2Contributions(baseName)
684 
685  # The constrained model needs the visit transform fit first; the chip
686  # transform is initialized from the singleFrame PhotoCalib, so it's close.
687  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
688  if self.config.photometryModel.startswith("constrained"):
689  # no line search: should be purely (or nearly) linear,
690  # and we want a large step size to initialize with.
691  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
692  self._logChi2AndValidate(associations, fit, model)
693  dumpMatrixFile = "" # so we don't redo the output on the next step
694 
695  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
696  self._logChi2AndValidate(associations, fit, model)
697 
698  fit.minimize("Fluxes") # no line search: always purely linear.
699  self._logChi2AndValidate(associations, fit, model)
700 
701  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
702  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
703 
704  model.freezeErrorTransform()
705  self.log.debug("Photometry error scales are frozen.")
706 
707  chi2 = self._iterate_fit(associations,
708  fit,
709  self.config.maxPhotometrySteps,
710  "photometry",
711  "Model Fluxes",
712  doRankUpdate=self.config.photometryDoRankUpdate,
713  doLineSearch=doLineSearch,
714  dataName=dataName)
715 
716  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
717  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
718  return Photometry(fit, model)
719 
720  def _fit_astrometry(self, associations, dataName=None):
721  """
722  Fit the astrometric data.
723 
724  Parameters
725  ----------
726  associations : lsst.jointcal.Associations
727  The star/reference star associations to fit.
728  dataName : str
729  Name of the data being processed (e.g. "1234_HSC-Y"), for
730  identifying debugging files.
731 
732  Returns
733  -------
734  namedtuple
735  fit : lsst.jointcal.AstrometryFit
736  The astrometric fitter used to perform the fit.
737  model : lsst.jointcal.AstrometryModel
738  The astrometric model that was fit.
739  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
740  The model for the sky to tangent plane projection that was used in the fit.
741  """
742 
743  self.log.info("=== Starting astrometric fitting...")
744 
745  associations.deprojectFittedStars()
746 
747  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
748  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
749  # them so carefully?
750  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
751 
752  if self.config.astrometryModel == "constrained":
753  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
754  sky_to_tan_projection,
755  chipOrder=self.config.astrometryChipOrder,
756  visitOrder=self.config.astrometryVisitOrder)
757  elif self.config.astrometryModel == "simple":
758  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
759  sky_to_tan_projection,
760  self.config.useInputWcs,
761  nNotFit=0,
762  order=self.config.astrometrySimpleOrder)
763 
764  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
765  self._logChi2AndValidate(associations, fit, model, "Initial")
766 
767  # TODO DM-12446: turn this into a "butler save" somehow.
768  # Save reference and measurement chi2 contributions for this data
769  if self.config.writeChi2ContributionFiles:
770  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
771  fit.saveChi2Contributions(baseName)
772 
773  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
774  # The constrained model needs the visit transform fit first; the chip
775  # transform is initialized from the detector's cameraGeom, so it's close.
776  if self.config.astrometryModel == "constrained":
777  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
778  self._logChi2AndValidate(associations, fit, model)
779  dumpMatrixFile = "" # so we don't redo the output on the next step
780 
781  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
782  self._logChi2AndValidate(associations, fit, model)
783 
784  fit.minimize("Positions")
785  self._logChi2AndValidate(associations, fit, model)
786 
787  fit.minimize("Distortions Positions")
788  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
789 
790  chi2 = self._iterate_fit(associations,
791  fit,
792  self.config.maxAstrometrySteps,
793  "astrometry",
794  "Distortions Positions",
795  doRankUpdate=self.config.astrometryDoRankUpdate,
796  dataName=dataName)
797 
798  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
799  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
800 
801  return Astrometry(fit, model, sky_to_tan_projection)
802 
803  def _check_stars(self, associations):
804  """Count measured and reference stars per ccd and warn/log them."""
805  for ccdImage in associations.getCcdImageList():
806  nMeasuredStars, nRefStars = ccdImage.countStars()
807  self.log.debug("ccdImage %s has %s measured and %s reference stars",
808  ccdImage.getName(), nMeasuredStars, nRefStars)
809  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
810  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
811  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
812  if nRefStars < self.config.minRefStarsPerCcd:
813  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
814  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
815 
816  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
817  dataName="",
818  doRankUpdate=True,
819  doLineSearch=False):
820  """Run fitter.minimize up to max_steps times, returning the final chi2.
821 
822  Parameters
823  ----------
824  associations : `lsst.jointcal.Associations`
825  The star/reference star associations to fit.
826  fitter : `lsst.jointcal.FitterBase`
827  The fitter to use for minimization.
828  max_steps : `int`
829  Maximum number of steps to run outlier rejection before declaring
830  convergence failure.
831  name : {'photometry' or 'astrometry'}
832  What type of data are we fitting (for logs and debugging files).
833  whatToFit : `str`
834  Passed to ``fitter.minimize()`` to define the parameters to fit.
835  dataName : str, optional
836  Descriptive name for this dataset (e.g. tract and filter),
837  for debugging.
838  doRankUpdate : bool, optional
839  Do an Eigen rank update during minimization, or recompute the full
840  matrix and gradient?
841  doLineSearch : bool, optional
842  Do a line search for the optimum step during minimization?
843 
844  Returns
845  -------
846  chi2: `lsst.jointcal.Chi2Statistic`
847  The final chi2 after the fit converges, or is forced to end.
848 
849  Raises
850  ------
851  FloatingPointError
852  Raised if the fitter fails with a non-finite value.
853  RuntimeError
854  Raised if the fitter fails for some other reason;
855  log messages will provide further details.
856  """
857  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
858  for i in range(max_steps):
859  result = fitter.minimize(whatToFit,
860  self.config.outlierRejectSigma,
861  doRankUpdate=doRankUpdate,
862  doLineSearch=doLineSearch,
863  dumpMatrixFile=dumpMatrixFile)
864  dumpMatrixFile = "" # clear it so we don't write the matrix again.
865  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel())
866 
867  if result == MinimizeResult.Converged:
868  if doRankUpdate:
869  self.log.debug("fit has converged - no more outliers - redo minimization "
870  "one more time in case we have lost accuracy in rank update.")
871  # Redo minimization one more time in case we have lost accuracy in rank update
872  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
873  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
874 
875  # log a message for a large final chi2, TODO: DM-15247 for something better
876  if chi2.chi2/chi2.ndof >= 4.0:
877  self.log.error("Potentially bad fit: High chi-squared/ndof.")
878 
879  break
880  elif result == MinimizeResult.Chi2Increased:
881  self.log.warn("still some outliers but chi2 increases - retry")
882  elif result == MinimizeResult.NonFinite:
883  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
884  # TODO DM-12446: turn this into a "butler save" somehow.
885  fitter.saveChi2Contributions(filename)
886  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
887  raise FloatingPointError(msg.format(filename))
888  elif result == MinimizeResult.Failed:
889  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
890  else:
891  raise RuntimeError("Unxepected return code from minimize().")
892  else:
893  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
894 
895  return chi2
896 
897  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
898  """
899  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
900 
901  Parameters
902  ----------
903  associations : lsst.jointcal.Associations
904  The star/reference star associations to fit.
905  model : lsst.jointcal.AstrometryModel
906  The astrometric model that was fit.
907  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
908  dict of ccdImage identifiers to dataRefs that were fit
909  """
910 
911  ccdImageList = associations.getCcdImageList()
912  for ccdImage in ccdImageList:
913  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
914  ccd = ccdImage.ccdId
915  visit = ccdImage.visit
916  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
917  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
918  skyWcs = model.makeSkyWcs(ccdImage)
919  try:
920  dataRef.put(skyWcs, 'jointcal_wcs')
921  except pexExceptions.Exception as e:
922  self.log.fatal('Failed to write updated Wcs: %s', str(e))
923  raise e
924 
925  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
926  """
927  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
928 
929  Parameters
930  ----------
931  associations : lsst.jointcal.Associations
932  The star/reference star associations to fit.
933  model : lsst.jointcal.PhotometryModel
934  The photoometric model that was fit.
935  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
936  dict of ccdImage identifiers to dataRefs that were fit
937  """
938 
939  ccdImageList = associations.getCcdImageList()
940  for ccdImage in ccdImageList:
941  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
942  ccd = ccdImage.ccdId
943  visit = ccdImage.visit
944  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
945  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
946  photoCalib = model.toPhotoCalib(ccdImage)
947  try:
948  dataRef.put(photoCalib, 'jointcal_photoCalib')
949  except pexExceptions.Exception as e:
950  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
951  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:413
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:347
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:630
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:68
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:720
def _check_star_lists(self, associations, name)
Definition: jointcal.py:610
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model")
Definition: jointcal.py:619
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:819
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:925
def _check_stars(self, associations)
Definition: jointcal.py:803
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:50
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:513
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:897
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:305