lsst.jointcal  16.0-29-g44d5b4c
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 
25 import lsst.utils
26 import lsst.pex.config as pexConfig
27 import lsst.pipe.base as pipeBase
28 import lsst.afw.image as afwImage
29 import lsst.afw.geom as afwGeom
30 import lsst.pex.exceptions as pexExceptions
31 import lsst.afw.table
33 from lsst.verify import Job, Measurement
34 
35 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
36 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
37 
38 from .dataIds import PerTractCcdDataIdContainer
39 
40 import lsst.jointcal
41 from lsst.jointcal import MinimizeResult
42 
43 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
44 
45 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
46 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
47 
48 
49 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
50 def add_measurement(job, name, value):
51  meas = Measurement(job.metrics[name], value)
52  job.measurements.insert(meas)
53 
54 
55 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
56  """Subclass of TaskRunner for jointcalTask
57 
58  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
59  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
60  single dataRef, are are called repeatedly). This class transforms the processed
61  arguments generated by the ArgumentParser into the arguments expected by
62  Jointcal.runDataRef().
63 
64  See pipeBase.TaskRunner for more information.
65  """
66 
67  @staticmethod
68  def getTargetList(parsedCmd, **kwargs):
69  """
70  Return a list of tuples per tract, each containing (dataRefs, kwargs).
71 
72  Jointcal operates on lists of dataRefs simultaneously.
73  """
74  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
75  kwargs['butler'] = parsedCmd.butler
76 
77  # organize data IDs by tract
78  refListDict = {}
79  for ref in parsedCmd.id.refList:
80  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
81  # we call runDataRef() once with each tract
82  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
83  return result
84 
85  def __call__(self, args):
86  """
87  Parameters
88  ----------
89  args
90  Arguments for Task.runDataRef()
91 
92  Returns
93  -------
94  pipe.base.Struct
95  if self.doReturnResults is False:
96 
97  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
98 
99  if self.doReturnResults is True:
100 
101  - ``result``: the result of calling jointcal.runDataRef()
102  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
103  """
104  exitStatus = 0 # exit status for shell
105 
106  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
107  dataRefList, kwargs = args
108  butler = kwargs.pop('butler')
109  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
110  result = None
111  try:
112  result = task.runDataRef(dataRefList, **kwargs)
113  exitStatus = result.exitStatus
114  job_path = butler.get('verify_job_filename')
115  result.job.write(job_path[0])
116  except Exception as e: # catch everything, sort it out later.
117  if self.doRaise:
118  raise e
119  else:
120  exitStatus = 1
121  eName = type(e).__name__
122  tract = dataRefList[0].dataId['tract']
123  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
124 
125  if self.doReturnResults:
126  return pipeBase.Struct(result=result, exitStatus=exitStatus)
127  else:
128  return pipeBase.Struct(exitStatus=exitStatus)
129 
130 
131 class JointcalConfig(pexConfig.Config):
132  """Config for JointcalTask"""
133 
134  doAstrometry = pexConfig.Field(
135  doc="Fit astrometry and write the fitted result.",
136  dtype=bool,
137  default=True
138  )
139  doPhotometry = pexConfig.Field(
140  doc="Fit photometry and write the fitted result.",
141  dtype=bool,
142  default=True
143  )
144  coaddName = pexConfig.Field(
145  doc="Type of coadd, typically deep or goodSeeing",
146  dtype=str,
147  default="deep"
148  )
149  positionErrorPedestal = pexConfig.Field(
150  doc="Systematic term to apply to the measured position error (pixels)",
151  dtype=float,
152  default=0.02,
153  )
154  photometryErrorPedestal = pexConfig.Field(
155  doc="Systematic term to apply to the measured error on flux or magnitude as a "
156  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
157  dtype=float,
158  default=0.0,
159  )
160  # TODO: DM-6885 matchCut should be an afw.geom.Angle
161  matchCut = pexConfig.Field(
162  doc="Matching radius between fitted and reference stars (arcseconds)",
163  dtype=float,
164  default=3.0,
165  )
166  minMeasurements = pexConfig.Field(
167  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
168  dtype=int,
169  default=2,
170  )
171  minMeasuredStarsPerCcd = pexConfig.Field(
172  doc="Minimum number of measuredStars per ccdImage before printing warnings",
173  dtype=int,
174  default=100,
175  )
176  minRefStarsPerCcd = pexConfig.Field(
177  doc="Minimum number of measuredStars per ccdImage before printing warnings",
178  dtype=int,
179  default=30,
180  )
181  allowLineSearch = pexConfig.Field(
182  doc="Allow a line search during minimization, if it is reasonable for the model"
183  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
184  dtype=bool,
185  default=False
186  )
187  astrometrySimpleOrder = pexConfig.Field(
188  doc="Polynomial order for fitting the simple astrometry model.",
189  dtype=int,
190  default=3,
191  )
192  astrometryChipOrder = pexConfig.Field(
193  doc="Order of the per-chip transform for the constrained astrometry model.",
194  dtype=int,
195  default=1,
196  )
197  astrometryVisitOrder = pexConfig.Field(
198  doc="Order of the per-visit transform for the constrained astrometry model.",
199  dtype=int,
200  default=5,
201  )
202  useInputWcs = pexConfig.Field(
203  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
204  dtype=bool,
205  default=True,
206  )
207  astrometryModel = pexConfig.ChoiceField(
208  doc="Type of model to fit to astrometry",
209  dtype=str,
210  default="constrained",
211  allowed={"simple": "One polynomial per ccd",
212  "constrained": "One polynomial per ccd, and one polynomial per visit"}
213  )
214  photometryModel = pexConfig.ChoiceField(
215  doc="Type of model to fit to photometry",
216  dtype=str,
217  default="constrainedMagnitude",
218  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
219  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
220  " fitting in flux space.",
221  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
222  " fitting in magnitude space.",
223  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
224  " fitting in magnitude space.",
225  }
226  )
227  photometryVisitOrder = pexConfig.Field(
228  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
229  dtype=int,
230  default=7,
231  )
232  photometryDoRankUpdate = pexConfig.Field(
233  doc="Do the rank update step during minimization. "
234  "Skipping this can help deal with models that are too non-linear.",
235  dtype=bool,
236  default=True,
237  )
238  astrometryDoRankUpdate = pexConfig.Field(
239  doc="Do the rank update step during minimization (should not change the astrometry fit). "
240  "Skipping this can help deal with models that are too non-linear.",
241  dtype=bool,
242  default=True,
243  )
244  outlierRejectSigma = pexConfig.Field(
245  doc="How many sigma to reject outliers at during minimization.",
246  dtype=float,
247  default=5.0,
248  )
249  maxPhotometrySteps = pexConfig.Field(
250  doc="Maximum number of minimize iterations to take when fitting photometry.",
251  dtype=int,
252  default=20,
253  )
254  maxAstrometrySteps = pexConfig.Field(
255  doc="Maximum number of minimize iterations to take when fitting photometry.",
256  dtype=int,
257  default=20,
258  )
259  astrometryRefObjLoader = pexConfig.ConfigurableField(
260  target=LoadIndexedReferenceObjectsTask,
261  doc="Reference object loader for astrometric fit",
262  )
263  photometryRefObjLoader = pexConfig.ConfigurableField(
264  target=LoadIndexedReferenceObjectsTask,
265  doc="Reference object loader for photometric fit",
266  )
267  sourceSelector = sourceSelectorRegistry.makeField(
268  doc="How to select sources for cross-matching",
269  default="astrometry"
270  )
271  writeInitMatrix = pexConfig.Field(
272  dtype=bool,
273  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
274  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
275  "Note that these files are the dense versions of the matrix, and so may be very large.",
276  default=False
277  )
278  writeChi2ContributionFiles = pexConfig.Field(
279  dtype=bool,
280  doc="Write initial/final fit files containing the contributions to chi2.",
281  default=False
282  )
283  sourceFluxType = pexConfig.Field(
284  dtype=str,
285  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
286  default='Calib'
287  )
288 
289  def setDefaults(self):
290  sourceSelector = self.sourceSelector["astrometry"]
291  sourceSelector.setDefaults()
292  # don't want to lose existing flags, just add to them.
293  sourceSelector.badFlags.extend(["slot_Shape_flag"])
294  # This should be used to set the FluxField value in jointcal::JointcalControl
295  sourceSelector.sourceFluxType = self.sourceFluxType
296 
297 
298 class JointcalTask(pipeBase.CmdLineTask):
299  """Jointly astrometrically and photometrically calibrate a group of images."""
300 
301  ConfigClass = JointcalConfig
302  RunnerClass = JointcalRunner
303  _DefaultName = "jointcal"
304 
305  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
306  """
307  Instantiate a JointcalTask.
308 
309  Parameters
310  ----------
311  butler : lsst.daf.persistence.Butler
312  The butler is passed to the refObjLoader constructor in case it is
313  needed. Ignored if the refObjLoader argument provides a loader directly.
314  Used to initialize the astrometry and photometry refObjLoaders.
315  profile_jointcal : bool
316  set to True to profile different stages of this jointcal run.
317  """
318  pipeBase.CmdLineTask.__init__(self, **kwargs)
319  self.profile_jointcal = profile_jointcal
320  self.makeSubtask("sourceSelector")
321  if self.config.doAstrometry:
322  self.makeSubtask('astrometryRefObjLoader', butler=butler)
323  else:
325  if self.config.doPhotometry:
326  self.makeSubtask('photometryRefObjLoader', butler=butler)
327  else:
329 
330  # To hold various computed metrics for use by tests
331  self.job = Job.load_metrics_package(subset='jointcal')
332 
333  # We don't need to persist config and metadata at this stage.
334  # In this way, we don't need to put a specific entry in the camera mapper policy file
335  def _getConfigName(self):
336  return None
337 
338  def _getMetadataName(self):
339  return None
340 
341  @classmethod
342  def _makeArgumentParser(cls):
343  """Create an argument parser"""
344  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
345  parser.add_argument("--profile_jointcal", default=False, action="store_true",
346  help="Profile steps of jointcal separately.")
347  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
348  ContainerClass=PerTractCcdDataIdContainer)
349  return parser
350 
351  def _build_ccdImage(self, dataRef, associations, jointcalControl):
352  """
353  Extract the necessary things from this dataRef to add a new ccdImage.
354 
355  Parameters
356  ----------
357  dataRef : lsst.daf.persistence.ButlerDataRef
358  dataRef to extract info from.
359  associations : lsst.jointcal.Associations
360  object to add the info to, to construct a new CcdImage
361  jointcalControl : jointcal.JointcalControl
362  control object for associations management
363 
364  Returns
365  ------
366  namedtuple
367  wcs : lsst.afw.geom.SkyWcs
368  the TAN WCS of this image, read from the calexp
369  key : namedtuple
370  a key to identify this dataRef by its visit and ccd ids
371  filter : str
372  this calexp's filter
373  """
374  if "visit" in dataRef.dataId.keys():
375  visit = dataRef.dataId["visit"]
376  else:
377  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
378 
379  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
380 
381  visitInfo = dataRef.get('calexp_visitInfo')
382  detector = dataRef.get('calexp_detector')
383  ccdId = detector.getId()
384  calib = dataRef.get('calexp_calib')
385  tanWcs = dataRef.get('calexp_wcs')
386  bbox = dataRef.get('calexp_bbox')
387  filt = dataRef.get('calexp_filter')
388  filterName = filt.getName()
389  fluxMag0 = calib.getFluxMag0()
390  # TODO: need to scale these until DM-10153 is completed and PhotoCalib has replaced Calib entirely
391  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
392  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
393  referenceFlux*fluxMag0[1]/fluxMag0[0]**2, bbox)
394 
395  goodSrc = self.sourceSelector.run(src)
396 
397  if len(goodSrc.sourceCat) == 0:
398  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
399  else:
400  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
401  associations.createCcdImage(goodSrc.sourceCat,
402  tanWcs,
403  visitInfo,
404  bbox,
405  filterName,
406  photoCalib,
407  detector,
408  visit,
409  ccdId,
410  jointcalControl)
411 
412  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
413  Key = collections.namedtuple('Key', ('visit', 'ccd'))
414  return Result(tanWcs, Key(visit, ccdId), filterName)
415 
416  @pipeBase.timeMethod
417  def runDataRef(self, dataRefs, profile_jointcal=False):
418  """
419  Jointly calibrate the astrometry and photometry across a set of images.
420 
421  Parameters
422  ----------
423  dataRefs : list of lsst.daf.persistence.ButlerDataRef
424  List of data references to the exposures to be fit.
425  profile_jointcal : bool
426  Profile the individual steps of jointcal.
427 
428  Returns
429  -------
430  pipe.base.Struct
431  struct containing:
432  * dataRefs: the provided data references that were fit (with updated WCSs)
433  * oldWcsList: the original WCS from each dataRef
434  * metrics: dictionary of internally-computed metrics for testing/validation.
435  """
436  if len(dataRefs) == 0:
437  raise ValueError('Need a non-empty list of data references!')
438 
439  exitStatus = 0 # exit status for shell
440 
441  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
442  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
443  associations = lsst.jointcal.Associations()
444 
445  visit_ccd_to_dataRef = {}
446  oldWcsList = []
447  filters = []
448  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
449  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
450  # We need the bounding-box of the focal plane for photometry visit models.
451  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
452  camera = dataRefs[0].get('camera', immediate=True)
453  self.focalPlaneBBox = camera.getFpBBox()
454  for ref in dataRefs:
455  result = self._build_ccdImage(ref, associations, jointcalControl)
456  oldWcsList.append(result.wcs)
457  visit_ccd_to_dataRef[result.key] = ref
458  filters.append(result.filter)
459  filters = collections.Counter(filters)
460 
461  associations.computeCommonTangentPoint()
462 
463  # Use external reference catalogs handled by LSST stack mechanism
464  # Get the bounding box overlapping all associated images
465  # ==> This is probably a bad idea to do it this way <== To be improved
466  bbox = associations.getRaDecBBox()
467  bboxCenter = bbox.getCenter()
468  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
469  bboxMax = bbox.getMax()
470  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
471  radius = center.separation(corner).asRadians()
472 
473  # Get astrometry_net_data path
474  anDir = lsst.utils.getPackageDir('astrometry_net_data')
475  if anDir is None:
476  raise RuntimeError("astrometry_net_data is not setup")
477 
478  # Determine a default filter associated with the catalog. See DM-9093
479  defaultFilter = filters.most_common(1)[0][0]
480  self.log.debug("Using %s band for reference flux", defaultFilter)
481 
482  # TODO: need a better way to get the tract.
483  tract = dataRefs[0].dataId['tract']
484 
485  if self.config.doAstrometry:
486  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
487  name="astrometry",
488  refObjLoader=self.astrometryRefObjLoader,
489  fit_function=self._fit_astrometry,
490  profile_jointcal=profile_jointcal,
491  tract=tract)
492  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
493  else:
494  astrometry = Astrometry(None, None, None)
495 
496  if self.config.doPhotometry:
497  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
498  name="photometry",
499  refObjLoader=self.photometryRefObjLoader,
500  fit_function=self._fit_photometry,
501  profile_jointcal=profile_jointcal,
502  tract=tract,
503  filters=filters,
504  reject_bad_fluxes=True)
505  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
506  else:
507  photometry = Photometry(None, None)
508 
509  return pipeBase.Struct(dataRefs=dataRefs,
510  oldWcsList=oldWcsList,
511  job=self.job,
512  astrometryRefObjLoader=self.astrometryRefObjLoader,
513  photometryRefObjLoader=self.photometryRefObjLoader,
514  defaultFilter=defaultFilter,
515  exitStatus=exitStatus)
516 
517  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
518  name="", refObjLoader=None, filters=[], fit_function=None,
519  tract=None, profile_jointcal=False, match_cut=3.0,
520  reject_bad_fluxes=False):
521  """Load reference catalog, perform the fit, and return the result.
522 
523  Parameters
524  ----------
525  associations : lsst.jointcal.Associations
526  The star/reference star associations to fit.
527  defaultFilter : str
528  filter to load from reference catalog.
529  center : lsst.afw.geom.SpherePoint
530  ICRS center of field to load from reference catalog.
531  radius : lsst.afw.geom.Angle
532  On-sky radius to load from reference catalog.
533  name : str
534  Name of thing being fit: "Astrometry" or "Photometry".
535  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
536  Reference object loader to load from for fit.
537  filters : list of str, optional
538  List of filters to load from the reference catalog.
539  fit_function : function
540  function to call to perform fit (takes associations object).
541  tract : str
542  Name of tract currently being fit.
543  profile_jointcal : bool, optional
544  Separately profile the fitting step.
545  match_cut : float, optional
546  Radius in arcseconds to find cross-catalog matches to during
547  associations.associateCatalogs.
548  reject_bad_fluxes : bool, optional
549  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
550 
551  Returns
552  -------
553  Result of `fit_function()`
554  """
555  self.log.info("====== Now processing %s...", name)
556  # TODO: this should not print "trying to invert a singular transformation:"
557  # if it does that, something's not right about the WCS...
558  associations.associateCatalogs(match_cut)
559  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
560  associations.fittedStarListSize())
561 
562  skyCircle = refObjLoader.loadSkyCircle(center,
563  afwGeom.Angle(radius, afwGeom.radians),
564  defaultFilter)
565 
566  # Need memory contiguity to get reference filters as a vector.
567  if not skyCircle.refCat.isContiguous():
568  refCat = skyCircle.refCat.copy(deep=True)
569  else:
570  refCat = skyCircle.refCat
571 
572  # load the reference catalog fluxes.
573  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
574  refFluxes = {}
575  refFluxErrs = {}
576  for filt in filters:
577  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
578  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
579  refFluxes[filt] = 1e9*refCat.get(filtKeys[0])
580  refFluxErrs[filt] = 1e9*refCat.get(filtKeys[1])
581 
582  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
583  refCat[skyCircle.fluxField] *= 1e9
584  try:
585  refCat[skyCircle.fluxField+'Err'] *= 1e9
586  except KeyError:
587  # not all existing refcats have an error field.
588  pass
589 
590  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
591  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
592  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
593  associations.refStarListSize())
594 
595  associations.prepareFittedStars(self.config.minMeasurements)
596 
597  self._check_star_lists(associations, name)
598  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
599  associations.nFittedStarsWithAssociatedRefStar())
600  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
601  associations.fittedStarListSize())
602  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
603  associations.nCcdImagesValidForFit())
604 
605  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
606  dataName = "{}_{}".format(tract, defaultFilter)
607  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
608  result = fit_function(associations, dataName)
609  # TODO DM-12446: turn this into a "butler save" somehow.
610  # Save reference and measurement chi2 contributions for this data
611  if self.config.writeChi2ContributionFiles:
612  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
613  result.fit.saveChi2Contributions(baseName)
614 
615  return result
616 
617  def _check_star_lists(self, associations, name):
618  # TODO: these should be len(blah), but we need this properly wrapped first.
619  if associations.nCcdImagesValidForFit() == 0:
620  raise RuntimeError('No images in the ccdImageList!')
621  if associations.fittedStarListSize() == 0:
622  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
623  if associations.refStarListSize() == 0:
624  raise RuntimeError('No stars in the {} reference star list!'.format(name))
625 
626  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model"):
627  """Compute chi2, log it, validate the model, and return chi2."""
628  chi2 = fit.computeChi2()
629  self.log.info("%s %s", chi2Label, chi2)
630  self._check_stars(associations)
631  if not np.isfinite(chi2.chi2):
632  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
633  if not model.validate(associations.getCcdImageList()):
634  raise ValueError("Model is not valid: check log messages for warnings.")
635  return chi2
636 
637  def _fit_photometry(self, associations, dataName=None):
638  """
639  Fit the photometric data.
640 
641  Parameters
642  ----------
643  associations : lsst.jointcal.Associations
644  The star/reference star associations to fit.
645  dataName : str
646  Name of the data being processed (e.g. "1234_HSC-Y"), for
647  identifying debugging files.
648 
649  Returns
650  -------
651  namedtuple
652  fit : lsst.jointcal.PhotometryFit
653  The photometric fitter used to perform the fit.
654  model : lsst.jointcal.PhotometryModel
655  The photometric model that was fit.
656  """
657  self.log.info("=== Starting photometric fitting...")
658 
659  # TODO: should use pex.config.RegistryField here (see DM-9195)
660  if self.config.photometryModel == "constrainedFlux":
661  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
662  self.focalPlaneBBox,
663  visitOrder=self.config.photometryVisitOrder,
664  errorPedestal=self.config.photometryErrorPedestal)
665  # potentially nonlinear problem, so we may need a line search to converge.
666  doLineSearch = self.config.allowLineSearch
667  elif self.config.photometryModel == "constrainedMagnitude":
668  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
669  self.focalPlaneBBox,
670  visitOrder=self.config.photometryVisitOrder,
671  errorPedestal=self.config.photometryErrorPedestal)
672  # potentially nonlinear problem, so we may need a line search to converge.
673  doLineSearch = self.config.allowLineSearch
674  elif self.config.photometryModel == "simpleFlux":
675  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
676  errorPedestal=self.config.photometryErrorPedestal)
677  doLineSearch = False # purely linear in model parameters, so no line search needed
678  elif self.config.photometryModel == "simpleMagnitude":
679  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
680  errorPedestal=self.config.photometryErrorPedestal)
681  doLineSearch = False # purely linear in model parameters, so no line search needed
682 
683  fit = lsst.jointcal.PhotometryFit(associations, model)
684  self._logChi2AndValidate(associations, fit, model, "Initialized")
685 
686  # TODO DM-12446: turn this into a "butler save" somehow.
687  # Save reference and measurement chi2 contributions for this data
688  if self.config.writeChi2ContributionFiles:
689  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
690  fit.saveChi2Contributions(baseName)
691 
692  # The constrained model needs the visit transform fit first; the chip
693  # transform is initialized from the singleFrame PhotoCalib, so it's close.
694  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
695  if self.config.photometryModel.startswith("constrained"):
696  # no line search: should be purely (or nearly) linear,
697  # and we want a large step size to initialize with.
698  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
699  self._logChi2AndValidate(associations, fit, model)
700  dumpMatrixFile = "" # so we don't redo the output on the next step
701 
702  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
703  self._logChi2AndValidate(associations, fit, model)
704 
705  fit.minimize("Fluxes") # no line search: always purely linear.
706  self._logChi2AndValidate(associations, fit, model)
707 
708  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
709  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
710 
711  model.freezeErrorTransform()
712  self.log.debug("Photometry error scales are frozen.")
713 
714  chi2 = self._iterate_fit(associations,
715  fit,
716  self.config.maxPhotometrySteps,
717  "photometry",
718  "Model Fluxes",
719  doRankUpdate=self.config.photometryDoRankUpdate,
720  doLineSearch=doLineSearch,
721  dataName=dataName)
722 
723  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
724  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
725  return Photometry(fit, model)
726 
727  def _fit_astrometry(self, associations, dataName=None):
728  """
729  Fit the astrometric data.
730 
731  Parameters
732  ----------
733  associations : lsst.jointcal.Associations
734  The star/reference star associations to fit.
735  dataName : str
736  Name of the data being processed (e.g. "1234_HSC-Y"), for
737  identifying debugging files.
738 
739  Returns
740  -------
741  namedtuple
742  fit : lsst.jointcal.AstrometryFit
743  The astrometric fitter used to perform the fit.
744  model : lsst.jointcal.AstrometryModel
745  The astrometric model that was fit.
746  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
747  The model for the sky to tangent plane projection that was used in the fit.
748  """
749 
750  self.log.info("=== Starting astrometric fitting...")
751 
752  associations.deprojectFittedStars()
753 
754  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
755  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
756  # them so carefully?
757  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
758 
759  if self.config.astrometryModel == "constrained":
760  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
761  sky_to_tan_projection,
762  chipOrder=self.config.astrometryChipOrder,
763  visitOrder=self.config.astrometryVisitOrder)
764  elif self.config.astrometryModel == "simple":
765  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
766  sky_to_tan_projection,
767  self.config.useInputWcs,
768  nNotFit=0,
769  order=self.config.astrometrySimpleOrder)
770 
771  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
772  self._logChi2AndValidate(associations, fit, model, "Initial")
773 
774  # TODO DM-12446: turn this into a "butler save" somehow.
775  # Save reference and measurement chi2 contributions for this data
776  if self.config.writeChi2ContributionFiles:
777  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
778  fit.saveChi2Contributions(baseName)
779 
780  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
781  # The constrained model needs the visit transform fit first; the chip
782  # transform is initialized from the detector's cameraGeom, so it's close.
783  if self.config.astrometryModel == "constrained":
784  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
785  self._logChi2AndValidate(associations, fit, model)
786  dumpMatrixFile = "" # so we don't redo the output on the next step
787 
788  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
789  self._logChi2AndValidate(associations, fit, model)
790 
791  fit.minimize("Positions")
792  self._logChi2AndValidate(associations, fit, model)
793 
794  fit.minimize("Distortions Positions")
795  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
796 
797  chi2 = self._iterate_fit(associations,
798  fit,
799  self.config.maxAstrometrySteps,
800  "astrometry",
801  "Distortions Positions",
802  doRankUpdate=self.config.astrometryDoRankUpdate,
803  dataName=dataName)
804 
805  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
806  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
807 
808  return Astrometry(fit, model, sky_to_tan_projection)
809 
810  def _check_stars(self, associations):
811  """Count measured and reference stars per ccd and warn/log them."""
812  for ccdImage in associations.getCcdImageList():
813  nMeasuredStars, nRefStars = ccdImage.countStars()
814  self.log.debug("ccdImage %s has %s measured and %s reference stars",
815  ccdImage.getName(), nMeasuredStars, nRefStars)
816  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
817  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
818  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
819  if nRefStars < self.config.minRefStarsPerCcd:
820  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
821  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
822 
823  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
824  dataName="",
825  doRankUpdate=True,
826  doLineSearch=False):
827  """Run fitter.minimize up to max_steps times, returning the final chi2.
828 
829  Parameters
830  ----------
831  associations : `lsst.jointcal.Associations`
832  The star/reference star associations to fit.
833  fitter : `lsst.jointcal.FitterBase`
834  The fitter to use for minimization.
835  max_steps : `int`
836  Maximum number of steps to run outlier rejection before declaring
837  convergence failure.
838  name : {'photometry' or 'astrometry'}
839  What type of data are we fitting (for logs and debugging files).
840  whatToFit : `str`
841  Passed to ``fitter.minimize()`` to define the parameters to fit.
842  dataName : str, optional
843  Descriptive name for this dataset (e.g. tract and filter),
844  for debugging.
845  doRankUpdate : bool, optional
846  Do an Eigen rank update during minimization, or recompute the full
847  matrix and gradient?
848  doLineSearch : bool, optional
849  Do a line search for the optimum step during minimization?
850 
851  Returns
852  -------
853  chi2: `lsst.jointcal.Chi2Statistic`
854  The final chi2 after the fit converges, or is forced to end.
855 
856  Raises
857  ------
858  FloatingPointError
859  Raised if the fitter fails with a non-finite value.
860  RuntimeError
861  Raised if the fitter fails for some other reason;
862  log messages will provide further details.
863  """
864  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
865  for i in range(max_steps):
866  result = fitter.minimize(whatToFit,
867  self.config.outlierRejectSigma,
868  doRankUpdate=doRankUpdate,
869  doLineSearch=doLineSearch,
870  dumpMatrixFile=dumpMatrixFile)
871  dumpMatrixFile = "" # clear it so we don't write the matrix again.
872  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel())
873 
874  if result == MinimizeResult.Converged:
875  if doRankUpdate:
876  self.log.debug("fit has converged - no more outliers - redo minimization "
877  "one more time in case we have lost accuracy in rank update.")
878  # Redo minimization one more time in case we have lost accuracy in rank update
879  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
880  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
881 
882  # log a message for a large final chi2, TODO: DM-15247 for something better
883  if chi2.chi2/chi2.ndof >= 4.0:
884  self.log.error("Potentially bad fit: High chi-squared/ndof.")
885 
886  break
887  elif result == MinimizeResult.Chi2Increased:
888  self.log.warn("still some outliers but chi2 increases - retry")
889  elif result == MinimizeResult.NonFinite:
890  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
891  # TODO DM-12446: turn this into a "butler save" somehow.
892  fitter.saveChi2Contributions(filename)
893  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
894  raise FloatingPointError(msg.format(filename))
895  elif result == MinimizeResult.Failed:
896  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
897  else:
898  raise RuntimeError("Unxepected return code from minimize().")
899  else:
900  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
901 
902  return chi2
903 
904  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
905  """
906  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
907 
908  Parameters
909  ----------
910  associations : lsst.jointcal.Associations
911  The star/reference star associations to fit.
912  model : lsst.jointcal.AstrometryModel
913  The astrometric model that was fit.
914  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
915  dict of ccdImage identifiers to dataRefs that were fit
916  """
917 
918  ccdImageList = associations.getCcdImageList()
919  for ccdImage in ccdImageList:
920  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
921  ccd = ccdImage.ccdId
922  visit = ccdImage.visit
923  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
924  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
925  skyWcs = model.makeSkyWcs(ccdImage)
926  try:
927  dataRef.put(skyWcs, 'jointcal_wcs')
928  except pexExceptions.Exception as e:
929  self.log.fatal('Failed to write updated Wcs: %s', str(e))
930  raise e
931 
932  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
933  """
934  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
935 
936  Parameters
937  ----------
938  associations : lsst.jointcal.Associations
939  The star/reference star associations to fit.
940  model : lsst.jointcal.PhotometryModel
941  The photoometric model that was fit.
942  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
943  dict of ccdImage identifiers to dataRefs that were fit
944  """
945 
946  ccdImageList = associations.getCcdImageList()
947  for ccdImage in ccdImageList:
948  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
949  ccd = ccdImage.ccdId
950  visit = ccdImage.visit
951  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
952  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
953  photoCalib = model.toPhotoCalib(ccdImage)
954  try:
955  dataRef.put(photoCalib, 'jointcal_photoCalib')
956  except pexExceptions.Exception as e:
957  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
958  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:417
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:351
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:637
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:68
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:727
def _check_star_lists(self, associations, name)
Definition: jointcal.py:617
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model")
Definition: jointcal.py:626
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:826
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:932
def _check_stars(self, associations)
Definition: jointcal.py:810
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:50
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:520
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:904
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:305