lsst.jointcal  16.0-5-g82b7855
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  allowLineSearch = pexConfig.Field(
156  doc="Allow a line search during minimization, if it is reasonable for the model"
157  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
158  dtype=bool,
159  default=False
160  )
161  astrometrySimpleOrder = pexConfig.Field(
162  doc="Polynomial order for fitting the simple astrometry model.",
163  dtype=int,
164  default=3,
165  )
166  astrometryChipOrder = pexConfig.Field(
167  doc="Order of the per-chip transform for the constrained astrometry model.",
168  dtype=int,
169  default=1,
170  )
171  astrometryVisitOrder = pexConfig.Field(
172  doc="Order of the per-visit transform for the constrained astrometry model.",
173  dtype=int,
174  default=5,
175  )
176  useInputWcs = pexConfig.Field(
177  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
178  dtype=bool,
179  default=True,
180  )
181  astrometryModel = pexConfig.ChoiceField(
182  doc="Type of model to fit to astrometry",
183  dtype=str,
184  default="simple",
185  allowed={"simple": "One polynomial per ccd",
186  "constrained": "One polynomial per ccd, and one polynomial per visit"}
187  )
188  photometryModel = pexConfig.ChoiceField(
189  doc="Type of model to fit to photometry",
190  dtype=str,
191  default="simple",
192  allowed={"simple": "One constant zeropoint per ccd and visit",
193  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
194  )
195  photometryVisitOrder = pexConfig.Field(
196  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
197  dtype=int,
198  default=7,
199  )
200  photometryDoRankUpdate = pexConfig.Field(
201  doc="Do the rank update step during minimization. "
202  "Skipping this can help deal with models that are too non-linear.",
203  dtype=bool,
204  default=True,
205  )
206  astrometryDoRankUpdate = pexConfig.Field(
207  doc="Do the rank update step during minimization (should not change the astrometry fit). "
208  "Skipping this can help deal with models that are too non-linear.",
209  dtype=bool,
210  default=True,
211  )
212  outlierRejectSigma = pexConfig.Field(
213  doc="How many sigma to reject outliers at during minimization.",
214  dtype=float,
215  default=5.0,
216  )
217  maxPhotometrySteps = pexConfig.Field(
218  doc="Maximum number of minimize iterations to take when fitting photometry.",
219  dtype=int,
220  default=20,
221  )
222  maxAstrometrySteps = pexConfig.Field(
223  doc="Maximum number of minimize iterations to take when fitting photometry.",
224  dtype=int,
225  default=20,
226  )
227  astrometryRefObjLoader = pexConfig.ConfigurableField(
228  target=LoadIndexedReferenceObjectsTask,
229  doc="Reference object loader for astrometric fit",
230  )
231  photometryRefObjLoader = pexConfig.ConfigurableField(
232  target=LoadIndexedReferenceObjectsTask,
233  doc="Reference object loader for photometric fit",
234  )
235  sourceSelector = sourceSelectorRegistry.makeField(
236  doc="How to select sources for cross-matching",
237  default="astrometry"
238  )
239  writeInitMatrix = pexConfig.Field(
240  dtype=bool,
241  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
242  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
243  "Note that these files are the dense versions of the matrix, and so may be very large.",
244  default=False
245  )
246  writeChi2ContributionFiles = pexConfig.Field(
247  dtype=bool,
248  doc="Write initial/final fit files containing the contributions to chi2.",
249  default=False
250  )
251  sourceFluxType = pexConfig.Field(
252  dtype=str,
253  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
254  default='Calib'
255  )
256 
257  def setDefaults(self):
258  sourceSelector = self.sourceSelector["astrometry"]
259  sourceSelector.setDefaults()
260  # don't want to lose existing flags, just add to them.
261  sourceSelector.badFlags.extend(["slot_Shape_flag"])
262  # This should be used to set the FluxField value in jointcal::JointcalControl
263  sourceSelector.sourceFluxType = self.sourceFluxType
264 
265 
266 class JointcalTask(pipeBase.CmdLineTask):
267  """Jointly astrometrically and photometrically calibrate a group of images."""
268 
269  ConfigClass = JointcalConfig
270  RunnerClass = JointcalRunner
271  _DefaultName = "jointcal"
272 
273  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
274  """
275  Instantiate a JointcalTask.
276 
277  Parameters
278  ----------
279  butler : lsst.daf.persistence.Butler
280  The butler is passed to the refObjLoader constructor in case it is
281  needed. Ignored if the refObjLoader argument provides a loader directly.
282  Used to initialize the astrometry and photometry refObjLoaders.
283  profile_jointcal : bool
284  set to True to profile different stages of this jointcal run.
285  """
286  pipeBase.CmdLineTask.__init__(self, **kwargs)
287  self.profile_jointcal = profile_jointcal
288  self.makeSubtask("sourceSelector")
289  if self.config.doAstrometry:
290  self.makeSubtask('astrometryRefObjLoader', butler=butler)
291  if self.config.doPhotometry:
292  self.makeSubtask('photometryRefObjLoader', butler=butler)
293 
294  # To hold various computed metrics for use by tests
295  self.job = Job.load_metrics_package(subset='jointcal')
296 
297  # We don't need to persist config and metadata at this stage.
298  # In this way, we don't need to put a specific entry in the camera mapper policy file
299  def _getConfigName(self):
300  return None
301 
302  def _getMetadataName(self):
303  return None
304 
305  @classmethod
306  def _makeArgumentParser(cls):
307  """Create an argument parser"""
308  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
309  parser.add_argument("--profile_jointcal", default=False, action="store_true",
310  help="Profile steps of jointcal separately.")
311  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
312  ContainerClass=PerTractCcdDataIdContainer)
313  return parser
314 
315  def _build_ccdImage(self, dataRef, associations, jointcalControl):
316  """
317  Extract the necessary things from this dataRef to add a new ccdImage.
318 
319  Parameters
320  ----------
321  dataRef : lsst.daf.persistence.ButlerDataRef
322  dataRef to extract info from.
323  associations : lsst.jointcal.Associations
324  object to add the info to, to construct a new CcdImage
325  jointcalControl : jointcal.JointcalControl
326  control object for associations management
327 
328  Returns
329  ------
330  namedtuple
331  wcs : lsst.afw.geom.SkyWcs
332  the TAN WCS of this image, read from the calexp
333  key : namedtuple
334  a key to identify this dataRef by its visit and ccd ids
335  filter : str
336  this calexp's filter
337  """
338  if "visit" in dataRef.dataId.keys():
339  visit = dataRef.dataId["visit"]
340  else:
341  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
342 
343  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
344 
345  visitInfo = dataRef.get('calexp_visitInfo')
346  detector = dataRef.get('calexp_detector')
347  ccdId = detector.getId()
348  calib = dataRef.get('calexp_calib')
349  tanWcs = dataRef.get('calexp_wcs')
350  bbox = dataRef.get('calexp_bbox')
351  filt = dataRef.get('calexp_filter')
352  filterName = filt.getName()
353  fluxMag0 = calib.getFluxMag0()
354  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
355 
356  goodSrc = self.sourceSelector.run(src)
357 
358  if len(goodSrc.sourceCat) == 0:
359  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
360  else:
361  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
362  associations.createCcdImage(goodSrc.sourceCat,
363  tanWcs,
364  visitInfo,
365  bbox,
366  filterName,
367  photoCalib,
368  detector,
369  visit,
370  ccdId,
371  jointcalControl)
372 
373  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
374  Key = collections.namedtuple('Key', ('visit', 'ccd'))
375  return Result(tanWcs, Key(visit, ccdId), filterName)
376 
377  @pipeBase.timeMethod
378  def run(self, dataRefs, profile_jointcal=False):
379  """
380  Jointly calibrate the astrometry and photometry across a set of images.
381 
382  Parameters
383  ----------
384  dataRefs : list of lsst.daf.persistence.ButlerDataRef
385  List of data references to the exposures to be fit.
386  profile_jointcal : bool
387  Profile the individual steps of jointcal.
388 
389  Returns
390  -------
391  pipe.base.Struct
392  struct containing:
393  * dataRefs: the provided data references that were fit (with updated WCSs)
394  * oldWcsList: the original WCS from each dataRef
395  * metrics: dictionary of internally-computed metrics for testing/validation.
396  """
397  if len(dataRefs) == 0:
398  raise ValueError('Need a non-empty list of data references!')
399 
400  exitStatus = 0 # exit status for shell
401 
402  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
403  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
404  associations = lsst.jointcal.Associations()
405 
406  visit_ccd_to_dataRef = {}
407  oldWcsList = []
408  filters = []
409  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
410  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
411  # We need the bounding-box of the focal plane for photometry visit models.
412  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
413  camera = dataRefs[0].get('camera', immediate=True)
414  self.focalPlaneBBox = camera.getFpBBox()
415  for ref in dataRefs:
416  result = self._build_ccdImage(ref, associations, jointcalControl)
417  oldWcsList.append(result.wcs)
418  visit_ccd_to_dataRef[result.key] = ref
419  filters.append(result.filter)
420  filters = collections.Counter(filters)
421 
422  associations.computeCommonTangentPoint()
423 
424  # Use external reference catalogs handled by LSST stack mechanism
425  # Get the bounding box overlapping all associated images
426  # ==> This is probably a bad idea to do it this way <== To be improved
427  bbox = associations.getRaDecBBox()
428  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
429  bboxCenter = bbox.getCenter()
430  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
431  bboxMax = bbox.getMax()
432  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
433  radius = center.separation(corner).asRadians()
434 
435  # Get astrometry_net_data path
436  anDir = lsst.utils.getPackageDir('astrometry_net_data')
437  if anDir is None:
438  raise RuntimeError("astrometry_net_data is not setup")
439 
440  # Determine a default filter associated with the catalog. See DM-9093
441  defaultFilter = filters.most_common(1)[0][0]
442  self.log.debug("Using %s band for reference flux", defaultFilter)
443 
444  # TODO: need a better way to get the tract.
445  tract = dataRefs[0].dataId['tract']
446 
447  if self.config.doAstrometry:
448  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
449  name="astrometry",
450  refObjLoader=self.astrometryRefObjLoader,
451  fit_function=self._fit_astrometry,
452  profile_jointcal=profile_jointcal,
453  tract=tract)
454  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
455  else:
456  astrometry = Astrometry(None, None, None)
457 
458  if self.config.doPhotometry:
459  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
460  name="photometry",
461  refObjLoader=self.photometryRefObjLoader,
462  fit_function=self._fit_photometry,
463  profile_jointcal=profile_jointcal,
464  tract=tract,
465  filters=filters,
466  reject_bad_fluxes=True)
467  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
468  else:
469  photometry = Photometry(None, None)
470 
471  return pipeBase.Struct(dataRefs=dataRefs,
472  oldWcsList=oldWcsList,
473  job=self.job,
474  exitStatus=exitStatus)
475 
476  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
477  name="", refObjLoader=None, filters=[], fit_function=None,
478  tract=None, profile_jointcal=False, match_cut=3.0,
479  reject_bad_fluxes=False):
480  """Load reference catalog, perform the fit, and return the result.
481 
482  Parameters
483  ----------
484  associations : lsst.jointcal.Associations
485  The star/reference star associations to fit.
486  defaultFilter : str
487  filter to load from reference catalog.
488  center : lsst.afw.geom.SpherePoint
489  ICRS center of field to load from reference catalog.
490  radius : lsst.afw.geom.Angle
491  On-sky radius to load from reference catalog.
492  name : str
493  Name of thing being fit: "Astrometry" or "Photometry".
494  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
495  Reference object loader to load from for fit.
496  filters : list of str, optional
497  List of filters to load from the reference catalog.
498  fit_function : function
499  function to call to perform fit (takes associations object).
500  tract : str
501  Name of tract currently being fit.
502  profile_jointcal : bool, optional
503  Separately profile the fitting step.
504  match_cut : float, optional
505  Radius in arcseconds to find cross-catalog matches to during
506  associations.associateCatalogs.
507  reject_bad_fluxes : bool, optional
508  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
509 
510  Returns
511  -------
512  Result of `fit_function()`
513  """
514  self.log.info("====== Now processing %s...", name)
515  # TODO: this should not print "trying to invert a singular transformation:"
516  # if it does that, something's not right about the WCS...
517  associations.associateCatalogs(match_cut)
518  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
519  associations.fittedStarListSize())
520 
521  skyCircle = refObjLoader.loadSkyCircle(center,
522  afwGeom.Angle(radius, afwGeom.radians),
523  defaultFilter)
524 
525  # Need memory contiguity to get reference filters as a vector.
526  if not skyCircle.refCat.isContiguous():
527  refCat = skyCircle.refCat.copy(deep=True)
528  else:
529  refCat = skyCircle.refCat
530 
531  # load the reference catalog fluxes.
532  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
533  refFluxes = {}
534  refFluxErrs = {}
535  for filt in filters:
536  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
537  refFluxes[filt] = refCat.get(filtKeys[0])
538  refFluxErrs[filt] = refCat.get(filtKeys[1])
539 
540  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
541  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
542  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
543  associations.refStarListSize())
544 
545  associations.prepareFittedStars(self.config.minMeasurements)
546 
547  self._check_star_lists(associations, name)
548  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
549  associations.nFittedStarsWithAssociatedRefStar())
550  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
551  associations.fittedStarListSize())
552  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
553  associations.nCcdImagesValidForFit())
554 
555  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
556  dataName = "{}_{}".format(tract, defaultFilter)
557  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
558  result = fit_function(associations, dataName)
559  # TODO DM-12446: turn this into a "butler save" somehow.
560  # Save reference and measurement chi2 contributions for this data
561  if self.config.writeChi2ContributionFiles:
562  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
563  result.fit.saveChi2Contributions(baseName)
564 
565  return result
566 
567  def _check_star_lists(self, associations, name):
568  # TODO: these should be len(blah), but we need this properly wrapped first.
569  if associations.nCcdImagesValidForFit() == 0:
570  raise RuntimeError('No images in the ccdImageList!')
571  if associations.fittedStarListSize() == 0:
572  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
573  if associations.refStarListSize() == 0:
574  raise RuntimeError('No stars in the {} reference star list!'.format(name))
575 
576  def _fit_photometry(self, associations, dataName=None):
577  """
578  Fit the photometric data.
579 
580  Parameters
581  ----------
582  associations : lsst.jointcal.Associations
583  The star/reference star associations to fit.
584  dataName : str
585  Name of the data being processed (e.g. "1234_HSC-Y"), for
586  identifying debugging files.
587 
588  Returns
589  -------
590  namedtuple
591  fit : lsst.jointcal.PhotometryFit
592  The photometric fitter used to perform the fit.
593  model : lsst.jointcal.PhotometryModel
594  The photometric model that was fit.
595  """
596  self.log.info("=== Starting photometric fitting...")
597 
598  # TODO: should use pex.config.RegistryField here (see DM-9195)
599  if self.config.photometryModel == "constrained":
600  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
601  self.focalPlaneBBox,
602  visitOrder=self.config.photometryVisitOrder)
603  # potentially nonlinear problem, so we may need a line search to converge.
604  doLineSearch = self.config.allowLineSearch
605  elif self.config.photometryModel == "simple":
606  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
607  doLineSearch = False # purely linear problem, so no line search needed
608 
609  fit = lsst.jointcal.PhotometryFit(associations, model)
610  chi2 = fit.computeChi2()
611  # TODO DM-12446: turn this into a "butler save" somehow.
612  # Save reference and measurement chi2 contributions for this data
613  if self.config.writeChi2ContributionFiles:
614  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
615  fit.saveChi2Contributions(baseName)
616 
617  if not np.isfinite(chi2.chi2):
618  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
619  self.log.info("Initialized: %s", str(chi2))
620  # The constrained model needs the visit transfo fit first; the chip
621  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
622  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
623  if self.config.photometryModel == "constrained":
624  # no line search: should be purely (or nearly) linear,
625  # and we want a large step size to initialize with.
626  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
627  chi2 = fit.computeChi2()
628  self.log.info(str(chi2))
629  dumpMatrixFile = "" # so we don't redo the output on the next step
630  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
631  chi2 = fit.computeChi2()
632  self.log.info(str(chi2))
633  fit.minimize("Fluxes") # no line search: always purely linear.
634  chi2 = fit.computeChi2()
635  self.log.info(str(chi2))
636  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
637  chi2 = fit.computeChi2()
638  if not np.isfinite(chi2.chi2):
639  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
640  self.log.info("Fit prepared with %s", str(chi2))
641 
642  model.freezeErrorTransform()
643  self.log.debug("Photometry error scales are frozen.")
644 
645  chi2 = self._iterate_fit(associations,
646  fit,
647  model,
648  self.config.maxPhotometrySteps,
649  "photometry",
650  "Model Fluxes",
651  doRankUpdate=self.config.photometryDoRankUpdate,
652  doLineSearch=doLineSearch)
653 
654  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
655  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
656  return Photometry(fit, model)
657 
658  def _fit_astrometry(self, associations, dataName=None):
659  """
660  Fit the astrometric data.
661 
662  Parameters
663  ----------
664  associations : lsst.jointcal.Associations
665  The star/reference star associations to fit.
666  dataName : str
667  Name of the data being processed (e.g. "1234_HSC-Y"), for
668  identifying debugging files.
669 
670  Returns
671  -------
672  namedtuple
673  fit : lsst.jointcal.AstrometryFit
674  The astrometric fitter used to perform the fit.
675  model : lsst.jointcal.AstrometryModel
676  The astrometric model that was fit.
677  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
678  The model for the sky to tangent plane projection that was used in the fit.
679  """
680 
681  self.log.info("=== Starting astrometric fitting...")
682 
683  associations.deprojectFittedStars()
684 
685  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
686  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
687  # them so carefully?
688  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
689 
690  if self.config.astrometryModel == "constrained":
691  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
692  sky_to_tan_projection,
693  chipOrder=self.config.astrometryChipOrder,
694  visitOrder=self.config.astrometryVisitOrder)
695  elif self.config.astrometryModel == "simple":
696  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
697  sky_to_tan_projection,
698  self.config.useInputWcs,
699  nNotFit=0,
700  order=self.config.astrometrySimpleOrder)
701 
702  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
703  chi2 = fit.computeChi2()
704  # TODO DM-12446: turn this into a "butler save" somehow.
705  # Save reference and measurement chi2 contributions for this data
706  if self.config.writeChi2ContributionFiles:
707  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
708  fit.saveChi2Contributions(baseName)
709 
710  if not np.isfinite(chi2.chi2):
711  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
712  self.log.info("Initialized: %s", str(chi2))
713  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
714  # The constrained model needs the visit transfo fit first; the chip
715  # transfo is initialized from the detector's cameraGeom, so it's close.
716  if self.config.astrometryModel == "constrained":
717  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
718  chi2 = fit.computeChi2()
719  self.log.info(str(chi2))
720  dumpMatrixFile = "" # so we don't redo the output on the next step
721  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
722  chi2 = fit.computeChi2()
723  self.log.info(str(chi2))
724  fit.minimize("Positions")
725  chi2 = fit.computeChi2()
726  self.log.info(str(chi2))
727  fit.minimize("Distortions Positions")
728  chi2 = fit.computeChi2()
729  self.log.info(str(chi2))
730  if not np.isfinite(chi2.chi2):
731  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
732  self.log.info("Fit prepared with %s", str(chi2))
733 
734  chi2 = self._iterate_fit(associations,
735  fit,
736  model,
737  self.config.maxAstrometrySteps,
738  "astrometry",
739  "Distortions Positions",
740  doRankUpdate=self.config.astrometryDoRankUpdate)
741 
742  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
743  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
744 
745  return Astrometry(fit, model, sky_to_tan_projection)
746 
747  def _check_stars(self, associations):
748  """Count measured and reference stars per ccd and warn/log them."""
749  for ccdImage in associations.getCcdImageList():
750  nMeasuredStars, nRefStars = ccdImage.countStars()
751  self.log.debug("ccdImage %s has %s measured and %s reference stars",
752  ccdImage.getName(), nMeasuredStars, nRefStars)
753  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
754  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
755  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
756  if nRefStars < self.config.minRefStarsPerCcd:
757  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
758  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
759 
760  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True,
761  doLineSearch=False):
762  """Run fit.minimize up to max_steps times, returning the final chi2."""
763 
764  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
765  for i in range(max_steps):
766  # outlier removal at 5 sigma.
767  r = fit.minimize(whatToFit,
768  self.config.outlierRejectSigma,
769  doRankUpdate=doRankUpdate,
770  doLineSearch=doLineSearch,
771  dumpMatrixFile=dumpMatrixFile)
772  dumpMatrixFile = "" # clear it so we don't write the matrix again.
773  chi2 = fit.computeChi2()
774  self._check_stars(associations)
775  if not np.isfinite(chi2.chi2):
776  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
777  self.log.info(str(chi2))
778  if r == MinimizeResult.Converged:
779  if doRankUpdate:
780  self.log.debug("fit has converged - no more outliers - redo minimization "
781  "one more time in case we have lost accuracy in rank update.")
782  # Redo minimization one more time in case we have lost accuracy in rank update
783  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
784  chi2 = fit.computeChi2()
785  self.log.info("Fit completed with: %s", str(chi2))
786  break
787  elif r == MinimizeResult.Chi2Increased:
788  self.log.warn("still some ouliers but chi2 increases - retry")
789  elif r == MinimizeResult.Failed:
790  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
791  else:
792  raise RuntimeError("Unxepected return code from minimize().")
793  else:
794  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
795 
796  return chi2
797 
798  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
799  """
800  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
801 
802  Parameters
803  ----------
804  associations : lsst.jointcal.Associations
805  The star/reference star associations to fit.
806  model : lsst.jointcal.AstrometryModel
807  The astrometric model that was fit.
808  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
809  dict of ccdImage identifiers to dataRefs that were fit
810  """
811 
812  ccdImageList = associations.getCcdImageList()
813  for ccdImage in ccdImageList:
814  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
815  ccd = ccdImage.ccdId
816  visit = ccdImage.visit
817  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
818  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
819  skyWcs = model.makeSkyWcs(ccdImage)
820  try:
821  dataRef.put(skyWcs, 'jointcal_wcs')
822  except pexExceptions.Exception as e:
823  self.log.fatal('Failed to write updated Wcs: %s', str(e))
824  raise e
825 
826  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
827  """
828  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
829 
830  Parameters
831  ----------
832  associations : lsst.jointcal.Associations
833  The star/reference star associations to fit.
834  model : lsst.jointcal.PhotometryModel
835  The photoometric model that was fit.
836  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
837  dict of ccdImage identifiers to dataRefs that were fit
838  """
839 
840  ccdImageList = associations.getCcdImageList()
841  for ccdImage in ccdImageList:
842  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
843  ccd = ccdImage.ccdId
844  visit = ccdImage.visit
845  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
846  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
847  photoCalib = model.toPhotoCalib(ccdImage)
848  try:
849  dataRef.put(photoCalib, 'jointcal_photoCalib')
850  except pexExceptions.Exception as e:
851  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
852  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:315
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:576
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:658
def _check_star_lists(self, associations, name)
Definition: jointcal.py:567
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:761
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:826
def _check_stars(self, associations)
Definition: jointcal.py:747
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:479
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:798
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:378
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:273