lsst.jointcal  15.0-9-g5661f8f+3
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  astrometrySimpleOrder = pexConfig.Field(
156  doc="Polynomial order for fitting the simple astrometry model.",
157  dtype=int,
158  default=3,
159  )
160  astrometryChipOrder = pexConfig.Field(
161  doc="Order of the per-chip transform for the constrained astrometry model.",
162  dtype=int,
163  default=1,
164  )
165  astrometryVisitOrder = pexConfig.Field(
166  doc="Order of the per-visit transform for the constrained astrometry model.",
167  dtype=int,
168  default=5,
169  )
170  useInputWcs = pexConfig.Field(
171  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
172  dtype=bool,
173  default=True,
174  )
175  astrometryModel = pexConfig.ChoiceField(
176  doc="Type of model to fit to astrometry",
177  dtype=str,
178  default="simple",
179  allowed={"simple": "One polynomial per ccd",
180  "constrained": "One polynomial per ccd, and one polynomial per visit"}
181  )
182  photometryModel = pexConfig.ChoiceField(
183  doc="Type of model to fit to photometry",
184  dtype=str,
185  default="simple",
186  allowed={"simple": "One constant zeropoint per ccd and visit",
187  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
188  )
189  photometryVisitOrder = pexConfig.Field(
190  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
191  dtype=int,
192  default=7,
193  )
194  astrometryRefObjLoader = pexConfig.ConfigurableField(
195  target=LoadIndexedReferenceObjectsTask,
196  doc="Reference object loader for astrometric fit",
197  )
198  photometryRefObjLoader = pexConfig.ConfigurableField(
199  target=LoadIndexedReferenceObjectsTask,
200  doc="Reference object loader for photometric fit",
201  )
202  sourceSelector = sourceSelectorRegistry.makeField(
203  doc="How to select sources for cross-matching",
204  default="astrometry"
205  )
206  writeChi2ContributionFiles = pexConfig.Field(
207  dtype=bool,
208  doc="Write initial/final fit files containing the contributions to chi2.",
209  default=False
210  )
211 
212  def setDefaults(self):
213  sourceSelector = self.sourceSelector["astrometry"]
214  sourceSelector.setDefaults()
215  # don't want to lose existing flags, just add to them.
216  sourceSelector.badFlags.extend(["slot_Shape_flag"])
217  # This should be used to set the FluxField value in jointcal::JointcalControl
218  sourceSelector.sourceFluxType = 'Calib'
219 
220 
221 class JointcalTask(pipeBase.CmdLineTask):
222  """Jointly astrometrically and photometrically calibrate a group of images."""
223 
224  ConfigClass = JointcalConfig
225  RunnerClass = JointcalRunner
226  _DefaultName = "jointcal"
227 
228  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
229  """
230  Instantiate a JointcalTask.
231 
232  Parameters
233  ----------
234  butler : lsst.daf.persistence.Butler
235  The butler is passed to the refObjLoader constructor in case it is
236  needed. Ignored if the refObjLoader argument provides a loader directly.
237  Used to initialize the astrometry and photometry refObjLoaders.
238  profile_jointcal : bool
239  set to True to profile different stages of this jointcal run.
240  """
241  pipeBase.CmdLineTask.__init__(self, **kwargs)
242  self.profile_jointcal = profile_jointcal
243  self.makeSubtask("sourceSelector")
244  if self.config.doAstrometry:
245  self.makeSubtask('astrometryRefObjLoader', butler=butler)
246  if self.config.doPhotometry:
247  self.makeSubtask('photometryRefObjLoader', butler=butler)
248 
249  # To hold various computed metrics for use by tests
250  self.job = Job.load_metrics_package(subset='jointcal')
251 
252  # We don't need to persist config and metadata at this stage.
253  # In this way, we don't need to put a specific entry in the camera mapper policy file
254  def _getConfigName(self):
255  return None
256 
257  def _getMetadataName(self):
258  return None
259 
260  @classmethod
261  def _makeArgumentParser(cls):
262  """Create an argument parser"""
263  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
264  parser.add_argument("--profile_jointcal", default=False, action="store_true",
265  help="Profile steps of jointcal separately.")
266  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
267  ContainerClass=PerTractCcdDataIdContainer)
268  return parser
269 
270  def _build_ccdImage(self, dataRef, associations, jointcalControl):
271  """
272  Extract the necessary things from this dataRef to add a new ccdImage.
273 
274  Parameters
275  ----------
276  dataRef : lsst.daf.persistence.ButlerDataRef
277  dataRef to extract info from.
278  associations : lsst.jointcal.Associations
279  object to add the info to, to construct a new CcdImage
280  jointcalControl : jointcal.JointcalControl
281  control object for associations management
282 
283  Returns
284  ------
285  namedtuple
286  wcs : lsst.afw.geom.SkyWcs
287  the TAN WCS of this image, read from the calexp
288  key : namedtuple
289  a key to identify this dataRef by its visit and ccd ids
290  filter : str
291  this calexp's filter
292  """
293  if "visit" in dataRef.dataId.keys():
294  visit = dataRef.dataId["visit"]
295  else:
296  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
297 
298  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
299 
300  visitInfo = dataRef.get('calexp_visitInfo')
301  detector = dataRef.get('calexp_detector')
302  ccdId = detector.getId()
303  calib = dataRef.get('calexp_calib')
304  tanWcs = dataRef.get('calexp_wcs')
305  bbox = dataRef.get('calexp_bbox')
306  filt = dataRef.get('calexp_filter')
307  filterName = filt.getName()
308  fluxMag0 = calib.getFluxMag0()
309  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
310 
311  goodSrc = self.sourceSelector.selectSources(src)
312 
313  if len(goodSrc.sourceCat) == 0:
314  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
315  else:
316  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
317  associations.createCcdImage(goodSrc.sourceCat,
318  tanWcs,
319  visitInfo,
320  bbox,
321  filterName,
322  photoCalib,
323  detector,
324  visit,
325  ccdId,
326  jointcalControl)
327 
328  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
329  Key = collections.namedtuple('Key', ('visit', 'ccd'))
330  return Result(tanWcs, Key(visit, ccdId), filterName)
331 
332  @pipeBase.timeMethod
333  def run(self, dataRefs, profile_jointcal=False):
334  """
335  Jointly calibrate the astrometry and photometry across a set of images.
336 
337  Parameters
338  ----------
339  dataRefs : list of lsst.daf.persistence.ButlerDataRef
340  List of data references to the exposures to be fit.
341  profile_jointcal : bool
342  Profile the individual steps of jointcal.
343 
344  Returns
345  -------
346  pipe.base.Struct
347  struct containing:
348  * dataRefs: the provided data references that were fit (with updated WCSs)
349  * oldWcsList: the original WCS from each dataRef
350  * metrics: dictionary of internally-computed metrics for testing/validation.
351  """
352  if len(dataRefs) == 0:
353  raise ValueError('Need a non-empty list of data references!')
354 
355  exitStatus = 0 # exit status for shell
356 
357  sourceFluxField = "slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
358  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
359  associations = lsst.jointcal.Associations()
360 
361  visit_ccd_to_dataRef = {}
362  oldWcsList = []
363  filters = []
364  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
365  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
366  # We need the bounding-box of the focal plane for photometry visit models.
367  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
368  camera = dataRefs[0].get('camera', immediate=True)
369  self.focalPlaneBBox = camera.getFpBBox()
370  for ref in dataRefs:
371  result = self._build_ccdImage(ref, associations, jointcalControl)
372  oldWcsList.append(result.wcs)
373  visit_ccd_to_dataRef[result.key] = ref
374  filters.append(result.filter)
375  filters = collections.Counter(filters)
376 
377  associations.computeCommonTangentPoint()
378 
379  # Use external reference catalogs handled by LSST stack mechanism
380  # Get the bounding box overlapping all associated images
381  # ==> This is probably a bad idea to do it this way <== To be improved
382  bbox = associations.getRaDecBBox()
383  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
384  bboxCenter = bbox.getCenter()
385  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
386  bboxMax = bbox.getMax()
387  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
388  radius = center.separation(corner).asRadians()
389 
390  # Get astrometry_net_data path
391  anDir = lsst.utils.getPackageDir('astrometry_net_data')
392  if anDir is None:
393  raise RuntimeError("astrometry_net_data is not setup")
394 
395  # Determine a default filter associated with the catalog. See DM-9093
396  defaultFilter = filters.most_common(1)[0][0]
397  self.log.debug("Using %s band for reference flux", defaultFilter)
398 
399  # TODO: need a better way to get the tract.
400  tract = dataRefs[0].dataId['tract']
401 
402  if self.config.doAstrometry:
403  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
404  name="astrometry",
405  refObjLoader=self.astrometryRefObjLoader,
406  fit_function=self._fit_astrometry,
407  profile_jointcal=profile_jointcal,
408  tract=tract)
409  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
410  else:
411  astrometry = Astrometry(None, None, None)
412 
413  if self.config.doPhotometry:
414  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
415  name="photometry",
416  refObjLoader=self.photometryRefObjLoader,
417  fit_function=self._fit_photometry,
418  profile_jointcal=profile_jointcal,
419  tract=tract,
420  filters=filters,
421  reject_bad_fluxes=True)
422  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
423  else:
424  photometry = Photometry(None, None)
425 
426  return pipeBase.Struct(dataRefs=dataRefs,
427  oldWcsList=oldWcsList,
428  job=self.job,
429  exitStatus=exitStatus)
430 
431  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
432  name="", refObjLoader=None, filters=[], fit_function=None,
433  tract=None, profile_jointcal=False, match_cut=3.0,
434  reject_bad_fluxes=False):
435  """Load reference catalog, perform the fit, and return the result.
436 
437  Parameters
438  ----------
439  associations : lsst.jointcal.Associations
440  The star/reference star associations to fit.
441  defaultFilter : str
442  filter to load from reference catalog.
443  center : lsst.afw.geom.SpherePoint
444  ICRS center of field to load from reference catalog.
445  radius : lsst.afw.geom.Angle
446  On-sky radius to load from reference catalog.
447  name : str
448  Name of thing being fit: "Astrometry" or "Photometry".
449  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
450  Reference object loader to load from for fit.
451  filters : list of str, optional
452  List of filters to load from the reference catalog.
453  fit_function : function
454  function to call to perform fit (takes associations object).
455  tract : str
456  Name of tract currently being fit.
457  profile_jointcal : bool, optional
458  Separately profile the fitting step.
459  match_cut : float, optional
460  Radius in arcseconds to find cross-catalog matches to during
461  associations.associateCatalogs.
462  reject_bad_fluxes : bool, optional
463  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
464 
465  Returns
466  -------
467  Result of `fit_function()`
468  """
469  self.log.info("====== Now processing %s...", name)
470  # TODO: this should not print "trying to invert a singular transformation:"
471  # if it does that, something's not right about the WCS...
472  associations.associateCatalogs(match_cut)
473  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
474  associations.fittedStarListSize())
475 
476  skyCircle = refObjLoader.loadSkyCircle(center,
477  afwGeom.Angle(radius, afwGeom.radians),
478  defaultFilter)
479 
480  # Need memory contiguity to get reference filters as a vector.
481  if not skyCircle.refCat.isContiguous():
482  refCat = skyCircle.refCat.copy(deep=True)
483  else:
484  refCat = skyCircle.refCat
485 
486  # load the reference catalog fluxes.
487  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
488  refFluxes = {}
489  refFluxErrs = {}
490  for filt in filters:
491  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
492  refFluxes[filt] = refCat.get(filtKeys[0])
493  refFluxErrs[filt] = refCat.get(filtKeys[1])
494 
495  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
496  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
497  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
498  associations.refStarListSize())
499 
500  associations.prepareFittedStars(self.config.minMeasurements)
501 
502  self._check_star_lists(associations, name)
503  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
504  associations.nFittedStarsWithAssociatedRefStar())
505  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
506  associations.fittedStarListSize())
507  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
508  associations.nCcdImagesValidForFit())
509 
510  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
511  dataName = "{}_{}".format(tract, defaultFilter)
512  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
513  result = fit_function(associations, dataName)
514  # TODO DM-12446: turn this into a "butler save" somehow.
515  # Save reference and measurement chi2 contributions for this data
516  if self.config.writeChi2ContributionFiles:
517  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
518  result.fit.saveChi2Contributions(baseName)
519 
520  return result
521 
522  def _check_star_lists(self, associations, name):
523  # TODO: these should be len(blah), but we need this properly wrapped first.
524  if associations.nCcdImagesValidForFit() == 0:
525  raise RuntimeError('No images in the ccdImageList!')
526  if associations.fittedStarListSize() == 0:
527  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
528  if associations.refStarListSize() == 0:
529  raise RuntimeError('No stars in the {} reference star list!'.format(name))
530 
531  def _fit_photometry(self, associations, dataName=None):
532  """
533  Fit the photometric data.
534 
535  Parameters
536  ----------
537  associations : lsst.jointcal.Associations
538  The star/reference star associations to fit.
539  dataName : str
540  Name of the data being processed (e.g. "1234_HSC-Y"), for
541  identifying debugging files.
542 
543  Returns
544  -------
545  namedtuple
546  fit : lsst.jointcal.PhotometryFit
547  The photometric fitter used to perform the fit.
548  model : lsst.jointcal.PhotometryModel
549  The photometric model that was fit.
550  """
551  self.log.info("=== Starting photometric fitting...")
552 
553  # TODO: should use pex.config.RegistryField here (see DM-9195)
554  if self.config.photometryModel == "constrained":
555  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
556  self.focalPlaneBBox,
557  visitOrder=self.config.photometryVisitOrder)
558  elif self.config.photometryModel == "simple":
559  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
560 
561  fit = lsst.jointcal.PhotometryFit(associations, model)
562  chi2 = fit.computeChi2()
563  # TODO DM-12446: turn this into a "butler save" somehow.
564  # Save reference and measurement chi2 contributions for this data
565  if self.config.writeChi2ContributionFiles:
566  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
567  fit.saveChi2Contributions(baseName)
568 
569  if not np.isfinite(chi2.chi2):
570  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
571  self.log.info("Initialized: %s", str(chi2))
572  # The constrained model needs the visit transfo fit first; the chip
573  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
574  if self.config.photometryModel == "constrained":
575  # TODO: (related to DM-8046): implement Visit/Chip choice
576  fit.minimize("ModelVisit")
577  chi2 = fit.computeChi2()
578  self.log.info(str(chi2))
579  fit.minimize("Model")
580  chi2 = fit.computeChi2()
581  self.log.info(str(chi2))
582  fit.minimize("Fluxes")
583  chi2 = fit.computeChi2()
584  self.log.info(str(chi2))
585  fit.minimize("Model Fluxes")
586  chi2 = fit.computeChi2()
587  if not np.isfinite(chi2.chi2):
588  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
589  self.log.info("Fit prepared with %s", str(chi2))
590 
591  model.freezeErrorTransform()
592  self.log.debug("Photometry error scales are frozen.")
593 
594  chi2 = self._iterate_fit(associations, fit, model, 20, "photometry", "Model Fluxes")
595 
596  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
597  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
598  return Photometry(fit, model)
599 
600  def _fit_astrometry(self, associations, dataName=None):
601  """
602  Fit the astrometric data.
603 
604  Parameters
605  ----------
606  associations : lsst.jointcal.Associations
607  The star/reference star associations to fit.
608  dataName : str
609  Name of the data being processed (e.g. "1234_HSC-Y"), for
610  identifying debugging files.
611 
612  Returns
613  -------
614  namedtuple
615  fit : lsst.jointcal.AstrometryFit
616  The astrometric fitter used to perform the fit.
617  model : lsst.jointcal.AstrometryModel
618  The astrometric model that was fit.
619  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
620  The model for the sky to tangent plane projection that was used in the fit.
621  """
622 
623  self.log.info("=== Starting astrometric fitting...")
624 
625  associations.deprojectFittedStars()
626 
627  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
628  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
629  # them so carefully?
630  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
631 
632  if self.config.astrometryModel == "constrained":
633  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
634  sky_to_tan_projection,
635  chipOrder=self.config.astrometryChipOrder,
636  visitOrder=self.config.astrometryVisitOrder)
637  elif self.config.astrometryModel == "simple":
638  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
639  sky_to_tan_projection,
640  self.config.useInputWcs,
641  nNotFit=0,
642  order=self.config.astrometrySimpleOrder)
643 
644  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
645  chi2 = fit.computeChi2()
646  # TODO DM-12446: turn this into a "butler save" somehow.
647  # Save reference and measurement chi2 contributions for this data
648  if self.config.writeChi2ContributionFiles:
649  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
650  fit.saveChi2Contributions(baseName)
651 
652  if not np.isfinite(chi2.chi2):
653  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
654  self.log.info("Initialized: %s", str(chi2))
655  # The constrained model needs the visit transfo fit first; the chip
656  # transfo is initialized from the detector's cameraGeom, so it's close.
657  if self.config.astrometryModel == "constrained":
658  fit.minimize("DistortionsVisit")
659  chi2 = fit.computeChi2()
660  self.log.info(str(chi2))
661  fit.minimize("Distortions")
662  chi2 = fit.computeChi2()
663  self.log.info(str(chi2))
664  fit.minimize("Positions")
665  chi2 = fit.computeChi2()
666  self.log.info(str(chi2))
667  fit.minimize("Distortions Positions")
668  chi2 = fit.computeChi2()
669  self.log.info(str(chi2))
670  if not np.isfinite(chi2.chi2):
671  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
672  self.log.info("Fit prepared with %s", str(chi2))
673 
674  chi2 = self._iterate_fit(associations, fit, model, 20, "astrometry", "Distortions Positions")
675 
676  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
677  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
678 
679  return Astrometry(fit, model, sky_to_tan_projection)
680 
681  def _check_stars(self, associations):
682  """Count measured and reference stars per ccd and warn/log them."""
683  for ccdImage in associations.getCcdImageList():
684  nMeasuredStars, nRefStars = ccdImage.countStars()
685  self.log.debug("ccdImage %s has %s measured and %s reference stars",
686  ccdImage.getName(), nMeasuredStars, nRefStars)
687  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
688  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
689  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
690  if nRefStars < self.config.minRefStarsPerCcd:
691  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
692  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
693 
694  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit):
695  """Run fit.minimize up to max_steps times, returning the final chi2."""
696 
697  for i in range(max_steps):
698  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
699  chi2 = fit.computeChi2()
700  self._check_stars(associations)
701  if not np.isfinite(chi2.chi2):
702  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
703  self.log.info(str(chi2))
704  if r == MinimizeResult.Converged:
705  self.log.debug("fit has converged - no more outliers - redo minimization "
706  "one more time in case we have lost accuracy in rank update.")
707  # Redo minimization one more time in case we have lost accuracy in rank update
708  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
709  chi2 = fit.computeChi2()
710  self.log.info("Fit completed with: %s", str(chi2))
711  break
712  elif r == MinimizeResult.Chi2Increased:
713  self.log.warn("still some ouliers but chi2 increases - retry")
714  elif r == MinimizeResult.Failed:
715  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
716  else:
717  raise RuntimeError("Unxepected return code from minimize().")
718  else:
719  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
720 
721  return chi2
722 
723  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
724  """
725  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
726 
727  Parameters
728  ----------
729  associations : lsst.jointcal.Associations
730  The star/reference star associations to fit.
731  model : lsst.jointcal.AstrometryModel
732  The astrometric model that was fit.
733  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
734  dict of ccdImage identifiers to dataRefs that were fit
735  """
736 
737  ccdImageList = associations.getCcdImageList()
738  for ccdImage in ccdImageList:
739  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
740  ccd = ccdImage.ccdId
741  visit = ccdImage.visit
742  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
743  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
744  skyWcs = model.makeSkyWcs(ccdImage)
745  try:
746  dataRef.put(skyWcs, 'jointcal_wcs')
747  except pexExceptions.Exception as e:
748  self.log.fatal('Failed to write updated Wcs: %s', str(e))
749  raise e
750 
751  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
752  """
753  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
754 
755  Parameters
756  ----------
757  associations : lsst.jointcal.Associations
758  The star/reference star associations to fit.
759  model : lsst.jointcal.PhotometryModel
760  The photoometric model that was fit.
761  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
762  dict of ccdImage identifiers to dataRefs that were fit
763  """
764 
765  ccdImageList = associations.getCcdImageList()
766  for ccdImage in ccdImageList:
767  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
768  ccd = ccdImage.ccdId
769  visit = ccdImage.visit
770  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
771  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
772  photoCalib = model.toPhotoCalib(ccdImage)
773  try:
774  dataRef.put(photoCalib, 'jointcal_photoCalib')
775  except pexExceptions.Exception as e:
776  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
777  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:270
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:531
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:600
def _check_star_lists(self, associations, name)
Definition: jointcal.py:522
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:751
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:694
def _check_stars(self, associations)
Definition: jointcal.py:681
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:434
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:723
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:333
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:228