lsst.jointcal  15.0-11-gda8ddd7+4
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  astrometrySimpleOrder = pexConfig.Field(
156  doc="Polynomial order for fitting the simple astrometry model.",
157  dtype=int,
158  default=3,
159  )
160  astrometryChipOrder = pexConfig.Field(
161  doc="Order of the per-chip transform for the constrained astrometry model.",
162  dtype=int,
163  default=1,
164  )
165  astrometryVisitOrder = pexConfig.Field(
166  doc="Order of the per-visit transform for the constrained astrometry model.",
167  dtype=int,
168  default=5,
169  )
170  useInputWcs = pexConfig.Field(
171  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
172  dtype=bool,
173  default=True,
174  )
175  astrometryModel = pexConfig.ChoiceField(
176  doc="Type of model to fit to astrometry",
177  dtype=str,
178  default="simple",
179  allowed={"simple": "One polynomial per ccd",
180  "constrained": "One polynomial per ccd, and one polynomial per visit"}
181  )
182  photometryModel = pexConfig.ChoiceField(
183  doc="Type of model to fit to photometry",
184  dtype=str,
185  default="simple",
186  allowed={"simple": "One constant zeropoint per ccd and visit",
187  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
188  )
189  photometryVisitOrder = pexConfig.Field(
190  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
191  dtype=int,
192  default=7,
193  )
194  astrometryRefObjLoader = pexConfig.ConfigurableField(
195  target=LoadIndexedReferenceObjectsTask,
196  doc="Reference object loader for astrometric fit",
197  )
198  photometryRefObjLoader = pexConfig.ConfigurableField(
199  target=LoadIndexedReferenceObjectsTask,
200  doc="Reference object loader for photometric fit",
201  )
202  sourceSelector = sourceSelectorRegistry.makeField(
203  doc="How to select sources for cross-matching",
204  default="astrometry"
205  )
206  writeChi2ContributionFiles = pexConfig.Field(
207  dtype=bool,
208  doc="Write initial/final fit files containing the contributions to chi2.",
209  default=False
210  )
211  sourceFluxType = pexConfig.Field(
212  dtype=str,
213  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
214  default='Calib'
215  )
216 
217  def setDefaults(self):
218  sourceSelector = self.sourceSelector["astrometry"]
219  sourceSelector.setDefaults()
220  # don't want to lose existing flags, just add to them.
221  sourceSelector.badFlags.extend(["slot_Shape_flag"])
222  # This should be used to set the FluxField value in jointcal::JointcalControl
223  sourceSelector.sourceFluxType = self.sourceFluxType
224 
225 
226 class JointcalTask(pipeBase.CmdLineTask):
227  """Jointly astrometrically and photometrically calibrate a group of images."""
228 
229  ConfigClass = JointcalConfig
230  RunnerClass = JointcalRunner
231  _DefaultName = "jointcal"
232 
233  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
234  """
235  Instantiate a JointcalTask.
236 
237  Parameters
238  ----------
239  butler : lsst.daf.persistence.Butler
240  The butler is passed to the refObjLoader constructor in case it is
241  needed. Ignored if the refObjLoader argument provides a loader directly.
242  Used to initialize the astrometry and photometry refObjLoaders.
243  profile_jointcal : bool
244  set to True to profile different stages of this jointcal run.
245  """
246  pipeBase.CmdLineTask.__init__(self, **kwargs)
247  self.profile_jointcal = profile_jointcal
248  self.makeSubtask("sourceSelector")
249  if self.config.doAstrometry:
250  self.makeSubtask('astrometryRefObjLoader', butler=butler)
251  if self.config.doPhotometry:
252  self.makeSubtask('photometryRefObjLoader', butler=butler)
253 
254  # To hold various computed metrics for use by tests
255  self.job = Job.load_metrics_package(subset='jointcal')
256 
257  # We don't need to persist config and metadata at this stage.
258  # In this way, we don't need to put a specific entry in the camera mapper policy file
259  def _getConfigName(self):
260  return None
261 
262  def _getMetadataName(self):
263  return None
264 
265  @classmethod
266  def _makeArgumentParser(cls):
267  """Create an argument parser"""
268  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
269  parser.add_argument("--profile_jointcal", default=False, action="store_true",
270  help="Profile steps of jointcal separately.")
271  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
272  ContainerClass=PerTractCcdDataIdContainer)
273  return parser
274 
275  def _build_ccdImage(self, dataRef, associations, jointcalControl):
276  """
277  Extract the necessary things from this dataRef to add a new ccdImage.
278 
279  Parameters
280  ----------
281  dataRef : lsst.daf.persistence.ButlerDataRef
282  dataRef to extract info from.
283  associations : lsst.jointcal.Associations
284  object to add the info to, to construct a new CcdImage
285  jointcalControl : jointcal.JointcalControl
286  control object for associations management
287 
288  Returns
289  ------
290  namedtuple
291  wcs : lsst.afw.geom.SkyWcs
292  the TAN WCS of this image, read from the calexp
293  key : namedtuple
294  a key to identify this dataRef by its visit and ccd ids
295  filter : str
296  this calexp's filter
297  """
298  if "visit" in dataRef.dataId.keys():
299  visit = dataRef.dataId["visit"]
300  else:
301  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
302 
303  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
304 
305  visitInfo = dataRef.get('calexp_visitInfo')
306  detector = dataRef.get('calexp_detector')
307  ccdId = detector.getId()
308  calib = dataRef.get('calexp_calib')
309  tanWcs = dataRef.get('calexp_wcs')
310  bbox = dataRef.get('calexp_bbox')
311  filt = dataRef.get('calexp_filter')
312  filterName = filt.getName()
313  fluxMag0 = calib.getFluxMag0()
314  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
315 
316  goodSrc = self.sourceSelector.run(src)
317 
318  if len(goodSrc.sourceCat) == 0:
319  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
320  else:
321  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
322  associations.createCcdImage(goodSrc.sourceCat,
323  tanWcs,
324  visitInfo,
325  bbox,
326  filterName,
327  photoCalib,
328  detector,
329  visit,
330  ccdId,
331  jointcalControl)
332 
333  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
334  Key = collections.namedtuple('Key', ('visit', 'ccd'))
335  return Result(tanWcs, Key(visit, ccdId), filterName)
336 
337  @pipeBase.timeMethod
338  def run(self, dataRefs, profile_jointcal=False):
339  """
340  Jointly calibrate the astrometry and photometry across a set of images.
341 
342  Parameters
343  ----------
344  dataRefs : list of lsst.daf.persistence.ButlerDataRef
345  List of data references to the exposures to be fit.
346  profile_jointcal : bool
347  Profile the individual steps of jointcal.
348 
349  Returns
350  -------
351  pipe.base.Struct
352  struct containing:
353  * dataRefs: the provided data references that were fit (with updated WCSs)
354  * oldWcsList: the original WCS from each dataRef
355  * metrics: dictionary of internally-computed metrics for testing/validation.
356  """
357  if len(dataRefs) == 0:
358  raise ValueError('Need a non-empty list of data references!')
359 
360  exitStatus = 0 # exit status for shell
361 
362  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
363  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
364  associations = lsst.jointcal.Associations()
365 
366  visit_ccd_to_dataRef = {}
367  oldWcsList = []
368  filters = []
369  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
370  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
371  # We need the bounding-box of the focal plane for photometry visit models.
372  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
373  camera = dataRefs[0].get('camera', immediate=True)
374  self.focalPlaneBBox = camera.getFpBBox()
375  for ref in dataRefs:
376  result = self._build_ccdImage(ref, associations, jointcalControl)
377  oldWcsList.append(result.wcs)
378  visit_ccd_to_dataRef[result.key] = ref
379  filters.append(result.filter)
380  filters = collections.Counter(filters)
381 
382  associations.computeCommonTangentPoint()
383 
384  # Use external reference catalogs handled by LSST stack mechanism
385  # Get the bounding box overlapping all associated images
386  # ==> This is probably a bad idea to do it this way <== To be improved
387  bbox = associations.getRaDecBBox()
388  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
389  bboxCenter = bbox.getCenter()
390  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
391  bboxMax = bbox.getMax()
392  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
393  radius = center.separation(corner).asRadians()
394 
395  # Get astrometry_net_data path
396  anDir = lsst.utils.getPackageDir('astrometry_net_data')
397  if anDir is None:
398  raise RuntimeError("astrometry_net_data is not setup")
399 
400  # Determine a default filter associated with the catalog. See DM-9093
401  defaultFilter = filters.most_common(1)[0][0]
402  self.log.debug("Using %s band for reference flux", defaultFilter)
403 
404  # TODO: need a better way to get the tract.
405  tract = dataRefs[0].dataId['tract']
406 
407  if self.config.doAstrometry:
408  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
409  name="astrometry",
410  refObjLoader=self.astrometryRefObjLoader,
411  fit_function=self._fit_astrometry,
412  profile_jointcal=profile_jointcal,
413  tract=tract)
414  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
415  else:
416  astrometry = Astrometry(None, None, None)
417 
418  if self.config.doPhotometry:
419  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
420  name="photometry",
421  refObjLoader=self.photometryRefObjLoader,
422  fit_function=self._fit_photometry,
423  profile_jointcal=profile_jointcal,
424  tract=tract,
425  filters=filters,
426  reject_bad_fluxes=True)
427  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
428  else:
429  photometry = Photometry(None, None)
430 
431  return pipeBase.Struct(dataRefs=dataRefs,
432  oldWcsList=oldWcsList,
433  job=self.job,
434  exitStatus=exitStatus)
435 
436  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
437  name="", refObjLoader=None, filters=[], fit_function=None,
438  tract=None, profile_jointcal=False, match_cut=3.0,
439  reject_bad_fluxes=False):
440  """Load reference catalog, perform the fit, and return the result.
441 
442  Parameters
443  ----------
444  associations : lsst.jointcal.Associations
445  The star/reference star associations to fit.
446  defaultFilter : str
447  filter to load from reference catalog.
448  center : lsst.afw.geom.SpherePoint
449  ICRS center of field to load from reference catalog.
450  radius : lsst.afw.geom.Angle
451  On-sky radius to load from reference catalog.
452  name : str
453  Name of thing being fit: "Astrometry" or "Photometry".
454  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
455  Reference object loader to load from for fit.
456  filters : list of str, optional
457  List of filters to load from the reference catalog.
458  fit_function : function
459  function to call to perform fit (takes associations object).
460  tract : str
461  Name of tract currently being fit.
462  profile_jointcal : bool, optional
463  Separately profile the fitting step.
464  match_cut : float, optional
465  Radius in arcseconds to find cross-catalog matches to during
466  associations.associateCatalogs.
467  reject_bad_fluxes : bool, optional
468  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
469 
470  Returns
471  -------
472  Result of `fit_function()`
473  """
474  self.log.info("====== Now processing %s...", name)
475  # TODO: this should not print "trying to invert a singular transformation:"
476  # if it does that, something's not right about the WCS...
477  associations.associateCatalogs(match_cut)
478  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
479  associations.fittedStarListSize())
480 
481  skyCircle = refObjLoader.loadSkyCircle(center,
482  afwGeom.Angle(radius, afwGeom.radians),
483  defaultFilter)
484 
485  # Need memory contiguity to get reference filters as a vector.
486  if not skyCircle.refCat.isContiguous():
487  refCat = skyCircle.refCat.copy(deep=True)
488  else:
489  refCat = skyCircle.refCat
490 
491  # load the reference catalog fluxes.
492  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
493  refFluxes = {}
494  refFluxErrs = {}
495  for filt in filters:
496  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
497  refFluxes[filt] = refCat.get(filtKeys[0])
498  refFluxErrs[filt] = refCat.get(filtKeys[1])
499 
500  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
501  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
502  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
503  associations.refStarListSize())
504 
505  associations.prepareFittedStars(self.config.minMeasurements)
506 
507  self._check_star_lists(associations, name)
508  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
509  associations.nFittedStarsWithAssociatedRefStar())
510  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
511  associations.fittedStarListSize())
512  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
513  associations.nCcdImagesValidForFit())
514 
515  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
516  dataName = "{}_{}".format(tract, defaultFilter)
517  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
518  result = fit_function(associations, dataName)
519  # TODO DM-12446: turn this into a "butler save" somehow.
520  # Save reference and measurement chi2 contributions for this data
521  if self.config.writeChi2ContributionFiles:
522  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
523  result.fit.saveChi2Contributions(baseName)
524 
525  return result
526 
527  def _check_star_lists(self, associations, name):
528  # TODO: these should be len(blah), but we need this properly wrapped first.
529  if associations.nCcdImagesValidForFit() == 0:
530  raise RuntimeError('No images in the ccdImageList!')
531  if associations.fittedStarListSize() == 0:
532  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
533  if associations.refStarListSize() == 0:
534  raise RuntimeError('No stars in the {} reference star list!'.format(name))
535 
536  def _fit_photometry(self, associations, dataName=None):
537  """
538  Fit the photometric data.
539 
540  Parameters
541  ----------
542  associations : lsst.jointcal.Associations
543  The star/reference star associations to fit.
544  dataName : str
545  Name of the data being processed (e.g. "1234_HSC-Y"), for
546  identifying debugging files.
547 
548  Returns
549  -------
550  namedtuple
551  fit : lsst.jointcal.PhotometryFit
552  The photometric fitter used to perform the fit.
553  model : lsst.jointcal.PhotometryModel
554  The photometric model that was fit.
555  """
556  self.log.info("=== Starting photometric fitting...")
557 
558  # TODO: should use pex.config.RegistryField here (see DM-9195)
559  if self.config.photometryModel == "constrained":
560  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
561  self.focalPlaneBBox,
562  visitOrder=self.config.photometryVisitOrder)
563  elif self.config.photometryModel == "simple":
564  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
565 
566  fit = lsst.jointcal.PhotometryFit(associations, model)
567  chi2 = fit.computeChi2()
568  # TODO DM-12446: turn this into a "butler save" somehow.
569  # Save reference and measurement chi2 contributions for this data
570  if self.config.writeChi2ContributionFiles:
571  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
572  fit.saveChi2Contributions(baseName)
573 
574  if not np.isfinite(chi2.chi2):
575  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
576  self.log.info("Initialized: %s", str(chi2))
577  # The constrained model needs the visit transfo fit first; the chip
578  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
579  if self.config.photometryModel == "constrained":
580  # TODO: (related to DM-8046): implement Visit/Chip choice
581  fit.minimize("ModelVisit")
582  chi2 = fit.computeChi2()
583  self.log.info(str(chi2))
584  fit.minimize("Model")
585  chi2 = fit.computeChi2()
586  self.log.info(str(chi2))
587  fit.minimize("Fluxes")
588  chi2 = fit.computeChi2()
589  self.log.info(str(chi2))
590  fit.minimize("Model Fluxes")
591  chi2 = fit.computeChi2()
592  if not np.isfinite(chi2.chi2):
593  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
594  self.log.info("Fit prepared with %s", str(chi2))
595 
596  model.freezeErrorTransform()
597  self.log.debug("Photometry error scales are frozen.")
598 
599  chi2 = self._iterate_fit(associations, fit, model, 20, "photometry", "Model Fluxes")
600 
601  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
602  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
603  return Photometry(fit, model)
604 
605  def _fit_astrometry(self, associations, dataName=None):
606  """
607  Fit the astrometric data.
608 
609  Parameters
610  ----------
611  associations : lsst.jointcal.Associations
612  The star/reference star associations to fit.
613  dataName : str
614  Name of the data being processed (e.g. "1234_HSC-Y"), for
615  identifying debugging files.
616 
617  Returns
618  -------
619  namedtuple
620  fit : lsst.jointcal.AstrometryFit
621  The astrometric fitter used to perform the fit.
622  model : lsst.jointcal.AstrometryModel
623  The astrometric model that was fit.
624  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
625  The model for the sky to tangent plane projection that was used in the fit.
626  """
627 
628  self.log.info("=== Starting astrometric fitting...")
629 
630  associations.deprojectFittedStars()
631 
632  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
633  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
634  # them so carefully?
635  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
636 
637  if self.config.astrometryModel == "constrained":
638  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
639  sky_to_tan_projection,
640  chipOrder=self.config.astrometryChipOrder,
641  visitOrder=self.config.astrometryVisitOrder)
642  elif self.config.astrometryModel == "simple":
643  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
644  sky_to_tan_projection,
645  self.config.useInputWcs,
646  nNotFit=0,
647  order=self.config.astrometrySimpleOrder)
648 
649  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
650  chi2 = fit.computeChi2()
651  # TODO DM-12446: turn this into a "butler save" somehow.
652  # Save reference and measurement chi2 contributions for this data
653  if self.config.writeChi2ContributionFiles:
654  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
655  fit.saveChi2Contributions(baseName)
656 
657  if not np.isfinite(chi2.chi2):
658  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
659  self.log.info("Initialized: %s", str(chi2))
660  # The constrained model needs the visit transfo fit first; the chip
661  # transfo is initialized from the detector's cameraGeom, so it's close.
662  if self.config.astrometryModel == "constrained":
663  fit.minimize("DistortionsVisit")
664  chi2 = fit.computeChi2()
665  self.log.info(str(chi2))
666  fit.minimize("Distortions")
667  chi2 = fit.computeChi2()
668  self.log.info(str(chi2))
669  fit.minimize("Positions")
670  chi2 = fit.computeChi2()
671  self.log.info(str(chi2))
672  fit.minimize("Distortions Positions")
673  chi2 = fit.computeChi2()
674  self.log.info(str(chi2))
675  if not np.isfinite(chi2.chi2):
676  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
677  self.log.info("Fit prepared with %s", str(chi2))
678 
679  chi2 = self._iterate_fit(associations, fit, model, 20, "astrometry", "Distortions Positions")
680 
681  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
682  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
683 
684  return Astrometry(fit, model, sky_to_tan_projection)
685 
686  def _check_stars(self, associations):
687  """Count measured and reference stars per ccd and warn/log them."""
688  for ccdImage in associations.getCcdImageList():
689  nMeasuredStars, nRefStars = ccdImage.countStars()
690  self.log.debug("ccdImage %s has %s measured and %s reference stars",
691  ccdImage.getName(), nMeasuredStars, nRefStars)
692  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
693  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
694  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
695  if nRefStars < self.config.minRefStarsPerCcd:
696  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
697  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
698 
699  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit):
700  """Run fit.minimize up to max_steps times, returning the final chi2."""
701 
702  for i in range(max_steps):
703  r = fit.minimize(whatToFit, 5) # outlier removal at 5 sigma.
704  chi2 = fit.computeChi2()
705  self._check_stars(associations)
706  if not np.isfinite(chi2.chi2):
707  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
708  self.log.info(str(chi2))
709  if r == MinimizeResult.Converged:
710  self.log.debug("fit has converged - no more outliers - redo minimization "
711  "one more time in case we have lost accuracy in rank update.")
712  # Redo minimization one more time in case we have lost accuracy in rank update
713  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
714  chi2 = fit.computeChi2()
715  self.log.info("Fit completed with: %s", str(chi2))
716  break
717  elif r == MinimizeResult.Chi2Increased:
718  self.log.warn("still some ouliers but chi2 increases - retry")
719  elif r == MinimizeResult.Failed:
720  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
721  else:
722  raise RuntimeError("Unxepected return code from minimize().")
723  else:
724  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
725 
726  return chi2
727 
728  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
729  """
730  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
731 
732  Parameters
733  ----------
734  associations : lsst.jointcal.Associations
735  The star/reference star associations to fit.
736  model : lsst.jointcal.AstrometryModel
737  The astrometric model that was fit.
738  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
739  dict of ccdImage identifiers to dataRefs that were fit
740  """
741 
742  ccdImageList = associations.getCcdImageList()
743  for ccdImage in ccdImageList:
744  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
745  ccd = ccdImage.ccdId
746  visit = ccdImage.visit
747  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
748  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
749  skyWcs = model.makeSkyWcs(ccdImage)
750  try:
751  dataRef.put(skyWcs, 'jointcal_wcs')
752  except pexExceptions.Exception as e:
753  self.log.fatal('Failed to write updated Wcs: %s', str(e))
754  raise e
755 
756  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
757  """
758  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
759 
760  Parameters
761  ----------
762  associations : lsst.jointcal.Associations
763  The star/reference star associations to fit.
764  model : lsst.jointcal.PhotometryModel
765  The photoometric model that was fit.
766  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
767  dict of ccdImage identifiers to dataRefs that were fit
768  """
769 
770  ccdImageList = associations.getCcdImageList()
771  for ccdImage in ccdImageList:
772  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
773  ccd = ccdImage.ccdId
774  visit = ccdImage.visit
775  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
776  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
777  photoCalib = model.toPhotoCalib(ccdImage)
778  try:
779  dataRef.put(photoCalib, 'jointcal_photoCalib')
780  except pexExceptions.Exception as e:
781  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
782  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:275
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:536
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:605
def _check_star_lists(self, associations, name)
Definition: jointcal.py:527
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:756
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit)
Definition: jointcal.py:699
def _check_stars(self, associations)
Definition: jointcal.py:686
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:439
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:728
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:338
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:233