lsst.jointcal  16.0-7-g6e35192
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  allowLineSearch = pexConfig.Field(
156  doc="Allow a line search during minimization, if it is reasonable for the model"
157  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
158  dtype=bool,
159  default=False
160  )
161  astrometrySimpleOrder = pexConfig.Field(
162  doc="Polynomial order for fitting the simple astrometry model.",
163  dtype=int,
164  default=3,
165  )
166  astrometryChipOrder = pexConfig.Field(
167  doc="Order of the per-chip transform for the constrained astrometry model.",
168  dtype=int,
169  default=1,
170  )
171  astrometryVisitOrder = pexConfig.Field(
172  doc="Order of the per-visit transform for the constrained astrometry model.",
173  dtype=int,
174  default=5,
175  )
176  useInputWcs = pexConfig.Field(
177  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
178  dtype=bool,
179  default=True,
180  )
181  astrometryModel = pexConfig.ChoiceField(
182  doc="Type of model to fit to astrometry",
183  dtype=str,
184  default="simple",
185  allowed={"simple": "One polynomial per ccd",
186  "constrained": "One polynomial per ccd, and one polynomial per visit"}
187  )
188  photometryModel = pexConfig.ChoiceField(
189  doc="Type of model to fit to photometry",
190  dtype=str,
191  default="simpleFlux",
192  allowed={"simpleFlux": "One constant zeropoint per ccd and visit",
193  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit",
194  "simpleMagnitude": "One constant zeropoint per ccd and visit"}
195  )
196  photometryVisitOrder = pexConfig.Field(
197  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
198  dtype=int,
199  default=7,
200  )
201  photometryDoRankUpdate = pexConfig.Field(
202  doc="Do the rank update step during minimization. "
203  "Skipping this can help deal with models that are too non-linear.",
204  dtype=bool,
205  default=True,
206  )
207  astrometryDoRankUpdate = pexConfig.Field(
208  doc="Do the rank update step during minimization (should not change the astrometry fit). "
209  "Skipping this can help deal with models that are too non-linear.",
210  dtype=bool,
211  default=True,
212  )
213  outlierRejectSigma = pexConfig.Field(
214  doc="How many sigma to reject outliers at during minimization.",
215  dtype=float,
216  default=5.0,
217  )
218  maxPhotometrySteps = pexConfig.Field(
219  doc="Maximum number of minimize iterations to take when fitting photometry.",
220  dtype=int,
221  default=20,
222  )
223  maxAstrometrySteps = pexConfig.Field(
224  doc="Maximum number of minimize iterations to take when fitting photometry.",
225  dtype=int,
226  default=20,
227  )
228  astrometryRefObjLoader = pexConfig.ConfigurableField(
229  target=LoadIndexedReferenceObjectsTask,
230  doc="Reference object loader for astrometric fit",
231  )
232  photometryRefObjLoader = pexConfig.ConfigurableField(
233  target=LoadIndexedReferenceObjectsTask,
234  doc="Reference object loader for photometric fit",
235  )
236  sourceSelector = sourceSelectorRegistry.makeField(
237  doc="How to select sources for cross-matching",
238  default="astrometry"
239  )
240  writeInitMatrix = pexConfig.Field(
241  dtype=bool,
242  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
243  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
244  "Note that these files are the dense versions of the matrix, and so may be very large.",
245  default=False
246  )
247  writeChi2ContributionFiles = pexConfig.Field(
248  dtype=bool,
249  doc="Write initial/final fit files containing the contributions to chi2.",
250  default=False
251  )
252  sourceFluxType = pexConfig.Field(
253  dtype=str,
254  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
255  default='Calib'
256  )
257 
258  def setDefaults(self):
259  sourceSelector = self.sourceSelector["astrometry"]
260  sourceSelector.setDefaults()
261  # don't want to lose existing flags, just add to them.
262  sourceSelector.badFlags.extend(["slot_Shape_flag"])
263  # This should be used to set the FluxField value in jointcal::JointcalControl
264  sourceSelector.sourceFluxType = self.sourceFluxType
265 
266 
267 class JointcalTask(pipeBase.CmdLineTask):
268  """Jointly astrometrically and photometrically calibrate a group of images."""
269 
270  ConfigClass = JointcalConfig
271  RunnerClass = JointcalRunner
272  _DefaultName = "jointcal"
273 
274  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
275  """
276  Instantiate a JointcalTask.
277 
278  Parameters
279  ----------
280  butler : lsst.daf.persistence.Butler
281  The butler is passed to the refObjLoader constructor in case it is
282  needed. Ignored if the refObjLoader argument provides a loader directly.
283  Used to initialize the astrometry and photometry refObjLoaders.
284  profile_jointcal : bool
285  set to True to profile different stages of this jointcal run.
286  """
287  pipeBase.CmdLineTask.__init__(self, **kwargs)
288  self.profile_jointcal = profile_jointcal
289  self.makeSubtask("sourceSelector")
290  if self.config.doAstrometry:
291  self.makeSubtask('astrometryRefObjLoader', butler=butler)
292  if self.config.doPhotometry:
293  self.makeSubtask('photometryRefObjLoader', butler=butler)
294 
295  # To hold various computed metrics for use by tests
296  self.job = Job.load_metrics_package(subset='jointcal')
297 
298  # We don't need to persist config and metadata at this stage.
299  # In this way, we don't need to put a specific entry in the camera mapper policy file
300  def _getConfigName(self):
301  return None
302 
303  def _getMetadataName(self):
304  return None
305 
306  @classmethod
307  def _makeArgumentParser(cls):
308  """Create an argument parser"""
309  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
310  parser.add_argument("--profile_jointcal", default=False, action="store_true",
311  help="Profile steps of jointcal separately.")
312  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
313  ContainerClass=PerTractCcdDataIdContainer)
314  return parser
315 
316  def _build_ccdImage(self, dataRef, associations, jointcalControl):
317  """
318  Extract the necessary things from this dataRef to add a new ccdImage.
319 
320  Parameters
321  ----------
322  dataRef : lsst.daf.persistence.ButlerDataRef
323  dataRef to extract info from.
324  associations : lsst.jointcal.Associations
325  object to add the info to, to construct a new CcdImage
326  jointcalControl : jointcal.JointcalControl
327  control object for associations management
328 
329  Returns
330  ------
331  namedtuple
332  wcs : lsst.afw.geom.SkyWcs
333  the TAN WCS of this image, read from the calexp
334  key : namedtuple
335  a key to identify this dataRef by its visit and ccd ids
336  filter : str
337  this calexp's filter
338  """
339  if "visit" in dataRef.dataId.keys():
340  visit = dataRef.dataId["visit"]
341  else:
342  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
343 
344  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
345 
346  visitInfo = dataRef.get('calexp_visitInfo')
347  detector = dataRef.get('calexp_detector')
348  ccdId = detector.getId()
349  calib = dataRef.get('calexp_calib')
350  tanWcs = dataRef.get('calexp_wcs')
351  bbox = dataRef.get('calexp_bbox')
352  filt = dataRef.get('calexp_filter')
353  filterName = filt.getName()
354  fluxMag0 = calib.getFluxMag0()
355  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
356 
357  goodSrc = self.sourceSelector.run(src)
358 
359  if len(goodSrc.sourceCat) == 0:
360  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
361  else:
362  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
363  associations.createCcdImage(goodSrc.sourceCat,
364  tanWcs,
365  visitInfo,
366  bbox,
367  filterName,
368  photoCalib,
369  detector,
370  visit,
371  ccdId,
372  jointcalControl)
373 
374  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
375  Key = collections.namedtuple('Key', ('visit', 'ccd'))
376  return Result(tanWcs, Key(visit, ccdId), filterName)
377 
378  @pipeBase.timeMethod
379  def run(self, dataRefs, profile_jointcal=False):
380  """
381  Jointly calibrate the astrometry and photometry across a set of images.
382 
383  Parameters
384  ----------
385  dataRefs : list of lsst.daf.persistence.ButlerDataRef
386  List of data references to the exposures to be fit.
387  profile_jointcal : bool
388  Profile the individual steps of jointcal.
389 
390  Returns
391  -------
392  pipe.base.Struct
393  struct containing:
394  * dataRefs: the provided data references that were fit (with updated WCSs)
395  * oldWcsList: the original WCS from each dataRef
396  * metrics: dictionary of internally-computed metrics for testing/validation.
397  """
398  if len(dataRefs) == 0:
399  raise ValueError('Need a non-empty list of data references!')
400 
401  exitStatus = 0 # exit status for shell
402 
403  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
404  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
405  associations = lsst.jointcal.Associations()
406 
407  visit_ccd_to_dataRef = {}
408  oldWcsList = []
409  filters = []
410  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
411  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
412  # We need the bounding-box of the focal plane for photometry visit models.
413  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
414  camera = dataRefs[0].get('camera', immediate=True)
415  self.focalPlaneBBox = camera.getFpBBox()
416  for ref in dataRefs:
417  result = self._build_ccdImage(ref, associations, jointcalControl)
418  oldWcsList.append(result.wcs)
419  visit_ccd_to_dataRef[result.key] = ref
420  filters.append(result.filter)
421  filters = collections.Counter(filters)
422 
423  associations.computeCommonTangentPoint()
424 
425  # Use external reference catalogs handled by LSST stack mechanism
426  # Get the bounding box overlapping all associated images
427  # ==> This is probably a bad idea to do it this way <== To be improved
428  bbox = associations.getRaDecBBox()
429  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
430  bboxCenter = bbox.getCenter()
431  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
432  bboxMax = bbox.getMax()
433  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
434  radius = center.separation(corner).asRadians()
435 
436  # Get astrometry_net_data path
437  anDir = lsst.utils.getPackageDir('astrometry_net_data')
438  if anDir is None:
439  raise RuntimeError("astrometry_net_data is not setup")
440 
441  # Determine a default filter associated with the catalog. See DM-9093
442  defaultFilter = filters.most_common(1)[0][0]
443  self.log.debug("Using %s band for reference flux", defaultFilter)
444 
445  # TODO: need a better way to get the tract.
446  tract = dataRefs[0].dataId['tract']
447 
448  if self.config.doAstrometry:
449  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
450  name="astrometry",
451  refObjLoader=self.astrometryRefObjLoader,
452  fit_function=self._fit_astrometry,
453  profile_jointcal=profile_jointcal,
454  tract=tract)
455  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
456  else:
457  astrometry = Astrometry(None, None, None)
458 
459  if self.config.doPhotometry:
460  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
461  name="photometry",
462  refObjLoader=self.photometryRefObjLoader,
463  fit_function=self._fit_photometry,
464  profile_jointcal=profile_jointcal,
465  tract=tract,
466  filters=filters,
467  reject_bad_fluxes=True)
468  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
469  else:
470  photometry = Photometry(None, None)
471 
472  return pipeBase.Struct(dataRefs=dataRefs,
473  oldWcsList=oldWcsList,
474  job=self.job,
475  exitStatus=exitStatus)
476 
477  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
478  name="", refObjLoader=None, filters=[], fit_function=None,
479  tract=None, profile_jointcal=False, match_cut=3.0,
480  reject_bad_fluxes=False):
481  """Load reference catalog, perform the fit, and return the result.
482 
483  Parameters
484  ----------
485  associations : lsst.jointcal.Associations
486  The star/reference star associations to fit.
487  defaultFilter : str
488  filter to load from reference catalog.
489  center : lsst.afw.geom.SpherePoint
490  ICRS center of field to load from reference catalog.
491  radius : lsst.afw.geom.Angle
492  On-sky radius to load from reference catalog.
493  name : str
494  Name of thing being fit: "Astrometry" or "Photometry".
495  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
496  Reference object loader to load from for fit.
497  filters : list of str, optional
498  List of filters to load from the reference catalog.
499  fit_function : function
500  function to call to perform fit (takes associations object).
501  tract : str
502  Name of tract currently being fit.
503  profile_jointcal : bool, optional
504  Separately profile the fitting step.
505  match_cut : float, optional
506  Radius in arcseconds to find cross-catalog matches to during
507  associations.associateCatalogs.
508  reject_bad_fluxes : bool, optional
509  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
510 
511  Returns
512  -------
513  Result of `fit_function()`
514  """
515  self.log.info("====== Now processing %s...", name)
516  # TODO: this should not print "trying to invert a singular transformation:"
517  # if it does that, something's not right about the WCS...
518  associations.associateCatalogs(match_cut)
519  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
520  associations.fittedStarListSize())
521 
522  skyCircle = refObjLoader.loadSkyCircle(center,
523  afwGeom.Angle(radius, afwGeom.radians),
524  defaultFilter)
525 
526  # Need memory contiguity to get reference filters as a vector.
527  if not skyCircle.refCat.isContiguous():
528  refCat = skyCircle.refCat.copy(deep=True)
529  else:
530  refCat = skyCircle.refCat
531 
532  # load the reference catalog fluxes.
533  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
534  refFluxes = {}
535  refFluxErrs = {}
536  for filt in filters:
537  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
538  refFluxes[filt] = refCat.get(filtKeys[0])
539  refFluxErrs[filt] = refCat.get(filtKeys[1])
540 
541  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
542  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
543  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
544  associations.refStarListSize())
545 
546  associations.prepareFittedStars(self.config.minMeasurements)
547 
548  self._check_star_lists(associations, name)
549  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
550  associations.nFittedStarsWithAssociatedRefStar())
551  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
552  associations.fittedStarListSize())
553  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
554  associations.nCcdImagesValidForFit())
555 
556  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
557  dataName = "{}_{}".format(tract, defaultFilter)
558  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
559  result = fit_function(associations, dataName)
560  # TODO DM-12446: turn this into a "butler save" somehow.
561  # Save reference and measurement chi2 contributions for this data
562  if self.config.writeChi2ContributionFiles:
563  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
564  result.fit.saveChi2Contributions(baseName)
565 
566  return result
567 
568  def _check_star_lists(self, associations, name):
569  # TODO: these should be len(blah), but we need this properly wrapped first.
570  if associations.nCcdImagesValidForFit() == 0:
571  raise RuntimeError('No images in the ccdImageList!')
572  if associations.fittedStarListSize() == 0:
573  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
574  if associations.refStarListSize() == 0:
575  raise RuntimeError('No stars in the {} reference star list!'.format(name))
576 
577  def _fit_photometry(self, associations, dataName=None):
578  """
579  Fit the photometric data.
580 
581  Parameters
582  ----------
583  associations : lsst.jointcal.Associations
584  The star/reference star associations to fit.
585  dataName : str
586  Name of the data being processed (e.g. "1234_HSC-Y"), for
587  identifying debugging files.
588 
589  Returns
590  -------
591  namedtuple
592  fit : lsst.jointcal.PhotometryFit
593  The photometric fitter used to perform the fit.
594  model : lsst.jointcal.PhotometryModel
595  The photometric model that was fit.
596  """
597  self.log.info("=== Starting photometric fitting...")
598 
599  # TODO: should use pex.config.RegistryField here (see DM-9195)
600  if self.config.photometryModel == "constrainedFlux":
601  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
602  self.focalPlaneBBox,
603  visitOrder=self.config.photometryVisitOrder)
604  # potentially nonlinear problem, so we may need a line search to converge.
605  doLineSearch = self.config.allowLineSearch
606  elif self.config.photometryModel == "simpleFlux":
607  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList())
608  doLineSearch = False # purely linear in model parameters, so no line search needed
609  elif self.config.photometryModel == "simpleMagnitude":
610  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList())
611  doLineSearch = False # purely linear in model parameters, so no line search needed
612 
613  fit = lsst.jointcal.PhotometryFit(associations, model)
614  chi2 = fit.computeChi2()
615  # TODO DM-12446: turn this into a "butler save" somehow.
616  # Save reference and measurement chi2 contributions for this data
617  if self.config.writeChi2ContributionFiles:
618  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
619  fit.saveChi2Contributions(baseName)
620 
621  if not np.isfinite(chi2.chi2):
622  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
623  self.log.info("Initialized: %s", str(chi2))
624  # The constrained model needs the visit transfo fit first; the chip
625  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
626  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
627  if self.config.photometryModel == "constrained":
628  # no line search: should be purely (or nearly) linear,
629  # and we want a large step size to initialize with.
630  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
631  chi2 = fit.computeChi2()
632  self.log.info(str(chi2))
633  dumpMatrixFile = "" # so we don't redo the output on the next step
634  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
635  chi2 = fit.computeChi2()
636  self.log.info(str(chi2))
637  fit.minimize("Fluxes") # no line search: always purely linear.
638  chi2 = fit.computeChi2()
639  self.log.info(str(chi2))
640  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
641  chi2 = fit.computeChi2()
642  if not np.isfinite(chi2.chi2):
643  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
644  self.log.info("Fit prepared with %s", str(chi2))
645 
646  model.freezeErrorTransform()
647  self.log.debug("Photometry error scales are frozen.")
648 
649  chi2 = self._iterate_fit(associations,
650  fit,
651  model,
652  self.config.maxPhotometrySteps,
653  "photometry",
654  "Model Fluxes",
655  doRankUpdate=self.config.photometryDoRankUpdate,
656  doLineSearch=doLineSearch)
657 
658  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
659  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
660  return Photometry(fit, model)
661 
662  def _fit_astrometry(self, associations, dataName=None):
663  """
664  Fit the astrometric data.
665 
666  Parameters
667  ----------
668  associations : lsst.jointcal.Associations
669  The star/reference star associations to fit.
670  dataName : str
671  Name of the data being processed (e.g. "1234_HSC-Y"), for
672  identifying debugging files.
673 
674  Returns
675  -------
676  namedtuple
677  fit : lsst.jointcal.AstrometryFit
678  The astrometric fitter used to perform the fit.
679  model : lsst.jointcal.AstrometryModel
680  The astrometric model that was fit.
681  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
682  The model for the sky to tangent plane projection that was used in the fit.
683  """
684 
685  self.log.info("=== Starting astrometric fitting...")
686 
687  associations.deprojectFittedStars()
688 
689  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
690  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
691  # them so carefully?
692  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
693 
694  if self.config.astrometryModel == "constrained":
695  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
696  sky_to_tan_projection,
697  chipOrder=self.config.astrometryChipOrder,
698  visitOrder=self.config.astrometryVisitOrder)
699  elif self.config.astrometryModel == "simple":
700  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
701  sky_to_tan_projection,
702  self.config.useInputWcs,
703  nNotFit=0,
704  order=self.config.astrometrySimpleOrder)
705 
706  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
707  chi2 = fit.computeChi2()
708  # TODO DM-12446: turn this into a "butler save" somehow.
709  # Save reference and measurement chi2 contributions for this data
710  if self.config.writeChi2ContributionFiles:
711  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
712  fit.saveChi2Contributions(baseName)
713 
714  if not np.isfinite(chi2.chi2):
715  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
716  self.log.info("Initialized: %s", str(chi2))
717  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
718  # The constrained model needs the visit transfo fit first; the chip
719  # transfo is initialized from the detector's cameraGeom, so it's close.
720  if self.config.astrometryModel == "constrained":
721  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
722  chi2 = fit.computeChi2()
723  self.log.info(str(chi2))
724  dumpMatrixFile = "" # so we don't redo the output on the next step
725  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
726  chi2 = fit.computeChi2()
727  self.log.info(str(chi2))
728  fit.minimize("Positions")
729  chi2 = fit.computeChi2()
730  self.log.info(str(chi2))
731  fit.minimize("Distortions Positions")
732  chi2 = fit.computeChi2()
733  self.log.info(str(chi2))
734  if not np.isfinite(chi2.chi2):
735  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
736  self.log.info("Fit prepared with %s", str(chi2))
737 
738  chi2 = self._iterate_fit(associations,
739  fit,
740  model,
741  self.config.maxAstrometrySteps,
742  "astrometry",
743  "Distortions Positions",
744  doRankUpdate=self.config.astrometryDoRankUpdate)
745 
746  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
747  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
748 
749  return Astrometry(fit, model, sky_to_tan_projection)
750 
751  def _check_stars(self, associations):
752  """Count measured and reference stars per ccd and warn/log them."""
753  for ccdImage in associations.getCcdImageList():
754  nMeasuredStars, nRefStars = ccdImage.countStars()
755  self.log.debug("ccdImage %s has %s measured and %s reference stars",
756  ccdImage.getName(), nMeasuredStars, nRefStars)
757  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
758  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
759  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
760  if nRefStars < self.config.minRefStarsPerCcd:
761  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
762  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
763 
764  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True,
765  doLineSearch=False):
766  """Run fit.minimize up to max_steps times, returning the final chi2."""
767 
768  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
769  for i in range(max_steps):
770  # outlier removal at 5 sigma.
771  r = fit.minimize(whatToFit,
772  self.config.outlierRejectSigma,
773  doRankUpdate=doRankUpdate,
774  doLineSearch=doLineSearch,
775  dumpMatrixFile=dumpMatrixFile)
776  dumpMatrixFile = "" # clear it so we don't write the matrix again.
777  chi2 = fit.computeChi2()
778  self._check_stars(associations)
779  if not np.isfinite(chi2.chi2):
780  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
781  self.log.info(str(chi2))
782  if r == MinimizeResult.Converged:
783  if doRankUpdate:
784  self.log.debug("fit has converged - no more outliers - redo minimization "
785  "one more time in case we have lost accuracy in rank update.")
786  # Redo minimization one more time in case we have lost accuracy in rank update
787  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
788  chi2 = fit.computeChi2()
789  self.log.info("Fit completed with: %s", str(chi2))
790  break
791  elif r == MinimizeResult.Chi2Increased:
792  self.log.warn("still some ouliers but chi2 increases - retry")
793  elif r == MinimizeResult.Failed:
794  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
795  else:
796  raise RuntimeError("Unxepected return code from minimize().")
797  else:
798  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
799 
800  return chi2
801 
802  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
803  """
804  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
805 
806  Parameters
807  ----------
808  associations : lsst.jointcal.Associations
809  The star/reference star associations to fit.
810  model : lsst.jointcal.AstrometryModel
811  The astrometric model that was fit.
812  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
813  dict of ccdImage identifiers to dataRefs that were fit
814  """
815 
816  ccdImageList = associations.getCcdImageList()
817  for ccdImage in ccdImageList:
818  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
819  ccd = ccdImage.ccdId
820  visit = ccdImage.visit
821  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
822  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
823  skyWcs = model.makeSkyWcs(ccdImage)
824  try:
825  dataRef.put(skyWcs, 'jointcal_wcs')
826  except pexExceptions.Exception as e:
827  self.log.fatal('Failed to write updated Wcs: %s', str(e))
828  raise e
829 
830  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
831  """
832  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
833 
834  Parameters
835  ----------
836  associations : lsst.jointcal.Associations
837  The star/reference star associations to fit.
838  model : lsst.jointcal.PhotometryModel
839  The photoometric model that was fit.
840  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
841  dict of ccdImage identifiers to dataRefs that were fit
842  """
843 
844  ccdImageList = associations.getCcdImageList()
845  for ccdImage in ccdImageList:
846  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
847  ccd = ccdImage.ccdId
848  visit = ccdImage.visit
849  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
850  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
851  photoCalib = model.toPhotoCalib(ccdImage)
852  try:
853  dataRef.put(photoCalib, 'jointcal_photoCalib')
854  except pexExceptions.Exception as e:
855  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
856  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:316
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:577
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:662
def _check_star_lists(self, associations, name)
Definition: jointcal.py:568
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:765
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:830
def _check_stars(self, associations)
Definition: jointcal.py:751
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:480
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:802
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:379
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:274