lsst.jointcal  15.0-17-g076ea75+13
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  astrometrySimpleOrder = pexConfig.Field(
156  doc="Polynomial order for fitting the simple astrometry model.",
157  dtype=int,
158  default=3,
159  )
160  astrometryChipOrder = pexConfig.Field(
161  doc="Order of the per-chip transform for the constrained astrometry model.",
162  dtype=int,
163  default=1,
164  )
165  astrometryVisitOrder = pexConfig.Field(
166  doc="Order of the per-visit transform for the constrained astrometry model.",
167  dtype=int,
168  default=5,
169  )
170  useInputWcs = pexConfig.Field(
171  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
172  dtype=bool,
173  default=True,
174  )
175  astrometryModel = pexConfig.ChoiceField(
176  doc="Type of model to fit to astrometry",
177  dtype=str,
178  default="simple",
179  allowed={"simple": "One polynomial per ccd",
180  "constrained": "One polynomial per ccd, and one polynomial per visit"}
181  )
182  photometryModel = pexConfig.ChoiceField(
183  doc="Type of model to fit to photometry",
184  dtype=str,
185  default="simple",
186  allowed={"simple": "One constant zeropoint per ccd and visit",
187  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
188  )
189  photometryVisitOrder = pexConfig.Field(
190  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
191  dtype=int,
192  default=7,
193  )
194  photometryDoRankUpdate = pexConfig.Field(
195  doc="Do the rank update step during minimization. "
196  "Skipping this can help deal with models that are too non-linear.",
197  dtype=bool,
198  default=True,
199  )
200  astrometryDoRankUpdate = pexConfig.Field(
201  doc="Do the rank update step during minimization (should not change the astrometry fit). "
202  "Skipping this can help deal with models that are too non-linear.",
203  dtype=bool,
204  default=True,
205  )
206  outlierRejectSigma = pexConfig.Field(
207  doc="How many sigma to reject outliers at during minimization.",
208  dtype=float,
209  default=5.0,
210  )
211  maxPhotometrySteps = pexConfig.Field(
212  doc="Maximum number of minimize iterations to take when fitting photometry.",
213  dtype=int,
214  default=20,
215  )
216  maxAstrometrySteps = pexConfig.Field(
217  doc="Maximum number of minimize iterations to take when fitting photometry.",
218  dtype=int,
219  default=20,
220  )
221  astrometryRefObjLoader = pexConfig.ConfigurableField(
222  target=LoadIndexedReferenceObjectsTask,
223  doc="Reference object loader for astrometric fit",
224  )
225  photometryRefObjLoader = pexConfig.ConfigurableField(
226  target=LoadIndexedReferenceObjectsTask,
227  doc="Reference object loader for photometric fit",
228  )
229  sourceSelector = sourceSelectorRegistry.makeField(
230  doc="How to select sources for cross-matching",
231  default="astrometry"
232  )
233  writeInitMatrix = pexConfig.Field(
234  dtype=bool,
235  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
236  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
237  "Note that these files are the dense versions of the matrix, and so may be very large.",
238  default=False
239  )
240  writeChi2ContributionFiles = pexConfig.Field(
241  dtype=bool,
242  doc="Write initial/final fit files containing the contributions to chi2.",
243  default=False
244  )
245  sourceFluxType = pexConfig.Field(
246  dtype=str,
247  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
248  default='Calib'
249  )
250 
251  def setDefaults(self):
252  sourceSelector = self.sourceSelector["astrometry"]
253  sourceSelector.setDefaults()
254  # don't want to lose existing flags, just add to them.
255  sourceSelector.badFlags.extend(["slot_Shape_flag"])
256  # This should be used to set the FluxField value in jointcal::JointcalControl
257  sourceSelector.sourceFluxType = self.sourceFluxType
258 
259 
260 class JointcalTask(pipeBase.CmdLineTask):
261  """Jointly astrometrically and photometrically calibrate a group of images."""
262 
263  ConfigClass = JointcalConfig
264  RunnerClass = JointcalRunner
265  _DefaultName = "jointcal"
266 
267  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
268  """
269  Instantiate a JointcalTask.
270 
271  Parameters
272  ----------
273  butler : lsst.daf.persistence.Butler
274  The butler is passed to the refObjLoader constructor in case it is
275  needed. Ignored if the refObjLoader argument provides a loader directly.
276  Used to initialize the astrometry and photometry refObjLoaders.
277  profile_jointcal : bool
278  set to True to profile different stages of this jointcal run.
279  """
280  pipeBase.CmdLineTask.__init__(self, **kwargs)
281  self.profile_jointcal = profile_jointcal
282  self.makeSubtask("sourceSelector")
283  if self.config.doAstrometry:
284  self.makeSubtask('astrometryRefObjLoader', butler=butler)
285  if self.config.doPhotometry:
286  self.makeSubtask('photometryRefObjLoader', butler=butler)
287 
288  # To hold various computed metrics for use by tests
289  self.job = Job.load_metrics_package(subset='jointcal')
290 
291  # We don't need to persist config and metadata at this stage.
292  # In this way, we don't need to put a specific entry in the camera mapper policy file
293  def _getConfigName(self):
294  return None
295 
296  def _getMetadataName(self):
297  return None
298 
299  @classmethod
300  def _makeArgumentParser(cls):
301  """Create an argument parser"""
302  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
303  parser.add_argument("--profile_jointcal", default=False, action="store_true",
304  help="Profile steps of jointcal separately.")
305  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
306  ContainerClass=PerTractCcdDataIdContainer)
307  return parser
308 
309  def _build_ccdImage(self, dataRef, associations, jointcalControl):
310  """
311  Extract the necessary things from this dataRef to add a new ccdImage.
312 
313  Parameters
314  ----------
315  dataRef : lsst.daf.persistence.ButlerDataRef
316  dataRef to extract info from.
317  associations : lsst.jointcal.Associations
318  object to add the info to, to construct a new CcdImage
319  jointcalControl : jointcal.JointcalControl
320  control object for associations management
321 
322  Returns
323  ------
324  namedtuple
325  wcs : lsst.afw.geom.SkyWcs
326  the TAN WCS of this image, read from the calexp
327  key : namedtuple
328  a key to identify this dataRef by its visit and ccd ids
329  filter : str
330  this calexp's filter
331  """
332  if "visit" in dataRef.dataId.keys():
333  visit = dataRef.dataId["visit"]
334  else:
335  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
336 
337  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
338 
339  visitInfo = dataRef.get('calexp_visitInfo')
340  detector = dataRef.get('calexp_detector')
341  ccdId = detector.getId()
342  calib = dataRef.get('calexp_calib')
343  tanWcs = dataRef.get('calexp_wcs')
344  bbox = dataRef.get('calexp_bbox')
345  filt = dataRef.get('calexp_filter')
346  filterName = filt.getName()
347  fluxMag0 = calib.getFluxMag0()
348  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
349 
350  goodSrc = self.sourceSelector.run(src)
351 
352  if len(goodSrc.sourceCat) == 0:
353  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
354  else:
355  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
356  associations.createCcdImage(goodSrc.sourceCat,
357  tanWcs,
358  visitInfo,
359  bbox,
360  filterName,
361  photoCalib,
362  detector,
363  visit,
364  ccdId,
365  jointcalControl)
366 
367  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
368  Key = collections.namedtuple('Key', ('visit', 'ccd'))
369  return Result(tanWcs, Key(visit, ccdId), filterName)
370 
371  @pipeBase.timeMethod
372  def run(self, dataRefs, profile_jointcal=False):
373  """
374  Jointly calibrate the astrometry and photometry across a set of images.
375 
376  Parameters
377  ----------
378  dataRefs : list of lsst.daf.persistence.ButlerDataRef
379  List of data references to the exposures to be fit.
380  profile_jointcal : bool
381  Profile the individual steps of jointcal.
382 
383  Returns
384  -------
385  pipe.base.Struct
386  struct containing:
387  * dataRefs: the provided data references that were fit (with updated WCSs)
388  * oldWcsList: the original WCS from each dataRef
389  * metrics: dictionary of internally-computed metrics for testing/validation.
390  """
391  if len(dataRefs) == 0:
392  raise ValueError('Need a non-empty list of data references!')
393 
394  exitStatus = 0 # exit status for shell
395 
396  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
397  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
398  associations = lsst.jointcal.Associations()
399 
400  visit_ccd_to_dataRef = {}
401  oldWcsList = []
402  filters = []
403  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
404  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
405  # We need the bounding-box of the focal plane for photometry visit models.
406  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
407  camera = dataRefs[0].get('camera', immediate=True)
408  self.focalPlaneBBox = camera.getFpBBox()
409  for ref in dataRefs:
410  result = self._build_ccdImage(ref, associations, jointcalControl)
411  oldWcsList.append(result.wcs)
412  visit_ccd_to_dataRef[result.key] = ref
413  filters.append(result.filter)
414  filters = collections.Counter(filters)
415 
416  associations.computeCommonTangentPoint()
417 
418  # Use external reference catalogs handled by LSST stack mechanism
419  # Get the bounding box overlapping all associated images
420  # ==> This is probably a bad idea to do it this way <== To be improved
421  bbox = associations.getRaDecBBox()
422  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
423  bboxCenter = bbox.getCenter()
424  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
425  bboxMax = bbox.getMax()
426  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
427  radius = center.separation(corner).asRadians()
428 
429  # Get astrometry_net_data path
430  anDir = lsst.utils.getPackageDir('astrometry_net_data')
431  if anDir is None:
432  raise RuntimeError("astrometry_net_data is not setup")
433 
434  # Determine a default filter associated with the catalog. See DM-9093
435  defaultFilter = filters.most_common(1)[0][0]
436  self.log.debug("Using %s band for reference flux", defaultFilter)
437 
438  # TODO: need a better way to get the tract.
439  tract = dataRefs[0].dataId['tract']
440 
441  if self.config.doAstrometry:
442  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
443  name="astrometry",
444  refObjLoader=self.astrometryRefObjLoader,
445  fit_function=self._fit_astrometry,
446  profile_jointcal=profile_jointcal,
447  tract=tract)
448  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
449  else:
450  astrometry = Astrometry(None, None, None)
451 
452  if self.config.doPhotometry:
453  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
454  name="photometry",
455  refObjLoader=self.photometryRefObjLoader,
456  fit_function=self._fit_photometry,
457  profile_jointcal=profile_jointcal,
458  tract=tract,
459  filters=filters,
460  reject_bad_fluxes=True)
461  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
462  else:
463  photometry = Photometry(None, None)
464 
465  return pipeBase.Struct(dataRefs=dataRefs,
466  oldWcsList=oldWcsList,
467  job=self.job,
468  exitStatus=exitStatus)
469 
470  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
471  name="", refObjLoader=None, filters=[], fit_function=None,
472  tract=None, profile_jointcal=False, match_cut=3.0,
473  reject_bad_fluxes=False):
474  """Load reference catalog, perform the fit, and return the result.
475 
476  Parameters
477  ----------
478  associations : lsst.jointcal.Associations
479  The star/reference star associations to fit.
480  defaultFilter : str
481  filter to load from reference catalog.
482  center : lsst.afw.geom.SpherePoint
483  ICRS center of field to load from reference catalog.
484  radius : lsst.afw.geom.Angle
485  On-sky radius to load from reference catalog.
486  name : str
487  Name of thing being fit: "Astrometry" or "Photometry".
488  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
489  Reference object loader to load from for fit.
490  filters : list of str, optional
491  List of filters to load from the reference catalog.
492  fit_function : function
493  function to call to perform fit (takes associations object).
494  tract : str
495  Name of tract currently being fit.
496  profile_jointcal : bool, optional
497  Separately profile the fitting step.
498  match_cut : float, optional
499  Radius in arcseconds to find cross-catalog matches to during
500  associations.associateCatalogs.
501  reject_bad_fluxes : bool, optional
502  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
503 
504  Returns
505  -------
506  Result of `fit_function()`
507  """
508  self.log.info("====== Now processing %s...", name)
509  # TODO: this should not print "trying to invert a singular transformation:"
510  # if it does that, something's not right about the WCS...
511  associations.associateCatalogs(match_cut)
512  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
513  associations.fittedStarListSize())
514 
515  skyCircle = refObjLoader.loadSkyCircle(center,
516  afwGeom.Angle(radius, afwGeom.radians),
517  defaultFilter)
518 
519  # Need memory contiguity to get reference filters as a vector.
520  if not skyCircle.refCat.isContiguous():
521  refCat = skyCircle.refCat.copy(deep=True)
522  else:
523  refCat = skyCircle.refCat
524 
525  # load the reference catalog fluxes.
526  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
527  refFluxes = {}
528  refFluxErrs = {}
529  for filt in filters:
530  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
531  refFluxes[filt] = refCat.get(filtKeys[0])
532  refFluxErrs[filt] = refCat.get(filtKeys[1])
533 
534  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
535  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
536  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
537  associations.refStarListSize())
538 
539  associations.prepareFittedStars(self.config.minMeasurements)
540 
541  self._check_star_lists(associations, name)
542  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
543  associations.nFittedStarsWithAssociatedRefStar())
544  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
545  associations.fittedStarListSize())
546  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
547  associations.nCcdImagesValidForFit())
548 
549  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
550  dataName = "{}_{}".format(tract, defaultFilter)
551  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
552  result = fit_function(associations, dataName)
553  # TODO DM-12446: turn this into a "butler save" somehow.
554  # Save reference and measurement chi2 contributions for this data
555  if self.config.writeChi2ContributionFiles:
556  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
557  result.fit.saveChi2Contributions(baseName)
558 
559  return result
560 
561  def _check_star_lists(self, associations, name):
562  # TODO: these should be len(blah), but we need this properly wrapped first.
563  if associations.nCcdImagesValidForFit() == 0:
564  raise RuntimeError('No images in the ccdImageList!')
565  if associations.fittedStarListSize() == 0:
566  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
567  if associations.refStarListSize() == 0:
568  raise RuntimeError('No stars in the {} reference star list!'.format(name))
569 
570  def _fit_photometry(self, associations, dataName=None):
571  """
572  Fit the photometric data.
573 
574  Parameters
575  ----------
576  associations : lsst.jointcal.Associations
577  The star/reference star associations to fit.
578  dataName : str
579  Name of the data being processed (e.g. "1234_HSC-Y"), for
580  identifying debugging files.
581 
582  Returns
583  -------
584  namedtuple
585  fit : lsst.jointcal.PhotometryFit
586  The photometric fitter used to perform the fit.
587  model : lsst.jointcal.PhotometryModel
588  The photometric model that was fit.
589  """
590  self.log.info("=== Starting photometric fitting...")
591 
592  # TODO: should use pex.config.RegistryField here (see DM-9195)
593  if self.config.photometryModel == "constrained":
594  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
595  self.focalPlaneBBox,
596  visitOrder=self.config.photometryVisitOrder)
597  elif self.config.photometryModel == "simple":
598  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
599 
600  fit = lsst.jointcal.PhotometryFit(associations, model)
601  chi2 = fit.computeChi2()
602  # TODO DM-12446: turn this into a "butler save" somehow.
603  # Save reference and measurement chi2 contributions for this data
604  if self.config.writeChi2ContributionFiles:
605  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
606  fit.saveChi2Contributions(baseName)
607 
608  if not np.isfinite(chi2.chi2):
609  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
610  self.log.info("Initialized: %s", str(chi2))
611  # The constrained model needs the visit transfo fit first; the chip
612  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
613  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
614  if self.config.photometryModel == "constrained":
615  # TODO: (related to DM-8046): implement Visit/Chip choice
616  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
617  chi2 = fit.computeChi2()
618  self.log.info(str(chi2))
619  dumpMatrixFile = "" # so we don't redo the output on the next step
620  fit.minimize("Model", dumpMatrixFile=dumpMatrixFile)
621  chi2 = fit.computeChi2()
622  self.log.info(str(chi2))
623  fit.minimize("Fluxes")
624  chi2 = fit.computeChi2()
625  self.log.info(str(chi2))
626  fit.minimize("Model Fluxes")
627  chi2 = fit.computeChi2()
628  if not np.isfinite(chi2.chi2):
629  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
630  self.log.info("Fit prepared with %s", str(chi2))
631 
632  model.freezeErrorTransform()
633  self.log.debug("Photometry error scales are frozen.")
634 
635  chi2 = self._iterate_fit(associations,
636  fit,
637  model,
638  self.config.maxPhotometrySteps,
639  "photometry",
640  "Model Fluxes",
641  doRankUpdate=self.config.photometryDoRankUpdate)
642 
643  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
644  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
645  return Photometry(fit, model)
646 
647  def _fit_astrometry(self, associations, dataName=None):
648  """
649  Fit the astrometric data.
650 
651  Parameters
652  ----------
653  associations : lsst.jointcal.Associations
654  The star/reference star associations to fit.
655  dataName : str
656  Name of the data being processed (e.g. "1234_HSC-Y"), for
657  identifying debugging files.
658 
659  Returns
660  -------
661  namedtuple
662  fit : lsst.jointcal.AstrometryFit
663  The astrometric fitter used to perform the fit.
664  model : lsst.jointcal.AstrometryModel
665  The astrometric model that was fit.
666  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
667  The model for the sky to tangent plane projection that was used in the fit.
668  """
669 
670  self.log.info("=== Starting astrometric fitting...")
671 
672  associations.deprojectFittedStars()
673 
674  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
675  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
676  # them so carefully?
677  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
678 
679  if self.config.astrometryModel == "constrained":
680  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
681  sky_to_tan_projection,
682  chipOrder=self.config.astrometryChipOrder,
683  visitOrder=self.config.astrometryVisitOrder)
684  elif self.config.astrometryModel == "simple":
685  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
686  sky_to_tan_projection,
687  self.config.useInputWcs,
688  nNotFit=0,
689  order=self.config.astrometrySimpleOrder)
690 
691  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
692  chi2 = fit.computeChi2()
693  # TODO DM-12446: turn this into a "butler save" somehow.
694  # Save reference and measurement chi2 contributions for this data
695  if self.config.writeChi2ContributionFiles:
696  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
697  fit.saveChi2Contributions(baseName)
698 
699  if not np.isfinite(chi2.chi2):
700  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
701  self.log.info("Initialized: %s", str(chi2))
702  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
703  # The constrained model needs the visit transfo fit first; the chip
704  # transfo is initialized from the detector's cameraGeom, so it's close.
705  if self.config.astrometryModel == "constrained":
706  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
707  chi2 = fit.computeChi2()
708  self.log.info(str(chi2))
709  dumpMatrixFile = "" # so we don't redo the output on the next step
710  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
711  chi2 = fit.computeChi2()
712  self.log.info(str(chi2))
713  fit.minimize("Positions")
714  chi2 = fit.computeChi2()
715  self.log.info(str(chi2))
716  fit.minimize("Distortions Positions")
717  chi2 = fit.computeChi2()
718  self.log.info(str(chi2))
719  if not np.isfinite(chi2.chi2):
720  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
721  self.log.info("Fit prepared with %s", str(chi2))
722 
723  chi2 = self._iterate_fit(associations,
724  fit,
725  model,
726  self.config.maxAstrometrySteps,
727  "astrometry",
728  "Distortions Positions",
729  doRankUpdate=self.config.astrometryDoRankUpdate)
730 
731  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
732  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
733 
734  return Astrometry(fit, model, sky_to_tan_projection)
735 
736  def _check_stars(self, associations):
737  """Count measured and reference stars per ccd and warn/log them."""
738  for ccdImage in associations.getCcdImageList():
739  nMeasuredStars, nRefStars = ccdImage.countStars()
740  self.log.debug("ccdImage %s has %s measured and %s reference stars",
741  ccdImage.getName(), nMeasuredStars, nRefStars)
742  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
743  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
744  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
745  if nRefStars < self.config.minRefStarsPerCcd:
746  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
747  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
748 
749  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True):
750  """Run fit.minimize up to max_steps times, returning the final chi2."""
751 
752  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
753  for i in range(max_steps):
754  # outlier removal at 5 sigma.
755  r = fit.minimize(whatToFit,
756  self.config.outlierRejectSigma,
757  doRankUpdate=doRankUpdate,
758  dumpMatrixFile=dumpMatrixFile)
759  dumpMatrixFile = "" # clear it so we don't write the matrix again.
760  chi2 = fit.computeChi2()
761  self._check_stars(associations)
762  if not np.isfinite(chi2.chi2):
763  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
764  self.log.info(str(chi2))
765  if r == MinimizeResult.Converged:
766  if doRankUpdate:
767  self.log.debug("fit has converged - no more outliers - redo minimization "
768  "one more time in case we have lost accuracy in rank update.")
769  # Redo minimization one more time in case we have lost accuracy in rank update
770  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
771  chi2 = fit.computeChi2()
772  self.log.info("Fit completed with: %s", str(chi2))
773  break
774  elif r == MinimizeResult.Chi2Increased:
775  self.log.warn("still some ouliers but chi2 increases - retry")
776  elif r == MinimizeResult.Failed:
777  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
778  else:
779  raise RuntimeError("Unxepected return code from minimize().")
780  else:
781  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
782 
783  return chi2
784 
785  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
786  """
787  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
788 
789  Parameters
790  ----------
791  associations : lsst.jointcal.Associations
792  The star/reference star associations to fit.
793  model : lsst.jointcal.AstrometryModel
794  The astrometric model that was fit.
795  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
796  dict of ccdImage identifiers to dataRefs that were fit
797  """
798 
799  ccdImageList = associations.getCcdImageList()
800  for ccdImage in ccdImageList:
801  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
802  ccd = ccdImage.ccdId
803  visit = ccdImage.visit
804  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
805  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
806  skyWcs = model.makeSkyWcs(ccdImage)
807  try:
808  dataRef.put(skyWcs, 'jointcal_wcs')
809  except pexExceptions.Exception as e:
810  self.log.fatal('Failed to write updated Wcs: %s', str(e))
811  raise e
812 
813  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
814  """
815  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
816 
817  Parameters
818  ----------
819  associations : lsst.jointcal.Associations
820  The star/reference star associations to fit.
821  model : lsst.jointcal.PhotometryModel
822  The photoometric model that was fit.
823  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
824  dict of ccdImage identifiers to dataRefs that were fit
825  """
826 
827  ccdImageList = associations.getCcdImageList()
828  for ccdImage in ccdImageList:
829  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
830  ccd = ccdImage.ccdId
831  visit = ccdImage.visit
832  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
833  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
834  photoCalib = model.toPhotoCalib(ccdImage)
835  try:
836  dataRef.put(photoCalib, 'jointcal_photoCalib')
837  except pexExceptions.Exception as e:
838  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
839  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:309
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:570
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:647
def _check_star_lists(self, associations, name)
Definition: jointcal.py:561
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:813
def _check_stars(self, associations)
Definition: jointcal.py:736
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:473
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:785
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:372
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:267
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True)
Definition: jointcal.py:749