lsst.jointcal  15.0-13-g7441b25+1
jointcal.py
Go to the documentation of this file.
1 # See COPYRIGHT file at the top of the source tree.
2 import collections
3 import numpy as np
4 
5 import lsst.utils
6 import lsst.pex.config as pexConfig
7 import lsst.pipe.base as pipeBase
8 import lsst.afw.image as afwImage
9 import lsst.afw.geom as afwGeom
10 import lsst.pex.exceptions as pexExceptions
11 import lsst.afw.table
13 from lsst.verify import Job, Measurement
14 
15 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask
16 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
17 
18 from .dataIds import PerTractCcdDataIdContainer
19 
20 import lsst.jointcal
21 from lsst.jointcal import MinimizeResult
22 
23 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
24 
25 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
26 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
27 
28 
29 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
30 def add_measurement(job, name, value):
31  meas = Measurement(job.metrics[name], value)
32  job.measurements.insert(meas)
33 
34 
35 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
36  """Subclass of TaskRunner for jointcalTask
37 
38  jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
39  extracted from the command line (whereas most CmdLineTasks' run methods take
40  single dataRef, are are called repeatedly). This class transforms the processed
41  arguments generated by the ArgumentParser into the arguments expected by
42  Jointcal.run().
43 
44  See pipeBase.TaskRunner for more information.
45  """
46 
47  @staticmethod
48  def getTargetList(parsedCmd, **kwargs):
49  """
50  Return a list of tuples per tract, each containing (dataRefs, kwargs).
51 
52  Jointcal operates on lists of dataRefs simultaneously.
53  """
54  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
55  kwargs['butler'] = parsedCmd.butler
56 
57  # organize data IDs by tract
58  refListDict = {}
59  for ref in parsedCmd.id.refList:
60  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
61  # we call run() once with each tract
62  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
63  return result
64 
65  def __call__(self, args):
66  """
67  Parameters
68  ----------
69  args
70  Arguments for Task.run()
71 
72  Returns
73  -------
74  pipe.base.Struct
75  if self.doReturnResults is False:
76 
77  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
78 
79  if self.doReturnResults is True:
80 
81  - ``result``: the result of calling jointcal.run()
82  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
83  """
84  exitStatus = 0 # exit status for shell
85 
86  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
87  dataRefList, kwargs = args
88  butler = kwargs.pop('butler')
89  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90  result = None
91  try:
92  result = task.run(dataRefList, **kwargs)
93  exitStatus = result.exitStatus
94  job_path = butler.get('verify_job_filename')
95  result.job.write(job_path[0])
96  except Exception as e: # catch everything, sort it out later.
97  if self.doRaise:
98  raise e
99  else:
100  exitStatus = 1
101  eName = type(e).__name__
102  tract = dataRefList[0].dataId['tract']
103  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
104 
105  if self.doReturnResults:
106  return pipeBase.Struct(result=result, exitStatus=exitStatus)
107  else:
108  return pipeBase.Struct(exitStatus=exitStatus)
109 
110 
111 class JointcalConfig(pexConfig.Config):
112  """Config for JointcalTask"""
113 
114  doAstrometry = pexConfig.Field(
115  doc="Fit astrometry and write the fitted result.",
116  dtype=bool,
117  default=True
118  )
119  doPhotometry = pexConfig.Field(
120  doc="Fit photometry and write the fitted result.",
121  dtype=bool,
122  default=True
123  )
124  coaddName = pexConfig.Field(
125  doc="Type of coadd, typically deep or goodSeeing",
126  dtype=str,
127  default="deep"
128  )
129  posError = pexConfig.Field(
130  doc="Constant term for error on position (in pixel unit)",
131  dtype=float,
132  default=0.02,
133  )
134  # TODO: DM-6885 matchCut should be an afw.geom.Angle
135  matchCut = pexConfig.Field(
136  doc="Matching radius between fitted and reference stars (arcseconds)",
137  dtype=float,
138  default=3.0,
139  )
140  minMeasurements = pexConfig.Field(
141  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
142  dtype=int,
143  default=2,
144  )
145  minMeasuredStarsPerCcd = pexConfig.Field(
146  doc="Minimum number of measuredStars per ccdImage before printing warnings",
147  dtype=int,
148  default=100,
149  )
150  minRefStarsPerCcd = pexConfig.Field(
151  doc="Minimum number of measuredStars per ccdImage before printing warnings",
152  dtype=int,
153  default=30,
154  )
155  astrometrySimpleOrder = pexConfig.Field(
156  doc="Polynomial order for fitting the simple astrometry model.",
157  dtype=int,
158  default=3,
159  )
160  astrometryChipOrder = pexConfig.Field(
161  doc="Order of the per-chip transform for the constrained astrometry model.",
162  dtype=int,
163  default=1,
164  )
165  astrometryVisitOrder = pexConfig.Field(
166  doc="Order of the per-visit transform for the constrained astrometry model.",
167  dtype=int,
168  default=5,
169  )
170  useInputWcs = pexConfig.Field(
171  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
172  dtype=bool,
173  default=True,
174  )
175  astrometryModel = pexConfig.ChoiceField(
176  doc="Type of model to fit to astrometry",
177  dtype=str,
178  default="simple",
179  allowed={"simple": "One polynomial per ccd",
180  "constrained": "One polynomial per ccd, and one polynomial per visit"}
181  )
182  photometryModel = pexConfig.ChoiceField(
183  doc="Type of model to fit to photometry",
184  dtype=str,
185  default="simple",
186  allowed={"simple": "One constant zeropoint per ccd and visit",
187  "constrained": "Constrained zeropoint per ccd, and one polynomial per visit"}
188  )
189  photometryVisitOrder = pexConfig.Field(
190  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
191  dtype=int,
192  default=7,
193  )
194  photometryDoRankUpdate = pexConfig.Field(
195  doc="Do the rank update step during minimization. "
196  "Skipping this can help deal with models that are too non-linear.",
197  dtype=bool,
198  default=True,
199  )
200  astrometryDoRankUpdate = pexConfig.Field(
201  doc="Do the rank update step during minimization (should not change the astrometry fit). "
202  "Skipping this can help deal with models that are too non-linear.",
203  dtype=bool,
204  default=True,
205  )
206  outlierRejectSigma = pexConfig.Field(
207  doc="How many sigma to reject outliers at during minimization.",
208  dtype=float,
209  default=5.0,
210  )
211  maxPhotometrySteps = pexConfig.Field(
212  doc="Maximum number of minimize iterations to take when fitting photometry.",
213  dtype=int,
214  default=20,
215  )
216  maxAstrometrySteps = pexConfig.Field(
217  doc="Maximum number of minimize iterations to take when fitting photometry.",
218  dtype=int,
219  default=20,
220  )
221  astrometryRefObjLoader = pexConfig.ConfigurableField(
222  target=LoadIndexedReferenceObjectsTask,
223  doc="Reference object loader for astrometric fit",
224  )
225  photometryRefObjLoader = pexConfig.ConfigurableField(
226  target=LoadIndexedReferenceObjectsTask,
227  doc="Reference object loader for photometric fit",
228  )
229  sourceSelector = sourceSelectorRegistry.makeField(
230  doc="How to select sources for cross-matching",
231  default="astrometry"
232  )
233  writeChi2ContributionFiles = pexConfig.Field(
234  dtype=bool,
235  doc="Write initial/final fit files containing the contributions to chi2.",
236  default=False
237  )
238  sourceFluxType = pexConfig.Field(
239  dtype=str,
240  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
241  default='Calib'
242  )
243 
244  def setDefaults(self):
245  sourceSelector = self.sourceSelector["astrometry"]
246  sourceSelector.setDefaults()
247  # don't want to lose existing flags, just add to them.
248  sourceSelector.badFlags.extend(["slot_Shape_flag"])
249  # This should be used to set the FluxField value in jointcal::JointcalControl
250  sourceSelector.sourceFluxType = self.sourceFluxType
251 
252 
253 class JointcalTask(pipeBase.CmdLineTask):
254  """Jointly astrometrically and photometrically calibrate a group of images."""
255 
256  ConfigClass = JointcalConfig
257  RunnerClass = JointcalRunner
258  _DefaultName = "jointcal"
259 
260  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
261  """
262  Instantiate a JointcalTask.
263 
264  Parameters
265  ----------
266  butler : lsst.daf.persistence.Butler
267  The butler is passed to the refObjLoader constructor in case it is
268  needed. Ignored if the refObjLoader argument provides a loader directly.
269  Used to initialize the astrometry and photometry refObjLoaders.
270  profile_jointcal : bool
271  set to True to profile different stages of this jointcal run.
272  """
273  pipeBase.CmdLineTask.__init__(self, **kwargs)
274  self.profile_jointcal = profile_jointcal
275  self.makeSubtask("sourceSelector")
276  if self.config.doAstrometry:
277  self.makeSubtask('astrometryRefObjLoader', butler=butler)
278  if self.config.doPhotometry:
279  self.makeSubtask('photometryRefObjLoader', butler=butler)
280 
281  # To hold various computed metrics for use by tests
282  self.job = Job.load_metrics_package(subset='jointcal')
283 
284  # We don't need to persist config and metadata at this stage.
285  # In this way, we don't need to put a specific entry in the camera mapper policy file
286  def _getConfigName(self):
287  return None
288 
289  def _getMetadataName(self):
290  return None
291 
292  @classmethod
293  def _makeArgumentParser(cls):
294  """Create an argument parser"""
295  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
296  parser.add_argument("--profile_jointcal", default=False, action="store_true",
297  help="Profile steps of jointcal separately.")
298  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
299  ContainerClass=PerTractCcdDataIdContainer)
300  return parser
301 
302  def _build_ccdImage(self, dataRef, associations, jointcalControl):
303  """
304  Extract the necessary things from this dataRef to add a new ccdImage.
305 
306  Parameters
307  ----------
308  dataRef : lsst.daf.persistence.ButlerDataRef
309  dataRef to extract info from.
310  associations : lsst.jointcal.Associations
311  object to add the info to, to construct a new CcdImage
312  jointcalControl : jointcal.JointcalControl
313  control object for associations management
314 
315  Returns
316  ------
317  namedtuple
318  wcs : lsst.afw.geom.SkyWcs
319  the TAN WCS of this image, read from the calexp
320  key : namedtuple
321  a key to identify this dataRef by its visit and ccd ids
322  filter : str
323  this calexp's filter
324  """
325  if "visit" in dataRef.dataId.keys():
326  visit = dataRef.dataId["visit"]
327  else:
328  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
329 
330  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
331 
332  visitInfo = dataRef.get('calexp_visitInfo')
333  detector = dataRef.get('calexp_detector')
334  ccdId = detector.getId()
335  calib = dataRef.get('calexp_calib')
336  tanWcs = dataRef.get('calexp_wcs')
337  bbox = dataRef.get('calexp_bbox')
338  filt = dataRef.get('calexp_filter')
339  filterName = filt.getName()
340  fluxMag0 = calib.getFluxMag0()
341  photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
342 
343  goodSrc = self.sourceSelector.run(src)
344 
345  if len(goodSrc.sourceCat) == 0:
346  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
347  else:
348  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
349  associations.createCcdImage(goodSrc.sourceCat,
350  tanWcs,
351  visitInfo,
352  bbox,
353  filterName,
354  photoCalib,
355  detector,
356  visit,
357  ccdId,
358  jointcalControl)
359 
360  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
361  Key = collections.namedtuple('Key', ('visit', 'ccd'))
362  return Result(tanWcs, Key(visit, ccdId), filterName)
363 
364  @pipeBase.timeMethod
365  def run(self, dataRefs, profile_jointcal=False):
366  """
367  Jointly calibrate the astrometry and photometry across a set of images.
368 
369  Parameters
370  ----------
371  dataRefs : list of lsst.daf.persistence.ButlerDataRef
372  List of data references to the exposures to be fit.
373  profile_jointcal : bool
374  Profile the individual steps of jointcal.
375 
376  Returns
377  -------
378  pipe.base.Struct
379  struct containing:
380  * dataRefs: the provided data references that were fit (with updated WCSs)
381  * oldWcsList: the original WCS from each dataRef
382  * metrics: dictionary of internally-computed metrics for testing/validation.
383  """
384  if len(dataRefs) == 0:
385  raise ValueError('Need a non-empty list of data references!')
386 
387  exitStatus = 0 # exit status for shell
388 
389  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
390  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
391  associations = lsst.jointcal.Associations()
392 
393  visit_ccd_to_dataRef = {}
394  oldWcsList = []
395  filters = []
396  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
397  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
398  # We need the bounding-box of the focal plane for photometry visit models.
399  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
400  camera = dataRefs[0].get('camera', immediate=True)
401  self.focalPlaneBBox = camera.getFpBBox()
402  for ref in dataRefs:
403  result = self._build_ccdImage(ref, associations, jointcalControl)
404  oldWcsList.append(result.wcs)
405  visit_ccd_to_dataRef[result.key] = ref
406  filters.append(result.filter)
407  filters = collections.Counter(filters)
408 
409  associations.computeCommonTangentPoint()
410 
411  # Use external reference catalogs handled by LSST stack mechanism
412  # Get the bounding box overlapping all associated images
413  # ==> This is probably a bad idea to do it this way <== To be improved
414  bbox = associations.getRaDecBBox()
415  # with Python 3 this can be simplified to afwGeom.SpherePoint(*bbox.getCenter(), afwGeom.degrees)
416  bboxCenter = bbox.getCenter()
417  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
418  bboxMax = bbox.getMax()
419  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
420  radius = center.separation(corner).asRadians()
421 
422  # Get astrometry_net_data path
423  anDir = lsst.utils.getPackageDir('astrometry_net_data')
424  if anDir is None:
425  raise RuntimeError("astrometry_net_data is not setup")
426 
427  # Determine a default filter associated with the catalog. See DM-9093
428  defaultFilter = filters.most_common(1)[0][0]
429  self.log.debug("Using %s band for reference flux", defaultFilter)
430 
431  # TODO: need a better way to get the tract.
432  tract = dataRefs[0].dataId['tract']
433 
434  if self.config.doAstrometry:
435  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
436  name="astrometry",
437  refObjLoader=self.astrometryRefObjLoader,
438  fit_function=self._fit_astrometry,
439  profile_jointcal=profile_jointcal,
440  tract=tract)
441  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
442  else:
443  astrometry = Astrometry(None, None, None)
444 
445  if self.config.doPhotometry:
446  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
447  name="photometry",
448  refObjLoader=self.photometryRefObjLoader,
449  fit_function=self._fit_photometry,
450  profile_jointcal=profile_jointcal,
451  tract=tract,
452  filters=filters,
453  reject_bad_fluxes=True)
454  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
455  else:
456  photometry = Photometry(None, None)
457 
458  return pipeBase.Struct(dataRefs=dataRefs,
459  oldWcsList=oldWcsList,
460  job=self.job,
461  exitStatus=exitStatus)
462 
463  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
464  name="", refObjLoader=None, filters=[], fit_function=None,
465  tract=None, profile_jointcal=False, match_cut=3.0,
466  reject_bad_fluxes=False):
467  """Load reference catalog, perform the fit, and return the result.
468 
469  Parameters
470  ----------
471  associations : lsst.jointcal.Associations
472  The star/reference star associations to fit.
473  defaultFilter : str
474  filter to load from reference catalog.
475  center : lsst.afw.geom.SpherePoint
476  ICRS center of field to load from reference catalog.
477  radius : lsst.afw.geom.Angle
478  On-sky radius to load from reference catalog.
479  name : str
480  Name of thing being fit: "Astrometry" or "Photometry".
481  refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
482  Reference object loader to load from for fit.
483  filters : list of str, optional
484  List of filters to load from the reference catalog.
485  fit_function : function
486  function to call to perform fit (takes associations object).
487  tract : str
488  Name of tract currently being fit.
489  profile_jointcal : bool, optional
490  Separately profile the fitting step.
491  match_cut : float, optional
492  Radius in arcseconds to find cross-catalog matches to during
493  associations.associateCatalogs.
494  reject_bad_fluxes : bool, optional
495  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
496 
497  Returns
498  -------
499  Result of `fit_function()`
500  """
501  self.log.info("====== Now processing %s...", name)
502  # TODO: this should not print "trying to invert a singular transformation:"
503  # if it does that, something's not right about the WCS...
504  associations.associateCatalogs(match_cut)
505  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
506  associations.fittedStarListSize())
507 
508  skyCircle = refObjLoader.loadSkyCircle(center,
509  afwGeom.Angle(radius, afwGeom.radians),
510  defaultFilter)
511 
512  # Need memory contiguity to get reference filters as a vector.
513  if not skyCircle.refCat.isContiguous():
514  refCat = skyCircle.refCat.copy(deep=True)
515  else:
516  refCat = skyCircle.refCat
517 
518  # load the reference catalog fluxes.
519  # TODO: Simon will file a ticket for making this better (and making it use the color terms)
520  refFluxes = {}
521  refFluxErrs = {}
522  for filt in filters:
523  filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
524  refFluxes[filt] = refCat.get(filtKeys[0])
525  refFluxErrs[filt] = refCat.get(filtKeys[1])
526 
527  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
528  skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
529  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
530  associations.refStarListSize())
531 
532  associations.prepareFittedStars(self.config.minMeasurements)
533 
534  self._check_star_lists(associations, name)
535  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
536  associations.nFittedStarsWithAssociatedRefStar())
537  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
538  associations.fittedStarListSize())
539  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
540  associations.nCcdImagesValidForFit())
541 
542  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
543  dataName = "{}_{}".format(tract, defaultFilter)
544  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
545  result = fit_function(associations, dataName)
546  # TODO DM-12446: turn this into a "butler save" somehow.
547  # Save reference and measurement chi2 contributions for this data
548  if self.config.writeChi2ContributionFiles:
549  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
550  result.fit.saveChi2Contributions(baseName)
551 
552  return result
553 
554  def _check_star_lists(self, associations, name):
555  # TODO: these should be len(blah), but we need this properly wrapped first.
556  if associations.nCcdImagesValidForFit() == 0:
557  raise RuntimeError('No images in the ccdImageList!')
558  if associations.fittedStarListSize() == 0:
559  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
560  if associations.refStarListSize() == 0:
561  raise RuntimeError('No stars in the {} reference star list!'.format(name))
562 
563  def _fit_photometry(self, associations, dataName=None):
564  """
565  Fit the photometric data.
566 
567  Parameters
568  ----------
569  associations : lsst.jointcal.Associations
570  The star/reference star associations to fit.
571  dataName : str
572  Name of the data being processed (e.g. "1234_HSC-Y"), for
573  identifying debugging files.
574 
575  Returns
576  -------
577  namedtuple
578  fit : lsst.jointcal.PhotometryFit
579  The photometric fitter used to perform the fit.
580  model : lsst.jointcal.PhotometryModel
581  The photometric model that was fit.
582  """
583  self.log.info("=== Starting photometric fitting...")
584 
585  # TODO: should use pex.config.RegistryField here (see DM-9195)
586  if self.config.photometryModel == "constrained":
587  model = lsst.jointcal.ConstrainedPhotometryModel(associations.getCcdImageList(),
588  self.focalPlaneBBox,
589  visitOrder=self.config.photometryVisitOrder)
590  elif self.config.photometryModel == "simple":
591  model = lsst.jointcal.SimplePhotometryModel(associations.getCcdImageList())
592 
593  fit = lsst.jointcal.PhotometryFit(associations, model)
594  chi2 = fit.computeChi2()
595  # TODO DM-12446: turn this into a "butler save" somehow.
596  # Save reference and measurement chi2 contributions for this data
597  if self.config.writeChi2ContributionFiles:
598  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
599  fit.saveChi2Contributions(baseName)
600 
601  if not np.isfinite(chi2.chi2):
602  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
603  self.log.info("Initialized: %s", str(chi2))
604  # The constrained model needs the visit transfo fit first; the chip
605  # transfo is initialized from the singleFrame PhotoCalib, so it's close.
606  if self.config.photometryModel == "constrained":
607  # TODO: (related to DM-8046): implement Visit/Chip choice
608  fit.minimize("ModelVisit")
609  chi2 = fit.computeChi2()
610  self.log.info(str(chi2))
611  fit.minimize("Model")
612  chi2 = fit.computeChi2()
613  self.log.info(str(chi2))
614  fit.minimize("Fluxes")
615  chi2 = fit.computeChi2()
616  self.log.info(str(chi2))
617  fit.minimize("Model Fluxes")
618  chi2 = fit.computeChi2()
619  if not np.isfinite(chi2.chi2):
620  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
621  self.log.info("Fit prepared with %s", str(chi2))
622 
623  model.freezeErrorTransform()
624  self.log.debug("Photometry error scales are frozen.")
625 
626  chi2 = self._iterate_fit(associations,
627  fit,
628  model,
629  self.config.maxPhotometrySteps,
630  "photometry",
631  "Model Fluxes",
632  doRankUpdate=self.config.photometryDoRankUpdate)
633 
634  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
635  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
636  return Photometry(fit, model)
637 
638  def _fit_astrometry(self, associations, dataName=None):
639  """
640  Fit the astrometric data.
641 
642  Parameters
643  ----------
644  associations : lsst.jointcal.Associations
645  The star/reference star associations to fit.
646  dataName : str
647  Name of the data being processed (e.g. "1234_HSC-Y"), for
648  identifying debugging files.
649 
650  Returns
651  -------
652  namedtuple
653  fit : lsst.jointcal.AstrometryFit
654  The astrometric fitter used to perform the fit.
655  model : lsst.jointcal.AstrometryModel
656  The astrometric model that was fit.
657  sky_to_tan_projection : lsst.jointcal.ProjectionHandler
658  The model for the sky to tangent plane projection that was used in the fit.
659  """
660 
661  self.log.info("=== Starting astrometric fitting...")
662 
663  associations.deprojectFittedStars()
664 
665  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
666  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
667  # them so carefully?
668  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
669 
670  if self.config.astrometryModel == "constrained":
671  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
672  sky_to_tan_projection,
673  chipOrder=self.config.astrometryChipOrder,
674  visitOrder=self.config.astrometryVisitOrder)
675  elif self.config.astrometryModel == "simple":
676  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
677  sky_to_tan_projection,
678  self.config.useInputWcs,
679  nNotFit=0,
680  order=self.config.astrometrySimpleOrder)
681 
682  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.posError)
683  chi2 = fit.computeChi2()
684  # TODO DM-12446: turn this into a "butler save" somehow.
685  # Save reference and measurement chi2 contributions for this data
686  if self.config.writeChi2ContributionFiles:
687  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
688  fit.saveChi2Contributions(baseName)
689 
690  if not np.isfinite(chi2.chi2):
691  raise FloatingPointError('Initial chi2 is invalid: %s'%chi2)
692  self.log.info("Initialized: %s", str(chi2))
693  # The constrained model needs the visit transfo fit first; the chip
694  # transfo is initialized from the detector's cameraGeom, so it's close.
695  if self.config.astrometryModel == "constrained":
696  fit.minimize("DistortionsVisit")
697  chi2 = fit.computeChi2()
698  self.log.info(str(chi2))
699  fit.minimize("Distortions")
700  chi2 = fit.computeChi2()
701  self.log.info(str(chi2))
702  fit.minimize("Positions")
703  chi2 = fit.computeChi2()
704  self.log.info(str(chi2))
705  fit.minimize("Distortions Positions")
706  chi2 = fit.computeChi2()
707  self.log.info(str(chi2))
708  if not np.isfinite(chi2.chi2):
709  raise FloatingPointError('Pre-iteration chi2 is invalid: %s'%chi2)
710  self.log.info("Fit prepared with %s", str(chi2))
711 
712  chi2 = self._iterate_fit(associations,
713  fit,
714  model,
715  self.config.maxAstrometrySteps,
716  "astrometry",
717  "Distortions Positions",
718  doRankUpdate=self.config.astrometryDoRankUpdate)
719 
720  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
721  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
722 
723  return Astrometry(fit, model, sky_to_tan_projection)
724 
725  def _check_stars(self, associations):
726  """Count measured and reference stars per ccd and warn/log them."""
727  for ccdImage in associations.getCcdImageList():
728  nMeasuredStars, nRefStars = ccdImage.countStars()
729  self.log.debug("ccdImage %s has %s measured and %s reference stars",
730  ccdImage.getName(), nMeasuredStars, nRefStars)
731  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
732  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
733  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
734  if nRefStars < self.config.minRefStarsPerCcd:
735  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
736  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
737 
738  def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True):
739  """Run fit.minimize up to max_steps times, returning the final chi2."""
740 
741  for i in range(max_steps):
742  # outlier removal at 5 sigma.
743  r = fit.minimize(whatToFit, self.config.outlierRejectSigma, doRankUpdate=doRankUpdate)
744  chi2 = fit.computeChi2()
745  self._check_stars(associations)
746  if not np.isfinite(chi2.chi2):
747  raise FloatingPointError('Fit iteration chi2 is invalid: %s'%chi2)
748  self.log.info(str(chi2))
749  if r == MinimizeResult.Converged:
750  if doRankUpdate:
751  self.log.debug("fit has converged - no more outliers - redo minimization "
752  "one more time in case we have lost accuracy in rank update.")
753  # Redo minimization one more time in case we have lost accuracy in rank update
754  r = fit.minimize(whatToFit, 5) # outliers removal at 5 sigma.
755  chi2 = fit.computeChi2()
756  self.log.info("Fit completed with: %s", str(chi2))
757  break
758  elif r == MinimizeResult.Chi2Increased:
759  self.log.warn("still some ouliers but chi2 increases - retry")
760  elif r == MinimizeResult.Failed:
761  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
762  else:
763  raise RuntimeError("Unxepected return code from minimize().")
764  else:
765  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
766 
767  return chi2
768 
769  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
770  """
771  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
772 
773  Parameters
774  ----------
775  associations : lsst.jointcal.Associations
776  The star/reference star associations to fit.
777  model : lsst.jointcal.AstrometryModel
778  The astrometric model that was fit.
779  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
780  dict of ccdImage identifiers to dataRefs that were fit
781  """
782 
783  ccdImageList = associations.getCcdImageList()
784  for ccdImage in ccdImageList:
785  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
786  ccd = ccdImage.ccdId
787  visit = ccdImage.visit
788  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
789  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
790  skyWcs = model.makeSkyWcs(ccdImage)
791  try:
792  dataRef.put(skyWcs, 'jointcal_wcs')
793  except pexExceptions.Exception as e:
794  self.log.fatal('Failed to write updated Wcs: %s', str(e))
795  raise e
796 
797  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
798  """
799  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
800 
801  Parameters
802  ----------
803  associations : lsst.jointcal.Associations
804  The star/reference star associations to fit.
805  model : lsst.jointcal.PhotometryModel
806  The photoometric model that was fit.
807  visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
808  dict of ccdImage identifiers to dataRefs that were fit
809  """
810 
811  ccdImageList = associations.getCcdImageList()
812  for ccdImage in ccdImageList:
813  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
814  ccd = ccdImage.ccdId
815  visit = ccdImage.visit
816  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
817  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
818  photoCalib = model.toPhotoCalib(ccdImage)
819  try:
820  dataRef.put(photoCalib, 'jointcal_photoCalib')
821  except pexExceptions.Exception as e:
822  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
823  raise e
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:302
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:563
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:48
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:638
def _check_star_lists(self, associations, name)
Definition: jointcal.py:554
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:30
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:797
def _check_stars(self, associations)
Definition: jointcal.py:725
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:21
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:55
Photometry model with constraints, .
def add_measurement(job, name, value)
Definition: jointcal.py:30
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:466
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:769
def run(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:365
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:260
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit, doRankUpdate=True)
Definition: jointcal.py:738