lsst.jointcal  17.0.1-7-g8a03198
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 import astropy.units as u
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 from lsst.afw.image import fluxErrFromABMagErr
30 import lsst.afw.geom as afwGeom
31 import lsst.pex.exceptions as pexExceptions
32 import lsst.afw.table
34 from lsst.pipe.tasks.colorterms import ColortermLibrary
35 from lsst.verify import Job, Measurement
36 
37 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask
38 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
39 
40 from .dataIds import PerTractCcdDataIdContainer
41 
42 import lsst.jointcal
43 from lsst.jointcal import MinimizeResult
44 
45 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
46 
47 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
48 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
49 
50 
51 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
52 def add_measurement(job, name, value):
53  meas = Measurement(job.metrics[name], value)
54  job.measurements.insert(meas)
55 
56 
57 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
58  """Subclass of TaskRunner for jointcalTask
59 
60  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
61  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
62  single dataRef, are are called repeatedly). This class transforms the processed
63  arguments generated by the ArgumentParser into the arguments expected by
64  Jointcal.runDataRef().
65 
66  See pipeBase.TaskRunner for more information.
67  """
68 
69  @staticmethod
70  def getTargetList(parsedCmd, **kwargs):
71  """
72  Return a list of tuples per tract, each containing (dataRefs, kwargs).
73 
74  Jointcal operates on lists of dataRefs simultaneously.
75  """
76  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
77  kwargs['butler'] = parsedCmd.butler
78 
79  # organize data IDs by tract
80  refListDict = {}
81  for ref in parsedCmd.id.refList:
82  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
83  # we call runDataRef() once with each tract
84  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
85  return result
86 
87  def __call__(self, args):
88  """
89  Parameters
90  ----------
91  args
92  Arguments for Task.runDataRef()
93 
94  Returns
95  -------
96  pipe.base.Struct
97  if self.doReturnResults is False:
98 
99  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
100 
101  if self.doReturnResults is True:
102 
103  - ``result``: the result of calling jointcal.runDataRef()
104  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
105  """
106  exitStatus = 0 # exit status for shell
107 
108  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
109  dataRefList, kwargs = args
110  butler = kwargs.pop('butler')
111  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
112  result = None
113  try:
114  result = task.runDataRef(dataRefList, **kwargs)
115  exitStatus = result.exitStatus
116  job_path = butler.get('verify_job_filename')
117  result.job.write(job_path[0])
118  except Exception as e: # catch everything, sort it out later.
119  if self.doRaise:
120  raise e
121  else:
122  exitStatus = 1
123  eName = type(e).__name__
124  tract = dataRefList[0].dataId['tract']
125  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
126 
127  # Put the butler back into kwargs for the other Tasks.
128  kwargs['butler'] = butler
129  if self.doReturnResults:
130  return pipeBase.Struct(result=result, exitStatus=exitStatus)
131  else:
132  return pipeBase.Struct(exitStatus=exitStatus)
133 
134 
135 class JointcalConfig(pexConfig.Config):
136  """Config for JointcalTask"""
137 
138  doAstrometry = pexConfig.Field(
139  doc="Fit astrometry and write the fitted result.",
140  dtype=bool,
141  default=True
142  )
143  doPhotometry = pexConfig.Field(
144  doc="Fit photometry and write the fitted result.",
145  dtype=bool,
146  default=True
147  )
148  coaddName = pexConfig.Field(
149  doc="Type of coadd, typically deep or goodSeeing",
150  dtype=str,
151  default="deep"
152  )
153  positionErrorPedestal = pexConfig.Field(
154  doc="Systematic term to apply to the measured position error (pixels)",
155  dtype=float,
156  default=0.02,
157  )
158  photometryErrorPedestal = pexConfig.Field(
159  doc="Systematic term to apply to the measured error on flux or magnitude as a "
160  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
161  dtype=float,
162  default=0.0,
163  )
164  # TODO: DM-6885 matchCut should be an afw.geom.Angle
165  matchCut = pexConfig.Field(
166  doc="Matching radius between fitted and reference stars (arcseconds)",
167  dtype=float,
168  default=3.0,
169  )
170  minMeasurements = pexConfig.Field(
171  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
172  dtype=int,
173  default=2,
174  )
175  minMeasuredStarsPerCcd = pexConfig.Field(
176  doc="Minimum number of measuredStars per ccdImage before printing warnings",
177  dtype=int,
178  default=100,
179  )
180  minRefStarsPerCcd = pexConfig.Field(
181  doc="Minimum number of measuredStars per ccdImage before printing warnings",
182  dtype=int,
183  default=30,
184  )
185  allowLineSearch = pexConfig.Field(
186  doc="Allow a line search during minimization, if it is reasonable for the model"
187  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
188  dtype=bool,
189  default=False
190  )
191  astrometrySimpleOrder = pexConfig.Field(
192  doc="Polynomial order for fitting the simple astrometry model.",
193  dtype=int,
194  default=3,
195  )
196  astrometryChipOrder = pexConfig.Field(
197  doc="Order of the per-chip transform for the constrained astrometry model.",
198  dtype=int,
199  default=1,
200  )
201  astrometryVisitOrder = pexConfig.Field(
202  doc="Order of the per-visit transform for the constrained astrometry model.",
203  dtype=int,
204  default=5,
205  )
206  useInputWcs = pexConfig.Field(
207  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
208  dtype=bool,
209  default=True,
210  )
211  astrometryModel = pexConfig.ChoiceField(
212  doc="Type of model to fit to astrometry",
213  dtype=str,
214  default="constrained",
215  allowed={"simple": "One polynomial per ccd",
216  "constrained": "One polynomial per ccd, and one polynomial per visit"}
217  )
218  photometryModel = pexConfig.ChoiceField(
219  doc="Type of model to fit to photometry",
220  dtype=str,
221  default="constrainedMagnitude",
222  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
223  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
224  " fitting in flux space.",
225  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
226  " fitting in magnitude space.",
227  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
228  " fitting in magnitude space.",
229  }
230  )
231  applyColorTerms = pexConfig.Field(
232  doc="Apply photometric color terms to reference stars?"
233  "Requires that colorterms be set to a ColortermLibrary",
234  dtype=bool,
235  default=False
236  )
237  colorterms = pexConfig.ConfigField(
238  doc="Library of photometric reference catalog name to color term dict.",
239  dtype=ColortermLibrary,
240  )
241  photometryVisitOrder = pexConfig.Field(
242  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
243  dtype=int,
244  default=7,
245  )
246  photometryDoRankUpdate = pexConfig.Field(
247  doc="Do the rank update step during minimization. "
248  "Skipping this can help deal with models that are too non-linear.",
249  dtype=bool,
250  default=True,
251  )
252  astrometryDoRankUpdate = pexConfig.Field(
253  doc="Do the rank update step during minimization (should not change the astrometry fit). "
254  "Skipping this can help deal with models that are too non-linear.",
255  dtype=bool,
256  default=True,
257  )
258  outlierRejectSigma = pexConfig.Field(
259  doc="How many sigma to reject outliers at during minimization.",
260  dtype=float,
261  default=5.0,
262  )
263  maxPhotometrySteps = pexConfig.Field(
264  doc="Maximum number of minimize iterations to take when fitting photometry.",
265  dtype=int,
266  default=20,
267  )
268  maxAstrometrySteps = pexConfig.Field(
269  doc="Maximum number of minimize iterations to take when fitting photometry.",
270  dtype=int,
271  default=20,
272  )
273  astrometryRefObjLoader = pexConfig.ConfigurableField(
274  target=LoadIndexedReferenceObjectsTask,
275  doc="Reference object loader for astrometric fit",
276  )
277  photometryRefObjLoader = pexConfig.ConfigurableField(
278  target=LoadIndexedReferenceObjectsTask,
279  doc="Reference object loader for photometric fit",
280  )
281  sourceSelector = sourceSelectorRegistry.makeField(
282  doc="How to select sources for cross-matching",
283  default="astrometry"
284  )
285  astrometryReferenceSelector = pexConfig.ConfigurableField(
286  target=ReferenceSourceSelectorTask,
287  doc="How to down-select the loaded astrometry reference catalog.",
288  )
289  photometryReferenceSelector = pexConfig.ConfigurableField(
290  target=ReferenceSourceSelectorTask,
291  doc="How to down-select the loaded photometry reference catalog.",
292  )
293  writeInitMatrix = pexConfig.Field(
294  dtype=bool,
295  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
296  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
297  "Note that these files are the dense versions of the matrix, and so may be very large.",
298  default=False
299  )
300  writeChi2FilesInitialFinal = pexConfig.Field(
301  dtype=bool,
302  doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.",
303  default=False
304  )
305  writeChi2FilesOuterLoop = pexConfig.Field(
306  dtype=bool,
307  doc="Write .csv files containing the contributions to chi2 for the outer fit loop.",
308  default=False
309  )
310  sourceFluxType = pexConfig.Field(
311  dtype=str,
312  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
313  default='Calib'
314  )
315 
316  def validate(self):
317  super().validate()
318  if self.applyColorTerms and len(self.colorterms.data) == 0:
319  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
320  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
321 
322  def setDefaults(self):
323  sourceSelector = self.sourceSelector["astrometry"]
324  sourceSelector.setDefaults()
325  # don't want to lose existing flags, just add to them.
326  sourceSelector.badFlags.extend(["slot_Shape_flag"])
327  # This should be used to set the FluxField value in jointcal::JointcalControl
328  sourceSelector.sourceFluxType = self.sourceFluxType
329 
330 
331 class JointcalTask(pipeBase.CmdLineTask):
332  """Jointly astrometrically and photometrically calibrate a group of images."""
333 
334  ConfigClass = JointcalConfig
335  RunnerClass = JointcalRunner
336  _DefaultName = "jointcal"
337 
338  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
339  """
340  Instantiate a JointcalTask.
341 
342  Parameters
343  ----------
344  butler : `lsst.daf.persistence.Butler`
345  The butler is passed to the refObjLoader constructor in case it is
346  needed. Ignored if the refObjLoader argument provides a loader directly.
347  Used to initialize the astrometry and photometry refObjLoaders.
348  profile_jointcal : `bool`
349  Set to True to profile different stages of this jointcal run.
350  """
351  pipeBase.CmdLineTask.__init__(self, **kwargs)
352  self.profile_jointcal = profile_jointcal
353  self.makeSubtask("sourceSelector")
354  if self.config.doAstrometry:
355  self.makeSubtask('astrometryRefObjLoader', butler=butler)
356  self.makeSubtask("astrometryReferenceSelector")
357  else:
359  if self.config.doPhotometry:
360  self.makeSubtask('photometryRefObjLoader', butler=butler)
361  self.makeSubtask("photometryReferenceSelector")
362  else:
364 
365  # To hold various computed metrics for use by tests
366  self.job = Job.load_metrics_package(subset='jointcal')
367 
368  # We don't currently need to persist the metadata.
369  # If we do in the future, we will have to add appropriate dataset templates
370  # to each obs package (the metadata template should look like `jointcal_wcs`).
371  def _getMetadataName(self):
372  return None
373 
374  @classmethod
375  def _makeArgumentParser(cls):
376  """Create an argument parser"""
377  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
378  parser.add_argument("--profile_jointcal", default=False, action="store_true",
379  help="Profile steps of jointcal separately.")
380  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
381  ContainerClass=PerTractCcdDataIdContainer)
382  return parser
383 
384  def _build_ccdImage(self, dataRef, associations, jointcalControl):
385  """
386  Extract the necessary things from this dataRef to add a new ccdImage.
387 
388  Parameters
389  ----------
390  dataRef : `lsst.daf.persistence.ButlerDataRef`
391  DataRef to extract info from.
392  associations : `lsst.jointcal.Associations`
393  Object to add the info to, to construct a new CcdImage
394  jointcalControl : `jointcal.JointcalControl`
395  Control object for associations management
396 
397  Returns
398  ------
399  namedtuple
400  ``wcs``
401  The TAN WCS of this image, read from the calexp
402  (`lsst.afw.geom.SkyWcs`).
403  ``key``
404  A key to identify this dataRef by its visit and ccd ids
405  (`namedtuple`).
406  ``filter``
407  This calexp's filter (`str`).
408  """
409  if "visit" in dataRef.dataId.keys():
410  visit = dataRef.dataId["visit"]
411  else:
412  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
413 
414  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
415 
416  visitInfo = dataRef.get('calexp_visitInfo')
417  detector = dataRef.get('calexp_detector')
418  ccdId = detector.getId()
419  photoCalib = dataRef.get('calexp_photoCalib')
420  tanWcs = dataRef.get('calexp_wcs')
421  bbox = dataRef.get('calexp_bbox')
422  filt = dataRef.get('calexp_filter')
423  filterName = filt.getName()
424 
425  goodSrc = self.sourceSelector.run(src)
426 
427  if len(goodSrc.sourceCat) == 0:
428  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
429  else:
430  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
431  associations.createCcdImage(goodSrc.sourceCat,
432  tanWcs,
433  visitInfo,
434  bbox,
435  filterName,
436  photoCalib,
437  detector,
438  visit,
439  ccdId,
440  jointcalControl)
441 
442  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
443  Key = collections.namedtuple('Key', ('visit', 'ccd'))
444  return Result(tanWcs, Key(visit, ccdId), filterName)
445 
446  @pipeBase.timeMethod
447  def runDataRef(self, dataRefs, profile_jointcal=False):
448  """
449  Jointly calibrate the astrometry and photometry across a set of images.
450 
451  Parameters
452  ----------
453  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
454  List of data references to the exposures to be fit.
455  profile_jointcal : `bool`
456  Profile the individual steps of jointcal.
457 
458  Returns
459  -------
460  result : `lsst.pipe.base.Struct`
461  Struct of metadata from the fit, containing:
462 
463  ``dataRefs``
464  The provided data references that were fit (with updated WCSs)
465  ``oldWcsList``
466  The original WCS from each dataRef
467  ``metrics``
468  Dictionary of internally-computed metrics for testing/validation.
469  """
470  if len(dataRefs) == 0:
471  raise ValueError('Need a non-empty list of data references!')
472 
473  exitStatus = 0 # exit status for shell
474 
475  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
476  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
477  associations = lsst.jointcal.Associations()
478 
479  visit_ccd_to_dataRef = {}
480  oldWcsList = []
481  filters = []
482  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
483  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
484  # We need the bounding-box of the focal plane for photometry visit models.
485  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
486  camera = dataRefs[0].get('camera', immediate=True)
487  self.focalPlaneBBox = camera.getFpBBox()
488  for ref in dataRefs:
489  result = self._build_ccdImage(ref, associations, jointcalControl)
490  oldWcsList.append(result.wcs)
491  visit_ccd_to_dataRef[result.key] = ref
492  filters.append(result.filter)
493  filters = collections.Counter(filters)
494 
495  associations.computeCommonTangentPoint()
496 
497  # Use external reference catalogs handled by LSST stack mechanism
498  # Get the bounding box overlapping all associated images
499  # ==> This is probably a bad idea to do it this way <== To be improved
500  bbox = associations.getRaDecBBox()
501  bboxCenter = bbox.getCenter()
502  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
503  bboxMax = bbox.getMax()
504  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
505  radius = center.separation(corner).asRadians()
506 
507  # Get astrometry_net_data path
508  anDir = lsst.utils.getPackageDir('astrometry_net_data')
509  if anDir is None:
510  raise RuntimeError("astrometry_net_data is not setup")
511 
512  # Determine a default filter associated with the catalog. See DM-9093
513  defaultFilter = filters.most_common(1)[0][0]
514  self.log.debug("Using %s band for reference flux", defaultFilter)
515 
516  # TODO: need a better way to get the tract.
517  tract = dataRefs[0].dataId['tract']
518 
519  if self.config.doAstrometry:
520  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
521  name="astrometry",
522  refObjLoader=self.astrometryRefObjLoader,
523  referenceSelector=self.astrometryReferenceSelector,
524  fit_function=self._fit_astrometry,
525  profile_jointcal=profile_jointcal,
526  tract=tract)
527  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
528  else:
529  astrometry = Astrometry(None, None, None)
530 
531  if self.config.doPhotometry:
532  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
533  name="photometry",
534  refObjLoader=self.photometryRefObjLoader,
535  referenceSelector=self.photometryReferenceSelector,
536  fit_function=self._fit_photometry,
537  profile_jointcal=profile_jointcal,
538  tract=tract,
539  filters=filters,
540  reject_bad_fluxes=True)
541  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
542  else:
543  photometry = Photometry(None, None)
544 
545  return pipeBase.Struct(dataRefs=dataRefs,
546  oldWcsList=oldWcsList,
547  job=self.job,
548  astrometryRefObjLoader=self.astrometryRefObjLoader,
549  photometryRefObjLoader=self.photometryRefObjLoader,
550  defaultFilter=defaultFilter,
551  exitStatus=exitStatus)
552 
553  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
554  name="", refObjLoader=None, referenceSelector=None,
555  filters=[], fit_function=None,
556  tract=None, profile_jointcal=False, match_cut=3.0,
557  reject_bad_fluxes=False):
558  """Load reference catalog, perform the fit, and return the result.
559 
560  Parameters
561  ----------
562  associations : `lsst.jointcal.Associations`
563  The star/reference star associations to fit.
564  defaultFilter : `str`
565  filter to load from reference catalog.
566  center : `lsst.afw.geom.SpherePoint`
567  ICRS center of field to load from reference catalog.
568  radius : `lsst.afw.geom.Angle`
569  On-sky radius to load from reference catalog.
570  name : `str`
571  Name of thing being fit: "Astrometry" or "Photometry".
572  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
573  Reference object loader to load from for fit.
574  filters : `list` of `str`, optional
575  List of filters to load from the reference catalog.
576  fit_function : callable
577  Function to call to perform fit (takes associations object).
578  tract : `str`
579  Name of tract currently being fit.
580  profile_jointcal : `bool`, optional
581  Separately profile the fitting step.
582  match_cut : `float`, optional
583  Radius in arcseconds to find cross-catalog matches to during
584  associations.associateCatalogs.
585  reject_bad_fluxes : `bool`, optional
586  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
587 
588  Returns
589  -------
590  result : `Photometry` or `Astrometry`
591  Result of `fit_function()`
592  """
593  self.log.info("====== Now processing %s...", name)
594  # TODO: this should not print "trying to invert a singular transformation:"
595  # if it does that, something's not right about the WCS...
596  associations.associateCatalogs(match_cut)
597  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
598  associations.fittedStarListSize())
599 
600  applyColorterms = False if name == "Astrometry" else self.config.applyColorTerms
601  if name == "Astrometry":
602  referenceSelector = self.config.astrometryReferenceSelector
603  elif name == "Photometry":
604  referenceSelector = self.config.photometryReferenceSelector
605  refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
606  center, radius, defaultFilter,
607  applyColorterms=applyColorterms)
608 
609  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
610  fluxField, reject_bad_fluxes)
611  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
612  associations.refStarListSize())
613 
614  associations.prepareFittedStars(self.config.minMeasurements)
615 
616  self._check_star_lists(associations, name)
617  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
618  associations.nFittedStarsWithAssociatedRefStar())
619  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
620  associations.fittedStarListSize())
621  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
622  associations.nCcdImagesValidForFit())
623 
624  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
625  dataName = "{}_{}".format(tract, defaultFilter)
626  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
627  result = fit_function(associations, dataName)
628  # TODO DM-12446: turn this into a "butler save" somehow.
629  # Save reference and measurement chi2 contributions for this data
630  if self.config.writeChi2FilesInitialFinal:
631  baseName = f"{name}_final_chi2-{dataName}"
632  result.fit.saveChi2Contributions(baseName+"{type}")
633 
634  return result
635 
636  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
637  applyColorterms=False):
638  """Load the necessary reference catalog sources, convert fluxes to
639  correct units, and apply color term corrections if requested.
640 
641  Parameters
642  ----------
643  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
644  The reference catalog loader to use to get the data.
645  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
646  Source selector to apply to loaded reference catalog.
647  center : `lsst.geom.SpherePoint`
648  The center around which to load sources.
649  radius : `lsst.geom.Angle`
650  The radius around ``center`` to load sources in.
651  filterName : `str`
652  The name of the camera filter to load fluxes for.
653  applyColorterms : `bool`
654  Apply colorterm corrections to the refcat for ``filterName``?
655 
656  Returns
657  -------
658  refCat : `lsst.afw.table.SimpleCatalog`
659  The loaded reference catalog.
660  fluxField : `str`
661  The name of the reference catalog flux field appropriate for ``filterName``.
662  """
663  skyCircle = refObjLoader.loadSkyCircle(center,
664  afwGeom.Angle(radius, afwGeom.radians),
665  filterName)
666 
667  selected = referenceSelector.run(skyCircle.refCat)
668  # Need memory contiguity to get reference filters as a vector.
669  if not selected.sourceCat.isContiguous():
670  refCat = selected.sourceCat.copy(deep=True)
671  else:
672  refCat = selected.sourceCat
673 
674  if applyColorterms:
675  try:
676  refCatName = refObjLoader.ref_dataset_name
677  except AttributeError:
678  # NOTE: we need this try:except: block in place until we've completely removed a.net support.
679  raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
680  self.log.info("Applying color terms for filterName=%r reference catalog=%s",
681  filterName, refCatName)
682  colorterm = self.config.colorterms.getColorterm(
683  filterName=filterName, photoCatName=refCatName, doRaise=True)
684 
685  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
686  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
687  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
688  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
689 
690  return refCat, skyCircle.fluxField
691 
692  def _check_star_lists(self, associations, name):
693  # TODO: these should be len(blah), but we need this properly wrapped first.
694  if associations.nCcdImagesValidForFit() == 0:
695  raise RuntimeError('No images in the ccdImageList!')
696  if associations.fittedStarListSize() == 0:
697  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
698  if associations.refStarListSize() == 0:
699  raise RuntimeError('No stars in the {} reference star list!'.format(name))
700 
701  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
702  writeChi2Name=None):
703  """Compute chi2, log it, validate the model, and return chi2.
704 
705  Parameters
706  ----------
707  associations : `lsst.jointcal.Associations`
708  The star/reference star associations to fit.
709  fit : `lsst.jointcal.FitterBase`
710  The fitter to use for minimization.
711  model : `lsst.jointcal.Model`
712  The model being fit.
713  chi2Label : str, optional
714  Label to describe the chi2 (e.g. "Initialized", "Final").
715  writeChi2Name : `str`, optional
716  Filename prefix to write the chi2 contributions to.
717  Do not supply an extension: an appropriate one will be added.
718 
719  Returns
720  -------
721  chi2: `lsst.jointcal.Chi2Accumulator`
722  The chi2 object for the current fitter and model.
723 
724  Raises
725  ------
726  FloatingPointError
727  Raised if chi2 is infinite or NaN.
728  ValueError
729  Raised if the model is not valid.
730  """
731  if writeChi2Name is not None:
732  fit.saveChi2Contributions(writeChi2Name+"{type}")
733  self.log.info("Wrote chi2 contributions files: %s", writeChi2Name)
734 
735  chi2 = fit.computeChi2()
736  self.log.info("%s %s", chi2Label, chi2)
737  self._check_stars(associations)
738  if not np.isfinite(chi2.chi2):
739  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
740  if not model.validate(associations.getCcdImageList()):
741  raise ValueError("Model is not valid: check log messages for warnings.")
742  return chi2
743 
744  def _fit_photometry(self, associations, dataName=None):
745  """
746  Fit the photometric data.
747 
748  Parameters
749  ----------
750  associations : `lsst.jointcal.Associations`
751  The star/reference star associations to fit.
752  dataName : `str`
753  Name of the data being processed (e.g. "1234_HSC-Y"), for
754  identifying debugging files.
755 
756  Returns
757  -------
758  fit_result : `namedtuple`
759  fit : `lsst.jointcal.PhotometryFit`
760  The photometric fitter used to perform the fit.
761  model : `lsst.jointcal.PhotometryModel`
762  The photometric model that was fit.
763  """
764  self.log.info("=== Starting photometric fitting...")
765 
766  # TODO: should use pex.config.RegistryField here (see DM-9195)
767  if self.config.photometryModel == "constrainedFlux":
768  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
769  self.focalPlaneBBox,
770  visitOrder=self.config.photometryVisitOrder,
771  errorPedestal=self.config.photometryErrorPedestal)
772  # potentially nonlinear problem, so we may need a line search to converge.
773  doLineSearch = self.config.allowLineSearch
774  elif self.config.photometryModel == "constrainedMagnitude":
775  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
776  self.focalPlaneBBox,
777  visitOrder=self.config.photometryVisitOrder,
778  errorPedestal=self.config.photometryErrorPedestal)
779  # potentially nonlinear problem, so we may need a line search to converge.
780  doLineSearch = self.config.allowLineSearch
781  elif self.config.photometryModel == "simpleFlux":
782  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
783  errorPedestal=self.config.photometryErrorPedestal)
784  doLineSearch = False # purely linear in model parameters, so no line search needed
785  elif self.config.photometryModel == "simpleMagnitude":
786  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
787  errorPedestal=self.config.photometryErrorPedestal)
788  doLineSearch = False # purely linear in model parameters, so no line search needed
789 
790  fit = lsst.jointcal.PhotometryFit(associations, model)
791  # TODO DM-12446: turn this into a "butler save" somehow.
792  # Save reference and measurement chi2 contributions for this data
793  if self.config.writeChi2FilesInitialFinal:
794  baseName = f"photometry_initial_chi2-{dataName}"
795  else:
796  baseName = None
797  self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
798 
799  def getChi2Name(whatToFit):
800  if self.config.writeChi2FilesOuterLoop:
801  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
802  else:
803  return None
804 
805  # The constrained model needs the visit transform fit first; the chip
806  # transform is initialized from the singleFrame PhotoCalib, so it's close.
807  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
808  if self.config.photometryModel.startswith("constrained"):
809  # no line search: should be purely (or nearly) linear,
810  # and we want a large step size to initialize with.
811  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
812  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("ModelVisit"))
813  dumpMatrixFile = "" # so we don't redo the output on the next step
814 
815  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
816  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Model"))
817 
818  fit.minimize("Fluxes") # no line search: always purely linear.
819  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Fluxes"))
820 
821  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
822  self._logChi2AndValidate(associations, fit, model, "Fit prepared",
823  writeChi2Name=getChi2Name("ModelFluxes"))
824 
825  model.freezeErrorTransform()
826  self.log.debug("Photometry error scales are frozen.")
827 
828  chi2 = self._iterate_fit(associations,
829  fit,
830  self.config.maxPhotometrySteps,
831  "photometry",
832  "Model Fluxes",
833  doRankUpdate=self.config.photometryDoRankUpdate,
834  doLineSearch=doLineSearch,
835  dataName=dataName)
836 
837  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
838  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
839  return Photometry(fit, model)
840 
841  def _fit_astrometry(self, associations, dataName=None):
842  """
843  Fit the astrometric data.
844 
845  Parameters
846  ----------
847  associations : `lsst.jointcal.Associations`
848  The star/reference star associations to fit.
849  dataName : `str`
850  Name of the data being processed (e.g. "1234_HSC-Y"), for
851  identifying debugging files.
852 
853  Returns
854  -------
855  fit_result : `namedtuple`
856  fit : `lsst.jointcal.AstrometryFit`
857  The astrometric fitter used to perform the fit.
858  model : `lsst.jointcal.AstrometryModel`
859  The astrometric model that was fit.
860  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
861  The model for the sky to tangent plane projection that was used in the fit.
862  """
863 
864  self.log.info("=== Starting astrometric fitting...")
865 
866  associations.deprojectFittedStars()
867 
868  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
869  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
870  # them so carefully?
871  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
872 
873  if self.config.astrometryModel == "constrained":
874  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
875  sky_to_tan_projection,
876  chipOrder=self.config.astrometryChipOrder,
877  visitOrder=self.config.astrometryVisitOrder)
878  elif self.config.astrometryModel == "simple":
879  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
880  sky_to_tan_projection,
881  self.config.useInputWcs,
882  nNotFit=0,
883  order=self.config.astrometrySimpleOrder)
884 
885  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
886  # TODO DM-12446: turn this into a "butler save" somehow.
887  # Save reference and measurement chi2 contributions for this data
888  if self.config.writeChi2FilesInitialFinal:
889  baseName = f"astrometry_initial_chi2-{dataName}"
890  else:
891  baseName = None
892  self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
893 
894  def getChi2Name(whatToFit):
895  if self.config.writeChi2FilesOuterLoop:
896  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
897  else:
898  return None
899 
900  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
901  # The constrained model needs the visit transform fit first; the chip
902  # transform is initialized from the detector's cameraGeom, so it's close.
903  if self.config.astrometryModel == "constrained":
904  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
905  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("DistortionsVisit"))
906  dumpMatrixFile = "" # so we don't redo the output on the next step
907 
908  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
909  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Distortions"))
910 
911  fit.minimize("Positions")
912  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Positions"))
913 
914  fit.minimize("Distortions Positions")
915  self._logChi2AndValidate(associations, fit, model, "Fit prepared",
916  writeChi2Name=getChi2Name("DistortionsPositions"))
917 
918  chi2 = self._iterate_fit(associations,
919  fit,
920  self.config.maxAstrometrySteps,
921  "astrometry",
922  "Distortions Positions",
923  doRankUpdate=self.config.astrometryDoRankUpdate,
924  dataName=dataName)
925 
926  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
927  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
928 
929  return Astrometry(fit, model, sky_to_tan_projection)
930 
931  def _check_stars(self, associations):
932  """Count measured and reference stars per ccd and warn/log them."""
933  for ccdImage in associations.getCcdImageList():
934  nMeasuredStars, nRefStars = ccdImage.countStars()
935  self.log.debug("ccdImage %s has %s measured and %s reference stars",
936  ccdImage.getName(), nMeasuredStars, nRefStars)
937  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
938  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
939  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
940  if nRefStars < self.config.minRefStarsPerCcd:
941  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
942  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
943 
944  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
945  dataName="",
946  doRankUpdate=True,
947  doLineSearch=False):
948  """Run fitter.minimize up to max_steps times, returning the final chi2.
949 
950  Parameters
951  ----------
952  associations : `lsst.jointcal.Associations`
953  The star/reference star associations to fit.
954  fitter : `lsst.jointcal.FitterBase`
955  The fitter to use for minimization.
956  max_steps : `int`
957  Maximum number of steps to run outlier rejection before declaring
958  convergence failure.
959  name : {'photometry' or 'astrometry'}
960  What type of data are we fitting (for logs and debugging files).
961  whatToFit : `str`
962  Passed to ``fitter.minimize()`` to define the parameters to fit.
963  dataName : `str`, optional
964  Descriptive name for this dataset (e.g. tract and filter),
965  for debugging.
966  doRankUpdate : `bool`, optional
967  Do an Eigen rank update during minimization, or recompute the full
968  matrix and gradient?
969  doLineSearch : `bool`, optional
970  Do a line search for the optimum step during minimization?
971 
972  Returns
973  -------
974  chi2: `lsst.jointcal.Chi2Statistic`
975  The final chi2 after the fit converges, or is forced to end.
976 
977  Raises
978  ------
979  FloatingPointError
980  Raised if the fitter fails with a non-finite value.
981  RuntimeError
982  Raised if the fitter fails for some other reason;
983  log messages will provide further details.
984  """
985  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
986  for i in range(max_steps):
987  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
988  result = fitter.minimize(whatToFit,
989  self.config.outlierRejectSigma,
990  doRankUpdate=doRankUpdate,
991  doLineSearch=doLineSearch,
992  dumpMatrixFile=dumpMatrixFile)
993  dumpMatrixFile = "" # clear it so we don't write the matrix again.
994  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
995  writeChi2Name=writeChi2Name)
996 
997  if result == MinimizeResult.Converged:
998  if doRankUpdate:
999  self.log.debug("fit has converged - no more outliers - redo minimization "
1000  "one more time in case we have lost accuracy in rank update.")
1001  # Redo minimization one more time in case we have lost accuracy in rank update
1002  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1003  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1004 
1005  # log a message for a large final chi2, TODO: DM-15247 for something better
1006  if chi2.chi2/chi2.ndof >= 4.0:
1007  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1008 
1009  break
1010  elif result == MinimizeResult.Chi2Increased:
1011  self.log.warn("still some outliers but chi2 increases - retry")
1012  elif result == MinimizeResult.NonFinite:
1013  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
1014  # TODO DM-12446: turn this into a "butler save" somehow.
1015  fitter.saveChi2Contributions(filename)
1016  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1017  raise FloatingPointError(msg.format(filename))
1018  elif result == MinimizeResult.Failed:
1019  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1020  else:
1021  raise RuntimeError("Unxepected return code from minimize().")
1022  else:
1023  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1024 
1025  return chi2
1026 
1027  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1028  """
1029  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1030 
1031  Parameters
1032  ----------
1033  associations : `lsst.jointcal.Associations`
1034  The star/reference star associations to fit.
1035  model : `lsst.jointcal.AstrometryModel`
1036  The astrometric model that was fit.
1037  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1038  Dict of ccdImage identifiers to dataRefs that were fit.
1039  """
1040 
1041  ccdImageList = associations.getCcdImageList()
1042  for ccdImage in ccdImageList:
1043  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1044  ccd = ccdImage.ccdId
1045  visit = ccdImage.visit
1046  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1047  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
1048  skyWcs = model.makeSkyWcs(ccdImage)
1049  try:
1050  dataRef.put(skyWcs, 'jointcal_wcs')
1051  except pexExceptions.Exception as e:
1052  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1053  raise e
1054 
1055  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1056  """
1057  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1058 
1059  Parameters
1060  ----------
1061  associations : `lsst.jointcal.Associations`
1062  The star/reference star associations to fit.
1063  model : `lsst.jointcal.PhotometryModel`
1064  The photoometric model that was fit.
1065  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1066  Dict of ccdImage identifiers to dataRefs that were fit.
1067  """
1068 
1069  ccdImageList = associations.getCcdImageList()
1070  for ccdImage in ccdImageList:
1071  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1072  ccd = ccdImage.ccdId
1073  visit = ccdImage.visit
1074  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1075  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1076  photoCalib = model.toPhotoCalib(ccdImage)
1077  try:
1078  dataRef.put(photoCalib, 'jointcal_photoCalib')
1079  except pexExceptions.Exception as e:
1080  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1081  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:447
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
Definition: jointcal.py:637
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:384
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:744
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:70
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:841
def _check_star_lists(self, associations, name)
Definition: jointcal.py:692
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", writeChi2Name=None)
Definition: jointcal.py:702
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:947
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1055
def _check_stars(self, associations)
Definition: jointcal.py:931
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:52
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:557
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1027
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:338