lsst.jointcal  17.0.1-3-g18e75bb+1
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 import astropy.units as u
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.afw.image as afwImage
30 from lsst.afw.image import fluxErrFromABMagErr
31 import lsst.afw.geom as afwGeom
32 import lsst.pex.exceptions as pexExceptions
33 import lsst.afw.table
35 from lsst.pipe.tasks.colorterms import ColortermLibrary
36 from lsst.verify import Job, Measurement
37 
38 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask
39 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40 
41 from .dataIds import PerTractCcdDataIdContainer
42 
43 import lsst.jointcal
44 from lsst.jointcal import MinimizeResult
45 
46 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
47 
48 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
49 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
50 
51 
52 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
53 def add_measurement(job, name, value):
54  meas = Measurement(job.metrics[name], value)
55  job.measurements.insert(meas)
56 
57 
58 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
59  """Subclass of TaskRunner for jointcalTask
60 
61  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
62  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
63  single dataRef, are are called repeatedly). This class transforms the processed
64  arguments generated by the ArgumentParser into the arguments expected by
65  Jointcal.runDataRef().
66 
67  See pipeBase.TaskRunner for more information.
68  """
69 
70  @staticmethod
71  def getTargetList(parsedCmd, **kwargs):
72  """
73  Return a list of tuples per tract, each containing (dataRefs, kwargs).
74 
75  Jointcal operates on lists of dataRefs simultaneously.
76  """
77  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
78  kwargs['butler'] = parsedCmd.butler
79 
80  # organize data IDs by tract
81  refListDict = {}
82  for ref in parsedCmd.id.refList:
83  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
84  # we call runDataRef() once with each tract
85  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
86  return result
87 
88  def __call__(self, args):
89  """
90  Parameters
91  ----------
92  args
93  Arguments for Task.runDataRef()
94 
95  Returns
96  -------
97  pipe.base.Struct
98  if self.doReturnResults is False:
99 
100  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
101 
102  if self.doReturnResults is True:
103 
104  - ``result``: the result of calling jointcal.runDataRef()
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106  """
107  exitStatus = 0 # exit status for shell
108 
109  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
110  dataRefList, kwargs = args
111  butler = kwargs.pop('butler')
112  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
113  result = None
114  try:
115  result = task.runDataRef(dataRefList, **kwargs)
116  exitStatus = result.exitStatus
117  job_path = butler.get('verify_job_filename')
118  result.job.write(job_path[0])
119  except Exception as e: # catch everything, sort it out later.
120  if self.doRaise:
121  raise e
122  else:
123  exitStatus = 1
124  eName = type(e).__name__
125  tract = dataRefList[0].dataId['tract']
126  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
127 
128  if self.doReturnResults:
129  return pipeBase.Struct(result=result, exitStatus=exitStatus)
130  else:
131  return pipeBase.Struct(exitStatus=exitStatus)
132 
133 
134 class JointcalConfig(pexConfig.Config):
135  """Config for JointcalTask"""
136 
137  doAstrometry = pexConfig.Field(
138  doc="Fit astrometry and write the fitted result.",
139  dtype=bool,
140  default=True
141  )
142  doPhotometry = pexConfig.Field(
143  doc="Fit photometry and write the fitted result.",
144  dtype=bool,
145  default=True
146  )
147  coaddName = pexConfig.Field(
148  doc="Type of coadd, typically deep or goodSeeing",
149  dtype=str,
150  default="deep"
151  )
152  positionErrorPedestal = pexConfig.Field(
153  doc="Systematic term to apply to the measured position error (pixels)",
154  dtype=float,
155  default=0.02,
156  )
157  photometryErrorPedestal = pexConfig.Field(
158  doc="Systematic term to apply to the measured error on flux or magnitude as a "
159  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
160  dtype=float,
161  default=0.0,
162  )
163  # TODO: DM-6885 matchCut should be an afw.geom.Angle
164  matchCut = pexConfig.Field(
165  doc="Matching radius between fitted and reference stars (arcseconds)",
166  dtype=float,
167  default=3.0,
168  )
169  minMeasurements = pexConfig.Field(
170  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
171  dtype=int,
172  default=2,
173  )
174  minMeasuredStarsPerCcd = pexConfig.Field(
175  doc="Minimum number of measuredStars per ccdImage before printing warnings",
176  dtype=int,
177  default=100,
178  )
179  minRefStarsPerCcd = pexConfig.Field(
180  doc="Minimum number of measuredStars per ccdImage before printing warnings",
181  dtype=int,
182  default=30,
183  )
184  allowLineSearch = pexConfig.Field(
185  doc="Allow a line search during minimization, if it is reasonable for the model"
186  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
187  dtype=bool,
188  default=False
189  )
190  astrometrySimpleOrder = pexConfig.Field(
191  doc="Polynomial order for fitting the simple astrometry model.",
192  dtype=int,
193  default=3,
194  )
195  astrometryChipOrder = pexConfig.Field(
196  doc="Order of the per-chip transform for the constrained astrometry model.",
197  dtype=int,
198  default=1,
199  )
200  astrometryVisitOrder = pexConfig.Field(
201  doc="Order of the per-visit transform for the constrained astrometry model.",
202  dtype=int,
203  default=5,
204  )
205  useInputWcs = pexConfig.Field(
206  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
207  dtype=bool,
208  default=True,
209  )
210  astrometryModel = pexConfig.ChoiceField(
211  doc="Type of model to fit to astrometry",
212  dtype=str,
213  default="constrained",
214  allowed={"simple": "One polynomial per ccd",
215  "constrained": "One polynomial per ccd, and one polynomial per visit"}
216  )
217  photometryModel = pexConfig.ChoiceField(
218  doc="Type of model to fit to photometry",
219  dtype=str,
220  default="constrainedMagnitude",
221  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
222  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
223  " fitting in flux space.",
224  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
225  " fitting in magnitude space.",
226  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
227  " fitting in magnitude space.",
228  }
229  )
230  applyColorTerms = pexConfig.Field(
231  doc="Apply photometric color terms to reference stars?"
232  "Requires that colorterms be set to a ColortermLibrary",
233  dtype=bool,
234  default=False
235  )
236  colorterms = pexConfig.ConfigField(
237  doc="Library of photometric reference catalog name to color term dict.",
238  dtype=ColortermLibrary,
239  )
240  photometryVisitOrder = pexConfig.Field(
241  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
242  dtype=int,
243  default=7,
244  )
245  photometryDoRankUpdate = pexConfig.Field(
246  doc="Do the rank update step during minimization. "
247  "Skipping this can help deal with models that are too non-linear.",
248  dtype=bool,
249  default=True,
250  )
251  astrometryDoRankUpdate = pexConfig.Field(
252  doc="Do the rank update step during minimization (should not change the astrometry fit). "
253  "Skipping this can help deal with models that are too non-linear.",
254  dtype=bool,
255  default=True,
256  )
257  outlierRejectSigma = pexConfig.Field(
258  doc="How many sigma to reject outliers at during minimization.",
259  dtype=float,
260  default=5.0,
261  )
262  maxPhotometrySteps = pexConfig.Field(
263  doc="Maximum number of minimize iterations to take when fitting photometry.",
264  dtype=int,
265  default=20,
266  )
267  maxAstrometrySteps = pexConfig.Field(
268  doc="Maximum number of minimize iterations to take when fitting photometry.",
269  dtype=int,
270  default=20,
271  )
272  astrometryRefObjLoader = pexConfig.ConfigurableField(
273  target=LoadIndexedReferenceObjectsTask,
274  doc="Reference object loader for astrometric fit",
275  )
276  photometryRefObjLoader = pexConfig.ConfigurableField(
277  target=LoadIndexedReferenceObjectsTask,
278  doc="Reference object loader for photometric fit",
279  )
280  sourceSelector = sourceSelectorRegistry.makeField(
281  doc="How to select sources for cross-matching",
282  default="astrometry"
283  )
284  astrometryReferenceSelector = pexConfig.ConfigurableField(
285  target=ReferenceSourceSelectorTask,
286  doc="How to down-select the loaded astrometry reference catalog.",
287  )
288  photometryReferenceSelector = pexConfig.ConfigurableField(
289  target=ReferenceSourceSelectorTask,
290  doc="How to down-select the loaded photometry reference catalog.",
291  )
292  writeInitMatrix = pexConfig.Field(
293  dtype=bool,
294  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
295  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
296  "Note that these files are the dense versions of the matrix, and so may be very large.",
297  default=False
298  )
299  writeChi2ContributionFiles = pexConfig.Field(
300  dtype=bool,
301  doc="Write initial/final fit files containing the contributions to chi2.",
302  default=False
303  )
304  sourceFluxType = pexConfig.Field(
305  dtype=str,
306  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
307  default='Calib'
308  )
309 
310  def validate(self):
311  super().validate()
312  if self.applyColorTerms and len(self.colorterms.data) == 0:
313  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
314  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
315 
316  def setDefaults(self):
317  sourceSelector = self.sourceSelector["astrometry"]
318  sourceSelector.setDefaults()
319  # don't want to lose existing flags, just add to them.
320  sourceSelector.badFlags.extend(["slot_Shape_flag"])
321  # This should be used to set the FluxField value in jointcal::JointcalControl
322  sourceSelector.sourceFluxType = self.sourceFluxType
323 
324 
325 class JointcalTask(pipeBase.CmdLineTask):
326  """Jointly astrometrically and photometrically calibrate a group of images."""
327 
328  ConfigClass = JointcalConfig
329  RunnerClass = JointcalRunner
330  _DefaultName = "jointcal"
331 
332  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
333  """
334  Instantiate a JointcalTask.
335 
336  Parameters
337  ----------
338  butler : `lsst.daf.persistence.Butler`
339  The butler is passed to the refObjLoader constructor in case it is
340  needed. Ignored if the refObjLoader argument provides a loader directly.
341  Used to initialize the astrometry and photometry refObjLoaders.
342  profile_jointcal : `bool`
343  Set to True to profile different stages of this jointcal run.
344  """
345  pipeBase.CmdLineTask.__init__(self, **kwargs)
346  self.profile_jointcal = profile_jointcal
347  self.makeSubtask("sourceSelector")
348  if self.config.doAstrometry:
349  self.makeSubtask('astrometryRefObjLoader', butler=butler)
350  self.makeSubtask("astrometryReferenceSelector")
351  else:
353  if self.config.doPhotometry:
354  self.makeSubtask('photometryRefObjLoader', butler=butler)
355  self.makeSubtask("photometryReferenceSelector")
356  else:
358 
359  # To hold various computed metrics for use by tests
360  self.job = Job.load_metrics_package(subset='jointcal')
361 
362  # We don't currently need to persist the metadata.
363  # If we do in the future, we will have to add appropriate dataset templates
364  # to each obs package (the metadata template should look like `jointcal_wcs`).
365  def _getMetadataName(self):
366  return None
367 
368  @classmethod
369  def _makeArgumentParser(cls):
370  """Create an argument parser"""
371  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
372  parser.add_argument("--profile_jointcal", default=False, action="store_true",
373  help="Profile steps of jointcal separately.")
374  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
375  ContainerClass=PerTractCcdDataIdContainer)
376  return parser
377 
378  def _build_ccdImage(self, dataRef, associations, jointcalControl):
379  """
380  Extract the necessary things from this dataRef to add a new ccdImage.
381 
382  Parameters
383  ----------
384  dataRef : `lsst.daf.persistence.ButlerDataRef`
385  DataRef to extract info from.
386  associations : `lsst.jointcal.Associations`
387  Object to add the info to, to construct a new CcdImage
388  jointcalControl : `jointcal.JointcalControl`
389  Control object for associations management
390 
391  Returns
392  ------
393  namedtuple
394  ``wcs``
395  The TAN WCS of this image, read from the calexp
396  (`lsst.afw.geom.SkyWcs`).
397  ``key``
398  A key to identify this dataRef by its visit and ccd ids
399  (`namedtuple`).
400  ``filter``
401  This calexp's filter (`str`).
402  """
403  if "visit" in dataRef.dataId.keys():
404  visit = dataRef.dataId["visit"]
405  else:
406  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
407 
408  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
409 
410  visitInfo = dataRef.get('calexp_visitInfo')
411  detector = dataRef.get('calexp_detector')
412  ccdId = detector.getId()
413  calib = dataRef.get('calexp_calib')
414  tanWcs = dataRef.get('calexp_wcs')
415  bbox = dataRef.get('calexp_bbox')
416  filt = dataRef.get('calexp_filter')
417  filterName = filt.getName()
418  fluxMag0 = calib.getFluxMag0()
419  # TODO: need to scale these until DM-10153 is completed and PhotoCalib has replaced Calib entirely
420  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
421  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
422  referenceFlux*fluxMag0[1]/fluxMag0[0]**2, bbox)
423 
424  goodSrc = self.sourceSelector.run(src)
425 
426  if len(goodSrc.sourceCat) == 0:
427  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
428  else:
429  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
430  associations.createCcdImage(goodSrc.sourceCat,
431  tanWcs,
432  visitInfo,
433  bbox,
434  filterName,
435  photoCalib,
436  detector,
437  visit,
438  ccdId,
439  jointcalControl)
440 
441  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
442  Key = collections.namedtuple('Key', ('visit', 'ccd'))
443  return Result(tanWcs, Key(visit, ccdId), filterName)
444 
445  @pipeBase.timeMethod
446  def runDataRef(self, dataRefs, profile_jointcal=False):
447  """
448  Jointly calibrate the astrometry and photometry across a set of images.
449 
450  Parameters
451  ----------
452  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
453  List of data references to the exposures to be fit.
454  profile_jointcal : `bool`
455  Profile the individual steps of jointcal.
456 
457  Returns
458  -------
459  result : `lsst.pipe.base.Struct`
460  Struct of metadata from the fit, containing:
461 
462  ``dataRefs``
463  The provided data references that were fit (with updated WCSs)
464  ``oldWcsList``
465  The original WCS from each dataRef
466  ``metrics``
467  Dictionary of internally-computed metrics for testing/validation.
468  """
469  if len(dataRefs) == 0:
470  raise ValueError('Need a non-empty list of data references!')
471 
472  exitStatus = 0 # exit status for shell
473 
474  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
475  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
476  associations = lsst.jointcal.Associations()
477 
478  visit_ccd_to_dataRef = {}
479  oldWcsList = []
480  filters = []
481  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
482  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
483  # We need the bounding-box of the focal plane for photometry visit models.
484  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
485  camera = dataRefs[0].get('camera', immediate=True)
486  self.focalPlaneBBox = camera.getFpBBox()
487  for ref in dataRefs:
488  result = self._build_ccdImage(ref, associations, jointcalControl)
489  oldWcsList.append(result.wcs)
490  visit_ccd_to_dataRef[result.key] = ref
491  filters.append(result.filter)
492  filters = collections.Counter(filters)
493 
494  associations.computeCommonTangentPoint()
495 
496  # Use external reference catalogs handled by LSST stack mechanism
497  # Get the bounding box overlapping all associated images
498  # ==> This is probably a bad idea to do it this way <== To be improved
499  bbox = associations.getRaDecBBox()
500  bboxCenter = bbox.getCenter()
501  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
502  bboxMax = bbox.getMax()
503  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
504  radius = center.separation(corner).asRadians()
505 
506  # Get astrometry_net_data path
507  anDir = lsst.utils.getPackageDir('astrometry_net_data')
508  if anDir is None:
509  raise RuntimeError("astrometry_net_data is not setup")
510 
511  # Determine a default filter associated with the catalog. See DM-9093
512  defaultFilter = filters.most_common(1)[0][0]
513  self.log.debug("Using %s band for reference flux", defaultFilter)
514 
515  # TODO: need a better way to get the tract.
516  tract = dataRefs[0].dataId['tract']
517 
518  if self.config.doAstrometry:
519  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
520  name="astrometry",
521  refObjLoader=self.astrometryRefObjLoader,
522  referenceSelector=self.astrometryReferenceSelector,
523  fit_function=self._fit_astrometry,
524  profile_jointcal=profile_jointcal,
525  tract=tract)
526  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
527  else:
528  astrometry = Astrometry(None, None, None)
529 
530  if self.config.doPhotometry:
531  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
532  name="photometry",
533  refObjLoader=self.photometryRefObjLoader,
534  referenceSelector=self.photometryReferenceSelector,
535  fit_function=self._fit_photometry,
536  profile_jointcal=profile_jointcal,
537  tract=tract,
538  filters=filters,
539  reject_bad_fluxes=True)
540  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
541  else:
542  photometry = Photometry(None, None)
543 
544  return pipeBase.Struct(dataRefs=dataRefs,
545  oldWcsList=oldWcsList,
546  job=self.job,
547  astrometryRefObjLoader=self.astrometryRefObjLoader,
548  photometryRefObjLoader=self.photometryRefObjLoader,
549  defaultFilter=defaultFilter,
550  exitStatus=exitStatus)
551 
552  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
553  name="", refObjLoader=None, referenceSelector=None,
554  filters=[], fit_function=None,
555  tract=None, profile_jointcal=False, match_cut=3.0,
556  reject_bad_fluxes=False):
557  """Load reference catalog, perform the fit, and return the result.
558 
559  Parameters
560  ----------
561  associations : `lsst.jointcal.Associations`
562  The star/reference star associations to fit.
563  defaultFilter : `str`
564  filter to load from reference catalog.
565  center : `lsst.afw.geom.SpherePoint`
566  ICRS center of field to load from reference catalog.
567  radius : `lsst.afw.geom.Angle`
568  On-sky radius to load from reference catalog.
569  name : `str`
570  Name of thing being fit: "Astrometry" or "Photometry".
571  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
572  Reference object loader to load from for fit.
573  filters : `list` of `str`, optional
574  List of filters to load from the reference catalog.
575  fit_function : callable
576  Function to call to perform fit (takes associations object).
577  tract : `str`
578  Name of tract currently being fit.
579  profile_jointcal : `bool`, optional
580  Separately profile the fitting step.
581  match_cut : `float`, optional
582  Radius in arcseconds to find cross-catalog matches to during
583  associations.associateCatalogs.
584  reject_bad_fluxes : `bool`, optional
585  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
586 
587  Returns
588  -------
589  result : `Photometry` or `Astrometry`
590  Result of `fit_function()`
591  """
592  self.log.info("====== Now processing %s...", name)
593  # TODO: this should not print "trying to invert a singular transformation:"
594  # if it does that, something's not right about the WCS...
595  associations.associateCatalogs(match_cut)
596  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
597  associations.fittedStarListSize())
598 
599  applyColorterms = False if name == "Astrometry" else self.config.applyColorTerms
600  if name == "Astrometry":
601  referenceSelector = self.config.astrometryReferenceSelector
602  elif name == "Photometry":
603  referenceSelector = self.config.photometryReferenceSelector
604  refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
605  center, radius, defaultFilter,
606  applyColorterms=applyColorterms)
607 
608  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
609  fluxField, reject_bad_fluxes)
610  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
611  associations.refStarListSize())
612 
613  associations.prepareFittedStars(self.config.minMeasurements)
614 
615  self._check_star_lists(associations, name)
616  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
617  associations.nFittedStarsWithAssociatedRefStar())
618  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
619  associations.fittedStarListSize())
620  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
621  associations.nCcdImagesValidForFit())
622 
623  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
624  dataName = "{}_{}".format(tract, defaultFilter)
625  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
626  result = fit_function(associations, dataName)
627  # TODO DM-12446: turn this into a "butler save" somehow.
628  # Save reference and measurement chi2 contributions for this data
629  if self.config.writeChi2ContributionFiles:
630  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
631  result.fit.saveChi2Contributions(baseName)
632 
633  return result
634 
635  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
636  applyColorterms=False):
637  """Load the necessary reference catalog sources, convert fluxes to
638  correct units, and apply color term corrections if requested.
639 
640  Parameters
641  ----------
642  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
643  The reference catalog loader to use to get the data.
644  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
645  Source selector to apply to loaded reference catalog.
646  center : `lsst.geom.SpherePoint`
647  The center around which to load sources.
648  radius : `lsst.geom.Angle`
649  The radius around ``center`` to load sources in.
650  filterName : `str`
651  The name of the camera filter to load fluxes for.
652  applyColorterms : `bool`
653  Apply colorterm corrections to the refcat for ``filterName``?
654 
655  Returns
656  -------
657  refCat : `lsst.afw.table.SimpleCatalog`
658  The loaded reference catalog.
659  fluxField : `str`
660  The name of the reference catalog flux field appropriate for ``filterName``.
661  """
662  skyCircle = refObjLoader.loadSkyCircle(center,
663  afwGeom.Angle(radius, afwGeom.radians),
664  filterName)
665 
666  selected = referenceSelector.run(skyCircle.refCat)
667  # Need memory contiguity to get reference filters as a vector.
668  if not selected.sourceCat.isContiguous():
669  refCat = selected.sourceCat.copy(deep=True)
670  else:
671  refCat = selected.sourceCat
672 
673  if applyColorterms:
674  try:
675  refCatName = refObjLoader.ref_dataset_name
676  except AttributeError:
677  # NOTE: we need this try:except: block in place until we've completely removed a.net support.
678  raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
679  self.log.info("Applying color terms for filterName=%r reference catalog=%s",
680  filterName, refCatName)
681  colorterm = self.config.colorterms.getColorterm(
682  filterName=filterName, photoCatName=refCatName, doRaise=True)
683 
684  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
685  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
686  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
687  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
688  else:
689  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
690  refCat[skyCircle.fluxField] *= 1e9
691  try:
692  refCat[skyCircle.fluxField+'Err'] *= 1e9
693  except KeyError:
694  # not all existing refcats have an error field.
695  pass
696 
697  return refCat, skyCircle.fluxField
698 
699  def _check_star_lists(self, associations, name):
700  # TODO: these should be len(blah), but we need this properly wrapped first.
701  if associations.nCcdImagesValidForFit() == 0:
702  raise RuntimeError('No images in the ccdImageList!')
703  if associations.fittedStarListSize() == 0:
704  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
705  if associations.refStarListSize() == 0:
706  raise RuntimeError('No stars in the {} reference star list!'.format(name))
707 
708  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model"):
709  """Compute chi2, log it, validate the model, and return chi2."""
710  chi2 = fit.computeChi2()
711  self.log.info("%s %s", chi2Label, chi2)
712  self._check_stars(associations)
713  if not np.isfinite(chi2.chi2):
714  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
715  if not model.validate(associations.getCcdImageList()):
716  raise ValueError("Model is not valid: check log messages for warnings.")
717  return chi2
718 
719  def _fit_photometry(self, associations, dataName=None):
720  """
721  Fit the photometric data.
722 
723  Parameters
724  ----------
725  associations : `lsst.jointcal.Associations`
726  The star/reference star associations to fit.
727  dataName : `str`
728  Name of the data being processed (e.g. "1234_HSC-Y"), for
729  identifying debugging files.
730 
731  Returns
732  -------
733  fit_result : `namedtuple`
734  fit : `lsst.jointcal.PhotometryFit`
735  The photometric fitter used to perform the fit.
736  model : `lsst.jointcal.PhotometryModel`
737  The photometric model that was fit.
738  """
739  self.log.info("=== Starting photometric fitting...")
740 
741  # TODO: should use pex.config.RegistryField here (see DM-9195)
742  if self.config.photometryModel == "constrainedFlux":
743  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
744  self.focalPlaneBBox,
745  visitOrder=self.config.photometryVisitOrder,
746  errorPedestal=self.config.photometryErrorPedestal)
747  # potentially nonlinear problem, so we may need a line search to converge.
748  doLineSearch = self.config.allowLineSearch
749  elif self.config.photometryModel == "constrainedMagnitude":
750  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
751  self.focalPlaneBBox,
752  visitOrder=self.config.photometryVisitOrder,
753  errorPedestal=self.config.photometryErrorPedestal)
754  # potentially nonlinear problem, so we may need a line search to converge.
755  doLineSearch = self.config.allowLineSearch
756  elif self.config.photometryModel == "simpleFlux":
757  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
758  errorPedestal=self.config.photometryErrorPedestal)
759  doLineSearch = False # purely linear in model parameters, so no line search needed
760  elif self.config.photometryModel == "simpleMagnitude":
761  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
762  errorPedestal=self.config.photometryErrorPedestal)
763  doLineSearch = False # purely linear in model parameters, so no line search needed
764 
765  fit = lsst.jointcal.PhotometryFit(associations, model)
766  self._logChi2AndValidate(associations, fit, model, "Initialized")
767 
768  # TODO DM-12446: turn this into a "butler save" somehow.
769  # Save reference and measurement chi2 contributions for this data
770  if self.config.writeChi2ContributionFiles:
771  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
772  fit.saveChi2Contributions(baseName)
773 
774  # The constrained model needs the visit transform fit first; the chip
775  # transform is initialized from the singleFrame PhotoCalib, so it's close.
776  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
777  if self.config.photometryModel.startswith("constrained"):
778  # no line search: should be purely (or nearly) linear,
779  # and we want a large step size to initialize with.
780  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
781  self._logChi2AndValidate(associations, fit, model)
782  dumpMatrixFile = "" # so we don't redo the output on the next step
783 
784  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
785  self._logChi2AndValidate(associations, fit, model)
786 
787  fit.minimize("Fluxes") # no line search: always purely linear.
788  self._logChi2AndValidate(associations, fit, model)
789 
790  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
791  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
792 
793  model.freezeErrorTransform()
794  self.log.debug("Photometry error scales are frozen.")
795 
796  chi2 = self._iterate_fit(associations,
797  fit,
798  self.config.maxPhotometrySteps,
799  "photometry",
800  "Model Fluxes",
801  doRankUpdate=self.config.photometryDoRankUpdate,
802  doLineSearch=doLineSearch,
803  dataName=dataName)
804 
805  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
806  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
807  return Photometry(fit, model)
808 
809  def _fit_astrometry(self, associations, dataName=None):
810  """
811  Fit the astrometric data.
812 
813  Parameters
814  ----------
815  associations : `lsst.jointcal.Associations`
816  The star/reference star associations to fit.
817  dataName : `str`
818  Name of the data being processed (e.g. "1234_HSC-Y"), for
819  identifying debugging files.
820 
821  Returns
822  -------
823  fit_result : `namedtuple`
824  fit : `lsst.jointcal.AstrometryFit`
825  The astrometric fitter used to perform the fit.
826  model : `lsst.jointcal.AstrometryModel`
827  The astrometric model that was fit.
828  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
829  The model for the sky to tangent plane projection that was used in the fit.
830  """
831 
832  self.log.info("=== Starting astrometric fitting...")
833 
834  associations.deprojectFittedStars()
835 
836  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
837  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
838  # them so carefully?
839  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
840 
841  if self.config.astrometryModel == "constrained":
842  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
843  sky_to_tan_projection,
844  chipOrder=self.config.astrometryChipOrder,
845  visitOrder=self.config.astrometryVisitOrder)
846  elif self.config.astrometryModel == "simple":
847  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
848  sky_to_tan_projection,
849  self.config.useInputWcs,
850  nNotFit=0,
851  order=self.config.astrometrySimpleOrder)
852 
853  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
854  self._logChi2AndValidate(associations, fit, model, "Initial")
855 
856  # TODO DM-12446: turn this into a "butler save" somehow.
857  # Save reference and measurement chi2 contributions for this data
858  if self.config.writeChi2ContributionFiles:
859  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
860  fit.saveChi2Contributions(baseName)
861 
862  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
863  # The constrained model needs the visit transform fit first; the chip
864  # transform is initialized from the detector's cameraGeom, so it's close.
865  if self.config.astrometryModel == "constrained":
866  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
867  self._logChi2AndValidate(associations, fit, model)
868  dumpMatrixFile = "" # so we don't redo the output on the next step
869 
870  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
871  self._logChi2AndValidate(associations, fit, model)
872 
873  fit.minimize("Positions")
874  self._logChi2AndValidate(associations, fit, model)
875 
876  fit.minimize("Distortions Positions")
877  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
878 
879  chi2 = self._iterate_fit(associations,
880  fit,
881  self.config.maxAstrometrySteps,
882  "astrometry",
883  "Distortions Positions",
884  doRankUpdate=self.config.astrometryDoRankUpdate,
885  dataName=dataName)
886 
887  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
888  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
889 
890  return Astrometry(fit, model, sky_to_tan_projection)
891 
892  def _check_stars(self, associations):
893  """Count measured and reference stars per ccd and warn/log them."""
894  for ccdImage in associations.getCcdImageList():
895  nMeasuredStars, nRefStars = ccdImage.countStars()
896  self.log.debug("ccdImage %s has %s measured and %s reference stars",
897  ccdImage.getName(), nMeasuredStars, nRefStars)
898  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
899  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
900  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
901  if nRefStars < self.config.minRefStarsPerCcd:
902  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
903  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
904 
905  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
906  dataName="",
907  doRankUpdate=True,
908  doLineSearch=False):
909  """Run fitter.minimize up to max_steps times, returning the final chi2.
910 
911  Parameters
912  ----------
913  associations : `lsst.jointcal.Associations`
914  The star/reference star associations to fit.
915  fitter : `lsst.jointcal.FitterBase`
916  The fitter to use for minimization.
917  max_steps : `int`
918  Maximum number of steps to run outlier rejection before declaring
919  convergence failure.
920  name : {'photometry' or 'astrometry'}
921  What type of data are we fitting (for logs and debugging files).
922  whatToFit : `str`
923  Passed to ``fitter.minimize()`` to define the parameters to fit.
924  dataName : `str`, optional
925  Descriptive name for this dataset (e.g. tract and filter),
926  for debugging.
927  doRankUpdate : `bool`, optional
928  Do an Eigen rank update during minimization, or recompute the full
929  matrix and gradient?
930  doLineSearch : `bool`, optional
931  Do a line search for the optimum step during minimization?
932 
933  Returns
934  -------
935  chi2: `lsst.jointcal.Chi2Statistic`
936  The final chi2 after the fit converges, or is forced to end.
937 
938  Raises
939  ------
940  FloatingPointError
941  Raised if the fitter fails with a non-finite value.
942  RuntimeError
943  Raised if the fitter fails for some other reason;
944  log messages will provide further details.
945  """
946  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
947  for i in range(max_steps):
948  result = fitter.minimize(whatToFit,
949  self.config.outlierRejectSigma,
950  doRankUpdate=doRankUpdate,
951  doLineSearch=doLineSearch,
952  dumpMatrixFile=dumpMatrixFile)
953  dumpMatrixFile = "" # clear it so we don't write the matrix again.
954  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel())
955 
956  if result == MinimizeResult.Converged:
957  if doRankUpdate:
958  self.log.debug("fit has converged - no more outliers - redo minimization "
959  "one more time in case we have lost accuracy in rank update.")
960  # Redo minimization one more time in case we have lost accuracy in rank update
961  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
962  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
963 
964  # log a message for a large final chi2, TODO: DM-15247 for something better
965  if chi2.chi2/chi2.ndof >= 4.0:
966  self.log.error("Potentially bad fit: High chi-squared/ndof.")
967 
968  break
969  elif result == MinimizeResult.Chi2Increased:
970  self.log.warn("still some outliers but chi2 increases - retry")
971  elif result == MinimizeResult.NonFinite:
972  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
973  # TODO DM-12446: turn this into a "butler save" somehow.
974  fitter.saveChi2Contributions(filename)
975  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
976  raise FloatingPointError(msg.format(filename))
977  elif result == MinimizeResult.Failed:
978  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
979  else:
980  raise RuntimeError("Unxepected return code from minimize().")
981  else:
982  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
983 
984  return chi2
985 
986  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
987  """
988  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
989 
990  Parameters
991  ----------
992  associations : `lsst.jointcal.Associations`
993  The star/reference star associations to fit.
994  model : `lsst.jointcal.AstrometryModel`
995  The astrometric model that was fit.
996  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
997  Dict of ccdImage identifiers to dataRefs that were fit.
998  """
999 
1000  ccdImageList = associations.getCcdImageList()
1001  for ccdImage in ccdImageList:
1002  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1003  ccd = ccdImage.ccdId
1004  visit = ccdImage.visit
1005  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1006  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
1007  skyWcs = model.makeSkyWcs(ccdImage)
1008  try:
1009  dataRef.put(skyWcs, 'jointcal_wcs')
1010  except pexExceptions.Exception as e:
1011  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1012  raise e
1013 
1014  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1015  """
1016  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1017 
1018  Parameters
1019  ----------
1020  associations : `lsst.jointcal.Associations`
1021  The star/reference star associations to fit.
1022  model : `lsst.jointcal.PhotometryModel`
1023  The photoometric model that was fit.
1024  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1025  Dict of ccdImage identifiers to dataRefs that were fit.
1026  """
1027 
1028  ccdImageList = associations.getCcdImageList()
1029  for ccdImage in ccdImageList:
1030  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1031  ccd = ccdImage.ccdId
1032  visit = ccdImage.visit
1033  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1034  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1035  photoCalib = model.toPhotoCalib(ccdImage)
1036  try:
1037  dataRef.put(photoCalib, 'jointcal_photoCalib')
1038  except pexExceptions.Exception as e:
1039  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1040  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:446
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
Definition: jointcal.py:636
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:378
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:719
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:71
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:809
def _check_star_lists(self, associations, name)
Definition: jointcal.py:699
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model")
Definition: jointcal.py:708
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:908
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1014
def _check_stars(self, associations)
Definition: jointcal.py:892
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:53
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:556
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:986
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:332