lsst.jointcal  17.0.1-5-g5a10bbc+1
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 import astropy.units as u
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.afw.image as afwImage
30 from lsst.afw.image import fluxErrFromABMagErr
31 import lsst.afw.geom as afwGeom
32 import lsst.pex.exceptions as pexExceptions
33 import lsst.afw.table
35 from lsst.pipe.tasks.colorterms import ColortermLibrary
36 from lsst.verify import Job, Measurement
37 
38 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask
39 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40 
41 from .dataIds import PerTractCcdDataIdContainer
42 
43 import lsst.jointcal
44 from lsst.jointcal import MinimizeResult
45 
46 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
47 
48 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
49 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
50 
51 
52 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
53 def add_measurement(job, name, value):
54  meas = Measurement(job.metrics[name], value)
55  job.measurements.insert(meas)
56 
57 
58 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
59  """Subclass of TaskRunner for jointcalTask
60 
61  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
62  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
63  single dataRef, are are called repeatedly). This class transforms the processed
64  arguments generated by the ArgumentParser into the arguments expected by
65  Jointcal.runDataRef().
66 
67  See pipeBase.TaskRunner for more information.
68  """
69 
70  @staticmethod
71  def getTargetList(parsedCmd, **kwargs):
72  """
73  Return a list of tuples per tract, each containing (dataRefs, kwargs).
74 
75  Jointcal operates on lists of dataRefs simultaneously.
76  """
77  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
78  kwargs['butler'] = parsedCmd.butler
79 
80  # organize data IDs by tract
81  refListDict = {}
82  for ref in parsedCmd.id.refList:
83  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
84  # we call runDataRef() once with each tract
85  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
86  return result
87 
88  def __call__(self, args):
89  """
90  Parameters
91  ----------
92  args
93  Arguments for Task.runDataRef()
94 
95  Returns
96  -------
97  pipe.base.Struct
98  if self.doReturnResults is False:
99 
100  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
101 
102  if self.doReturnResults is True:
103 
104  - ``result``: the result of calling jointcal.runDataRef()
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106  """
107  exitStatus = 0 # exit status for shell
108 
109  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
110  dataRefList, kwargs = args
111  butler = kwargs.pop('butler')
112  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
113  result = None
114  try:
115  result = task.runDataRef(dataRefList, **kwargs)
116  exitStatus = result.exitStatus
117  job_path = butler.get('verify_job_filename')
118  result.job.write(job_path[0])
119  except Exception as e: # catch everything, sort it out later.
120  if self.doRaise:
121  raise e
122  else:
123  exitStatus = 1
124  eName = type(e).__name__
125  tract = dataRefList[0].dataId['tract']
126  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
127 
128  # Put the butler back into kwargs for the other Tasks.
129  kwargs['butler'] = butler
130  if self.doReturnResults:
131  return pipeBase.Struct(result=result, exitStatus=exitStatus)
132  else:
133  return pipeBase.Struct(exitStatus=exitStatus)
134 
135 
136 class JointcalConfig(pexConfig.Config):
137  """Config for JointcalTask"""
138 
139  doAstrometry = pexConfig.Field(
140  doc="Fit astrometry and write the fitted result.",
141  dtype=bool,
142  default=True
143  )
144  doPhotometry = pexConfig.Field(
145  doc="Fit photometry and write the fitted result.",
146  dtype=bool,
147  default=True
148  )
149  coaddName = pexConfig.Field(
150  doc="Type of coadd, typically deep or goodSeeing",
151  dtype=str,
152  default="deep"
153  )
154  positionErrorPedestal = pexConfig.Field(
155  doc="Systematic term to apply to the measured position error (pixels)",
156  dtype=float,
157  default=0.02,
158  )
159  photometryErrorPedestal = pexConfig.Field(
160  doc="Systematic term to apply to the measured error on flux or magnitude as a "
161  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
162  dtype=float,
163  default=0.0,
164  )
165  # TODO: DM-6885 matchCut should be an afw.geom.Angle
166  matchCut = pexConfig.Field(
167  doc="Matching radius between fitted and reference stars (arcseconds)",
168  dtype=float,
169  default=3.0,
170  )
171  minMeasurements = pexConfig.Field(
172  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
173  dtype=int,
174  default=2,
175  )
176  minMeasuredStarsPerCcd = pexConfig.Field(
177  doc="Minimum number of measuredStars per ccdImage before printing warnings",
178  dtype=int,
179  default=100,
180  )
181  minRefStarsPerCcd = pexConfig.Field(
182  doc="Minimum number of measuredStars per ccdImage before printing warnings",
183  dtype=int,
184  default=30,
185  )
186  allowLineSearch = pexConfig.Field(
187  doc="Allow a line search during minimization, if it is reasonable for the model"
188  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
189  dtype=bool,
190  default=False
191  )
192  astrometrySimpleOrder = pexConfig.Field(
193  doc="Polynomial order for fitting the simple astrometry model.",
194  dtype=int,
195  default=3,
196  )
197  astrometryChipOrder = pexConfig.Field(
198  doc="Order of the per-chip transform for the constrained astrometry model.",
199  dtype=int,
200  default=1,
201  )
202  astrometryVisitOrder = pexConfig.Field(
203  doc="Order of the per-visit transform for the constrained astrometry model.",
204  dtype=int,
205  default=5,
206  )
207  useInputWcs = pexConfig.Field(
208  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
209  dtype=bool,
210  default=True,
211  )
212  astrometryModel = pexConfig.ChoiceField(
213  doc="Type of model to fit to astrometry",
214  dtype=str,
215  default="constrained",
216  allowed={"simple": "One polynomial per ccd",
217  "constrained": "One polynomial per ccd, and one polynomial per visit"}
218  )
219  photometryModel = pexConfig.ChoiceField(
220  doc="Type of model to fit to photometry",
221  dtype=str,
222  default="constrainedMagnitude",
223  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
224  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
225  " fitting in flux space.",
226  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
227  " fitting in magnitude space.",
228  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
229  " fitting in magnitude space.",
230  }
231  )
232  applyColorTerms = pexConfig.Field(
233  doc="Apply photometric color terms to reference stars?"
234  "Requires that colorterms be set to a ColortermLibrary",
235  dtype=bool,
236  default=False
237  )
238  colorterms = pexConfig.ConfigField(
239  doc="Library of photometric reference catalog name to color term dict.",
240  dtype=ColortermLibrary,
241  )
242  photometryVisitOrder = pexConfig.Field(
243  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
244  dtype=int,
245  default=7,
246  )
247  photometryDoRankUpdate = pexConfig.Field(
248  doc="Do the rank update step during minimization. "
249  "Skipping this can help deal with models that are too non-linear.",
250  dtype=bool,
251  default=True,
252  )
253  astrometryDoRankUpdate = pexConfig.Field(
254  doc="Do the rank update step during minimization (should not change the astrometry fit). "
255  "Skipping this can help deal with models that are too non-linear.",
256  dtype=bool,
257  default=True,
258  )
259  outlierRejectSigma = pexConfig.Field(
260  doc="How many sigma to reject outliers at during minimization.",
261  dtype=float,
262  default=5.0,
263  )
264  maxPhotometrySteps = pexConfig.Field(
265  doc="Maximum number of minimize iterations to take when fitting photometry.",
266  dtype=int,
267  default=20,
268  )
269  maxAstrometrySteps = pexConfig.Field(
270  doc="Maximum number of minimize iterations to take when fitting photometry.",
271  dtype=int,
272  default=20,
273  )
274  astrometryRefObjLoader = pexConfig.ConfigurableField(
275  target=LoadIndexedReferenceObjectsTask,
276  doc="Reference object loader for astrometric fit",
277  )
278  photometryRefObjLoader = pexConfig.ConfigurableField(
279  target=LoadIndexedReferenceObjectsTask,
280  doc="Reference object loader for photometric fit",
281  )
282  sourceSelector = sourceSelectorRegistry.makeField(
283  doc="How to select sources for cross-matching",
284  default="astrometry"
285  )
286  astrometryReferenceSelector = pexConfig.ConfigurableField(
287  target=ReferenceSourceSelectorTask,
288  doc="How to down-select the loaded astrometry reference catalog.",
289  )
290  photometryReferenceSelector = pexConfig.ConfigurableField(
291  target=ReferenceSourceSelectorTask,
292  doc="How to down-select the loaded photometry reference catalog.",
293  )
294  writeInitMatrix = pexConfig.Field(
295  dtype=bool,
296  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
297  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
298  "Note that these files are the dense versions of the matrix, and so may be very large.",
299  default=False
300  )
301  writeChi2FilesInitialFinal = pexConfig.Field(
302  dtype=bool,
303  doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.",
304  default=False
305  )
306  writeChi2FilesOuterLoop = pexConfig.Field(
307  dtype=bool,
308  doc="Write .csv files containing the contributions to chi2 for the outer fit loop.",
309  default=False
310  )
311  sourceFluxType = pexConfig.Field(
312  dtype=str,
313  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
314  default='Calib'
315  )
316 
317  def validate(self):
318  super().validate()
319  if self.applyColorTerms and len(self.colorterms.data) == 0:
320  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
321  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
322 
323  def setDefaults(self):
324  sourceSelector = self.sourceSelector["astrometry"]
325  sourceSelector.setDefaults()
326  # don't want to lose existing flags, just add to them.
327  sourceSelector.badFlags.extend(["slot_Shape_flag"])
328  # This should be used to set the FluxField value in jointcal::JointcalControl
329  sourceSelector.sourceFluxType = self.sourceFluxType
330 
331 
332 class JointcalTask(pipeBase.CmdLineTask):
333  """Jointly astrometrically and photometrically calibrate a group of images."""
334 
335  ConfigClass = JointcalConfig
336  RunnerClass = JointcalRunner
337  _DefaultName = "jointcal"
338 
339  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
340  """
341  Instantiate a JointcalTask.
342 
343  Parameters
344  ----------
345  butler : `lsst.daf.persistence.Butler`
346  The butler is passed to the refObjLoader constructor in case it is
347  needed. Ignored if the refObjLoader argument provides a loader directly.
348  Used to initialize the astrometry and photometry refObjLoaders.
349  profile_jointcal : `bool`
350  Set to True to profile different stages of this jointcal run.
351  """
352  pipeBase.CmdLineTask.__init__(self, **kwargs)
353  self.profile_jointcal = profile_jointcal
354  self.makeSubtask("sourceSelector")
355  if self.config.doAstrometry:
356  self.makeSubtask('astrometryRefObjLoader', butler=butler)
357  self.makeSubtask("astrometryReferenceSelector")
358  else:
360  if self.config.doPhotometry:
361  self.makeSubtask('photometryRefObjLoader', butler=butler)
362  self.makeSubtask("photometryReferenceSelector")
363  else:
365 
366  # To hold various computed metrics for use by tests
367  self.job = Job.load_metrics_package(subset='jointcal')
368 
369  # We don't currently need to persist the metadata.
370  # If we do in the future, we will have to add appropriate dataset templates
371  # to each obs package (the metadata template should look like `jointcal_wcs`).
372  def _getMetadataName(self):
373  return None
374 
375  @classmethod
376  def _makeArgumentParser(cls):
377  """Create an argument parser"""
378  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
379  parser.add_argument("--profile_jointcal", default=False, action="store_true",
380  help="Profile steps of jointcal separately.")
381  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
382  ContainerClass=PerTractCcdDataIdContainer)
383  return parser
384 
385  def _build_ccdImage(self, dataRef, associations, jointcalControl):
386  """
387  Extract the necessary things from this dataRef to add a new ccdImage.
388 
389  Parameters
390  ----------
391  dataRef : `lsst.daf.persistence.ButlerDataRef`
392  DataRef to extract info from.
393  associations : `lsst.jointcal.Associations`
394  Object to add the info to, to construct a new CcdImage
395  jointcalControl : `jointcal.JointcalControl`
396  Control object for associations management
397 
398  Returns
399  ------
400  namedtuple
401  ``wcs``
402  The TAN WCS of this image, read from the calexp
403  (`lsst.afw.geom.SkyWcs`).
404  ``key``
405  A key to identify this dataRef by its visit and ccd ids
406  (`namedtuple`).
407  ``filter``
408  This calexp's filter (`str`).
409  """
410  if "visit" in dataRef.dataId.keys():
411  visit = dataRef.dataId["visit"]
412  else:
413  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
414 
415  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
416 
417  visitInfo = dataRef.get('calexp_visitInfo')
418  detector = dataRef.get('calexp_detector')
419  ccdId = detector.getId()
420  calib = dataRef.get('calexp_calib')
421  tanWcs = dataRef.get('calexp_wcs')
422  bbox = dataRef.get('calexp_bbox')
423  filt = dataRef.get('calexp_filter')
424  filterName = filt.getName()
425  fluxMag0 = calib.getFluxMag0()
426  # TODO: need to scale these until DM-10153 is completed and PhotoCalib has replaced Calib entirely
427  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
428  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
429  referenceFlux*fluxMag0[1]/fluxMag0[0]**2, bbox)
430 
431  goodSrc = self.sourceSelector.run(src)
432 
433  if len(goodSrc.sourceCat) == 0:
434  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
435  else:
436  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
437  associations.createCcdImage(goodSrc.sourceCat,
438  tanWcs,
439  visitInfo,
440  bbox,
441  filterName,
442  photoCalib,
443  detector,
444  visit,
445  ccdId,
446  jointcalControl)
447 
448  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
449  Key = collections.namedtuple('Key', ('visit', 'ccd'))
450  return Result(tanWcs, Key(visit, ccdId), filterName)
451 
452  @pipeBase.timeMethod
453  def runDataRef(self, dataRefs, profile_jointcal=False):
454  """
455  Jointly calibrate the astrometry and photometry across a set of images.
456 
457  Parameters
458  ----------
459  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
460  List of data references to the exposures to be fit.
461  profile_jointcal : `bool`
462  Profile the individual steps of jointcal.
463 
464  Returns
465  -------
466  result : `lsst.pipe.base.Struct`
467  Struct of metadata from the fit, containing:
468 
469  ``dataRefs``
470  The provided data references that were fit (with updated WCSs)
471  ``oldWcsList``
472  The original WCS from each dataRef
473  ``metrics``
474  Dictionary of internally-computed metrics for testing/validation.
475  """
476  if len(dataRefs) == 0:
477  raise ValueError('Need a non-empty list of data references!')
478 
479  exitStatus = 0 # exit status for shell
480 
481  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
482  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
483  associations = lsst.jointcal.Associations()
484 
485  visit_ccd_to_dataRef = {}
486  oldWcsList = []
487  filters = []
488  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
489  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
490  # We need the bounding-box of the focal plane for photometry visit models.
491  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
492  camera = dataRefs[0].get('camera', immediate=True)
493  self.focalPlaneBBox = camera.getFpBBox()
494  for ref in dataRefs:
495  result = self._build_ccdImage(ref, associations, jointcalControl)
496  oldWcsList.append(result.wcs)
497  visit_ccd_to_dataRef[result.key] = ref
498  filters.append(result.filter)
499  filters = collections.Counter(filters)
500 
501  associations.computeCommonTangentPoint()
502 
503  # Use external reference catalogs handled by LSST stack mechanism
504  # Get the bounding box overlapping all associated images
505  # ==> This is probably a bad idea to do it this way <== To be improved
506  bbox = associations.getRaDecBBox()
507  bboxCenter = bbox.getCenter()
508  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
509  bboxMax = bbox.getMax()
510  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
511  radius = center.separation(corner).asRadians()
512 
513  # Get astrometry_net_data path
514  anDir = lsst.utils.getPackageDir('astrometry_net_data')
515  if anDir is None:
516  raise RuntimeError("astrometry_net_data is not setup")
517 
518  # Determine a default filter associated with the catalog. See DM-9093
519  defaultFilter = filters.most_common(1)[0][0]
520  self.log.debug("Using %s band for reference flux", defaultFilter)
521 
522  # TODO: need a better way to get the tract.
523  tract = dataRefs[0].dataId['tract']
524 
525  if self.config.doAstrometry:
526  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
527  name="astrometry",
528  refObjLoader=self.astrometryRefObjLoader,
529  referenceSelector=self.astrometryReferenceSelector,
530  fit_function=self._fit_astrometry,
531  profile_jointcal=profile_jointcal,
532  tract=tract)
533  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
534  else:
535  astrometry = Astrometry(None, None, None)
536 
537  if self.config.doPhotometry:
538  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
539  name="photometry",
540  refObjLoader=self.photometryRefObjLoader,
541  referenceSelector=self.photometryReferenceSelector,
542  fit_function=self._fit_photometry,
543  profile_jointcal=profile_jointcal,
544  tract=tract,
545  filters=filters,
546  reject_bad_fluxes=True)
547  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
548  else:
549  photometry = Photometry(None, None)
550 
551  return pipeBase.Struct(dataRefs=dataRefs,
552  oldWcsList=oldWcsList,
553  job=self.job,
554  astrometryRefObjLoader=self.astrometryRefObjLoader,
555  photometryRefObjLoader=self.photometryRefObjLoader,
556  defaultFilter=defaultFilter,
557  exitStatus=exitStatus)
558 
559  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
560  name="", refObjLoader=None, referenceSelector=None,
561  filters=[], fit_function=None,
562  tract=None, profile_jointcal=False, match_cut=3.0,
563  reject_bad_fluxes=False):
564  """Load reference catalog, perform the fit, and return the result.
565 
566  Parameters
567  ----------
568  associations : `lsst.jointcal.Associations`
569  The star/reference star associations to fit.
570  defaultFilter : `str`
571  filter to load from reference catalog.
572  center : `lsst.afw.geom.SpherePoint`
573  ICRS center of field to load from reference catalog.
574  radius : `lsst.afw.geom.Angle`
575  On-sky radius to load from reference catalog.
576  name : `str`
577  Name of thing being fit: "Astrometry" or "Photometry".
578  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
579  Reference object loader to load from for fit.
580  filters : `list` of `str`, optional
581  List of filters to load from the reference catalog.
582  fit_function : callable
583  Function to call to perform fit (takes associations object).
584  tract : `str`
585  Name of tract currently being fit.
586  profile_jointcal : `bool`, optional
587  Separately profile the fitting step.
588  match_cut : `float`, optional
589  Radius in arcseconds to find cross-catalog matches to during
590  associations.associateCatalogs.
591  reject_bad_fluxes : `bool`, optional
592  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
593 
594  Returns
595  -------
596  result : `Photometry` or `Astrometry`
597  Result of `fit_function()`
598  """
599  self.log.info("====== Now processing %s...", name)
600  # TODO: this should not print "trying to invert a singular transformation:"
601  # if it does that, something's not right about the WCS...
602  associations.associateCatalogs(match_cut)
603  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
604  associations.fittedStarListSize())
605 
606  applyColorterms = False if name == "Astrometry" else self.config.applyColorTerms
607  if name == "Astrometry":
608  referenceSelector = self.config.astrometryReferenceSelector
609  elif name == "Photometry":
610  referenceSelector = self.config.photometryReferenceSelector
611  refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
612  center, radius, defaultFilter,
613  applyColorterms=applyColorterms)
614 
615  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
616  fluxField, reject_bad_fluxes)
617  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
618  associations.refStarListSize())
619 
620  associations.prepareFittedStars(self.config.minMeasurements)
621 
622  self._check_star_lists(associations, name)
623  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
624  associations.nFittedStarsWithAssociatedRefStar())
625  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
626  associations.fittedStarListSize())
627  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
628  associations.nCcdImagesValidForFit())
629 
630  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
631  dataName = "{}_{}".format(tract, defaultFilter)
632  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
633  result = fit_function(associations, dataName)
634  # TODO DM-12446: turn this into a "butler save" somehow.
635  # Save reference and measurement chi2 contributions for this data
636  if self.config.writeChi2FilesInitialFinal:
637  baseName = f"{name}_final_chi2-{dataName}"
638  result.fit.saveChi2Contributions(baseName+"{type}")
639 
640  return result
641 
642  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
643  applyColorterms=False):
644  """Load the necessary reference catalog sources, convert fluxes to
645  correct units, and apply color term corrections if requested.
646 
647  Parameters
648  ----------
649  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
650  The reference catalog loader to use to get the data.
651  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
652  Source selector to apply to loaded reference catalog.
653  center : `lsst.geom.SpherePoint`
654  The center around which to load sources.
655  radius : `lsst.geom.Angle`
656  The radius around ``center`` to load sources in.
657  filterName : `str`
658  The name of the camera filter to load fluxes for.
659  applyColorterms : `bool`
660  Apply colorterm corrections to the refcat for ``filterName``?
661 
662  Returns
663  -------
664  refCat : `lsst.afw.table.SimpleCatalog`
665  The loaded reference catalog.
666  fluxField : `str`
667  The name of the reference catalog flux field appropriate for ``filterName``.
668  """
669  skyCircle = refObjLoader.loadSkyCircle(center,
670  afwGeom.Angle(radius, afwGeom.radians),
671  filterName)
672 
673  selected = referenceSelector.run(skyCircle.refCat)
674  # Need memory contiguity to get reference filters as a vector.
675  if not selected.sourceCat.isContiguous():
676  refCat = selected.sourceCat.copy(deep=True)
677  else:
678  refCat = selected.sourceCat
679 
680  if applyColorterms:
681  try:
682  refCatName = refObjLoader.ref_dataset_name
683  except AttributeError:
684  # NOTE: we need this try:except: block in place until we've completely removed a.net support.
685  raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
686  self.log.info("Applying color terms for filterName=%r reference catalog=%s",
687  filterName, refCatName)
688  colorterm = self.config.colorterms.getColorterm(
689  filterName=filterName, photoCatName=refCatName, doRaise=True)
690 
691  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
692  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
693  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
694  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
695  else:
696  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
697  refCat[skyCircle.fluxField] *= 1e9
698  try:
699  refCat[skyCircle.fluxField+'Err'] *= 1e9
700  except KeyError:
701  # not all existing refcats have an error field.
702  pass
703 
704  return refCat, skyCircle.fluxField
705 
706  def _check_star_lists(self, associations, name):
707  # TODO: these should be len(blah), but we need this properly wrapped first.
708  if associations.nCcdImagesValidForFit() == 0:
709  raise RuntimeError('No images in the ccdImageList!')
710  if associations.fittedStarListSize() == 0:
711  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
712  if associations.refStarListSize() == 0:
713  raise RuntimeError('No stars in the {} reference star list!'.format(name))
714 
715  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
716  writeChi2Name=None):
717  """Compute chi2, log it, validate the model, and return chi2.
718 
719  Parameters
720  ----------
721  associations : `lsst.jointcal.Associations`
722  The star/reference star associations to fit.
723  fit : `lsst.jointcal.FitterBase`
724  The fitter to use for minimization.
725  model : `lsst.jointcal.Model`
726  The model being fit.
727  chi2Label : str, optional
728  Label to describe the chi2 (e.g. "Initialized", "Final").
729  writeChi2Name : `str`, optional
730  Filename prefix to write the chi2 contributions to.
731  Do not supply an extension: an appropriate one will be added.
732 
733  Returns
734  -------
735  chi2: `lsst.jointcal.Chi2Accumulator`
736  The chi2 object for the current fitter and model.
737 
738  Raises
739  ------
740  FloatingPointError
741  Raised if chi2 is infinite or NaN.
742  ValueError
743  Raised if the model is not valid.
744  """
745  if writeChi2Name is not None:
746  fit.saveChi2Contributions(writeChi2Name+"{type}")
747  self.log.info("Wrote chi2 contributions files: %s", writeChi2Name)
748 
749  chi2 = fit.computeChi2()
750  self.log.info("%s %s", chi2Label, chi2)
751  self._check_stars(associations)
752  if not np.isfinite(chi2.chi2):
753  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
754  if not model.validate(associations.getCcdImageList()):
755  raise ValueError("Model is not valid: check log messages for warnings.")
756  return chi2
757 
758  def _fit_photometry(self, associations, dataName=None):
759  """
760  Fit the photometric data.
761 
762  Parameters
763  ----------
764  associations : `lsst.jointcal.Associations`
765  The star/reference star associations to fit.
766  dataName : `str`
767  Name of the data being processed (e.g. "1234_HSC-Y"), for
768  identifying debugging files.
769 
770  Returns
771  -------
772  fit_result : `namedtuple`
773  fit : `lsst.jointcal.PhotometryFit`
774  The photometric fitter used to perform the fit.
775  model : `lsst.jointcal.PhotometryModel`
776  The photometric model that was fit.
777  """
778  self.log.info("=== Starting photometric fitting...")
779 
780  # TODO: should use pex.config.RegistryField here (see DM-9195)
781  if self.config.photometryModel == "constrainedFlux":
782  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
783  self.focalPlaneBBox,
784  visitOrder=self.config.photometryVisitOrder,
785  errorPedestal=self.config.photometryErrorPedestal)
786  # potentially nonlinear problem, so we may need a line search to converge.
787  doLineSearch = self.config.allowLineSearch
788  elif self.config.photometryModel == "constrainedMagnitude":
789  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
790  self.focalPlaneBBox,
791  visitOrder=self.config.photometryVisitOrder,
792  errorPedestal=self.config.photometryErrorPedestal)
793  # potentially nonlinear problem, so we may need a line search to converge.
794  doLineSearch = self.config.allowLineSearch
795  elif self.config.photometryModel == "simpleFlux":
796  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
797  errorPedestal=self.config.photometryErrorPedestal)
798  doLineSearch = False # purely linear in model parameters, so no line search needed
799  elif self.config.photometryModel == "simpleMagnitude":
800  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
801  errorPedestal=self.config.photometryErrorPedestal)
802  doLineSearch = False # purely linear in model parameters, so no line search needed
803 
804  fit = lsst.jointcal.PhotometryFit(associations, model)
805  # TODO DM-12446: turn this into a "butler save" somehow.
806  # Save reference and measurement chi2 contributions for this data
807  if self.config.writeChi2FilesInitialFinal:
808  baseName = f"photometry_initial_chi2-{dataName}"
809  else:
810  baseName = None
811  self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
812 
813  def getChi2Name(whatToFit):
814  if self.config.writeChi2FilesOuterLoop:
815  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
816  else:
817  return None
818 
819  # The constrained model needs the visit transform fit first; the chip
820  # transform is initialized from the singleFrame PhotoCalib, so it's close.
821  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
822  if self.config.photometryModel.startswith("constrained"):
823  # no line search: should be purely (or nearly) linear,
824  # and we want a large step size to initialize with.
825  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
826  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("ModelVisit"))
827  dumpMatrixFile = "" # so we don't redo the output on the next step
828 
829  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
830  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Model"))
831 
832  fit.minimize("Fluxes") # no line search: always purely linear.
833  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Fluxes"))
834 
835  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
836  self._logChi2AndValidate(associations, fit, model, "Fit prepared",
837  writeChi2Name=getChi2Name("ModelFluxes"))
838 
839  model.freezeErrorTransform()
840  self.log.debug("Photometry error scales are frozen.")
841 
842  chi2 = self._iterate_fit(associations,
843  fit,
844  self.config.maxPhotometrySteps,
845  "photometry",
846  "Model Fluxes",
847  doRankUpdate=self.config.photometryDoRankUpdate,
848  doLineSearch=doLineSearch,
849  dataName=dataName)
850 
851  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
852  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
853  return Photometry(fit, model)
854 
855  def _fit_astrometry(self, associations, dataName=None):
856  """
857  Fit the astrometric data.
858 
859  Parameters
860  ----------
861  associations : `lsst.jointcal.Associations`
862  The star/reference star associations to fit.
863  dataName : `str`
864  Name of the data being processed (e.g. "1234_HSC-Y"), for
865  identifying debugging files.
866 
867  Returns
868  -------
869  fit_result : `namedtuple`
870  fit : `lsst.jointcal.AstrometryFit`
871  The astrometric fitter used to perform the fit.
872  model : `lsst.jointcal.AstrometryModel`
873  The astrometric model that was fit.
874  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
875  The model for the sky to tangent plane projection that was used in the fit.
876  """
877 
878  self.log.info("=== Starting astrometric fitting...")
879 
880  associations.deprojectFittedStars()
881 
882  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
883  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
884  # them so carefully?
885  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
886 
887  if self.config.astrometryModel == "constrained":
888  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
889  sky_to_tan_projection,
890  chipOrder=self.config.astrometryChipOrder,
891  visitOrder=self.config.astrometryVisitOrder)
892  elif self.config.astrometryModel == "simple":
893  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
894  sky_to_tan_projection,
895  self.config.useInputWcs,
896  nNotFit=0,
897  order=self.config.astrometrySimpleOrder)
898 
899  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
900  # TODO DM-12446: turn this into a "butler save" somehow.
901  # Save reference and measurement chi2 contributions for this data
902  if self.config.writeChi2FilesInitialFinal:
903  baseName = f"astrometry_initial_chi2-{dataName}"
904  else:
905  baseName = None
906  self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
907 
908  def getChi2Name(whatToFit):
909  if self.config.writeChi2FilesOuterLoop:
910  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
911  else:
912  return None
913 
914  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
915  # The constrained model needs the visit transform fit first; the chip
916  # transform is initialized from the detector's cameraGeom, so it's close.
917  if self.config.astrometryModel == "constrained":
918  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
919  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("DistortionsVisit"))
920  dumpMatrixFile = "" # so we don't redo the output on the next step
921 
922  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
923  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Distortions"))
924 
925  fit.minimize("Positions")
926  self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Positions"))
927 
928  fit.minimize("Distortions Positions")
929  self._logChi2AndValidate(associations, fit, model, "Fit prepared",
930  writeChi2Name=getChi2Name("DistortionsPositions"))
931 
932  chi2 = self._iterate_fit(associations,
933  fit,
934  self.config.maxAstrometrySteps,
935  "astrometry",
936  "Distortions Positions",
937  doRankUpdate=self.config.astrometryDoRankUpdate,
938  dataName=dataName)
939 
940  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
941  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
942 
943  return Astrometry(fit, model, sky_to_tan_projection)
944 
945  def _check_stars(self, associations):
946  """Count measured and reference stars per ccd and warn/log them."""
947  for ccdImage in associations.getCcdImageList():
948  nMeasuredStars, nRefStars = ccdImage.countStars()
949  self.log.debug("ccdImage %s has %s measured and %s reference stars",
950  ccdImage.getName(), nMeasuredStars, nRefStars)
951  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
952  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
953  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
954  if nRefStars < self.config.minRefStarsPerCcd:
955  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
956  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
957 
958  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
959  dataName="",
960  doRankUpdate=True,
961  doLineSearch=False):
962  """Run fitter.minimize up to max_steps times, returning the final chi2.
963 
964  Parameters
965  ----------
966  associations : `lsst.jointcal.Associations`
967  The star/reference star associations to fit.
968  fitter : `lsst.jointcal.FitterBase`
969  The fitter to use for minimization.
970  max_steps : `int`
971  Maximum number of steps to run outlier rejection before declaring
972  convergence failure.
973  name : {'photometry' or 'astrometry'}
974  What type of data are we fitting (for logs and debugging files).
975  whatToFit : `str`
976  Passed to ``fitter.minimize()`` to define the parameters to fit.
977  dataName : `str`, optional
978  Descriptive name for this dataset (e.g. tract and filter),
979  for debugging.
980  doRankUpdate : `bool`, optional
981  Do an Eigen rank update during minimization, or recompute the full
982  matrix and gradient?
983  doLineSearch : `bool`, optional
984  Do a line search for the optimum step during minimization?
985 
986  Returns
987  -------
988  chi2: `lsst.jointcal.Chi2Statistic`
989  The final chi2 after the fit converges, or is forced to end.
990 
991  Raises
992  ------
993  FloatingPointError
994  Raised if the fitter fails with a non-finite value.
995  RuntimeError
996  Raised if the fitter fails for some other reason;
997  log messages will provide further details.
998  """
999  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
1000  for i in range(max_steps):
1001  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1002  result = fitter.minimize(whatToFit,
1003  self.config.outlierRejectSigma,
1004  doRankUpdate=doRankUpdate,
1005  doLineSearch=doLineSearch,
1006  dumpMatrixFile=dumpMatrixFile)
1007  dumpMatrixFile = "" # clear it so we don't write the matrix again.
1008  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1009  writeChi2Name=writeChi2Name)
1010 
1011  if result == MinimizeResult.Converged:
1012  if doRankUpdate:
1013  self.log.debug("fit has converged - no more outliers - redo minimization "
1014  "one more time in case we have lost accuracy in rank update.")
1015  # Redo minimization one more time in case we have lost accuracy in rank update
1016  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1017  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1018 
1019  # log a message for a large final chi2, TODO: DM-15247 for something better
1020  if chi2.chi2/chi2.ndof >= 4.0:
1021  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1022 
1023  break
1024  elif result == MinimizeResult.Chi2Increased:
1025  self.log.warn("still some outliers but chi2 increases - retry")
1026  elif result == MinimizeResult.NonFinite:
1027  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
1028  # TODO DM-12446: turn this into a "butler save" somehow.
1029  fitter.saveChi2Contributions(filename)
1030  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1031  raise FloatingPointError(msg.format(filename))
1032  elif result == MinimizeResult.Failed:
1033  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1034  else:
1035  raise RuntimeError("Unxepected return code from minimize().")
1036  else:
1037  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1038 
1039  return chi2
1040 
1041  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1042  """
1043  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1044 
1045  Parameters
1046  ----------
1047  associations : `lsst.jointcal.Associations`
1048  The star/reference star associations to fit.
1049  model : `lsst.jointcal.AstrometryModel`
1050  The astrometric model that was fit.
1051  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1052  Dict of ccdImage identifiers to dataRefs that were fit.
1053  """
1054 
1055  ccdImageList = associations.getCcdImageList()
1056  for ccdImage in ccdImageList:
1057  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1058  ccd = ccdImage.ccdId
1059  visit = ccdImage.visit
1060  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1061  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
1062  skyWcs = model.makeSkyWcs(ccdImage)
1063  try:
1064  dataRef.put(skyWcs, 'jointcal_wcs')
1065  except pexExceptions.Exception as e:
1066  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1067  raise e
1068 
1069  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1070  """
1071  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1072 
1073  Parameters
1074  ----------
1075  associations : `lsst.jointcal.Associations`
1076  The star/reference star associations to fit.
1077  model : `lsst.jointcal.PhotometryModel`
1078  The photoometric model that was fit.
1079  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1080  Dict of ccdImage identifiers to dataRefs that were fit.
1081  """
1082 
1083  ccdImageList = associations.getCcdImageList()
1084  for ccdImage in ccdImageList:
1085  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1086  ccd = ccdImage.ccdId
1087  visit = ccdImage.visit
1088  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1089  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1090  photoCalib = model.toPhotoCalib(ccdImage)
1091  try:
1092  dataRef.put(photoCalib, 'jointcal_photoCalib')
1093  except pexExceptions.Exception as e:
1094  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1095  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:453
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
Definition: jointcal.py:643
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:385
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:758
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:71
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:855
def _check_star_lists(self, associations, name)
Definition: jointcal.py:706
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", writeChi2Name=None)
Definition: jointcal.py:716
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:961
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1069
def _check_stars(self, associations)
Definition: jointcal.py:945
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:53
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:563
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1041
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:339