lsst.jointcal  17.0.1-4-gfa71e81+2
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import collections
23 import numpy as np
24 import astropy.units as u
25 
26 import lsst.utils
27 import lsst.pex.config as pexConfig
28 import lsst.pipe.base as pipeBase
29 import lsst.afw.image as afwImage
30 from lsst.afw.image import fluxErrFromABMagErr
31 import lsst.afw.geom as afwGeom
32 import lsst.pex.exceptions as pexExceptions
33 import lsst.afw.table
35 from lsst.pipe.tasks.colorterms import ColortermLibrary
36 from lsst.verify import Job, Measurement
37 
38 from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask
39 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
40 
41 from .dataIds import PerTractCcdDataIdContainer
42 
43 import lsst.jointcal
44 from lsst.jointcal import MinimizeResult
45 
46 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
47 
48 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
49 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
50 
51 
52 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
53 def add_measurement(job, name, value):
54  meas = Measurement(job.metrics[name], value)
55  job.measurements.insert(meas)
56 
57 
58 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
59  """Subclass of TaskRunner for jointcalTask
60 
61  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
62  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
63  single dataRef, are are called repeatedly). This class transforms the processed
64  arguments generated by the ArgumentParser into the arguments expected by
65  Jointcal.runDataRef().
66 
67  See pipeBase.TaskRunner for more information.
68  """
69 
70  @staticmethod
71  def getTargetList(parsedCmd, **kwargs):
72  """
73  Return a list of tuples per tract, each containing (dataRefs, kwargs).
74 
75  Jointcal operates on lists of dataRefs simultaneously.
76  """
77  kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
78  kwargs['butler'] = parsedCmd.butler
79 
80  # organize data IDs by tract
81  refListDict = {}
82  for ref in parsedCmd.id.refList:
83  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
84  # we call runDataRef() once with each tract
85  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
86  return result
87 
88  def __call__(self, args):
89  """
90  Parameters
91  ----------
92  args
93  Arguments for Task.runDataRef()
94 
95  Returns
96  -------
97  pipe.base.Struct
98  if self.doReturnResults is False:
99 
100  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
101 
102  if self.doReturnResults is True:
103 
104  - ``result``: the result of calling jointcal.runDataRef()
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106  """
107  exitStatus = 0 # exit status for shell
108 
109  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
110  dataRefList, kwargs = args
111  butler = kwargs.pop('butler')
112  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
113  result = None
114  try:
115  result = task.runDataRef(dataRefList, **kwargs)
116  exitStatus = result.exitStatus
117  job_path = butler.get('verify_job_filename')
118  result.job.write(job_path[0])
119  except Exception as e: # catch everything, sort it out later.
120  if self.doRaise:
121  raise e
122  else:
123  exitStatus = 1
124  eName = type(e).__name__
125  tract = dataRefList[0].dataId['tract']
126  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
127 
128  # Put the butler back into kwargs for the other Tasks.
129  kwargs['butler'] = butler
130  if self.doReturnResults:
131  return pipeBase.Struct(result=result, exitStatus=exitStatus)
132  else:
133  return pipeBase.Struct(exitStatus=exitStatus)
134 
135 
136 class JointcalConfig(pexConfig.Config):
137  """Config for JointcalTask"""
138 
139  doAstrometry = pexConfig.Field(
140  doc="Fit astrometry and write the fitted result.",
141  dtype=bool,
142  default=True
143  )
144  doPhotometry = pexConfig.Field(
145  doc="Fit photometry and write the fitted result.",
146  dtype=bool,
147  default=True
148  )
149  coaddName = pexConfig.Field(
150  doc="Type of coadd, typically deep or goodSeeing",
151  dtype=str,
152  default="deep"
153  )
154  positionErrorPedestal = pexConfig.Field(
155  doc="Systematic term to apply to the measured position error (pixels)",
156  dtype=float,
157  default=0.02,
158  )
159  photometryErrorPedestal = pexConfig.Field(
160  doc="Systematic term to apply to the measured error on flux or magnitude as a "
161  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
162  dtype=float,
163  default=0.0,
164  )
165  # TODO: DM-6885 matchCut should be an afw.geom.Angle
166  matchCut = pexConfig.Field(
167  doc="Matching radius between fitted and reference stars (arcseconds)",
168  dtype=float,
169  default=3.0,
170  )
171  minMeasurements = pexConfig.Field(
172  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
173  dtype=int,
174  default=2,
175  )
176  minMeasuredStarsPerCcd = pexConfig.Field(
177  doc="Minimum number of measuredStars per ccdImage before printing warnings",
178  dtype=int,
179  default=100,
180  )
181  minRefStarsPerCcd = pexConfig.Field(
182  doc="Minimum number of measuredStars per ccdImage before printing warnings",
183  dtype=int,
184  default=30,
185  )
186  allowLineSearch = pexConfig.Field(
187  doc="Allow a line search during minimization, if it is reasonable for the model"
188  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
189  dtype=bool,
190  default=False
191  )
192  astrometrySimpleOrder = pexConfig.Field(
193  doc="Polynomial order for fitting the simple astrometry model.",
194  dtype=int,
195  default=3,
196  )
197  astrometryChipOrder = pexConfig.Field(
198  doc="Order of the per-chip transform for the constrained astrometry model.",
199  dtype=int,
200  default=1,
201  )
202  astrometryVisitOrder = pexConfig.Field(
203  doc="Order of the per-visit transform for the constrained astrometry model.",
204  dtype=int,
205  default=5,
206  )
207  useInputWcs = pexConfig.Field(
208  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
209  dtype=bool,
210  default=True,
211  )
212  astrometryModel = pexConfig.ChoiceField(
213  doc="Type of model to fit to astrometry",
214  dtype=str,
215  default="constrained",
216  allowed={"simple": "One polynomial per ccd",
217  "constrained": "One polynomial per ccd, and one polynomial per visit"}
218  )
219  photometryModel = pexConfig.ChoiceField(
220  doc="Type of model to fit to photometry",
221  dtype=str,
222  default="constrainedMagnitude",
223  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
224  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
225  " fitting in flux space.",
226  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
227  " fitting in magnitude space.",
228  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
229  " fitting in magnitude space.",
230  }
231  )
232  applyColorTerms = pexConfig.Field(
233  doc="Apply photometric color terms to reference stars?"
234  "Requires that colorterms be set to a ColortermLibrary",
235  dtype=bool,
236  default=False
237  )
238  colorterms = pexConfig.ConfigField(
239  doc="Library of photometric reference catalog name to color term dict.",
240  dtype=ColortermLibrary,
241  )
242  photometryVisitOrder = pexConfig.Field(
243  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
244  dtype=int,
245  default=7,
246  )
247  photometryDoRankUpdate = pexConfig.Field(
248  doc="Do the rank update step during minimization. "
249  "Skipping this can help deal with models that are too non-linear.",
250  dtype=bool,
251  default=True,
252  )
253  astrometryDoRankUpdate = pexConfig.Field(
254  doc="Do the rank update step during minimization (should not change the astrometry fit). "
255  "Skipping this can help deal with models that are too non-linear.",
256  dtype=bool,
257  default=True,
258  )
259  outlierRejectSigma = pexConfig.Field(
260  doc="How many sigma to reject outliers at during minimization.",
261  dtype=float,
262  default=5.0,
263  )
264  maxPhotometrySteps = pexConfig.Field(
265  doc="Maximum number of minimize iterations to take when fitting photometry.",
266  dtype=int,
267  default=20,
268  )
269  maxAstrometrySteps = pexConfig.Field(
270  doc="Maximum number of minimize iterations to take when fitting photometry.",
271  dtype=int,
272  default=20,
273  )
274  astrometryRefObjLoader = pexConfig.ConfigurableField(
275  target=LoadIndexedReferenceObjectsTask,
276  doc="Reference object loader for astrometric fit",
277  )
278  photometryRefObjLoader = pexConfig.ConfigurableField(
279  target=LoadIndexedReferenceObjectsTask,
280  doc="Reference object loader for photometric fit",
281  )
282  sourceSelector = sourceSelectorRegistry.makeField(
283  doc="How to select sources for cross-matching",
284  default="astrometry"
285  )
286  astrometryReferenceSelector = pexConfig.ConfigurableField(
287  target=ReferenceSourceSelectorTask,
288  doc="How to down-select the loaded astrometry reference catalog.",
289  )
290  photometryReferenceSelector = pexConfig.ConfigurableField(
291  target=ReferenceSourceSelectorTask,
292  doc="How to down-select the loaded photometry reference catalog.",
293  )
294  writeInitMatrix = pexConfig.Field(
295  dtype=bool,
296  doc="Write the pre/post-initialization Hessian and gradient to text files, for debugging."
297  "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory."
298  "Note that these files are the dense versions of the matrix, and so may be very large.",
299  default=False
300  )
301  writeChi2ContributionFiles = pexConfig.Field(
302  dtype=bool,
303  doc="Write initial/final fit files containing the contributions to chi2.",
304  default=False
305  )
306  sourceFluxType = pexConfig.Field(
307  dtype=str,
308  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
309  default='Calib'
310  )
311 
312  def validate(self):
313  super().validate()
314  if self.applyColorTerms and len(self.colorterms.data) == 0:
315  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
316  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
317 
318  def setDefaults(self):
319  sourceSelector = self.sourceSelector["astrometry"]
320  sourceSelector.setDefaults()
321  # don't want to lose existing flags, just add to them.
322  sourceSelector.badFlags.extend(["slot_Shape_flag"])
323  # This should be used to set the FluxField value in jointcal::JointcalControl
324  sourceSelector.sourceFluxType = self.sourceFluxType
325 
326 
327 class JointcalTask(pipeBase.CmdLineTask):
328  """Jointly astrometrically and photometrically calibrate a group of images."""
329 
330  ConfigClass = JointcalConfig
331  RunnerClass = JointcalRunner
332  _DefaultName = "jointcal"
333 
334  def __init__(self, butler=None, profile_jointcal=False, **kwargs):
335  """
336  Instantiate a JointcalTask.
337 
338  Parameters
339  ----------
340  butler : `lsst.daf.persistence.Butler`
341  The butler is passed to the refObjLoader constructor in case it is
342  needed. Ignored if the refObjLoader argument provides a loader directly.
343  Used to initialize the astrometry and photometry refObjLoaders.
344  profile_jointcal : `bool`
345  Set to True to profile different stages of this jointcal run.
346  """
347  pipeBase.CmdLineTask.__init__(self, **kwargs)
348  self.profile_jointcal = profile_jointcal
349  self.makeSubtask("sourceSelector")
350  if self.config.doAstrometry:
351  self.makeSubtask('astrometryRefObjLoader', butler=butler)
352  self.makeSubtask("astrometryReferenceSelector")
353  else:
355  if self.config.doPhotometry:
356  self.makeSubtask('photometryRefObjLoader', butler=butler)
357  self.makeSubtask("photometryReferenceSelector")
358  else:
360 
361  # To hold various computed metrics for use by tests
362  self.job = Job.load_metrics_package(subset='jointcal')
363 
364  # We don't currently need to persist the metadata.
365  # If we do in the future, we will have to add appropriate dataset templates
366  # to each obs package (the metadata template should look like `jointcal_wcs`).
367  def _getMetadataName(self):
368  return None
369 
370  @classmethod
371  def _makeArgumentParser(cls):
372  """Create an argument parser"""
373  parser = pipeBase.ArgumentParser(name=cls._DefaultName)
374  parser.add_argument("--profile_jointcal", default=False, action="store_true",
375  help="Profile steps of jointcal separately.")
376  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
377  ContainerClass=PerTractCcdDataIdContainer)
378  return parser
379 
380  def _build_ccdImage(self, dataRef, associations, jointcalControl):
381  """
382  Extract the necessary things from this dataRef to add a new ccdImage.
383 
384  Parameters
385  ----------
386  dataRef : `lsst.daf.persistence.ButlerDataRef`
387  DataRef to extract info from.
388  associations : `lsst.jointcal.Associations`
389  Object to add the info to, to construct a new CcdImage
390  jointcalControl : `jointcal.JointcalControl`
391  Control object for associations management
392 
393  Returns
394  ------
395  namedtuple
396  ``wcs``
397  The TAN WCS of this image, read from the calexp
398  (`lsst.afw.geom.SkyWcs`).
399  ``key``
400  A key to identify this dataRef by its visit and ccd ids
401  (`namedtuple`).
402  ``filter``
403  This calexp's filter (`str`).
404  """
405  if "visit" in dataRef.dataId.keys():
406  visit = dataRef.dataId["visit"]
407  else:
408  visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
409 
410  src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
411 
412  visitInfo = dataRef.get('calexp_visitInfo')
413  detector = dataRef.get('calexp_detector')
414  ccdId = detector.getId()
415  calib = dataRef.get('calexp_calib')
416  tanWcs = dataRef.get('calexp_wcs')
417  bbox = dataRef.get('calexp_bbox')
418  filt = dataRef.get('calexp_filter')
419  filterName = filt.getName()
420  fluxMag0 = calib.getFluxMag0()
421  # TODO: need to scale these until DM-10153 is completed and PhotoCalib has replaced Calib entirely
422  referenceFlux = 1e23 * 10**(48.6 / -2.5) * 1e9
423  photoCalib = afwImage.PhotoCalib(referenceFlux/fluxMag0[0],
424  referenceFlux*fluxMag0[1]/fluxMag0[0]**2, bbox)
425 
426  goodSrc = self.sourceSelector.run(src)
427 
428  if len(goodSrc.sourceCat) == 0:
429  self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
430  else:
431  self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
432  associations.createCcdImage(goodSrc.sourceCat,
433  tanWcs,
434  visitInfo,
435  bbox,
436  filterName,
437  photoCalib,
438  detector,
439  visit,
440  ccdId,
441  jointcalControl)
442 
443  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
444  Key = collections.namedtuple('Key', ('visit', 'ccd'))
445  return Result(tanWcs, Key(visit, ccdId), filterName)
446 
447  @pipeBase.timeMethod
448  def runDataRef(self, dataRefs, profile_jointcal=False):
449  """
450  Jointly calibrate the astrometry and photometry across a set of images.
451 
452  Parameters
453  ----------
454  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
455  List of data references to the exposures to be fit.
456  profile_jointcal : `bool`
457  Profile the individual steps of jointcal.
458 
459  Returns
460  -------
461  result : `lsst.pipe.base.Struct`
462  Struct of metadata from the fit, containing:
463 
464  ``dataRefs``
465  The provided data references that were fit (with updated WCSs)
466  ``oldWcsList``
467  The original WCS from each dataRef
468  ``metrics``
469  Dictionary of internally-computed metrics for testing/validation.
470  """
471  if len(dataRefs) == 0:
472  raise ValueError('Need a non-empty list of data references!')
473 
474  exitStatus = 0 # exit status for shell
475 
476  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
477  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
478  associations = lsst.jointcal.Associations()
479 
480  visit_ccd_to_dataRef = {}
481  oldWcsList = []
482  filters = []
483  load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
484  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
485  # We need the bounding-box of the focal plane for photometry visit models.
486  # NOTE: we only need to read it once, because its the same for all exposures of a camera.
487  camera = dataRefs[0].get('camera', immediate=True)
488  self.focalPlaneBBox = camera.getFpBBox()
489  for ref in dataRefs:
490  result = self._build_ccdImage(ref, associations, jointcalControl)
491  oldWcsList.append(result.wcs)
492  visit_ccd_to_dataRef[result.key] = ref
493  filters.append(result.filter)
494  filters = collections.Counter(filters)
495 
496  associations.computeCommonTangentPoint()
497 
498  # Use external reference catalogs handled by LSST stack mechanism
499  # Get the bounding box overlapping all associated images
500  # ==> This is probably a bad idea to do it this way <== To be improved
501  bbox = associations.getRaDecBBox()
502  bboxCenter = bbox.getCenter()
503  center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
504  bboxMax = bbox.getMax()
505  corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
506  radius = center.separation(corner).asRadians()
507 
508  # Get astrometry_net_data path
509  anDir = lsst.utils.getPackageDir('astrometry_net_data')
510  if anDir is None:
511  raise RuntimeError("astrometry_net_data is not setup")
512 
513  # Determine a default filter associated with the catalog. See DM-9093
514  defaultFilter = filters.most_common(1)[0][0]
515  self.log.debug("Using %s band for reference flux", defaultFilter)
516 
517  # TODO: need a better way to get the tract.
518  tract = dataRefs[0].dataId['tract']
519 
520  if self.config.doAstrometry:
521  astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
522  name="astrometry",
523  refObjLoader=self.astrometryRefObjLoader,
524  referenceSelector=self.astrometryReferenceSelector,
525  fit_function=self._fit_astrometry,
526  profile_jointcal=profile_jointcal,
527  tract=tract)
528  self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
529  else:
530  astrometry = Astrometry(None, None, None)
531 
532  if self.config.doPhotometry:
533  photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
534  name="photometry",
535  refObjLoader=self.photometryRefObjLoader,
536  referenceSelector=self.photometryReferenceSelector,
537  fit_function=self._fit_photometry,
538  profile_jointcal=profile_jointcal,
539  tract=tract,
540  filters=filters,
541  reject_bad_fluxes=True)
542  self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
543  else:
544  photometry = Photometry(None, None)
545 
546  return pipeBase.Struct(dataRefs=dataRefs,
547  oldWcsList=oldWcsList,
548  job=self.job,
549  astrometryRefObjLoader=self.astrometryRefObjLoader,
550  photometryRefObjLoader=self.photometryRefObjLoader,
551  defaultFilter=defaultFilter,
552  exitStatus=exitStatus)
553 
554  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
555  name="", refObjLoader=None, referenceSelector=None,
556  filters=[], fit_function=None,
557  tract=None, profile_jointcal=False, match_cut=3.0,
558  reject_bad_fluxes=False):
559  """Load reference catalog, perform the fit, and return the result.
560 
561  Parameters
562  ----------
563  associations : `lsst.jointcal.Associations`
564  The star/reference star associations to fit.
565  defaultFilter : `str`
566  filter to load from reference catalog.
567  center : `lsst.afw.geom.SpherePoint`
568  ICRS center of field to load from reference catalog.
569  radius : `lsst.afw.geom.Angle`
570  On-sky radius to load from reference catalog.
571  name : `str`
572  Name of thing being fit: "Astrometry" or "Photometry".
573  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
574  Reference object loader to load from for fit.
575  filters : `list` of `str`, optional
576  List of filters to load from the reference catalog.
577  fit_function : callable
578  Function to call to perform fit (takes associations object).
579  tract : `str`
580  Name of tract currently being fit.
581  profile_jointcal : `bool`, optional
582  Separately profile the fitting step.
583  match_cut : `float`, optional
584  Radius in arcseconds to find cross-catalog matches to during
585  associations.associateCatalogs.
586  reject_bad_fluxes : `bool`, optional
587  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
588 
589  Returns
590  -------
591  result : `Photometry` or `Astrometry`
592  Result of `fit_function()`
593  """
594  self.log.info("====== Now processing %s...", name)
595  # TODO: this should not print "trying to invert a singular transformation:"
596  # if it does that, something's not right about the WCS...
597  associations.associateCatalogs(match_cut)
598  add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
599  associations.fittedStarListSize())
600 
601  applyColorterms = False if name == "Astrometry" else self.config.applyColorTerms
602  if name == "Astrometry":
603  referenceSelector = self.config.astrometryReferenceSelector
604  elif name == "Photometry":
605  referenceSelector = self.config.photometryReferenceSelector
606  refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
607  center, radius, defaultFilter,
608  applyColorterms=applyColorterms)
609 
610  associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
611  fluxField, reject_bad_fluxes)
612  add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
613  associations.refStarListSize())
614 
615  associations.prepareFittedStars(self.config.minMeasurements)
616 
617  self._check_star_lists(associations, name)
618  add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
619  associations.nFittedStarsWithAssociatedRefStar())
620  add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
621  associations.fittedStarListSize())
622  add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
623  associations.nCcdImagesValidForFit())
624 
625  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
626  dataName = "{}_{}".format(tract, defaultFilter)
627  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
628  result = fit_function(associations, dataName)
629  # TODO DM-12446: turn this into a "butler save" somehow.
630  # Save reference and measurement chi2 contributions for this data
631  if self.config.writeChi2ContributionFiles:
632  baseName = "{}_final_chi2-{}.csv".format(name, dataName)
633  result.fit.saveChi2Contributions(baseName)
634 
635  return result
636 
637  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
638  applyColorterms=False):
639  """Load the necessary reference catalog sources, convert fluxes to
640  correct units, and apply color term corrections if requested.
641 
642  Parameters
643  ----------
644  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
645  The reference catalog loader to use to get the data.
646  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
647  Source selector to apply to loaded reference catalog.
648  center : `lsst.geom.SpherePoint`
649  The center around which to load sources.
650  radius : `lsst.geom.Angle`
651  The radius around ``center`` to load sources in.
652  filterName : `str`
653  The name of the camera filter to load fluxes for.
654  applyColorterms : `bool`
655  Apply colorterm corrections to the refcat for ``filterName``?
656 
657  Returns
658  -------
659  refCat : `lsst.afw.table.SimpleCatalog`
660  The loaded reference catalog.
661  fluxField : `str`
662  The name of the reference catalog flux field appropriate for ``filterName``.
663  """
664  skyCircle = refObjLoader.loadSkyCircle(center,
665  afwGeom.Angle(radius, afwGeom.radians),
666  filterName)
667 
668  selected = referenceSelector.run(skyCircle.refCat)
669  # Need memory contiguity to get reference filters as a vector.
670  if not selected.sourceCat.isContiguous():
671  refCat = selected.sourceCat.copy(deep=True)
672  else:
673  refCat = selected.sourceCat
674 
675  if applyColorterms:
676  try:
677  refCatName = refObjLoader.ref_dataset_name
678  except AttributeError:
679  # NOTE: we need this try:except: block in place until we've completely removed a.net support.
680  raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
681  self.log.info("Applying color terms for filterName=%r reference catalog=%s",
682  filterName, refCatName)
683  colorterm = self.config.colorterms.getColorterm(
684  filterName=filterName, photoCatName=refCatName, doRaise=True)
685 
686  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
687  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
688  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
689  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
690  else:
691  # TODO: need to scale these until RFC-549 is completed and refcats return nanojansky
692  refCat[skyCircle.fluxField] *= 1e9
693  try:
694  refCat[skyCircle.fluxField+'Err'] *= 1e9
695  except KeyError:
696  # not all existing refcats have an error field.
697  pass
698 
699  return refCat, skyCircle.fluxField
700 
701  def _check_star_lists(self, associations, name):
702  # TODO: these should be len(blah), but we need this properly wrapped first.
703  if associations.nCcdImagesValidForFit() == 0:
704  raise RuntimeError('No images in the ccdImageList!')
705  if associations.fittedStarListSize() == 0:
706  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
707  if associations.refStarListSize() == 0:
708  raise RuntimeError('No stars in the {} reference star list!'.format(name))
709 
710  def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model"):
711  """Compute chi2, log it, validate the model, and return chi2."""
712  chi2 = fit.computeChi2()
713  self.log.info("%s %s", chi2Label, chi2)
714  self._check_stars(associations)
715  if not np.isfinite(chi2.chi2):
716  raise FloatingPointError('%s chi2 is invalid: %s', chi2Label, chi2)
717  if not model.validate(associations.getCcdImageList()):
718  raise ValueError("Model is not valid: check log messages for warnings.")
719  return chi2
720 
721  def _fit_photometry(self, associations, dataName=None):
722  """
723  Fit the photometric data.
724 
725  Parameters
726  ----------
727  associations : `lsst.jointcal.Associations`
728  The star/reference star associations to fit.
729  dataName : `str`
730  Name of the data being processed (e.g. "1234_HSC-Y"), for
731  identifying debugging files.
732 
733  Returns
734  -------
735  fit_result : `namedtuple`
736  fit : `lsst.jointcal.PhotometryFit`
737  The photometric fitter used to perform the fit.
738  model : `lsst.jointcal.PhotometryModel`
739  The photometric model that was fit.
740  """
741  self.log.info("=== Starting photometric fitting...")
742 
743  # TODO: should use pex.config.RegistryField here (see DM-9195)
744  if self.config.photometryModel == "constrainedFlux":
745  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
746  self.focalPlaneBBox,
747  visitOrder=self.config.photometryVisitOrder,
748  errorPedestal=self.config.photometryErrorPedestal)
749  # potentially nonlinear problem, so we may need a line search to converge.
750  doLineSearch = self.config.allowLineSearch
751  elif self.config.photometryModel == "constrainedMagnitude":
752  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
753  self.focalPlaneBBox,
754  visitOrder=self.config.photometryVisitOrder,
755  errorPedestal=self.config.photometryErrorPedestal)
756  # potentially nonlinear problem, so we may need a line search to converge.
757  doLineSearch = self.config.allowLineSearch
758  elif self.config.photometryModel == "simpleFlux":
759  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
760  errorPedestal=self.config.photometryErrorPedestal)
761  doLineSearch = False # purely linear in model parameters, so no line search needed
762  elif self.config.photometryModel == "simpleMagnitude":
763  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
764  errorPedestal=self.config.photometryErrorPedestal)
765  doLineSearch = False # purely linear in model parameters, so no line search needed
766 
767  fit = lsst.jointcal.PhotometryFit(associations, model)
768  self._logChi2AndValidate(associations, fit, model, "Initialized")
769 
770  # TODO DM-12446: turn this into a "butler save" somehow.
771  # Save reference and measurement chi2 contributions for this data
772  if self.config.writeChi2ContributionFiles:
773  baseName = "photometry_initial_chi2-{}.csv".format(dataName)
774  fit.saveChi2Contributions(baseName)
775 
776  # The constrained model needs the visit transform fit first; the chip
777  # transform is initialized from the singleFrame PhotoCalib, so it's close.
778  dumpMatrixFile = "photometry_preinit" if self.config.writeInitMatrix else ""
779  if self.config.photometryModel.startswith("constrained"):
780  # no line search: should be purely (or nearly) linear,
781  # and we want a large step size to initialize with.
782  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
783  self._logChi2AndValidate(associations, fit, model)
784  dumpMatrixFile = "" # so we don't redo the output on the next step
785 
786  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
787  self._logChi2AndValidate(associations, fit, model)
788 
789  fit.minimize("Fluxes") # no line search: always purely linear.
790  self._logChi2AndValidate(associations, fit, model)
791 
792  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
793  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
794 
795  model.freezeErrorTransform()
796  self.log.debug("Photometry error scales are frozen.")
797 
798  chi2 = self._iterate_fit(associations,
799  fit,
800  self.config.maxPhotometrySteps,
801  "photometry",
802  "Model Fluxes",
803  doRankUpdate=self.config.photometryDoRankUpdate,
804  doLineSearch=doLineSearch,
805  dataName=dataName)
806 
807  add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
808  add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
809  return Photometry(fit, model)
810 
811  def _fit_astrometry(self, associations, dataName=None):
812  """
813  Fit the astrometric data.
814 
815  Parameters
816  ----------
817  associations : `lsst.jointcal.Associations`
818  The star/reference star associations to fit.
819  dataName : `str`
820  Name of the data being processed (e.g. "1234_HSC-Y"), for
821  identifying debugging files.
822 
823  Returns
824  -------
825  fit_result : `namedtuple`
826  fit : `lsst.jointcal.AstrometryFit`
827  The astrometric fitter used to perform the fit.
828  model : `lsst.jointcal.AstrometryModel`
829  The astrometric model that was fit.
830  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
831  The model for the sky to tangent plane projection that was used in the fit.
832  """
833 
834  self.log.info("=== Starting astrometric fitting...")
835 
836  associations.deprojectFittedStars()
837 
838  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
839  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
840  # them so carefully?
841  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
842 
843  if self.config.astrometryModel == "constrained":
844  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
845  sky_to_tan_projection,
846  chipOrder=self.config.astrometryChipOrder,
847  visitOrder=self.config.astrometryVisitOrder)
848  elif self.config.astrometryModel == "simple":
849  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
850  sky_to_tan_projection,
851  self.config.useInputWcs,
852  nNotFit=0,
853  order=self.config.astrometrySimpleOrder)
854 
855  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
856  self._logChi2AndValidate(associations, fit, model, "Initial")
857 
858  # TODO DM-12446: turn this into a "butler save" somehow.
859  # Save reference and measurement chi2 contributions for this data
860  if self.config.writeChi2ContributionFiles:
861  baseName = "astrometry_initial_chi2-{}.csv".format(dataName)
862  fit.saveChi2Contributions(baseName)
863 
864  dumpMatrixFile = "astrometry_preinit" if self.config.writeInitMatrix else ""
865  # The constrained model needs the visit transform fit first; the chip
866  # transform is initialized from the detector's cameraGeom, so it's close.
867  if self.config.astrometryModel == "constrained":
868  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
869  self._logChi2AndValidate(associations, fit, model)
870  dumpMatrixFile = "" # so we don't redo the output on the next step
871 
872  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
873  self._logChi2AndValidate(associations, fit, model)
874 
875  fit.minimize("Positions")
876  self._logChi2AndValidate(associations, fit, model)
877 
878  fit.minimize("Distortions Positions")
879  self._logChi2AndValidate(associations, fit, model, "Fit prepared")
880 
881  chi2 = self._iterate_fit(associations,
882  fit,
883  self.config.maxAstrometrySteps,
884  "astrometry",
885  "Distortions Positions",
886  doRankUpdate=self.config.astrometryDoRankUpdate,
887  dataName=dataName)
888 
889  add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
890  add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
891 
892  return Astrometry(fit, model, sky_to_tan_projection)
893 
894  def _check_stars(self, associations):
895  """Count measured and reference stars per ccd and warn/log them."""
896  for ccdImage in associations.getCcdImageList():
897  nMeasuredStars, nRefStars = ccdImage.countStars()
898  self.log.debug("ccdImage %s has %s measured and %s reference stars",
899  ccdImage.getName(), nMeasuredStars, nRefStars)
900  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
901  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
902  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
903  if nRefStars < self.config.minRefStarsPerCcd:
904  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
905  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
906 
907  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
908  dataName="",
909  doRankUpdate=True,
910  doLineSearch=False):
911  """Run fitter.minimize up to max_steps times, returning the final chi2.
912 
913  Parameters
914  ----------
915  associations : `lsst.jointcal.Associations`
916  The star/reference star associations to fit.
917  fitter : `lsst.jointcal.FitterBase`
918  The fitter to use for minimization.
919  max_steps : `int`
920  Maximum number of steps to run outlier rejection before declaring
921  convergence failure.
922  name : {'photometry' or 'astrometry'}
923  What type of data are we fitting (for logs and debugging files).
924  whatToFit : `str`
925  Passed to ``fitter.minimize()`` to define the parameters to fit.
926  dataName : `str`, optional
927  Descriptive name for this dataset (e.g. tract and filter),
928  for debugging.
929  doRankUpdate : `bool`, optional
930  Do an Eigen rank update during minimization, or recompute the full
931  matrix and gradient?
932  doLineSearch : `bool`, optional
933  Do a line search for the optimum step during minimization?
934 
935  Returns
936  -------
937  chi2: `lsst.jointcal.Chi2Statistic`
938  The final chi2 after the fit converges, or is forced to end.
939 
940  Raises
941  ------
942  FloatingPointError
943  Raised if the fitter fails with a non-finite value.
944  RuntimeError
945  Raised if the fitter fails for some other reason;
946  log messages will provide further details.
947  """
948  dumpMatrixFile = "%s_postinit" % name if self.config.writeInitMatrix else ""
949  for i in range(max_steps):
950  result = fitter.minimize(whatToFit,
951  self.config.outlierRejectSigma,
952  doRankUpdate=doRankUpdate,
953  doLineSearch=doLineSearch,
954  dumpMatrixFile=dumpMatrixFile)
955  dumpMatrixFile = "" # clear it so we don't write the matrix again.
956  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel())
957 
958  if result == MinimizeResult.Converged:
959  if doRankUpdate:
960  self.log.debug("fit has converged - no more outliers - redo minimization "
961  "one more time in case we have lost accuracy in rank update.")
962  # Redo minimization one more time in case we have lost accuracy in rank update
963  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
964  chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
965 
966  # log a message for a large final chi2, TODO: DM-15247 for something better
967  if chi2.chi2/chi2.ndof >= 4.0:
968  self.log.error("Potentially bad fit: High chi-squared/ndof.")
969 
970  break
971  elif result == MinimizeResult.Chi2Increased:
972  self.log.warn("still some outliers but chi2 increases - retry")
973  elif result == MinimizeResult.NonFinite:
974  filename = "{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
975  # TODO DM-12446: turn this into a "butler save" somehow.
976  fitter.saveChi2Contributions(filename)
977  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
978  raise FloatingPointError(msg.format(filename))
979  elif result == MinimizeResult.Failed:
980  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
981  else:
982  raise RuntimeError("Unxepected return code from minimize().")
983  else:
984  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
985 
986  return chi2
987 
988  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
989  """
990  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
991 
992  Parameters
993  ----------
994  associations : `lsst.jointcal.Associations`
995  The star/reference star associations to fit.
996  model : `lsst.jointcal.AstrometryModel`
997  The astrometric model that was fit.
998  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
999  Dict of ccdImage identifiers to dataRefs that were fit.
1000  """
1001 
1002  ccdImageList = associations.getCcdImageList()
1003  for ccdImage in ccdImageList:
1004  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1005  ccd = ccdImage.ccdId
1006  visit = ccdImage.visit
1007  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1008  self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
1009  skyWcs = model.makeSkyWcs(ccdImage)
1010  try:
1011  dataRef.put(skyWcs, 'jointcal_wcs')
1012  except pexExceptions.Exception as e:
1013  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1014  raise e
1015 
1016  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1017  """
1018  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1019 
1020  Parameters
1021  ----------
1022  associations : `lsst.jointcal.Associations`
1023  The star/reference star associations to fit.
1024  model : `lsst.jointcal.PhotometryModel`
1025  The photoometric model that was fit.
1026  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1027  Dict of ccdImage identifiers to dataRefs that were fit.
1028  """
1029 
1030  ccdImageList = associations.getCcdImageList()
1031  for ccdImage in ccdImageList:
1032  # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1033  ccd = ccdImage.ccdId
1034  visit = ccdImage.visit
1035  dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1036  self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1037  photoCalib = model.toPhotoCalib(ccdImage)
1038  try:
1039  dataRef.put(photoCalib, 'jointcal_photoCalib')
1040  except pexExceptions.Exception as e:
1041  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1042  raise e
def runDataRef(self, dataRefs, profile_jointcal=False)
Definition: jointcal.py:448
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
Definition: jointcal.py:638
def _build_ccdImage(self, dataRef, associations, jointcalControl)
Definition: jointcal.py:380
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:721
def getTargetList(parsedCmd, kwargs)
Definition: jointcal.py:71
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:811
def _check_star_lists(self, associations, name)
Definition: jointcal.py:701
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:53
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model")
Definition: jointcal.py:710
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:910
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1016
def _check_stars(self, associations)
Definition: jointcal.py:894
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
def add_measurement(job, name, value)
Definition: jointcal.py:53
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
Definition: jointcal.py:558
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:988
def __init__(self, butler=None, profile_jointcal=False, kwargs)
Definition: jointcal.py:334