lsst.jointcal  21.0.0-19-g8edf490+60c609c61a
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import dataclasses
23 import collections
24 import os
25 
26 import astropy.time
27 import numpy as np
28 import astropy.units as u
29 
30 import lsst.geom
31 import lsst.utils
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 from lsst.afw.image import fluxErrFromABMagErr
35 import lsst.pex.exceptions as pexExceptions
37 import lsst.afw.table
38 import lsst.log
39 from lsst.obs.base import Instrument
40 from lsst.pipe.tasks.colorterms import ColortermLibrary
41 from lsst.verify import Job, Measurement
42 
43 from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44  ReferenceObjectLoader)
45 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46 
47 from .dataIds import PerTractCcdDataIdContainer
48 
49 import lsst.jointcal
50 from lsst.jointcal import MinimizeResult
51 
52 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
53 
54 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
56 
57 
58 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59 def add_measurement(job, name, value):
60  meas = Measurement(job.metrics[name], value)
61  job.measurements.insert(meas)
62 
63 
64 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65  """Subclass of TaskRunner for jointcalTask (gen2)
66 
67  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69  single dataRef, are are called repeatedly). This class transforms the processed
70  arguments generated by the ArgumentParser into the arguments expected by
71  Jointcal.runDataRef().
72 
73  See pipeBase.TaskRunner for more information.
74  """
75 
76  @staticmethod
77  def getTargetList(parsedCmd, **kwargs):
78  """
79  Return a list of tuples per tract, each containing (dataRefs, kwargs).
80 
81  Jointcal operates on lists of dataRefs simultaneously.
82  """
83  kwargs['butler'] = parsedCmd.butler
84 
85  # organize data IDs by tract
86  refListDict = {}
87  for ref in parsedCmd.id.refList:
88  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89  # we call runDataRef() once with each tract
90  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91  return result
92 
93  def __call__(self, args):
94  """
95  Parameters
96  ----------
97  args
98  Arguments for Task.runDataRef()
99 
100  Returns
101  -------
102  pipe.base.Struct
103  if self.doReturnResults is False:
104 
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106 
107  if self.doReturnResults is True:
108 
109  - ``result``: the result of calling jointcal.runDataRef()
110  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111  """
112  exitStatus = 0 # exit status for shell
113 
114  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115  dataRefList, kwargs = args
116  butler = kwargs.pop('butler')
117  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118  result = None
119  try:
120  result = task.runDataRef(dataRefList, **kwargs)
121  exitStatus = result.exitStatus
122  job_path = butler.get('verify_job_filename')
123  result.job.write(job_path[0])
124  except Exception as e: # catch everything, sort it out later.
125  if self.doRaise:
126  raise e
127  else:
128  exitStatus = 1
129  eName = type(e).__name__
130  tract = dataRefList[0].dataId['tract']
131  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
132 
133  # Put the butler back into kwargs for the other Tasks.
134  kwargs['butler'] = butler
135  if self.doReturnResults:
136  return pipeBase.Struct(result=result, exitStatus=exitStatus)
137  else:
138  return pipeBase.Struct(exitStatus=exitStatus)
139 
140 
141 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142  """Lookup function that asserts/hopes that a static calibration dataset
143  exists in a particular collection, since this task can't provide a single
144  date/time to use to search for one properly.
145 
146  This is mostly useful for the ``camera`` dataset, in cases where the task's
147  quantum dimensions do *not* include something temporal, like ``exposure``
148  or ``visit``.
149 
150  Parameters
151  ----------
152  datasetType : `lsst.daf.butler.DatasetType`
153  Type of dataset being searched for.
154  registry : `lsst.daf.butler.Registry`
155  Data repository registry to search.
156  quantumDataId : `lsst.daf.butler.DataCoordinate`
157  Data ID of the quantum this camera should match.
158  collections : `Iterable` [ `str` ]
159  Collections that should be searched - but this lookup function works
160  by ignoring this in favor of a more-or-less hard-coded value.
161 
162  Returns
163  -------
164  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165  Iterator over dataset references; should have only one element.
166 
167  Notes
168  -----
169  This implementation duplicates one in fgcmcal, and is at least quite
170  similar to another in cp_pipe. This duplicate has the most documentation.
171  Fixing this is DM-29661.
172  """
173  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175  return registry.queryDatasets(datasetType,
176  dataId=quantumDataId,
177  collections=[unboundedCollection],
178  findFirst=True)
179 
180 
181 def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182  """Lookup function that finds all refcats for all visits that overlap a
183  tract, rather than just the refcats that directly overlap the tract.
184 
185  Parameters
186  ----------
187  datasetType : `lsst.daf.butler.DatasetType`
188  Type of dataset being searched for.
189  registry : `lsst.daf.butler.Registry`
190  Data repository registry to search.
191  quantumDataId : `lsst.daf.butler.DataCoordinate`
192  Data ID of the quantum; expected to be something we can use as a
193  constraint to query for overlapping visits.
194  collections : `Iterable` [ `str` ]
195  Collections to search.
196 
197  Returns
198  -------
199  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200  Iterator over refcat references.
201  """
202  refs = set()
203  # Use .expanded() on the query methods below because we need data IDs with
204  # regions, both in the outer loop over visits (queryDatasets will expand
205  # any data ID we give it, but doing it up-front in bulk is much more
206  # efficient) and in the data IDs of the DatasetRefs this function yields
207  # (because the RefCatLoader relies on them to do some of its own
208  # filtering).
209  for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210  refs.update(
211  registry.queryDatasets(
212  datasetType,
213  collections=collections,
214  dataId=visit_data_id,
215  findFirst=True,
216  ).expanded()
217  )
218  yield from refs
219 
220 
221 class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222  dimensions=("skymap", "tract", "instrument", "physical_filter")):
223  """Middleware input/output connections for jointcal data."""
224  inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225  doc="The camera instrument that took these observations.",
226  name="camera",
227  storageClass="Camera",
228  dimensions=("instrument",),
229  isCalibration=True,
230  lookupFunction=lookupStaticCalibrations,
231  )
232  inputSourceTableVisit = pipeBase.connectionTypes.Input(
233  doc="Source table in parquet format, per visit",
234  name="sourceTable_visit",
235  storageClass="DataFrame",
236  dimensions=("instrument", "visit"),
237  deferLoad=True,
238  multiple=True,
239  )
240  inputVisitSummary = pipeBase.connectionTypes.Input(
241  doc=("Per-visit consolidated exposure metadata built from calexps. "
242  "These catalogs use detector id for the id and must be sorted for "
243  "fast lookups of a detector."),
244  name="visitSummary",
245  storageClass="ExposureCatalog",
246  dimensions=("instrument", "visit"),
247  deferLoad=True,
248  multiple=True,
249  )
250  astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251  doc="The astrometry reference catalog to match to loaded input catalog sources.",
252  name="gaia_dr2_20200414",
253  storageClass="SimpleCatalog",
254  dimensions=("skypix",),
255  deferLoad=True,
256  multiple=True,
257  lookupFunction=lookupVisitRefCats,
258  )
259  photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260  doc="The photometry reference catalog to match to loaded input catalog sources.",
261  name="ps1_pv3_3pi_20170110",
262  storageClass="SimpleCatalog",
263  dimensions=("skypix",),
264  deferLoad=True,
265  multiple=True,
266  lookupFunction=lookupVisitRefCats,
267  )
268 
269  outputWcs = pipeBase.connectionTypes.Output(
270  doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271  " These catalogs only contain entries for detectors with an output, and use"
272  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273  name="jointcalSkyWcsCatalog",
274  storageClass="ExposureCatalog",
275  dimensions=("instrument", "visit", "skymap", "tract"),
276  multiple=True
277  )
278  outputPhotoCalib = pipeBase.connectionTypes.Output(
279  doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280  " These catalogs only contain entries for detectors with an output, and use"
281  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282  name="jointcalPhotoCalibCatalog",
283  storageClass="ExposureCatalog",
284  dimensions=("instrument", "visit", "skymap", "tract"),
285  multiple=True
286  )
287 
288  def __init__(self, *, config=None):
289  super().__init__(config=config)
290  # When we are only doing one of astrometry or photometry, we don't
291  # need the reference catalog or produce the outputs for the other.
292  # This informs the middleware of that when the QuantumGraph is
293  # generated, so we don't block on getting something we won't need or
294  # create an expectation that downstream tasks will be able to consume
295  # something we won't produce.
296  if not config.doAstrometry:
297  self.prerequisiteInputs.remove("astrometryRefCat")
298  self.outputs.remove("outputWcs")
299  if not config.doPhotometry:
300  self.prerequisiteInputs.remove("photometryRefCat")
301  self.outputs.remove("outputPhotoCalib")
302 
303 
304 class JointcalConfig(pipeBase.PipelineTaskConfig,
305  pipelineConnections=JointcalTaskConnections):
306  """Configuration for JointcalTask"""
307 
308  doAstrometry = pexConfig.Field(
309  doc="Fit astrometry and write the fitted result.",
310  dtype=bool,
311  default=True
312  )
313  doPhotometry = pexConfig.Field(
314  doc="Fit photometry and write the fitted result.",
315  dtype=bool,
316  default=True
317  )
318  coaddName = pexConfig.Field(
319  doc="Type of coadd, typically deep or goodSeeing",
320  dtype=str,
321  default="deep"
322  )
323  # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
324  sourceFluxType = pexConfig.Field(
325  dtype=str,
326  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
327  default='Calib'
328  )
329  positionErrorPedestal = pexConfig.Field(
330  doc="Systematic term to apply to the measured position error (pixels)",
331  dtype=float,
332  default=0.02,
333  )
334  photometryErrorPedestal = pexConfig.Field(
335  doc="Systematic term to apply to the measured error on flux or magnitude as a "
336  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
337  dtype=float,
338  default=0.0,
339  )
340  # TODO: DM-6885 matchCut should be an geom.Angle
341  matchCut = pexConfig.Field(
342  doc="Matching radius between fitted and reference stars (arcseconds)",
343  dtype=float,
344  default=3.0,
345  )
346  minMeasurements = pexConfig.Field(
347  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
348  dtype=int,
349  default=2,
350  )
351  minMeasuredStarsPerCcd = pexConfig.Field(
352  doc="Minimum number of measuredStars per ccdImage before printing warnings",
353  dtype=int,
354  default=100,
355  )
356  minRefStarsPerCcd = pexConfig.Field(
357  doc="Minimum number of measuredStars per ccdImage before printing warnings",
358  dtype=int,
359  default=30,
360  )
361  allowLineSearch = pexConfig.Field(
362  doc="Allow a line search during minimization, if it is reasonable for the model"
363  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
364  dtype=bool,
365  default=False
366  )
367  astrometrySimpleOrder = pexConfig.Field(
368  doc="Polynomial order for fitting the simple astrometry model.",
369  dtype=int,
370  default=3,
371  )
372  astrometryChipOrder = pexConfig.Field(
373  doc="Order of the per-chip transform for the constrained astrometry model.",
374  dtype=int,
375  default=1,
376  )
377  astrometryVisitOrder = pexConfig.Field(
378  doc="Order of the per-visit transform for the constrained astrometry model.",
379  dtype=int,
380  default=5,
381  )
382  useInputWcs = pexConfig.Field(
383  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
384  dtype=bool,
385  default=True,
386  )
387  astrometryModel = pexConfig.ChoiceField(
388  doc="Type of model to fit to astrometry",
389  dtype=str,
390  default="constrained",
391  allowed={"simple": "One polynomial per ccd",
392  "constrained": "One polynomial per ccd, and one polynomial per visit"}
393  )
394  photometryModel = pexConfig.ChoiceField(
395  doc="Type of model to fit to photometry",
396  dtype=str,
397  default="constrainedMagnitude",
398  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
399  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
400  " fitting in flux space.",
401  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
402  " fitting in magnitude space.",
403  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
404  " fitting in magnitude space.",
405  }
406  )
407  applyColorTerms = pexConfig.Field(
408  doc="Apply photometric color terms to reference stars?"
409  "Requires that colorterms be set to a ColortermLibrary",
410  dtype=bool,
411  default=False
412  )
413  colorterms = pexConfig.ConfigField(
414  doc="Library of photometric reference catalog name to color term dict.",
415  dtype=ColortermLibrary,
416  )
417  photometryVisitOrder = pexConfig.Field(
418  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
419  dtype=int,
420  default=7,
421  )
422  photometryDoRankUpdate = pexConfig.Field(
423  doc=("Do the rank update step during minimization. "
424  "Skipping this can help deal with models that are too non-linear."),
425  dtype=bool,
426  default=True,
427  )
428  astrometryDoRankUpdate = pexConfig.Field(
429  doc=("Do the rank update step during minimization (should not change the astrometry fit). "
430  "Skipping this can help deal with models that are too non-linear."),
431  dtype=bool,
432  default=True,
433  )
434  outlierRejectSigma = pexConfig.Field(
435  doc="How many sigma to reject outliers at during minimization.",
436  dtype=float,
437  default=5.0,
438  )
439  astrometryOutlierRelativeTolerance = pexConfig.Field(
440  doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
441  "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
442  "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
443  "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
444  "close to the solution found when removing all outliers."),
445  dtype=float,
446  default=0,
447  )
448  maxPhotometrySteps = pexConfig.Field(
449  doc="Maximum number of minimize iterations to take when fitting photometry.",
450  dtype=int,
451  default=20,
452  )
453  maxAstrometrySteps = pexConfig.Field(
454  doc="Maximum number of minimize iterations to take when fitting photometry.",
455  dtype=int,
456  default=20,
457  )
458  astrometryRefObjLoader = pexConfig.ConfigurableField(
459  target=LoadIndexedReferenceObjectsTask,
460  doc="Reference object loader for astrometric fit",
461  )
462  photometryRefObjLoader = pexConfig.ConfigurableField(
463  target=LoadIndexedReferenceObjectsTask,
464  doc="Reference object loader for photometric fit",
465  )
466  sourceSelector = sourceSelectorRegistry.makeField(
467  doc="How to select sources for cross-matching",
468  default="astrometry"
469  )
470  astrometryReferenceSelector = pexConfig.ConfigurableField(
471  target=ReferenceSourceSelectorTask,
472  doc="How to down-select the loaded astrometry reference catalog.",
473  )
474  photometryReferenceSelector = pexConfig.ConfigurableField(
475  target=ReferenceSourceSelectorTask,
476  doc="How to down-select the loaded photometry reference catalog.",
477  )
478  astrometryReferenceErr = pexConfig.Field(
479  doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
480  "If None, then raise an exception if the reference catalog is missing coordinate errors. "
481  "If specified, overrides any existing `coord_*Err` values."),
482  dtype=float,
483  default=None,
484  optional=True
485  )
486 
487  # configs for outputting debug information
488  writeInitMatrix = pexConfig.Field(
489  dtype=bool,
490  doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
491  "Output files will be written to `config.debugOutputPath` and will "
492  "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
493  "Note that these files are the dense versions of the matrix, and so may be very large."),
494  default=False
495  )
496  writeChi2FilesInitialFinal = pexConfig.Field(
497  dtype=bool,
498  doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
499  "Output files will be written to `config.debugOutputPath` and will "
500  "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER1."),
501  default=False
502  )
503  writeChi2FilesOuterLoop = pexConfig.Field(
504  dtype=bool,
505  doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
506  "Output files will be written to `config.debugOutputPath` and will "
507  "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
508  default=False
509  )
510  writeInitialModel = pexConfig.Field(
511  dtype=bool,
512  doc=("Write the pre-initialization model to text files, for debugging. "
513  "Output files will be written to `config.debugOutputPath` and will be "
514  "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
515  default=False
516  )
517  debugOutputPath = pexConfig.Field(
518  dtype=str,
519  default=".",
520  doc=("Path to write debug output files to. Used by "
521  "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
522  )
523  detailedProfile = pexConfig.Field(
524  dtype=bool,
525  default=False,
526  doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
527  )
528 
529  def validate(self):
530  super().validate()
531  if self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms and len(self.colortermscolorterms.data) == 0:
532  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
533  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
534  if self.doAstrometrydoAstrometry and not self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms:
535  msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
536  "applyColorTerms=True will be ignored.")
537  lsst.log.warn(msg)
538 
539  def setDefaults(self):
540  # Use science source selector which can filter on extendedness, SNR, and whether blended
541  self.sourceSelectorsourceSelector.name = 'science'
542  # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
543  self.sourceSelectorsourceSelector['science'].doUnresolved = True
544  # with dependable signal to noise ratio.
545  self.sourceSelectorsourceSelector['science'].doSignalToNoise = True
546  # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
547  # and S/N > 10 to use sources that are not too faint, and thus better measured.
548  self.sourceSelectorsourceSelector['science'].signalToNoise.minimum = 10.
549  # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
550  fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
551  self.sourceSelectorsourceSelector['science'].signalToNoise.fluxField = fluxField
552  self.sourceSelectorsourceSelector['science'].signalToNoise.errField = fluxField + "Err"
553  # Do not trust blended sources' aperture fluxes which also depend on seeing
554  self.sourceSelectorsourceSelector['science'].doIsolated = True
555  # Do not trust either flux or centroid measurements with flags,
556  # chosen from the usual QA flags for stars)
557  self.sourceSelectorsourceSelector['science'].doFlags = True
558  badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
559  'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
560  'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
561  self.sourceSelectorsourceSelector['science'].flags.bad = badFlags
562 
563  # Default to Gaia-DR2 (with proper motions) for astrometry and
564  # PS1-DR1 for photometry, with a reasonable initial filterMap.
565  self.astrometryRefObjLoaderastrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
566  self.astrometryRefObjLoaderastrometryRefObjLoader.requireProperMotion = True
567  self.astrometryRefObjLoaderastrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
568  self.photometryRefObjLoaderphotometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
569 
570 
571 def writeModel(model, filename, log):
572  """Write model to outfile."""
573  with open(filename, "w") as file:
574  file.write(repr(model))
575  log.info("Wrote %s to file: %s", model, filename)
576 
577 
578 @dataclasses.dataclass
580  """The input data jointcal needs for each detector/visit."""
581  visit: int
582  """The visit identifier of this exposure."""
584  """The catalog derived from this exposure."""
585  visitInfo: lsst.afw.image.VisitInfo
586  """The VisitInfo of this exposure."""
588  """The detector of this exposure."""
589  photoCalib: lsst.afw.image.PhotoCalib
590  """The photometric calibration of this exposure."""
592  """The WCS of this exposure."""
593  bbox: lsst.geom.Box2I
594  """The bounding box of this exposure."""
596  """The filter of this exposure."""
597 
598 
599 class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
600  """Astrometricly and photometricly calibrate across multiple visits of the
601  same field.
602 
603  Parameters
604  ----------
605  butler : `lsst.daf.persistence.Butler`
606  The butler is passed to the refObjLoader constructor in case it is
607  needed. Ignored if the refObjLoader argument provides a loader directly.
608  Used to initialize the astrometry and photometry refObjLoaders.
609  initInputs : `dict`, optional
610  Dictionary used to initialize PipelineTasks (empty for jointcal).
611  """
612 
613  ConfigClass = JointcalConfig
614  RunnerClass = JointcalRunner
615  _DefaultName = "jointcal"
616 
617  def __init__(self, butler=None, initInputs=None, **kwargs):
618  super().__init__(**kwargs)
619  self.makeSubtask("sourceSelector")
620  if self.config.doAstrometry:
621  if initInputs is None:
622  # gen3 middleware does refcat things internally (and will not have a butler here)
623  self.makeSubtask('astrometryRefObjLoader', butler=butler)
624  self.makeSubtask("astrometryReferenceSelector")
625  else:
626  self.astrometryRefObjLoaderastrometryRefObjLoader = None
627  if self.config.doPhotometry:
628  if initInputs is None:
629  # gen3 middleware does refcat things internally (and will not have a butler here)
630  self.makeSubtask('photometryRefObjLoader', butler=butler)
631  self.makeSubtask("photometryReferenceSelector")
632  else:
633  self.photometryRefObjLoaderphotometryRefObjLoader = None
634 
635  # To hold various computed metrics for use by tests
636  self.jobjob = Job.load_metrics_package(subset='jointcal')
637 
638  def runQuantum(self, butlerQC, inputRefs, outputRefs):
639  # We override runQuantum to set up the refObjLoaders and write the
640  # outputs to the correct refs.
641  inputs = butlerQC.get(inputRefs)
642  # We want the tract number for writing debug files
643  tract = butlerQC.quantum.dataId['tract']
644  if self.config.doAstrometry:
645  self.astrometryRefObjLoaderastrometryRefObjLoader = ReferenceObjectLoader(
646  dataIds=[ref.datasetRef.dataId
647  for ref in inputRefs.astrometryRefCat],
648  refCats=inputs.pop('astrometryRefCat'),
649  config=self.config.astrometryRefObjLoader,
650  log=self.log)
651  if self.config.doPhotometry:
652  self.photometryRefObjLoaderphotometryRefObjLoader = ReferenceObjectLoader(
653  dataIds=[ref.datasetRef.dataId
654  for ref in inputRefs.photometryRefCat],
655  refCats=inputs.pop('photometryRefCat'),
656  config=self.config.photometryRefObjLoader,
657  log=self.log)
658  outputs = self.runrun(**inputs, tract=tract)
659  if self.config.doAstrometry:
660  self._put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
661  inputs['inputCamera'], "setWcs")
662  if self.config.doPhotometry:
663  self._put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
664  inputs['inputCamera'], "setPhotoCalib")
665 
666  def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
667  """Persist the output datasets to their appropriate datarefs.
668 
669  Parameters
670  ----------
671  butlerQC : `ButlerQuantumContext`
672  A butler which is specialized to operate in the context of a
673  `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
674  outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
675  `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
676  The fitted objects to persist.
677  outputRefs : `list` [`OutputQuantizedConnection`]
678  The DatasetRefs to persist the data to.
679  camera : `lsst.afw.cameraGeom.Camera`
680  The camera for this instrument, to get detector ids from.
681  setter : `str`
682  The method to call on the ExposureCatalog to set each output.
683  """
685  schema.addField('visit', type='I', doc='Visit number')
686 
687  def new_catalog(visit, size):
688  """Return an catalog ready to be filled with appropriate output."""
689  catalog = lsst.afw.table.ExposureCatalog(schema)
690  catalog.resize(size)
691  catalog['visit'] = visit
692  metadata = lsst.daf.base.PropertyList()
693  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
694  metadata.add("COMMENT", "Only detectors with data have entries.")
695  return catalog
696 
697  # count how many detectors have output for each visit
698  detectors_per_visit = collections.defaultdict(int)
699  for key in outputs:
700  # key is (visit, detector_id), and we only need visit here
701  detectors_per_visit[key[0]] += 1
702 
703  for ref in outputRefs:
704  visit = ref.dataId['visit']
705  catalog = new_catalog(visit, detectors_per_visit[visit])
706  # Iterate over every detector and skip the ones we don't have output for.
707  i = 0
708  for detector in camera:
709  detectorId = detector.getId()
710  key = (ref.dataId['visit'], detectorId)
711  if key not in outputs:
712  # skip detectors we don't have output for
713  self.log.debug("No %s output for detector %s in visit %s",
714  setter[3:], detectorId, visit)
715  continue
716 
717  catalog[i].setId(detectorId)
718  getattr(catalog[i], setter)(outputs[key])
719  i += 1
720 
721  catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
722  butlerQC.put(catalog, ref)
723  self.log.info("Wrote %s detectors to %s", i, ref)
724 
725  def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
726  # Docstring inherited.
727 
728  # We take values out of the Parquet table, and put them in "flux_",
729  # and the config.sourceFluxType field is used during that extraction,
730  # so just use "flux" here.
731  sourceFluxField = "flux"
732  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
733  associations = lsst.jointcal.Associations()
734  self.focalPlaneBBoxfocalPlaneBBox = inputCamera.getFpBBox()
735  oldWcsList, bands = self._load_data_load_data(inputSourceTableVisit,
736  inputVisitSummary,
737  associations,
738  jointcalControl,
739  inputCamera)
740 
741  boundingCircle, center, radius, defaultFilter = self._prep_sky_prep_sky(associations, bands)
742  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
743 
744  if self.config.doAstrometry:
745  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
746  name="astrometry",
747  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
748  referenceSelector=self.astrometryReferenceSelector,
749  fit_function=self._fit_astrometry_fit_astrometry,
750  tract=tract,
751  epoch=epoch)
752  astrometry_output = self._make_output_make_output(associations.getCcdImageList(),
753  astrometry.model,
754  "makeSkyWcs")
755  else:
756  astrometry_output = None
757 
758  if self.config.doPhotometry:
759  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
760  name="photometry",
761  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
762  referenceSelector=self.photometryReferenceSelector,
763  fit_function=self._fit_photometry_fit_photometry,
764  tract=tract,
765  epoch=epoch,
766  reject_bad_fluxes=True)
767  photometry_output = self._make_output_make_output(associations.getCcdImageList(),
768  photometry.model,
769  "toPhotoCalib")
770  else:
771  photometry_output = None
772 
773  return pipeBase.Struct(outputWcs=astrometry_output,
774  outputPhotoCalib=photometry_output,
775  job=self.jobjob,
776  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
777  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader)
778 
779  def _make_schema_table(self):
780  """Return an afw SourceTable to use as a base for creating the
781  SourceCatalog to insert values from the dataFrame into.
782 
783  Returns
784  -------
785  table : `lsst.afw.table.SourceTable`
786  Table with schema and slots to use to make SourceCatalogs.
787  """
789  schema.addField("centroid_x", "D")
790  schema.addField("centroid_y", "D")
791  schema.addField("centroid_xErr", "F")
792  schema.addField("centroid_yErr", "F")
793  schema.addField("shape_xx", "D")
794  schema.addField("shape_yy", "D")
795  schema.addField("shape_xy", "D")
796  schema.addField("flux_instFlux", "D")
797  schema.addField("flux_instFluxErr", "D")
798  table = lsst.afw.table.SourceTable.make(schema)
799  table.defineCentroid("centroid")
800  table.defineShape("shape")
801  return table
802 
803  def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
804  """Return an afw SourceCatalog extracted from a visit-level dataframe,
805  limited to just one detector.
806 
807  Parameters
808  ----------
809  table : `lsst.afw.table.SourceTable`
810  Table factory to use to make the SourceCatalog that will be
811  populated with data from ``visitCatalog``.
812  visitCatalog : `pandas.DataFrame`
813  DataFrame to extract a detector catalog from.
814  detectorId : `int`
815  Numeric id of the detector to extract from ``visitCatalog``.
816 
817  Returns
818  -------
819  catalog : `lsst.afw.table.SourceCatalog`
820  Detector-level catalog extracted from ``visitCatalog``.
821  """
822  # map from dataFrame column to afw table column
823  mapping = {'sourceId': 'id',
824  'x': 'centroid_x',
825  'y': 'centroid_y',
826  'xErr': 'centroid_xErr',
827  'yErr': 'centroid_yErr',
828  'Ixx': 'shape_xx',
829  'Iyy': 'shape_yy',
830  'Ixy': 'shape_xy',
831  f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
832  f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
833  }
834  # If the DataFrame we're reading was generated by a task running with
835  # Gen2 middleware, the column for the detector will be "ccd" for at
836  # least HSC (who knows what it might be in general!); that would be
837  # true even if the data repo is later converted to Gen3, because that
838  # doesn't touch the files themselves. In Gen3, the column for the
839  # detector will always be "detector".
840  detector_column = "detector" if "detector" in visitCatalog.columns else "ccd"
841  catalog = lsst.afw.table.SourceCatalog(table)
842  matched = visitCatalog[detector_column] == detectorId
843  catalog.resize(sum(matched))
844  view = visitCatalog.loc[matched]
845  for dfCol, afwCol in mapping.items():
846  catalog[afwCol] = view[dfCol]
847 
848  self.log.debug("%d sources selected in visit %d detector %d",
849  len(catalog),
850  view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
851  detectorId)
852  return catalog
853 
854  def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
855  jointcalControl, camera):
856  """Read the data that jointcal needs to run. (Gen3 version)
857 
858  Modifies ``associations`` in-place with the loaded data.
859 
860  Parameters
861  ----------
862  inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
863  References to visit-level DataFrames to load the catalog data from.
864  inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
865  Visit-level exposure summary catalog with metadata.
866  associations : `lsst.jointcal.Associations`
867  Object to add the loaded data to by constructing new CcdImages.
868  jointcalControl : `jointcal.JointcalControl`
869  Control object for C++ associations management.
870  camera : `lsst.afw.cameraGeom.Camera`
871  Camera object for detector geometry.
872 
873  Returns
874  -------
875  oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
876  The original WCS of the input data, to aid in writing tests.
877  bands : `list` [`str`]
878  The filter bands of each input dataset.
879  """
880  oldWcsList = []
881  filters = []
882  load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
883  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
884  table = self._make_schema_table_make_schema_table() # every detector catalog has the same layout
885  # No guarantee that the input is in the same order of visits, so we have to map one of them.
886  catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
887  detectorDict = {detector.getId(): detector for detector in camera}
888 
889  for visitSummaryRef in inputVisitSummary:
890  visitSummary = visitSummaryRef.get()
891  visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
892  selected = self.sourceSelector.run(visitCatalog)
893 
894  # Build a CcdImage for each detector in this visit.
895  detectors = {id: index for index, id in enumerate(visitSummary['id'])}
896  for id, index in detectors.items():
897  catalog = self._extract_detector_catalog_from_visit_catalog_extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
898  data = self._make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
899  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
900  if result is not None:
901  oldWcsList.append(result.wcs)
902  # A visit has only one band, so we can just use the first.
903  filters.append(data.filter)
904  if len(filters) == 0:
905  raise RuntimeError("No data to process: did source selector remove all sources?")
906  filters = collections.Counter(filters)
907 
908  return oldWcsList, filters
909 
910  def _make_one_input_data(self, visitRecord, catalog, detectorDict):
911  """Return a data structure for this detector+visit."""
912  return JointcalInputData(visit=visitRecord['visit'],
913  catalog=catalog,
914  visitInfo=visitRecord.getVisitInfo(),
915  detector=detectorDict[visitRecord.getId()],
916  photoCalib=visitRecord.getPhotoCalib(),
917  wcs=visitRecord.getWcs(),
918  bbox=visitRecord.getBBox(),
919  # ExposureRecord doesn't have a FilterLabel yet,
920  # so we have to make one.
921  filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
922  physical=visitRecord['physical_filter']))
923 
924  # We don't currently need to persist the metadata.
925  # If we do in the future, we will have to add appropriate dataset templates
926  # to each obs package (the metadata template should look like `jointcal_wcs`).
927  def _getMetadataName(self):
928  return None
929 
930  @classmethod
931  def _makeArgumentParser(cls):
932  """Create an argument parser"""
933  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
934  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
935  ContainerClass=PerTractCcdDataIdContainer)
936  return parser
937 
938  def _build_ccdImage(self, data, associations, jointcalControl):
939  """
940  Extract the necessary things from this catalog+metadata to add a new
941  ccdImage.
942 
943  Parameters
944  ----------
945  data : `JointcalInputData`
946  The loaded input data.
947  associations : `lsst.jointcal.Associations`
948  Object to add the info to, to construct a new CcdImage
949  jointcalControl : `jointcal.JointcalControl`
950  Control object for associations management
951 
952  Returns
953  ------
954  namedtuple or `None`
955  ``wcs``
956  The TAN WCS of this image, read from the calexp
957  (`lsst.afw.geom.SkyWcs`).
958  ``key``
959  A key to identify this dataRef by its visit and ccd ids
960  (`namedtuple`).
961  `None`
962  if there are no sources in the loaded catalog.
963  """
964  if len(data.catalog) == 0:
965  self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
966  return None
967 
968  associations.createCcdImage(data.catalog,
969  data.wcs,
970  data.visitInfo,
971  data.bbox,
972  data.filter.physicalLabel,
973  data.photoCalib,
974  data.detector,
975  data.visit,
976  data.detector.getId(),
977  jointcalControl)
978 
979  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
980  Key = collections.namedtuple('Key', ('visit', 'ccd'))
981  return Result(data.wcs, Key(data.visit, data.detector.getId()))
982 
983  def _readDataId(self, butler, dataId):
984  """Read all of the data for one dataId from the butler. (gen2 version)"""
985  # Not all instruments have `visit` in their dataIds.
986  if "visit" in dataId.keys():
987  visit = dataId["visit"]
988  else:
989  visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
990  detector = butler.get('calexp_detector', dataId=dataId)
991 
992  catalog = butler.get('src',
993  flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
994  dataId=dataId)
995  goodSrc = self.sourceSelector.run(catalog)
996  self.log.debug("%d sources selected in visit %d detector %d",
997  len(goodSrc.sourceCat),
998  visit,
999  detector.getId())
1000  return JointcalInputData(visit=visit,
1001  catalog=goodSrc.sourceCat,
1002  visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1003  detector=detector,
1004  photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1005  wcs=butler.get('calexp_wcs', dataId=dataId),
1006  bbox=butler.get('calexp_bbox', dataId=dataId),
1007  filter=butler.get('calexp_filterLabel', dataId=dataId))
1008 
1009  def loadData(self, dataRefs, associations, jointcalControl):
1010  """Read the data that jointcal needs to run. (Gen2 version)"""
1011  visit_ccd_to_dataRef = {}
1012  oldWcsList = []
1013  filters = []
1014  load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1015  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1016  # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1017  camera = dataRefs[0].get('camera', immediate=True)
1018  self.focalPlaneBBoxfocalPlaneBBox = camera.getFpBBox()
1019  for dataRef in dataRefs:
1020  data = self._readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1021  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1022  if result is None:
1023  continue
1024  oldWcsList.append(result.wcs)
1025  visit_ccd_to_dataRef[result.key] = dataRef
1026  filters.append(data.filter)
1027  if len(filters) == 0:
1028  raise RuntimeError("No data to process: did source selector remove all sources?")
1029  filters = collections.Counter(filters)
1030 
1031  return oldWcsList, filters, visit_ccd_to_dataRef
1032 
1033  def _getDebugPath(self, filename):
1034  """Constructs a path to filename using the configured debug path.
1035  """
1036  return os.path.join(self.config.debugOutputPath, filename)
1037 
1038  def _prep_sky(self, associations, filters):
1039  """Prepare on-sky and other data that must be computed after data has
1040  been read.
1041  """
1042  associations.computeCommonTangentPoint()
1043 
1044  boundingCircle = associations.computeBoundingCircle()
1045  center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1046  radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1047 
1048  self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1049 
1050  # Determine a default filter band associated with the catalog. See DM-9093
1051  defaultFilter = filters.most_common(1)[0][0]
1052  self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1053 
1054  return boundingCircle, center, radius, defaultFilter
1055 
1056  @pipeBase.timeMethod
1057  def runDataRef(self, dataRefs):
1058  """
1059  Jointly calibrate the astrometry and photometry across a set of images.
1060 
1061  NOTE: this is for gen2 middleware only.
1062 
1063  Parameters
1064  ----------
1065  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1066  List of data references to the exposures to be fit.
1067 
1068  Returns
1069  -------
1070  result : `lsst.pipe.base.Struct`
1071  Struct of metadata from the fit, containing:
1072 
1073  ``dataRefs``
1074  The provided data references that were fit (with updated WCSs)
1075  ``oldWcsList``
1076  The original WCS from each dataRef
1077  ``metrics``
1078  Dictionary of internally-computed metrics for testing/validation.
1079  """
1080  if len(dataRefs) == 0:
1081  raise ValueError('Need a non-empty list of data references!')
1082 
1083  exitStatus = 0 # exit status for shell
1084 
1085  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1086  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1087  associations = lsst.jointcal.Associations()
1088 
1089  oldWcsList, filters, visit_ccd_to_dataRef = self.loadDataloadData(dataRefs,
1090  associations,
1091  jointcalControl)
1092 
1093  boundingCircle, center, radius, defaultFilter = self._prep_sky_prep_sky(associations, filters)
1094  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
1095 
1096  tract = dataRefs[0].dataId['tract']
1097 
1098  if self.config.doAstrometry:
1099  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1100  name="astrometry",
1101  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1102  referenceSelector=self.astrometryReferenceSelector,
1103  fit_function=self._fit_astrometry_fit_astrometry,
1104  tract=tract,
1105  epoch=epoch)
1106  self._write_astrometry_results_write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1107  else:
1108  astrometry = Astrometry(None, None, None)
1109 
1110  if self.config.doPhotometry:
1111  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1112  name="photometry",
1113  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1114  referenceSelector=self.photometryReferenceSelector,
1115  fit_function=self._fit_photometry_fit_photometry,
1116  tract=tract,
1117  epoch=epoch,
1118  reject_bad_fluxes=True)
1119  self._write_photometry_results_write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1120  else:
1121  photometry = Photometry(None, None)
1122 
1123  return pipeBase.Struct(dataRefs=dataRefs,
1124  oldWcsList=oldWcsList,
1125  job=self.jobjob,
1126  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1127  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1128  defaultFilter=defaultFilter,
1129  epoch=epoch,
1130  exitStatus=exitStatus)
1131 
1132  def _get_refcat_coordinate_error_override(self, refCat, name):
1133  """Check whether we should override the refcat coordinate errors, and
1134  return the overridden error if necessary.
1135 
1136  Parameters
1137  ----------
1138  refCat : `lsst.afw.table.SimpleCatalog`
1139  The reference catalog to check for a ``coord_raErr`` field.
1140  name : `str`
1141  Whether we are doing "astrometry" or "photometry".
1142 
1143  Returns
1144  -------
1145  refCoordErr : `float`
1146  The refcat coordinate error to use, or NaN if we are not overriding
1147  those fields.
1148 
1149  Raises
1150  ------
1151  lsst.pex.config.FieldValidationError
1152  Raised if the refcat does not contain coordinate errors and
1153  ``config.astrometryReferenceErr`` is not set.
1154  """
1155  # This value doesn't matter for photometry, so just set something to
1156  # keep old refcats from causing problems.
1157  if name.lower() == "photometry":
1158  if 'coord_raErr' not in refCat.schema:
1159  return 100
1160  else:
1161  return float('nan')
1162 
1163  if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1164  msg = ("Reference catalog does not contain coordinate errors, "
1165  "and config.astrometryReferenceErr not supplied.")
1166  raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1167  self.config,
1168  msg)
1169 
1170  if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1171  self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1172  self.config.astrometryReferenceErr)
1173 
1174  if self.config.astrometryReferenceErr is None:
1175  return float('nan')
1176  else:
1177  return self.config.astrometryReferenceErr
1178 
1179  def _compute_proper_motion_epoch(self, ccdImageList):
1180  """Return the proper motion correction epoch of the provided images.
1181 
1182  Parameters
1183  ----------
1184  ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1185  The images to compute the appropriate epoch for.
1186 
1187  Returns
1188  -------
1189  epoch : `astropy.time.Time`
1190  The date to use for proper motion corrections.
1191  """
1192  mjds = [ccdImage.getMjd() for ccdImage in ccdImageList]
1193  return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai")
1194 
1195  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1196  tract="", match_cut=3.0,
1197  reject_bad_fluxes=False, *,
1198  name="", refObjLoader=None, referenceSelector=None,
1199  fit_function=None, epoch=None):
1200  """Load reference catalog, perform the fit, and return the result.
1201 
1202  Parameters
1203  ----------
1204  associations : `lsst.jointcal.Associations`
1205  The star/reference star associations to fit.
1206  defaultFilter : `lsst.afw.image.FilterLabel`
1207  filter to load from reference catalog.
1208  center : `lsst.geom.SpherePoint`
1209  ICRS center of field to load from reference catalog.
1210  radius : `lsst.geom.Angle`
1211  On-sky radius to load from reference catalog.
1212  name : `str`
1213  Name of thing being fit: "astrometry" or "photometry".
1214  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1215  Reference object loader to use to load a reference catalog.
1216  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1217  Selector to use to pick objects from the loaded reference catalog.
1218  fit_function : callable
1219  Function to call to perform fit (takes Associations object).
1220  tract : `str`, optional
1221  Name of tract currently being fit.
1222  match_cut : `float`, optional
1223  Radius in arcseconds to find cross-catalog matches to during
1224  associations.associateCatalogs.
1225  reject_bad_fluxes : `bool`, optional
1226  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1227  epoch : `astropy.time.Time`, optional
1228  Epoch to which to correct refcat proper motion and parallax,
1229  or `None` to not apply such corrections.
1230 
1231  Returns
1232  -------
1233  result : `Photometry` or `Astrometry`
1234  Result of `fit_function()`
1235  """
1236  self.log.info("====== Now processing %s...", name)
1237  # TODO: this should not print "trying to invert a singular transformation:"
1238  # if it does that, something's not right about the WCS...
1239  associations.associateCatalogs(match_cut)
1240  add_measurement(self.jobjob, 'jointcal.associated_%s_fittedStars' % name,
1241  associations.fittedStarListSize())
1242 
1243  applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1244  refCat, fluxField = self._load_reference_catalog_load_reference_catalog(refObjLoader, referenceSelector,
1245  center, radius, defaultFilter,
1246  applyColorterms=applyColorterms,
1247  epoch=epoch)
1248  refCoordErr = self._get_refcat_coordinate_error_override_get_refcat_coordinate_error_override(refCat, name)
1249 
1250  associations.collectRefStars(refCat,
1251  self.config.matchCut*lsst.geom.arcseconds,
1252  fluxField,
1253  refCoordinateErr=refCoordErr,
1254  rejectBadFluxes=reject_bad_fluxes)
1255  add_measurement(self.jobjob, 'jointcal.collected_%s_refStars' % name,
1256  associations.refStarListSize())
1257 
1258  associations.prepareFittedStars(self.config.minMeasurements)
1259 
1260  self._check_star_lists_check_star_lists(associations, name)
1261  add_measurement(self.jobjob, 'jointcal.selected_%s_refStars' % name,
1262  associations.nFittedStarsWithAssociatedRefStar())
1263  add_measurement(self.jobjob, 'jointcal.selected_%s_fittedStars' % name,
1264  associations.fittedStarListSize())
1265  add_measurement(self.jobjob, 'jointcal.selected_%s_ccdImages' % name,
1266  associations.nCcdImagesValidForFit())
1267 
1268  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1269  dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1270  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1271  result = fit_function(associations, dataName)
1272  # TODO DM-12446: turn this into a "butler save" somehow.
1273  # Save reference and measurement chi2 contributions for this data
1274  if self.config.writeChi2FilesInitialFinal:
1275  baseName = self._getDebugPath_getDebugPath(f"{name}_final_chi2-{dataName}")
1276  result.fit.saveChi2Contributions(baseName+"{type}")
1277  self.log.info("Wrote chi2 contributions files: %s", baseName)
1278 
1279  return result
1280 
1281  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1282  applyColorterms=False, epoch=None):
1283  """Load the necessary reference catalog sources, convert fluxes to
1284  correct units, and apply color term corrections if requested.
1285 
1286  Parameters
1287  ----------
1288  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1289  The reference catalog loader to use to get the data.
1290  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1291  Source selector to apply to loaded reference catalog.
1292  center : `lsst.geom.SpherePoint`
1293  The center around which to load sources.
1294  radius : `lsst.geom.Angle`
1295  The radius around ``center`` to load sources in.
1296  filterLabel : `lsst.afw.image.FilterLabel`
1297  The camera filter to load fluxes for.
1298  applyColorterms : `bool`
1299  Apply colorterm corrections to the refcat for ``filterName``?
1300  epoch : `astropy.time.Time`, optional
1301  Epoch to which to correct refcat proper motion and parallax,
1302  or `None` to not apply such corrections.
1303 
1304  Returns
1305  -------
1306  refCat : `lsst.afw.table.SimpleCatalog`
1307  The loaded reference catalog.
1308  fluxField : `str`
1309  The name of the reference catalog flux field appropriate for ``filterName``.
1310  """
1311  skyCircle = refObjLoader.loadSkyCircle(center,
1312  radius,
1313  filterLabel.bandLabel,
1314  epoch=epoch)
1315 
1316  selected = referenceSelector.run(skyCircle.refCat)
1317  # Need memory contiguity to get reference filters as a vector.
1318  if not selected.sourceCat.isContiguous():
1319  refCat = selected.sourceCat.copy(deep=True)
1320  else:
1321  refCat = selected.sourceCat
1322 
1323  if applyColorterms:
1324  refCatName = refObjLoader.ref_dataset_name
1325  self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1326  filterLabel.physicalLabel, refCatName)
1327  colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1328  refCatName,
1329  doRaise=True)
1330 
1331  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1332  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1333  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1334  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1335 
1336  return refCat, skyCircle.fluxField
1337 
1338  def _check_star_lists(self, associations, name):
1339  # TODO: these should be len(blah), but we need this properly wrapped first.
1340  if associations.nCcdImagesValidForFit() == 0:
1341  raise RuntimeError('No images in the ccdImageList!')
1342  if associations.fittedStarListSize() == 0:
1343  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1344  if associations.refStarListSize() == 0:
1345  raise RuntimeError('No stars in the {} reference star list!'.format(name))
1346 
1347  def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1348  """Compute chi2, log it, validate the model, and return chi2.
1349 
1350  Parameters
1351  ----------
1352  associations : `lsst.jointcal.Associations`
1353  The star/reference star associations to fit.
1354  fit : `lsst.jointcal.FitterBase`
1355  The fitter to use for minimization.
1356  model : `lsst.jointcal.Model`
1357  The model being fit.
1358  chi2Label : `str`
1359  Label to describe the chi2 (e.g. "Initialized", "Final").
1360  writeChi2Name : `str`, optional
1361  Filename prefix to write the chi2 contributions to.
1362  Do not supply an extension: an appropriate one will be added.
1363 
1364  Returns
1365  -------
1366  chi2: `lsst.jointcal.Chi2Accumulator`
1367  The chi2 object for the current fitter and model.
1368 
1369  Raises
1370  ------
1371  FloatingPointError
1372  Raised if chi2 is infinite or NaN.
1373  ValueError
1374  Raised if the model is not valid.
1375  """
1376  if writeChi2Name is not None:
1377  fullpath = self._getDebugPath_getDebugPath(writeChi2Name)
1378  fit.saveChi2Contributions(fullpath+"{type}")
1379  self.log.info("Wrote chi2 contributions files: %s", fullpath)
1380 
1381  chi2 = fit.computeChi2()
1382  self.log.info("%s %s", chi2Label, chi2)
1383  self._check_stars_check_stars(associations)
1384  if not np.isfinite(chi2.chi2):
1385  raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1386  if not model.validate(associations.getCcdImageList(), chi2.ndof):
1387  raise ValueError("Model is not valid: check log messages for warnings.")
1388  return chi2
1389 
1390  def _fit_photometry(self, associations, dataName=None):
1391  """
1392  Fit the photometric data.
1393 
1394  Parameters
1395  ----------
1396  associations : `lsst.jointcal.Associations`
1397  The star/reference star associations to fit.
1398  dataName : `str`
1399  Name of the data being processed (e.g. "1234_HSC-Y"), for
1400  identifying debugging files.
1401 
1402  Returns
1403  -------
1404  fit_result : `namedtuple`
1405  fit : `lsst.jointcal.PhotometryFit`
1406  The photometric fitter used to perform the fit.
1407  model : `lsst.jointcal.PhotometryModel`
1408  The photometric model that was fit.
1409  """
1410  self.log.info("=== Starting photometric fitting...")
1411 
1412  # TODO: should use pex.config.RegistryField here (see DM-9195)
1413  if self.config.photometryModel == "constrainedFlux":
1414  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1415  self.focalPlaneBBoxfocalPlaneBBox,
1416  visitOrder=self.config.photometryVisitOrder,
1417  errorPedestal=self.config.photometryErrorPedestal)
1418  # potentially nonlinear problem, so we may need a line search to converge.
1419  doLineSearch = self.config.allowLineSearch
1420  elif self.config.photometryModel == "constrainedMagnitude":
1421  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1422  self.focalPlaneBBoxfocalPlaneBBox,
1423  visitOrder=self.config.photometryVisitOrder,
1424  errorPedestal=self.config.photometryErrorPedestal)
1425  # potentially nonlinear problem, so we may need a line search to converge.
1426  doLineSearch = self.config.allowLineSearch
1427  elif self.config.photometryModel == "simpleFlux":
1428  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1429  errorPedestal=self.config.photometryErrorPedestal)
1430  doLineSearch = False # purely linear in model parameters, so no line search needed
1431  elif self.config.photometryModel == "simpleMagnitude":
1432  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1433  errorPedestal=self.config.photometryErrorPedestal)
1434  doLineSearch = False # purely linear in model parameters, so no line search needed
1435 
1436  fit = lsst.jointcal.PhotometryFit(associations, model)
1437  # TODO DM-12446: turn this into a "butler save" somehow.
1438  # Save reference and measurement chi2 contributions for this data
1439  if self.config.writeChi2FilesInitialFinal:
1440  baseName = f"photometry_initial_chi2-{dataName}"
1441  else:
1442  baseName = None
1443  if self.config.writeInitialModel:
1444  fullpath = self._getDebugPath_getDebugPath(f"initial_photometry_model-{dataName}.txt")
1445  writeModel(model, fullpath, self.log)
1446  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1447 
1448  def getChi2Name(whatToFit):
1449  if self.config.writeChi2FilesOuterLoop:
1450  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1451  else:
1452  return None
1453 
1454  # The constrained model needs the visit transform fit first; the chip
1455  # transform is initialized from the singleFrame PhotoCalib, so it's close.
1456  if self.config.writeInitMatrix:
1457  dumpMatrixFile = self._getDebugPath_getDebugPath(f"photometry_preinit-{dataName}")
1458  else:
1459  dumpMatrixFile = ""
1460  if self.config.photometryModel.startswith("constrained"):
1461  # no line search: should be purely (or nearly) linear,
1462  # and we want a large step size to initialize with.
1463  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1464  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1465  writeChi2Name=getChi2Name("ModelVisit"))
1466  dumpMatrixFile = "" # so we don't redo the output on the next step
1467 
1468  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1469  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Model",
1470  writeChi2Name=getChi2Name("Model"))
1471 
1472  fit.minimize("Fluxes") # no line search: always purely linear.
1473  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1474  writeChi2Name=getChi2Name("Fluxes"))
1475 
1476  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1477  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1478  writeChi2Name=getChi2Name("ModelFluxes"))
1479 
1480  model.freezeErrorTransform()
1481  self.log.debug("Photometry error scales are frozen.")
1482 
1483  chi2 = self._iterate_fit_iterate_fit(associations,
1484  fit,
1485  self.config.maxPhotometrySteps,
1486  "photometry",
1487  "Model Fluxes",
1488  doRankUpdate=self.config.photometryDoRankUpdate,
1489  doLineSearch=doLineSearch,
1490  dataName=dataName)
1491 
1492  add_measurement(self.jobjob, 'jointcal.photometry_final_chi2', chi2.chi2)
1493  add_measurement(self.jobjob, 'jointcal.photometry_final_ndof', chi2.ndof)
1494  return Photometry(fit, model)
1495 
1496  def _fit_astrometry(self, associations, dataName=None):
1497  """
1498  Fit the astrometric data.
1499 
1500  Parameters
1501  ----------
1502  associations : `lsst.jointcal.Associations`
1503  The star/reference star associations to fit.
1504  dataName : `str`
1505  Name of the data being processed (e.g. "1234_HSC-Y"), for
1506  identifying debugging files.
1507 
1508  Returns
1509  -------
1510  fit_result : `namedtuple`
1511  fit : `lsst.jointcal.AstrometryFit`
1512  The astrometric fitter used to perform the fit.
1513  model : `lsst.jointcal.AstrometryModel`
1514  The astrometric model that was fit.
1515  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1516  The model for the sky to tangent plane projection that was used in the fit.
1517  """
1518 
1519  self.log.info("=== Starting astrometric fitting...")
1520 
1521  associations.deprojectFittedStars()
1522 
1523  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1524  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1525  # them so carefully?
1526  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1527 
1528  if self.config.astrometryModel == "constrained":
1529  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1530  sky_to_tan_projection,
1531  chipOrder=self.config.astrometryChipOrder,
1532  visitOrder=self.config.astrometryVisitOrder)
1533  elif self.config.astrometryModel == "simple":
1534  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1535  sky_to_tan_projection,
1536  self.config.useInputWcs,
1537  nNotFit=0,
1538  order=self.config.astrometrySimpleOrder)
1539 
1540  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1541  # TODO DM-12446: turn this into a "butler save" somehow.
1542  # Save reference and measurement chi2 contributions for this data
1543  if self.config.writeChi2FilesInitialFinal:
1544  baseName = f"astrometry_initial_chi2-{dataName}"
1545  else:
1546  baseName = None
1547  if self.config.writeInitialModel:
1548  fullpath = self._getDebugPath_getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1549  writeModel(model, fullpath, self.log)
1550  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1551 
1552  def getChi2Name(whatToFit):
1553  if self.config.writeChi2FilesOuterLoop:
1554  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1555  else:
1556  return None
1557 
1558  if self.config.writeInitMatrix:
1559  dumpMatrixFile = self._getDebugPath_getDebugPath(f"astrometry_preinit-{dataName}")
1560  else:
1561  dumpMatrixFile = ""
1562  # The constrained model needs the visit transform fit first; the chip
1563  # transform is initialized from the detector's cameraGeom, so it's close.
1564  if self.config.astrometryModel == "constrained":
1565  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1566  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1567  writeChi2Name=getChi2Name("DistortionsVisit"))
1568  dumpMatrixFile = "" # so we don't redo the output on the next step
1569 
1570  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1571  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1572  writeChi2Name=getChi2Name("Distortions"))
1573 
1574  fit.minimize("Positions")
1575  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Positions",
1576  writeChi2Name=getChi2Name("Positions"))
1577 
1578  fit.minimize("Distortions Positions")
1579  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1580  writeChi2Name=getChi2Name("DistortionsPositions"))
1581 
1582  chi2 = self._iterate_fit_iterate_fit(associations,
1583  fit,
1584  self.config.maxAstrometrySteps,
1585  "astrometry",
1586  "Distortions Positions",
1587  sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1588  doRankUpdate=self.config.astrometryDoRankUpdate,
1589  dataName=dataName)
1590 
1591  add_measurement(self.jobjob, 'jointcal.astrometry_final_chi2', chi2.chi2)
1592  add_measurement(self.jobjob, 'jointcal.astrometry_final_ndof', chi2.ndof)
1593 
1594  return Astrometry(fit, model, sky_to_tan_projection)
1595 
1596  def _check_stars(self, associations):
1597  """Count measured and reference stars per ccd and warn/log them."""
1598  for ccdImage in associations.getCcdImageList():
1599  nMeasuredStars, nRefStars = ccdImage.countStars()
1600  self.log.debug("ccdImage %s has %s measured and %s reference stars",
1601  ccdImage.getName(), nMeasuredStars, nRefStars)
1602  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1603  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1604  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1605  if nRefStars < self.config.minRefStarsPerCcd:
1606  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1607  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1608 
1609  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1610  dataName="",
1611  sigmaRelativeTolerance=0,
1612  doRankUpdate=True,
1613  doLineSearch=False):
1614  """Run fitter.minimize up to max_steps times, returning the final chi2.
1615 
1616  Parameters
1617  ----------
1618  associations : `lsst.jointcal.Associations`
1619  The star/reference star associations to fit.
1620  fitter : `lsst.jointcal.FitterBase`
1621  The fitter to use for minimization.
1622  max_steps : `int`
1623  Maximum number of steps to run outlier rejection before declaring
1624  convergence failure.
1625  name : {'photometry' or 'astrometry'}
1626  What type of data are we fitting (for logs and debugging files).
1627  whatToFit : `str`
1628  Passed to ``fitter.minimize()`` to define the parameters to fit.
1629  dataName : `str`, optional
1630  Descriptive name for this dataset (e.g. tract and filter),
1631  for debugging.
1632  sigmaRelativeTolerance : `float`, optional
1633  Convergence tolerance for the fractional change in the chi2 cut
1634  level for determining outliers. If set to zero, iterations will
1635  continue until there are no outliers.
1636  doRankUpdate : `bool`, optional
1637  Do an Eigen rank update during minimization, or recompute the full
1638  matrix and gradient?
1639  doLineSearch : `bool`, optional
1640  Do a line search for the optimum step during minimization?
1641 
1642  Returns
1643  -------
1644  chi2: `lsst.jointcal.Chi2Statistic`
1645  The final chi2 after the fit converges, or is forced to end.
1646 
1647  Raises
1648  ------
1649  FloatingPointError
1650  Raised if the fitter fails with a non-finite value.
1651  RuntimeError
1652  Raised if the fitter fails for some other reason;
1653  log messages will provide further details.
1654  """
1655  if self.config.writeInitMatrix:
1656  dumpMatrixFile = self._getDebugPath_getDebugPath(f"{name}_postinit-{dataName}")
1657  else:
1658  dumpMatrixFile = ""
1659  oldChi2 = lsst.jointcal.Chi2Statistic()
1660  oldChi2.chi2 = float("inf")
1661  for i in range(max_steps):
1662  if self.config.writeChi2FilesOuterLoop:
1663  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1664  else:
1665  writeChi2Name = None
1666  result = fitter.minimize(whatToFit,
1667  self.config.outlierRejectSigma,
1668  sigmaRelativeTolerance=sigmaRelativeTolerance,
1669  doRankUpdate=doRankUpdate,
1670  doLineSearch=doLineSearch,
1671  dumpMatrixFile=dumpMatrixFile)
1672  dumpMatrixFile = "" # clear it so we don't write the matrix again.
1673  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1674  f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1675 
1676  if result == MinimizeResult.Converged:
1677  if doRankUpdate:
1678  self.log.debug("fit has converged - no more outliers - redo minimization "
1679  "one more time in case we have lost accuracy in rank update.")
1680  # Redo minimization one more time in case we have lost accuracy in rank update
1681  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1682  sigmaRelativeTolerance=sigmaRelativeTolerance)
1683  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1684 
1685  # log a message for a large final chi2, TODO: DM-15247 for something better
1686  if chi2.chi2/chi2.ndof >= 4.0:
1687  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1688 
1689  break
1690  elif result == MinimizeResult.Chi2Increased:
1691  self.log.warn("Still some outliers remaining but chi2 increased - retry")
1692  # Check whether the increase was large enough to cause trouble.
1693  chi2Ratio = chi2.chi2 / oldChi2.chi2
1694  if chi2Ratio > 1.5:
1695  self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1696  chi2.chi2, oldChi2.chi2, chi2Ratio)
1697  # Based on a variety of HSC jointcal logs (see DM-25779), it
1698  # appears that chi2 increases more than a factor of ~2 always
1699  # result in the fit diverging rapidly and ending at chi2 > 1e10.
1700  # Using 10 as the "failure" threshold gives some room between
1701  # leaving a warning and bailing early.
1702  if chi2Ratio > 10:
1703  msg = ("Large chi2 increase between steps: fit likely cannot converge."
1704  " Try setting one or more of the `writeChi2*` config fields and looking"
1705  " at how individual star chi2-values evolve during the fit.")
1706  raise RuntimeError(msg)
1707  oldChi2 = chi2
1708  elif result == MinimizeResult.NonFinite:
1709  filename = self._getDebugPath_getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1710  # TODO DM-12446: turn this into a "butler save" somehow.
1711  fitter.saveChi2Contributions(filename+"{type}")
1712  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1713  raise FloatingPointError(msg.format(filename))
1714  elif result == MinimizeResult.Failed:
1715  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1716  else:
1717  raise RuntimeError("Unxepected return code from minimize().")
1718  else:
1719  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1720 
1721  return chi2
1722 
1723  def _make_output(self, ccdImageList, model, func):
1724  """Return the internal jointcal models converted to the afw
1725  structures that will be saved to disk.
1726 
1727  Parameters
1728  ----------
1729  ccdImageList : `lsst.jointcal.CcdImageList`
1730  The list of CcdImages to get the output for.
1731  model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1732  The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1733  func : `str`
1734  The name of the function to call on ``model`` to get the converted
1735  structure. Must accept an `lsst.jointcal.CcdImage`.
1736 
1737  Returns
1738  -------
1739  output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1740  `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1741  The data to be saved, keyed on (visit, detector).
1742  """
1743  output = {}
1744  for ccdImage in ccdImageList:
1745  ccd = ccdImage.ccdId
1746  visit = ccdImage.visit
1747  self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1748  output[(visit, ccd)] = getattr(model, func)(ccdImage)
1749  return output
1750 
1751  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1752  """
1753  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1754 
1755  Parameters
1756  ----------
1757  associations : `lsst.jointcal.Associations`
1758  The star/reference star associations to fit.
1759  model : `lsst.jointcal.AstrometryModel`
1760  The astrometric model that was fit.
1761  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1762  Dict of ccdImage identifiers to dataRefs that were fit.
1763  """
1764  ccdImageList = associations.getCcdImageList()
1765  output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1766  for key, skyWcs in output.items():
1767  dataRef = visit_ccd_to_dataRef[key]
1768  try:
1769  dataRef.put(skyWcs, 'jointcal_wcs')
1770  except pexExceptions.Exception as e:
1771  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1772  raise e
1773 
1774  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1775  """
1776  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1777 
1778  Parameters
1779  ----------
1780  associations : `lsst.jointcal.Associations`
1781  The star/reference star associations to fit.
1782  model : `lsst.jointcal.PhotometryModel`
1783  The photoometric model that was fit.
1784  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1785  Dict of ccdImage identifiers to dataRefs that were fit.
1786  """
1787 
1788  ccdImageList = associations.getCcdImageList()
1789  output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1790  for key, photoCalib in output.items():
1791  dataRef = visit_ccd_to_dataRef[key]
1792  try:
1793  dataRef.put(photoCalib, 'jointcal_photoCalib')
1794  except pexExceptions.Exception as e:
1795  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1796  raise e
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:54
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
Definition: jointcal.py:77
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:855
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:1179
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:1132
def _getDebugPath(self, filename)
Definition: jointcal.py:1033
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1338
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:910
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1613
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1751
def _check_stars(self, associations)
Definition: jointcal.py:1596
def _prep_sky(self, associations, filters)
Definition: jointcal.py:1038
def _readDataId(self, butler, dataId)
Definition: jointcal.py:983
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:638
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:1199
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:938
def runDataRef(self, dataRefs)
Definition: jointcal.py:1057
def __init__(self, butler=None, initInputs=None, **kwargs)
Definition: jointcal.py:617
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId)
Definition: jointcal.py:803
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:725
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1390
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1774
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1347
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1723
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:666
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1282
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1496
def loadData(self, dataRefs, associations, jointcalControl)
Definition: jointcal.py:1009
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:141
def add_measurement(job, name, value)
Definition: jointcal.py:59
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:181
def writeModel(model, filename, log)
Definition: jointcal.py:571