lsst.jointcal  22.0.1-14-gc97e25b+d548b459a8
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import dataclasses
23 import collections
24 import os
25 
26 import astropy.time
27 import numpy as np
28 import astropy.units as u
29 
30 import lsst.geom
31 import lsst.utils
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 from lsst.afw.image import fluxErrFromABMagErr
35 import lsst.pex.exceptions as pexExceptions
37 import lsst.afw.table
38 import lsst.log
39 from lsst.obs.base import Instrument
40 from lsst.pipe.tasks.colorterms import ColortermLibrary
41 from lsst.verify import Job, Measurement
42 
43 from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44  ReferenceObjectLoader)
45 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46 
47 from .dataIds import PerTractCcdDataIdContainer
48 
49 import lsst.jointcal
50 from lsst.jointcal import MinimizeResult
51 
52 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
53 
54 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
56 
57 
58 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59 def add_measurement(job, name, value):
60  meas = Measurement(job.metrics[name], value)
61  job.measurements.insert(meas)
62 
63 
64 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65  """Subclass of TaskRunner for jointcalTask (gen2)
66 
67  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69  single dataRef, are are called repeatedly). This class transforms the processed
70  arguments generated by the ArgumentParser into the arguments expected by
71  Jointcal.runDataRef().
72 
73  See pipeBase.TaskRunner for more information.
74  """
75 
76  @staticmethod
77  def getTargetList(parsedCmd, **kwargs):
78  """
79  Return a list of tuples per tract, each containing (dataRefs, kwargs).
80 
81  Jointcal operates on lists of dataRefs simultaneously.
82  """
83  kwargs['butler'] = parsedCmd.butler
84 
85  # organize data IDs by tract
86  refListDict = {}
87  for ref in parsedCmd.id.refList:
88  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89  # we call runDataRef() once with each tract
90  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91  return result
92 
93  def __call__(self, args):
94  """
95  Parameters
96  ----------
97  args
98  Arguments for Task.runDataRef()
99 
100  Returns
101  -------
102  pipe.base.Struct
103  if self.doReturnResults is False:
104 
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106 
107  if self.doReturnResults is True:
108 
109  - ``result``: the result of calling jointcal.runDataRef()
110  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111  """
112  exitStatus = 0 # exit status for shell
113 
114  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115  dataRefList, kwargs = args
116  butler = kwargs.pop('butler')
117  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118  result = None
119  try:
120  result = task.runDataRef(dataRefList, **kwargs)
121  exitStatus = result.exitStatus
122  job_path = butler.get('verify_job_filename')
123  result.job.write(job_path[0])
124  except Exception as e: # catch everything, sort it out later.
125  if self.doRaise:
126  raise e
127  else:
128  exitStatus = 1
129  eName = type(e).__name__
130  tract = dataRefList[0].dataId['tract']
131  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
132 
133  # Put the butler back into kwargs for the other Tasks.
134  kwargs['butler'] = butler
135  if self.doReturnResults:
136  return pipeBase.Struct(result=result, exitStatus=exitStatus)
137  else:
138  return pipeBase.Struct(exitStatus=exitStatus)
139 
140 
141 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142  """Lookup function that asserts/hopes that a static calibration dataset
143  exists in a particular collection, since this task can't provide a single
144  date/time to use to search for one properly.
145 
146  This is mostly useful for the ``camera`` dataset, in cases where the task's
147  quantum dimensions do *not* include something temporal, like ``exposure``
148  or ``visit``.
149 
150  Parameters
151  ----------
152  datasetType : `lsst.daf.butler.DatasetType`
153  Type of dataset being searched for.
154  registry : `lsst.daf.butler.Registry`
155  Data repository registry to search.
156  quantumDataId : `lsst.daf.butler.DataCoordinate`
157  Data ID of the quantum this camera should match.
158  collections : `Iterable` [ `str` ]
159  Collections that should be searched - but this lookup function works
160  by ignoring this in favor of a more-or-less hard-coded value.
161 
162  Returns
163  -------
164  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165  Iterator over dataset references; should have only one element.
166 
167  Notes
168  -----
169  This implementation duplicates one in fgcmcal, and is at least quite
170  similar to another in cp_pipe. This duplicate has the most documentation.
171  Fixing this is DM-29661.
172  """
173  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175  return registry.queryDatasets(datasetType,
176  dataId=quantumDataId,
177  collections=[unboundedCollection],
178  findFirst=True)
179 
180 
181 def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182  """Lookup function that finds all refcats for all visits that overlap a
183  tract, rather than just the refcats that directly overlap the tract.
184 
185  Parameters
186  ----------
187  datasetType : `lsst.daf.butler.DatasetType`
188  Type of dataset being searched for.
189  registry : `lsst.daf.butler.Registry`
190  Data repository registry to search.
191  quantumDataId : `lsst.daf.butler.DataCoordinate`
192  Data ID of the quantum; expected to be something we can use as a
193  constraint to query for overlapping visits.
194  collections : `Iterable` [ `str` ]
195  Collections to search.
196 
197  Returns
198  -------
199  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200  Iterator over refcat references.
201  """
202  refs = set()
203  # Use .expanded() on the query methods below because we need data IDs with
204  # regions, both in the outer loop over visits (queryDatasets will expand
205  # any data ID we give it, but doing it up-front in bulk is much more
206  # efficient) and in the data IDs of the DatasetRefs this function yields
207  # (because the RefCatLoader relies on them to do some of its own
208  # filtering).
209  for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210  refs.update(
211  registry.queryDatasets(
212  datasetType,
213  collections=collections,
214  dataId=visit_data_id,
215  findFirst=True,
216  ).expanded()
217  )
218  yield from refs
219 
220 
221 class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222  dimensions=("skymap", "tract", "instrument", "physical_filter")):
223  """Middleware input/output connections for jointcal data."""
224  inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225  doc="The camera instrument that took these observations.",
226  name="camera",
227  storageClass="Camera",
228  dimensions=("instrument",),
229  isCalibration=True,
230  lookupFunction=lookupStaticCalibrations,
231  )
232  inputSourceTableVisit = pipeBase.connectionTypes.Input(
233  doc="Source table in parquet format, per visit",
234  name="sourceTable_visit",
235  storageClass="DataFrame",
236  dimensions=("instrument", "visit"),
237  deferLoad=True,
238  multiple=True,
239  )
240  inputVisitSummary = pipeBase.connectionTypes.Input(
241  doc=("Per-visit consolidated exposure metadata built from calexps. "
242  "These catalogs use detector id for the id and must be sorted for "
243  "fast lookups of a detector."),
244  name="visitSummary",
245  storageClass="ExposureCatalog",
246  dimensions=("instrument", "visit"),
247  deferLoad=True,
248  multiple=True,
249  )
250  astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251  doc="The astrometry reference catalog to match to loaded input catalog sources.",
252  name="gaia_dr2_20200414",
253  storageClass="SimpleCatalog",
254  dimensions=("skypix",),
255  deferLoad=True,
256  multiple=True,
257  lookupFunction=lookupVisitRefCats,
258  )
259  photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260  doc="The photometry reference catalog to match to loaded input catalog sources.",
261  name="ps1_pv3_3pi_20170110",
262  storageClass="SimpleCatalog",
263  dimensions=("skypix",),
264  deferLoad=True,
265  multiple=True,
266  lookupFunction=lookupVisitRefCats,
267  )
268 
269  outputWcs = pipeBase.connectionTypes.Output(
270  doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271  " These catalogs only contain entries for detectors with an output, and use"
272  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273  name="jointcalSkyWcsCatalog",
274  storageClass="ExposureCatalog",
275  dimensions=("instrument", "visit", "skymap", "tract"),
276  multiple=True
277  )
278  outputPhotoCalib = pipeBase.connectionTypes.Output(
279  doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280  " These catalogs only contain entries for detectors with an output, and use"
281  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282  name="jointcalPhotoCalibCatalog",
283  storageClass="ExposureCatalog",
284  dimensions=("instrument", "visit", "skymap", "tract"),
285  multiple=True
286  )
287 
288  def __init__(self, *, config=None):
289  super().__init__(config=config)
290  # When we are only doing one of astrometry or photometry, we don't
291  # need the reference catalog or produce the outputs for the other.
292  # This informs the middleware of that when the QuantumGraph is
293  # generated, so we don't block on getting something we won't need or
294  # create an expectation that downstream tasks will be able to consume
295  # something we won't produce.
296  if not config.doAstrometry:
297  self.prerequisiteInputs.remove("astrometryRefCat")
298  self.outputs.remove("outputWcs")
299  if not config.doPhotometry:
300  self.prerequisiteInputs.remove("photometryRefCat")
301  self.outputs.remove("outputPhotoCalib")
302 
303 
304 class JointcalConfig(pipeBase.PipelineTaskConfig,
305  pipelineConnections=JointcalTaskConnections):
306  """Configuration for JointcalTask"""
307 
308  doAstrometry = pexConfig.Field(
309  doc="Fit astrometry and write the fitted result.",
310  dtype=bool,
311  default=True
312  )
313  doPhotometry = pexConfig.Field(
314  doc="Fit photometry and write the fitted result.",
315  dtype=bool,
316  default=True
317  )
318  coaddName = pexConfig.Field(
319  doc="Type of coadd, typically deep or goodSeeing",
320  dtype=str,
321  default="deep"
322  )
323  # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
324  sourceFluxType = pexConfig.Field(
325  dtype=str,
326  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
327  default='Calib'
328  )
329  positionErrorPedestal = pexConfig.Field(
330  doc="Systematic term to apply to the measured position error (pixels)",
331  dtype=float,
332  default=0.02,
333  )
334  photometryErrorPedestal = pexConfig.Field(
335  doc="Systematic term to apply to the measured error on flux or magnitude as a "
336  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
337  dtype=float,
338  default=0.0,
339  )
340  # TODO: DM-6885 matchCut should be an geom.Angle
341  matchCut = pexConfig.Field(
342  doc="Matching radius between fitted and reference stars (arcseconds)",
343  dtype=float,
344  default=3.0,
345  )
346  minMeasurements = pexConfig.Field(
347  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
348  dtype=int,
349  default=2,
350  )
351  minMeasuredStarsPerCcd = pexConfig.Field(
352  doc="Minimum number of measuredStars per ccdImage before printing warnings",
353  dtype=int,
354  default=100,
355  )
356  minRefStarsPerCcd = pexConfig.Field(
357  doc="Minimum number of measuredStars per ccdImage before printing warnings",
358  dtype=int,
359  default=30,
360  )
361  allowLineSearch = pexConfig.Field(
362  doc="Allow a line search during minimization, if it is reasonable for the model"
363  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
364  dtype=bool,
365  default=False
366  )
367  astrometrySimpleOrder = pexConfig.Field(
368  doc="Polynomial order for fitting the simple astrometry model.",
369  dtype=int,
370  default=3,
371  )
372  astrometryChipOrder = pexConfig.Field(
373  doc="Order of the per-chip transform for the constrained astrometry model.",
374  dtype=int,
375  default=1,
376  )
377  astrometryVisitOrder = pexConfig.Field(
378  doc="Order of the per-visit transform for the constrained astrometry model.",
379  dtype=int,
380  default=5,
381  )
382  useInputWcs = pexConfig.Field(
383  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
384  dtype=bool,
385  default=True,
386  )
387  astrometryModel = pexConfig.ChoiceField(
388  doc="Type of model to fit to astrometry",
389  dtype=str,
390  default="constrained",
391  allowed={"simple": "One polynomial per ccd",
392  "constrained": "One polynomial per ccd, and one polynomial per visit"}
393  )
394  photometryModel = pexConfig.ChoiceField(
395  doc="Type of model to fit to photometry",
396  dtype=str,
397  default="constrainedMagnitude",
398  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
399  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
400  " fitting in flux space.",
401  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
402  " fitting in magnitude space.",
403  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
404  " fitting in magnitude space.",
405  }
406  )
407  applyColorTerms = pexConfig.Field(
408  doc="Apply photometric color terms to reference stars?"
409  "Requires that colorterms be set to a ColortermLibrary",
410  dtype=bool,
411  default=False
412  )
413  colorterms = pexConfig.ConfigField(
414  doc="Library of photometric reference catalog name to color term dict.",
415  dtype=ColortermLibrary,
416  )
417  photometryVisitOrder = pexConfig.Field(
418  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
419  dtype=int,
420  default=7,
421  )
422  photometryDoRankUpdate = pexConfig.Field(
423  doc=("Do the rank update step during minimization. "
424  "Skipping this can help deal with models that are too non-linear."),
425  dtype=bool,
426  default=True,
427  )
428  astrometryDoRankUpdate = pexConfig.Field(
429  doc=("Do the rank update step during minimization (should not change the astrometry fit). "
430  "Skipping this can help deal with models that are too non-linear."),
431  dtype=bool,
432  default=True,
433  )
434  outlierRejectSigma = pexConfig.Field(
435  doc="How many sigma to reject outliers at during minimization.",
436  dtype=float,
437  default=5.0,
438  )
439  astrometryOutlierRelativeTolerance = pexConfig.Field(
440  doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
441  "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
442  "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
443  "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
444  "close to the solution found when removing all outliers."),
445  dtype=float,
446  default=0,
447  )
448  maxPhotometrySteps = pexConfig.Field(
449  doc="Maximum number of minimize iterations to take when fitting photometry.",
450  dtype=int,
451  default=20,
452  )
453  maxAstrometrySteps = pexConfig.Field(
454  doc="Maximum number of minimize iterations to take when fitting astrometry.",
455  dtype=int,
456  default=20,
457  )
458  astrometryRefObjLoader = pexConfig.ConfigurableField(
459  target=LoadIndexedReferenceObjectsTask,
460  doc="Reference object loader for astrometric fit",
461  )
462  photometryRefObjLoader = pexConfig.ConfigurableField(
463  target=LoadIndexedReferenceObjectsTask,
464  doc="Reference object loader for photometric fit",
465  )
466  sourceSelector = sourceSelectorRegistry.makeField(
467  doc="How to select sources for cross-matching",
468  default="astrometry"
469  )
470  astrometryReferenceSelector = pexConfig.ConfigurableField(
471  target=ReferenceSourceSelectorTask,
472  doc="How to down-select the loaded astrometry reference catalog.",
473  )
474  photometryReferenceSelector = pexConfig.ConfigurableField(
475  target=ReferenceSourceSelectorTask,
476  doc="How to down-select the loaded photometry reference catalog.",
477  )
478  astrometryReferenceErr = pexConfig.Field(
479  doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
480  "If None, then raise an exception if the reference catalog is missing coordinate errors. "
481  "If specified, overrides any existing `coord_*Err` values."),
482  dtype=float,
483  default=None,
484  optional=True
485  )
486 
487  # configs for outputting debug information
488  writeInitMatrix = pexConfig.Field(
489  dtype=bool,
490  doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
491  "Output files will be written to `config.debugOutputPath` and will "
492  "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
493  "Note that these files are the dense versions of the matrix, and so may be very large."),
494  default=False
495  )
496  writeChi2FilesInitialFinal = pexConfig.Field(
497  dtype=bool,
498  doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
499  "Output files will be written to `config.debugOutputPath` and will "
500  "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
501  default=False
502  )
503  writeChi2FilesOuterLoop = pexConfig.Field(
504  dtype=bool,
505  doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
506  "Output files will be written to `config.debugOutputPath` and will "
507  "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
508  default=False
509  )
510  writeInitialModel = pexConfig.Field(
511  dtype=bool,
512  doc=("Write the pre-initialization model to text files, for debugging. "
513  "Output files will be written to `config.debugOutputPath` and will be "
514  "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
515  default=False
516  )
517  debugOutputPath = pexConfig.Field(
518  dtype=str,
519  default=".",
520  doc=("Path to write debug output files to. Used by "
521  "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
522  )
523  detailedProfile = pexConfig.Field(
524  dtype=bool,
525  default=False,
526  doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
527  )
528 
529  def validate(self):
530  super().validate()
531  if self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms and len(self.colortermscolorterms.data) == 0:
532  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
533  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
534  if self.doAstrometrydoAstrometry and not self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms:
535  msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
536  "applyColorTerms=True will be ignored.")
537  lsst.log.warn(msg)
538 
539  def setDefaults(self):
540  # Use science source selector which can filter on extendedness, SNR, and whether blended
541  self.sourceSelectorsourceSelector.name = 'science'
542  # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
543  self.sourceSelectorsourceSelector['science'].doUnresolved = True
544  # with dependable signal to noise ratio.
545  self.sourceSelectorsourceSelector['science'].doSignalToNoise = True
546  # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
547  # and S/N > 10 to use sources that are not too faint, and thus better measured.
548  self.sourceSelectorsourceSelector['science'].signalToNoise.minimum = 10.
549  # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
550  fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
551  self.sourceSelectorsourceSelector['science'].signalToNoise.fluxField = fluxField
552  self.sourceSelectorsourceSelector['science'].signalToNoise.errField = fluxField + "Err"
553  # Do not trust blended sources' aperture fluxes which also depend on seeing
554  self.sourceSelectorsourceSelector['science'].doIsolated = True
555  # Do not trust either flux or centroid measurements with flags,
556  # chosen from the usual QA flags for stars)
557  self.sourceSelectorsourceSelector['science'].doFlags = True
558  badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
559  'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
560  'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
561  self.sourceSelectorsourceSelector['science'].flags.bad = badFlags
562 
563  # Default to Gaia-DR2 (with proper motions) for astrometry and
564  # PS1-DR1 for photometry, with a reasonable initial filterMap.
565  self.astrometryRefObjLoaderastrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
566  self.astrometryRefObjLoaderastrometryRefObjLoader.requireProperMotion = True
567  self.astrometryRefObjLoaderastrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
568  self.photometryRefObjLoaderphotometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
569 
570 
571 def writeModel(model, filename, log):
572  """Write model to outfile."""
573  with open(filename, "w") as file:
574  file.write(repr(model))
575  log.info("Wrote %s to file: %s", model, filename)
576 
577 
578 @dataclasses.dataclass
580  """The input data jointcal needs for each detector/visit."""
581  visit: int
582  """The visit identifier of this exposure."""
584  """The catalog derived from this exposure."""
585  visitInfo: lsst.afw.image.VisitInfo
586  """The VisitInfo of this exposure."""
588  """The detector of this exposure."""
589  photoCalib: lsst.afw.image.PhotoCalib
590  """The photometric calibration of this exposure."""
592  """The WCS of this exposure."""
593  bbox: lsst.geom.Box2I
594  """The bounding box of this exposure."""
596  """The filter of this exposure."""
597 
598 
599 class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
600  """Astrometricly and photometricly calibrate across multiple visits of the
601  same field.
602 
603  Parameters
604  ----------
605  butler : `lsst.daf.persistence.Butler`
606  The butler is passed to the refObjLoader constructor in case it is
607  needed. Ignored if the refObjLoader argument provides a loader directly.
608  Used to initialize the astrometry and photometry refObjLoaders.
609  initInputs : `dict`, optional
610  Dictionary used to initialize PipelineTasks (empty for jointcal).
611  """
612 
613  ConfigClass = JointcalConfig
614  RunnerClass = JointcalRunner
615  _DefaultName = "jointcal"
616 
617  def __init__(self, butler=None, initInputs=None, **kwargs):
618  super().__init__(**kwargs)
619  self.makeSubtask("sourceSelector")
620  if self.config.doAstrometry:
621  if initInputs is None:
622  # gen3 middleware does refcat things internally (and will not have a butler here)
623  self.makeSubtask('astrometryRefObjLoader', butler=butler)
624  self.makeSubtask("astrometryReferenceSelector")
625  else:
626  self.astrometryRefObjLoaderastrometryRefObjLoader = None
627  if self.config.doPhotometry:
628  if initInputs is None:
629  # gen3 middleware does refcat things internally (and will not have a butler here)
630  self.makeSubtask('photometryRefObjLoader', butler=butler)
631  self.makeSubtask("photometryReferenceSelector")
632  else:
633  self.photometryRefObjLoaderphotometryRefObjLoader = None
634 
635  # To hold various computed metrics for use by tests
636  self.jobjob = Job.load_metrics_package(subset='jointcal')
637 
638  def runQuantum(self, butlerQC, inputRefs, outputRefs):
639  # We override runQuantum to set up the refObjLoaders and write the
640  # outputs to the correct refs.
641  inputs = butlerQC.get(inputRefs)
642  # We want the tract number for writing debug files
643  tract = butlerQC.quantum.dataId['tract']
644  if self.config.doAstrometry:
645  self.astrometryRefObjLoaderastrometryRefObjLoader = ReferenceObjectLoader(
646  dataIds=[ref.datasetRef.dataId
647  for ref in inputRefs.astrometryRefCat],
648  refCats=inputs.pop('astrometryRefCat'),
649  config=self.config.astrometryRefObjLoader,
650  log=self.log)
651  if self.config.doPhotometry:
652  self.photometryRefObjLoaderphotometryRefObjLoader = ReferenceObjectLoader(
653  dataIds=[ref.datasetRef.dataId
654  for ref in inputRefs.photometryRefCat],
655  refCats=inputs.pop('photometryRefCat'),
656  config=self.config.photometryRefObjLoader,
657  log=self.log)
658  outputs = self.runrun(**inputs, tract=tract)
659  if self.config.doAstrometry:
660  self._put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
661  inputs['inputCamera'], "setWcs")
662  if self.config.doPhotometry:
663  self._put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
664  inputs['inputCamera'], "setPhotoCalib")
665 
666  def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
667  """Persist the output datasets to their appropriate datarefs.
668 
669  Parameters
670  ----------
671  butlerQC : `ButlerQuantumContext`
672  A butler which is specialized to operate in the context of a
673  `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
674  outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
675  `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
676  The fitted objects to persist.
677  outputRefs : `list` [`OutputQuantizedConnection`]
678  The DatasetRefs to persist the data to.
679  camera : `lsst.afw.cameraGeom.Camera`
680  The camera for this instrument, to get detector ids from.
681  setter : `str`
682  The method to call on the ExposureCatalog to set each output.
683  """
685  schema.addField('visit', type='I', doc='Visit number')
686 
687  def new_catalog(visit, size):
688  """Return an catalog ready to be filled with appropriate output."""
689  catalog = lsst.afw.table.ExposureCatalog(schema)
690  catalog.resize(size)
691  catalog['visit'] = visit
692  metadata = lsst.daf.base.PropertyList()
693  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
694  metadata.add("COMMENT", "Only detectors with data have entries.")
695  return catalog
696 
697  # count how many detectors have output for each visit
698  detectors_per_visit = collections.defaultdict(int)
699  for key in outputs:
700  # key is (visit, detector_id), and we only need visit here
701  detectors_per_visit[key[0]] += 1
702 
703  for ref in outputRefs:
704  visit = ref.dataId['visit']
705  catalog = new_catalog(visit, detectors_per_visit[visit])
706  # Iterate over every detector and skip the ones we don't have output for.
707  i = 0
708  for detector in camera:
709  detectorId = detector.getId()
710  key = (ref.dataId['visit'], detectorId)
711  if key not in outputs:
712  # skip detectors we don't have output for
713  self.log.debug("No %s output for detector %s in visit %s",
714  setter[3:], detectorId, visit)
715  continue
716 
717  catalog[i].setId(detectorId)
718  getattr(catalog[i], setter)(outputs[key])
719  i += 1
720 
721  catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
722  butlerQC.put(catalog, ref)
723  self.log.info("Wrote %s detectors to %s", i, ref)
724 
725  def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
726  # Docstring inherited.
727 
728  # We take values out of the Parquet table, and put them in "flux_",
729  # and the config.sourceFluxType field is used during that extraction,
730  # so just use "flux" here.
731  sourceFluxField = "flux"
732  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
733  associations = lsst.jointcal.Associations()
734  self.focalPlaneBBoxfocalPlaneBBox = inputCamera.getFpBBox()
735  oldWcsList, bands = self._load_data_load_data(inputSourceTableVisit,
736  inputVisitSummary,
737  associations,
738  jointcalControl,
739  inputCamera)
740 
741  boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, bands)
742 
743  if self.config.doAstrometry:
744  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
745  name="astrometry",
746  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
747  referenceSelector=self.astrometryReferenceSelector,
748  fit_function=self._fit_astrometry_fit_astrometry,
749  tract=tract,
750  epoch=epoch)
751  astrometry_output = self._make_output_make_output(associations.getCcdImageList(),
752  astrometry.model,
753  "makeSkyWcs")
754  else:
755  astrometry_output = None
756 
757  if self.config.doPhotometry:
758  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
759  name="photometry",
760  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
761  referenceSelector=self.photometryReferenceSelector,
762  fit_function=self._fit_photometry_fit_photometry,
763  tract=tract,
764  epoch=epoch,
765  reject_bad_fluxes=True)
766  photometry_output = self._make_output_make_output(associations.getCcdImageList(),
767  photometry.model,
768  "toPhotoCalib")
769  else:
770  photometry_output = None
771 
772  return pipeBase.Struct(outputWcs=astrometry_output,
773  outputPhotoCalib=photometry_output,
774  job=self.jobjob,
775  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
776  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader)
777 
778  def _make_schema_table(self):
779  """Return an afw SourceTable to use as a base for creating the
780  SourceCatalog to insert values from the dataFrame into.
781 
782  Returns
783  -------
784  table : `lsst.afw.table.SourceTable`
785  Table with schema and slots to use to make SourceCatalogs.
786  """
788  schema.addField("centroid_x", "D")
789  schema.addField("centroid_y", "D")
790  schema.addField("centroid_xErr", "F")
791  schema.addField("centroid_yErr", "F")
792  schema.addField("shape_xx", "D")
793  schema.addField("shape_yy", "D")
794  schema.addField("shape_xy", "D")
795  schema.addField("flux_instFlux", "D")
796  schema.addField("flux_instFluxErr", "D")
797  table = lsst.afw.table.SourceTable.make(schema)
798  table.defineCentroid("centroid")
799  table.defineShape("shape")
800  return table
801 
802  def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
803  """Return an afw SourceCatalog extracted from a visit-level dataframe,
804  limited to just one detector.
805 
806  Parameters
807  ----------
808  table : `lsst.afw.table.SourceTable`
809  Table factory to use to make the SourceCatalog that will be
810  populated with data from ``visitCatalog``.
811  visitCatalog : `pandas.DataFrame`
812  DataFrame to extract a detector catalog from.
813  detectorId : `int`
814  Numeric id of the detector to extract from ``visitCatalog``.
815 
816  Returns
817  -------
818  catalog : `lsst.afw.table.SourceCatalog`
819  Detector-level catalog extracted from ``visitCatalog``.
820  """
821  # map from dataFrame column to afw table column
822  mapping = {'sourceId': 'id',
823  'x': 'centroid_x',
824  'y': 'centroid_y',
825  'xErr': 'centroid_xErr',
826  'yErr': 'centroid_yErr',
827  'Ixx': 'shape_xx',
828  'Iyy': 'shape_yy',
829  'Ixy': 'shape_xy',
830  f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
831  f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
832  }
833  # If the DataFrame we're reading was generated by a task running with
834  # Gen2 middleware, the column for the detector will be "ccd" for at
835  # least HSC (who knows what it might be in general!); that would be
836  # true even if the data repo is later converted to Gen3, because that
837  # doesn't touch the files themselves. In Gen3, the column for the
838  # detector will always be "detector".
839  detector_column = "detector" if "detector" in visitCatalog.columns else "ccd"
840  catalog = lsst.afw.table.SourceCatalog(table)
841  matched = visitCatalog[detector_column] == detectorId
842  catalog.resize(sum(matched))
843  view = visitCatalog.loc[matched]
844  for dfCol, afwCol in mapping.items():
845  catalog[afwCol] = view[dfCol]
846 
847  self.log.debug("%d sources selected in visit %d detector %d",
848  len(catalog),
849  view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
850  detectorId)
851  return catalog
852 
853  def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
854  jointcalControl, camera):
855  """Read the data that jointcal needs to run. (Gen3 version)
856 
857  Modifies ``associations`` in-place with the loaded data.
858 
859  Parameters
860  ----------
861  inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
862  References to visit-level DataFrames to load the catalog data from.
863  inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
864  Visit-level exposure summary catalog with metadata.
865  associations : `lsst.jointcal.Associations`
866  Object to add the loaded data to by constructing new CcdImages.
867  jointcalControl : `jointcal.JointcalControl`
868  Control object for C++ associations management.
869  camera : `lsst.afw.cameraGeom.Camera`
870  Camera object for detector geometry.
871 
872  Returns
873  -------
874  oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
875  The original WCS of the input data, to aid in writing tests.
876  bands : `list` [`str`]
877  The filter bands of each input dataset.
878  """
879  oldWcsList = []
880  filters = []
881  load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
882  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
883  table = self._make_schema_table_make_schema_table() # every detector catalog has the same layout
884  # No guarantee that the input is in the same order of visits, so we have to map one of them.
885  catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
886  detectorDict = {detector.getId(): detector for detector in camera}
887 
888  for visitSummaryRef in inputVisitSummary:
889  visitSummary = visitSummaryRef.get()
890  visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
891  selected = self.sourceSelector.run(visitCatalog)
892 
893  # Build a CcdImage for each detector in this visit.
894  detectors = {id: index for index, id in enumerate(visitSummary['id'])}
895  for id, index in detectors.items():
896  catalog = self._extract_detector_catalog_from_visit_catalog_extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
897  data = self._make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
898  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
899  if result is not None:
900  oldWcsList.append(result.wcs)
901  # A visit has only one band, so we can just use the first.
902  filters.append(data.filter)
903  if len(filters) == 0:
904  raise RuntimeError("No data to process: did source selector remove all sources?")
905  filters = collections.Counter(filters)
906 
907  return oldWcsList, filters
908 
909  def _make_one_input_data(self, visitRecord, catalog, detectorDict):
910  """Return a data structure for this detector+visit."""
911  return JointcalInputData(visit=visitRecord['visit'],
912  catalog=catalog,
913  visitInfo=visitRecord.getVisitInfo(),
914  detector=detectorDict[visitRecord.getId()],
915  photoCalib=visitRecord.getPhotoCalib(),
916  wcs=visitRecord.getWcs(),
917  bbox=visitRecord.getBBox(),
918  # ExposureRecord doesn't have a FilterLabel yet,
919  # so we have to make one.
920  filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
921  physical=visitRecord['physical_filter']))
922 
923  # We don't currently need to persist the metadata.
924  # If we do in the future, we will have to add appropriate dataset templates
925  # to each obs package (the metadata template should look like `jointcal_wcs`).
926  def _getMetadataName(self):
927  return None
928 
929  @classmethod
930  def _makeArgumentParser(cls):
931  """Create an argument parser"""
932  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
933  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
934  ContainerClass=PerTractCcdDataIdContainer)
935  return parser
936 
937  def _build_ccdImage(self, data, associations, jointcalControl):
938  """
939  Extract the necessary things from this catalog+metadata to add a new
940  ccdImage.
941 
942  Parameters
943  ----------
944  data : `JointcalInputData`
945  The loaded input data.
946  associations : `lsst.jointcal.Associations`
947  Object to add the info to, to construct a new CcdImage
948  jointcalControl : `jointcal.JointcalControl`
949  Control object for associations management
950 
951  Returns
952  ------
953  namedtuple or `None`
954  ``wcs``
955  The TAN WCS of this image, read from the calexp
956  (`lsst.afw.geom.SkyWcs`).
957  ``key``
958  A key to identify this dataRef by its visit and ccd ids
959  (`namedtuple`).
960  `None`
961  if there are no sources in the loaded catalog.
962  """
963  if len(data.catalog) == 0:
964  self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
965  return None
966 
967  associations.createCcdImage(data.catalog,
968  data.wcs,
969  data.visitInfo,
970  data.bbox,
971  data.filter.physicalLabel,
972  data.photoCalib,
973  data.detector,
974  data.visit,
975  data.detector.getId(),
976  jointcalControl)
977 
978  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
979  Key = collections.namedtuple('Key', ('visit', 'ccd'))
980  return Result(data.wcs, Key(data.visit, data.detector.getId()))
981 
982  def _readDataId(self, butler, dataId):
983  """Read all of the data for one dataId from the butler. (gen2 version)"""
984  # Not all instruments have `visit` in their dataIds.
985  if "visit" in dataId.keys():
986  visit = dataId["visit"]
987  else:
988  visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
989  detector = butler.get('calexp_detector', dataId=dataId)
990 
991  catalog = butler.get('src',
992  flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
993  dataId=dataId)
994  goodSrc = self.sourceSelector.run(catalog)
995  self.log.debug("%d sources selected in visit %d detector %d",
996  len(goodSrc.sourceCat),
997  visit,
998  detector.getId())
999  return JointcalInputData(visit=visit,
1000  catalog=goodSrc.sourceCat,
1001  visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1002  detector=detector,
1003  photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1004  wcs=butler.get('calexp_wcs', dataId=dataId),
1005  bbox=butler.get('calexp_bbox', dataId=dataId),
1006  filter=butler.get('calexp_filterLabel', dataId=dataId))
1007 
1008  def loadData(self, dataRefs, associations, jointcalControl):
1009  """Read the data that jointcal needs to run. (Gen2 version)"""
1010  visit_ccd_to_dataRef = {}
1011  oldWcsList = []
1012  filters = []
1013  load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1014  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1015  # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1016  camera = dataRefs[0].get('camera', immediate=True)
1017  self.focalPlaneBBoxfocalPlaneBBox = camera.getFpBBox()
1018  for dataRef in dataRefs:
1019  data = self._readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1020  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1021  if result is None:
1022  continue
1023  oldWcsList.append(result.wcs)
1024  visit_ccd_to_dataRef[result.key] = dataRef
1025  filters.append(data.filter)
1026  if len(filters) == 0:
1027  raise RuntimeError("No data to process: did source selector remove all sources?")
1028  filters = collections.Counter(filters)
1029 
1030  return oldWcsList, filters, visit_ccd_to_dataRef
1031 
1032  def _getDebugPath(self, filename):
1033  """Constructs a path to filename using the configured debug path.
1034  """
1035  return os.path.join(self.config.debugOutputPath, filename)
1036 
1037  def _prep_sky(self, associations, filters):
1038  """Prepare on-sky and other data that must be computed after data has
1039  been read.
1040  """
1041  associations.computeCommonTangentPoint()
1042 
1043  boundingCircle = associations.computeBoundingCircle()
1044  center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1045  radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1046 
1047  self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1048 
1049  # Determine a default filter band associated with the catalog. See DM-9093
1050  defaultFilter = filters.most_common(1)[0][0]
1051  self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1052 
1053  # compute and set the reference epoch of the observations, for proper motion corrections
1054  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
1055  associations.setEpoch(epoch.jyear)
1056 
1057  return boundingCircle, center, radius, defaultFilter, epoch
1058 
1059  @pipeBase.timeMethod
1060  def runDataRef(self, dataRefs):
1061  """
1062  Jointly calibrate the astrometry and photometry across a set of images.
1063 
1064  NOTE: this is for gen2 middleware only.
1065 
1066  Parameters
1067  ----------
1068  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1069  List of data references to the exposures to be fit.
1070 
1071  Returns
1072  -------
1073  result : `lsst.pipe.base.Struct`
1074  Struct of metadata from the fit, containing:
1075 
1076  ``dataRefs``
1077  The provided data references that were fit (with updated WCSs)
1078  ``oldWcsList``
1079  The original WCS from each dataRef
1080  ``metrics``
1081  Dictionary of internally-computed metrics for testing/validation.
1082  """
1083  if len(dataRefs) == 0:
1084  raise ValueError('Need a non-empty list of data references!')
1085 
1086  exitStatus = 0 # exit status for shell
1087 
1088  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1089  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1090  associations = lsst.jointcal.Associations()
1091 
1092  oldWcsList, filters, visit_ccd_to_dataRef = self.loadDataloadData(dataRefs,
1093  associations,
1094  jointcalControl)
1095 
1096  boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, filters)
1097 
1098  tract = dataRefs[0].dataId['tract']
1099 
1100  if self.config.doAstrometry:
1101  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1102  name="astrometry",
1103  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1104  referenceSelector=self.astrometryReferenceSelector,
1105  fit_function=self._fit_astrometry_fit_astrometry,
1106  tract=tract,
1107  epoch=epoch)
1108  self._write_astrometry_results_write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1109  else:
1110  astrometry = Astrometry(None, None, None)
1111 
1112  if self.config.doPhotometry:
1113  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1114  name="photometry",
1115  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1116  referenceSelector=self.photometryReferenceSelector,
1117  fit_function=self._fit_photometry_fit_photometry,
1118  tract=tract,
1119  epoch=epoch,
1120  reject_bad_fluxes=True)
1121  self._write_photometry_results_write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1122  else:
1123  photometry = Photometry(None, None)
1124 
1125  return pipeBase.Struct(dataRefs=dataRefs,
1126  oldWcsList=oldWcsList,
1127  job=self.jobjob,
1128  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1129  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1130  defaultFilter=defaultFilter,
1131  epoch=epoch,
1132  exitStatus=exitStatus)
1133 
1134  def _get_refcat_coordinate_error_override(self, refCat, name):
1135  """Check whether we should override the refcat coordinate errors, and
1136  return the overridden error if necessary.
1137 
1138  Parameters
1139  ----------
1140  refCat : `lsst.afw.table.SimpleCatalog`
1141  The reference catalog to check for a ``coord_raErr`` field.
1142  name : `str`
1143  Whether we are doing "astrometry" or "photometry".
1144 
1145  Returns
1146  -------
1147  refCoordErr : `float`
1148  The refcat coordinate error to use, or NaN if we are not overriding
1149  those fields.
1150 
1151  Raises
1152  ------
1153  lsst.pex.config.FieldValidationError
1154  Raised if the refcat does not contain coordinate errors and
1155  ``config.astrometryReferenceErr`` is not set.
1156  """
1157  # This value doesn't matter for photometry, so just set something to
1158  # keep old refcats from causing problems.
1159  if name.lower() == "photometry":
1160  if 'coord_raErr' not in refCat.schema:
1161  return 100
1162  else:
1163  return float('nan')
1164 
1165  if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1166  msg = ("Reference catalog does not contain coordinate errors, "
1167  "and config.astrometryReferenceErr not supplied.")
1168  raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1169  self.config,
1170  msg)
1171 
1172  if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1173  self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1174  self.config.astrometryReferenceErr)
1175 
1176  if self.config.astrometryReferenceErr is None:
1177  return float('nan')
1178  else:
1179  return self.config.astrometryReferenceErr
1180 
1181  def _compute_proper_motion_epoch(self, ccdImageList):
1182  """Return the proper motion correction epoch of the provided images.
1183 
1184  Parameters
1185  ----------
1186  ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1187  The images to compute the appropriate epoch for.
1188 
1189  Returns
1190  -------
1191  epoch : `astropy.time.Time`
1192  The date to use for proper motion corrections.
1193  """
1194  return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
1195  format="jyear",
1196  scale="tai")
1197 
1198  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1199  tract="", match_cut=3.0,
1200  reject_bad_fluxes=False, *,
1201  name="", refObjLoader=None, referenceSelector=None,
1202  fit_function=None, epoch=None):
1203  """Load reference catalog, perform the fit, and return the result.
1204 
1205  Parameters
1206  ----------
1207  associations : `lsst.jointcal.Associations`
1208  The star/reference star associations to fit.
1209  defaultFilter : `lsst.afw.image.FilterLabel`
1210  filter to load from reference catalog.
1211  center : `lsst.geom.SpherePoint`
1212  ICRS center of field to load from reference catalog.
1213  radius : `lsst.geom.Angle`
1214  On-sky radius to load from reference catalog.
1215  name : `str`
1216  Name of thing being fit: "astrometry" or "photometry".
1217  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1218  Reference object loader to use to load a reference catalog.
1219  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1220  Selector to use to pick objects from the loaded reference catalog.
1221  fit_function : callable
1222  Function to call to perform fit (takes Associations object).
1223  tract : `str`, optional
1224  Name of tract currently being fit.
1225  match_cut : `float`, optional
1226  Radius in arcseconds to find cross-catalog matches to during
1227  associations.associateCatalogs.
1228  reject_bad_fluxes : `bool`, optional
1229  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1230  epoch : `astropy.time.Time`, optional
1231  Epoch to which to correct refcat proper motion and parallax,
1232  or `None` to not apply such corrections.
1233 
1234  Returns
1235  -------
1236  result : `Photometry` or `Astrometry`
1237  Result of `fit_function()`
1238  """
1239  self.log.info("====== Now processing %s...", name)
1240  # TODO: this should not print "trying to invert a singular transformation:"
1241  # if it does that, something's not right about the WCS...
1242  associations.associateCatalogs(match_cut)
1243  add_measurement(self.jobjob, 'jointcal.associated_%s_fittedStars' % name,
1244  associations.fittedStarListSize())
1245 
1246  applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1247  refCat, fluxField = self._load_reference_catalog_load_reference_catalog(refObjLoader, referenceSelector,
1248  center, radius, defaultFilter,
1249  applyColorterms=applyColorterms,
1250  epoch=epoch)
1251  refCoordErr = self._get_refcat_coordinate_error_override_get_refcat_coordinate_error_override(refCat, name)
1252 
1253  associations.collectRefStars(refCat,
1254  self.config.matchCut*lsst.geom.arcseconds,
1255  fluxField,
1256  refCoordinateErr=refCoordErr,
1257  rejectBadFluxes=reject_bad_fluxes)
1258  add_measurement(self.jobjob, 'jointcal.collected_%s_refStars' % name,
1259  associations.refStarListSize())
1260 
1261  associations.prepareFittedStars(self.config.minMeasurements)
1262 
1263  self._check_star_lists_check_star_lists(associations, name)
1264  add_measurement(self.jobjob, 'jointcal.selected_%s_refStars' % name,
1265  associations.nFittedStarsWithAssociatedRefStar())
1266  add_measurement(self.jobjob, 'jointcal.selected_%s_fittedStars' % name,
1267  associations.fittedStarListSize())
1268  add_measurement(self.jobjob, 'jointcal.selected_%s_ccdImages' % name,
1269  associations.nCcdImagesValidForFit())
1270 
1271  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1272  dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1273  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1274  result = fit_function(associations, dataName)
1275  # TODO DM-12446: turn this into a "butler save" somehow.
1276  # Save reference and measurement chi2 contributions for this data
1277  if self.config.writeChi2FilesInitialFinal:
1278  baseName = self._getDebugPath_getDebugPath(f"{name}_final_chi2-{dataName}")
1279  result.fit.saveChi2Contributions(baseName+"{type}")
1280  self.log.info("Wrote chi2 contributions files: %s", baseName)
1281 
1282  return result
1283 
1284  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1285  applyColorterms=False, epoch=None):
1286  """Load the necessary reference catalog sources, convert fluxes to
1287  correct units, and apply color term corrections if requested.
1288 
1289  Parameters
1290  ----------
1291  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1292  The reference catalog loader to use to get the data.
1293  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1294  Source selector to apply to loaded reference catalog.
1295  center : `lsst.geom.SpherePoint`
1296  The center around which to load sources.
1297  radius : `lsst.geom.Angle`
1298  The radius around ``center`` to load sources in.
1299  filterLabel : `lsst.afw.image.FilterLabel`
1300  The camera filter to load fluxes for.
1301  applyColorterms : `bool`
1302  Apply colorterm corrections to the refcat for ``filterName``?
1303  epoch : `astropy.time.Time`, optional
1304  Epoch to which to correct refcat proper motion and parallax,
1305  or `None` to not apply such corrections.
1306 
1307  Returns
1308  -------
1309  refCat : `lsst.afw.table.SimpleCatalog`
1310  The loaded reference catalog.
1311  fluxField : `str`
1312  The name of the reference catalog flux field appropriate for ``filterName``.
1313  """
1314  skyCircle = refObjLoader.loadSkyCircle(center,
1315  radius,
1316  filterLabel.bandLabel,
1317  epoch=epoch)
1318 
1319  selected = referenceSelector.run(skyCircle.refCat)
1320  # Need memory contiguity to get reference filters as a vector.
1321  if not selected.sourceCat.isContiguous():
1322  refCat = selected.sourceCat.copy(deep=True)
1323  else:
1324  refCat = selected.sourceCat
1325 
1326  if applyColorterms:
1327  refCatName = refObjLoader.ref_dataset_name
1328  self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1329  filterLabel.physicalLabel, refCatName)
1330  colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1331  refCatName,
1332  doRaise=True)
1333 
1334  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1335  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1336  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1337  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1338 
1339  return refCat, skyCircle.fluxField
1340 
1341  def _check_star_lists(self, associations, name):
1342  # TODO: these should be len(blah), but we need this properly wrapped first.
1343  if associations.nCcdImagesValidForFit() == 0:
1344  raise RuntimeError('No images in the ccdImageList!')
1345  if associations.fittedStarListSize() == 0:
1346  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1347  if associations.refStarListSize() == 0:
1348  raise RuntimeError('No stars in the {} reference star list!'.format(name))
1349 
1350  def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1351  """Compute chi2, log it, validate the model, and return chi2.
1352 
1353  Parameters
1354  ----------
1355  associations : `lsst.jointcal.Associations`
1356  The star/reference star associations to fit.
1357  fit : `lsst.jointcal.FitterBase`
1358  The fitter to use for minimization.
1359  model : `lsst.jointcal.Model`
1360  The model being fit.
1361  chi2Label : `str`
1362  Label to describe the chi2 (e.g. "Initialized", "Final").
1363  writeChi2Name : `str`, optional
1364  Filename prefix to write the chi2 contributions to.
1365  Do not supply an extension: an appropriate one will be added.
1366 
1367  Returns
1368  -------
1369  chi2: `lsst.jointcal.Chi2Accumulator`
1370  The chi2 object for the current fitter and model.
1371 
1372  Raises
1373  ------
1374  FloatingPointError
1375  Raised if chi2 is infinite or NaN.
1376  ValueError
1377  Raised if the model is not valid.
1378  """
1379  if writeChi2Name is not None:
1380  fullpath = self._getDebugPath_getDebugPath(writeChi2Name)
1381  fit.saveChi2Contributions(fullpath+"{type}")
1382  self.log.info("Wrote chi2 contributions files: %s", fullpath)
1383 
1384  chi2 = fit.computeChi2()
1385  self.log.info("%s %s", chi2Label, chi2)
1386  self._check_stars_check_stars(associations)
1387  if not np.isfinite(chi2.chi2):
1388  raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1389  if not model.validate(associations.getCcdImageList(), chi2.ndof):
1390  raise ValueError("Model is not valid: check log messages for warnings.")
1391  return chi2
1392 
1393  def _fit_photometry(self, associations, dataName=None):
1394  """
1395  Fit the photometric data.
1396 
1397  Parameters
1398  ----------
1399  associations : `lsst.jointcal.Associations`
1400  The star/reference star associations to fit.
1401  dataName : `str`
1402  Name of the data being processed (e.g. "1234_HSC-Y"), for
1403  identifying debugging files.
1404 
1405  Returns
1406  -------
1407  fit_result : `namedtuple`
1408  fit : `lsst.jointcal.PhotometryFit`
1409  The photometric fitter used to perform the fit.
1410  model : `lsst.jointcal.PhotometryModel`
1411  The photometric model that was fit.
1412  """
1413  self.log.info("=== Starting photometric fitting...")
1414 
1415  # TODO: should use pex.config.RegistryField here (see DM-9195)
1416  if self.config.photometryModel == "constrainedFlux":
1417  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1418  self.focalPlaneBBoxfocalPlaneBBox,
1419  visitOrder=self.config.photometryVisitOrder,
1420  errorPedestal=self.config.photometryErrorPedestal)
1421  # potentially nonlinear problem, so we may need a line search to converge.
1422  doLineSearch = self.config.allowLineSearch
1423  elif self.config.photometryModel == "constrainedMagnitude":
1424  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1425  self.focalPlaneBBoxfocalPlaneBBox,
1426  visitOrder=self.config.photometryVisitOrder,
1427  errorPedestal=self.config.photometryErrorPedestal)
1428  # potentially nonlinear problem, so we may need a line search to converge.
1429  doLineSearch = self.config.allowLineSearch
1430  elif self.config.photometryModel == "simpleFlux":
1431  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1432  errorPedestal=self.config.photometryErrorPedestal)
1433  doLineSearch = False # purely linear in model parameters, so no line search needed
1434  elif self.config.photometryModel == "simpleMagnitude":
1435  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1436  errorPedestal=self.config.photometryErrorPedestal)
1437  doLineSearch = False # purely linear in model parameters, so no line search needed
1438 
1439  fit = lsst.jointcal.PhotometryFit(associations, model)
1440  # TODO DM-12446: turn this into a "butler save" somehow.
1441  # Save reference and measurement chi2 contributions for this data
1442  if self.config.writeChi2FilesInitialFinal:
1443  baseName = f"photometry_initial_chi2-{dataName}"
1444  else:
1445  baseName = None
1446  if self.config.writeInitialModel:
1447  fullpath = self._getDebugPath_getDebugPath(f"initial_photometry_model-{dataName}.txt")
1448  writeModel(model, fullpath, self.log)
1449  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1450 
1451  def getChi2Name(whatToFit):
1452  if self.config.writeChi2FilesOuterLoop:
1453  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1454  else:
1455  return None
1456 
1457  # The constrained model needs the visit transform fit first; the chip
1458  # transform is initialized from the singleFrame PhotoCalib, so it's close.
1459  if self.config.writeInitMatrix:
1460  dumpMatrixFile = self._getDebugPath_getDebugPath(f"photometry_preinit-{dataName}")
1461  else:
1462  dumpMatrixFile = ""
1463  if self.config.photometryModel.startswith("constrained"):
1464  # no line search: should be purely (or nearly) linear,
1465  # and we want a large step size to initialize with.
1466  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1467  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1468  writeChi2Name=getChi2Name("ModelVisit"))
1469  dumpMatrixFile = "" # so we don't redo the output on the next step
1470 
1471  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1472  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Model",
1473  writeChi2Name=getChi2Name("Model"))
1474 
1475  fit.minimize("Fluxes") # no line search: always purely linear.
1476  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1477  writeChi2Name=getChi2Name("Fluxes"))
1478 
1479  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1480  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1481  writeChi2Name=getChi2Name("ModelFluxes"))
1482 
1483  model.freezeErrorTransform()
1484  self.log.debug("Photometry error scales are frozen.")
1485 
1486  chi2 = self._iterate_fit_iterate_fit(associations,
1487  fit,
1488  self.config.maxPhotometrySteps,
1489  "photometry",
1490  "Model Fluxes",
1491  doRankUpdate=self.config.photometryDoRankUpdate,
1492  doLineSearch=doLineSearch,
1493  dataName=dataName)
1494 
1495  add_measurement(self.jobjob, 'jointcal.photometry_final_chi2', chi2.chi2)
1496  add_measurement(self.jobjob, 'jointcal.photometry_final_ndof', chi2.ndof)
1497  return Photometry(fit, model)
1498 
1499  def _fit_astrometry(self, associations, dataName=None):
1500  """
1501  Fit the astrometric data.
1502 
1503  Parameters
1504  ----------
1505  associations : `lsst.jointcal.Associations`
1506  The star/reference star associations to fit.
1507  dataName : `str`
1508  Name of the data being processed (e.g. "1234_HSC-Y"), for
1509  identifying debugging files.
1510 
1511  Returns
1512  -------
1513  fit_result : `namedtuple`
1514  fit : `lsst.jointcal.AstrometryFit`
1515  The astrometric fitter used to perform the fit.
1516  model : `lsst.jointcal.AstrometryModel`
1517  The astrometric model that was fit.
1518  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1519  The model for the sky to tangent plane projection that was used in the fit.
1520  """
1521 
1522  self.log.info("=== Starting astrometric fitting...")
1523 
1524  associations.deprojectFittedStars()
1525 
1526  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1527  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1528  # them so carefully?
1529  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1530 
1531  if self.config.astrometryModel == "constrained":
1532  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1533  sky_to_tan_projection,
1534  chipOrder=self.config.astrometryChipOrder,
1535  visitOrder=self.config.astrometryVisitOrder)
1536  elif self.config.astrometryModel == "simple":
1537  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1538  sky_to_tan_projection,
1539  self.config.useInputWcs,
1540  nNotFit=0,
1541  order=self.config.astrometrySimpleOrder)
1542 
1543  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1544  # TODO DM-12446: turn this into a "butler save" somehow.
1545  # Save reference and measurement chi2 contributions for this data
1546  if self.config.writeChi2FilesInitialFinal:
1547  baseName = f"astrometry_initial_chi2-{dataName}"
1548  else:
1549  baseName = None
1550  if self.config.writeInitialModel:
1551  fullpath = self._getDebugPath_getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1552  writeModel(model, fullpath, self.log)
1553  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1554 
1555  def getChi2Name(whatToFit):
1556  if self.config.writeChi2FilesOuterLoop:
1557  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1558  else:
1559  return None
1560 
1561  if self.config.writeInitMatrix:
1562  dumpMatrixFile = self._getDebugPath_getDebugPath(f"astrometry_preinit-{dataName}")
1563  else:
1564  dumpMatrixFile = ""
1565  # The constrained model needs the visit transform fit first; the chip
1566  # transform is initialized from the detector's cameraGeom, so it's close.
1567  if self.config.astrometryModel == "constrained":
1568  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1569  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1570  writeChi2Name=getChi2Name("DistortionsVisit"))
1571  dumpMatrixFile = "" # so we don't redo the output on the next step
1572 
1573  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1574  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1575  writeChi2Name=getChi2Name("Distortions"))
1576 
1577  fit.minimize("Positions")
1578  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Positions",
1579  writeChi2Name=getChi2Name("Positions"))
1580 
1581  fit.minimize("Distortions Positions")
1582  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1583  writeChi2Name=getChi2Name("DistortionsPositions"))
1584 
1585  chi2 = self._iterate_fit_iterate_fit(associations,
1586  fit,
1587  self.config.maxAstrometrySteps,
1588  "astrometry",
1589  "Distortions Positions",
1590  sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1591  doRankUpdate=self.config.astrometryDoRankUpdate,
1592  dataName=dataName)
1593 
1594  add_measurement(self.jobjob, 'jointcal.astrometry_final_chi2', chi2.chi2)
1595  add_measurement(self.jobjob, 'jointcal.astrometry_final_ndof', chi2.ndof)
1596 
1597  return Astrometry(fit, model, sky_to_tan_projection)
1598 
1599  def _check_stars(self, associations):
1600  """Count measured and reference stars per ccd and warn/log them."""
1601  for ccdImage in associations.getCcdImageList():
1602  nMeasuredStars, nRefStars = ccdImage.countStars()
1603  self.log.debug("ccdImage %s has %s measured and %s reference stars",
1604  ccdImage.getName(), nMeasuredStars, nRefStars)
1605  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1606  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1607  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1608  if nRefStars < self.config.minRefStarsPerCcd:
1609  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1610  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1611 
1612  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1613  dataName="",
1614  sigmaRelativeTolerance=0,
1615  doRankUpdate=True,
1616  doLineSearch=False):
1617  """Run fitter.minimize up to max_steps times, returning the final chi2.
1618 
1619  Parameters
1620  ----------
1621  associations : `lsst.jointcal.Associations`
1622  The star/reference star associations to fit.
1623  fitter : `lsst.jointcal.FitterBase`
1624  The fitter to use for minimization.
1625  max_steps : `int`
1626  Maximum number of steps to run outlier rejection before declaring
1627  convergence failure.
1628  name : {'photometry' or 'astrometry'}
1629  What type of data are we fitting (for logs and debugging files).
1630  whatToFit : `str`
1631  Passed to ``fitter.minimize()`` to define the parameters to fit.
1632  dataName : `str`, optional
1633  Descriptive name for this dataset (e.g. tract and filter),
1634  for debugging.
1635  sigmaRelativeTolerance : `float`, optional
1636  Convergence tolerance for the fractional change in the chi2 cut
1637  level for determining outliers. If set to zero, iterations will
1638  continue until there are no outliers.
1639  doRankUpdate : `bool`, optional
1640  Do an Eigen rank update during minimization, or recompute the full
1641  matrix and gradient?
1642  doLineSearch : `bool`, optional
1643  Do a line search for the optimum step during minimization?
1644 
1645  Returns
1646  -------
1647  chi2: `lsst.jointcal.Chi2Statistic`
1648  The final chi2 after the fit converges, or is forced to end.
1649 
1650  Raises
1651  ------
1652  FloatingPointError
1653  Raised if the fitter fails with a non-finite value.
1654  RuntimeError
1655  Raised if the fitter fails for some other reason;
1656  log messages will provide further details.
1657  """
1658  if self.config.writeInitMatrix:
1659  dumpMatrixFile = self._getDebugPath_getDebugPath(f"{name}_postinit-{dataName}")
1660  else:
1661  dumpMatrixFile = ""
1662  oldChi2 = lsst.jointcal.Chi2Statistic()
1663  oldChi2.chi2 = float("inf")
1664  for i in range(max_steps):
1665  if self.config.writeChi2FilesOuterLoop:
1666  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1667  else:
1668  writeChi2Name = None
1669  result = fitter.minimize(whatToFit,
1670  self.config.outlierRejectSigma,
1671  sigmaRelativeTolerance=sigmaRelativeTolerance,
1672  doRankUpdate=doRankUpdate,
1673  doLineSearch=doLineSearch,
1674  dumpMatrixFile=dumpMatrixFile)
1675  dumpMatrixFile = "" # clear it so we don't write the matrix again.
1676  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1677  f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1678 
1679  if result == MinimizeResult.Converged:
1680  if doRankUpdate:
1681  self.log.debug("fit has converged - no more outliers - redo minimization "
1682  "one more time in case we have lost accuracy in rank update.")
1683  # Redo minimization one more time in case we have lost accuracy in rank update
1684  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1685  sigmaRelativeTolerance=sigmaRelativeTolerance)
1686  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1687 
1688  # log a message for a large final chi2, TODO: DM-15247 for something better
1689  if chi2.chi2/chi2.ndof >= 4.0:
1690  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1691 
1692  break
1693  elif result == MinimizeResult.Chi2Increased:
1694  self.log.warn("Still some outliers remaining but chi2 increased - retry")
1695  # Check whether the increase was large enough to cause trouble.
1696  chi2Ratio = chi2.chi2 / oldChi2.chi2
1697  if chi2Ratio > 1.5:
1698  self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1699  chi2.chi2, oldChi2.chi2, chi2Ratio)
1700  # Based on a variety of HSC jointcal logs (see DM-25779), it
1701  # appears that chi2 increases more than a factor of ~2 always
1702  # result in the fit diverging rapidly and ending at chi2 > 1e10.
1703  # Using 10 as the "failure" threshold gives some room between
1704  # leaving a warning and bailing early.
1705  if chi2Ratio > 10:
1706  msg = ("Large chi2 increase between steps: fit likely cannot converge."
1707  " Try setting one or more of the `writeChi2*` config fields and looking"
1708  " at how individual star chi2-values evolve during the fit.")
1709  raise RuntimeError(msg)
1710  oldChi2 = chi2
1711  elif result == MinimizeResult.NonFinite:
1712  filename = self._getDebugPath_getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1713  # TODO DM-12446: turn this into a "butler save" somehow.
1714  fitter.saveChi2Contributions(filename+"{type}")
1715  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1716  raise FloatingPointError(msg.format(filename))
1717  elif result == MinimizeResult.Failed:
1718  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1719  else:
1720  raise RuntimeError("Unxepected return code from minimize().")
1721  else:
1722  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1723 
1724  return chi2
1725 
1726  def _make_output(self, ccdImageList, model, func):
1727  """Return the internal jointcal models converted to the afw
1728  structures that will be saved to disk.
1729 
1730  Parameters
1731  ----------
1732  ccdImageList : `lsst.jointcal.CcdImageList`
1733  The list of CcdImages to get the output for.
1734  model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1735  The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1736  func : `str`
1737  The name of the function to call on ``model`` to get the converted
1738  structure. Must accept an `lsst.jointcal.CcdImage`.
1739 
1740  Returns
1741  -------
1742  output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1743  `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1744  The data to be saved, keyed on (visit, detector).
1745  """
1746  output = {}
1747  for ccdImage in ccdImageList:
1748  ccd = ccdImage.ccdId
1749  visit = ccdImage.visit
1750  self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1751  output[(visit, ccd)] = getattr(model, func)(ccdImage)
1752  return output
1753 
1754  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1755  """
1756  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1757 
1758  Parameters
1759  ----------
1760  associations : `lsst.jointcal.Associations`
1761  The star/reference star associations to fit.
1762  model : `lsst.jointcal.AstrometryModel`
1763  The astrometric model that was fit.
1764  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1765  Dict of ccdImage identifiers to dataRefs that were fit.
1766  """
1767  ccdImageList = associations.getCcdImageList()
1768  output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1769  for key, skyWcs in output.items():
1770  dataRef = visit_ccd_to_dataRef[key]
1771  try:
1772  dataRef.put(skyWcs, 'jointcal_wcs')
1773  except pexExceptions.Exception as e:
1774  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1775  raise e
1776 
1777  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1778  """
1779  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1780 
1781  Parameters
1782  ----------
1783  associations : `lsst.jointcal.Associations`
1784  The star/reference star associations to fit.
1785  model : `lsst.jointcal.PhotometryModel`
1786  The photoometric model that was fit.
1787  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1788  Dict of ccdImage identifiers to dataRefs that were fit.
1789  """
1790 
1791  ccdImageList = associations.getCcdImageList()
1792  output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1793  for key, photoCalib in output.items():
1794  dataRef = visit_ccd_to_dataRef[key]
1795  try:
1796  dataRef.put(photoCalib, 'jointcal_photoCalib')
1797  except pexExceptions.Exception as e:
1798  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1799  raise e
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:54
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
Definition: jointcal.py:77
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:854
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:1181
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:1134
def _getDebugPath(self, filename)
Definition: jointcal.py:1032
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1341
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:909
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1616
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1754
def _check_stars(self, associations)
Definition: jointcal.py:1599
def _prep_sky(self, associations, filters)
Definition: jointcal.py:1037
def _readDataId(self, butler, dataId)
Definition: jointcal.py:982
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:638
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:1202
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:937
def runDataRef(self, dataRefs)
Definition: jointcal.py:1060
def __init__(self, butler=None, initInputs=None, **kwargs)
Definition: jointcal.py:617
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId)
Definition: jointcal.py:802
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:725
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1393
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1777
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1350
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1726
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:666
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1285
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1499
def loadData(self, dataRefs, associations, jointcalControl)
Definition: jointcal.py:1008
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:141
def add_measurement(job, name, value)
Definition: jointcal.py:59
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:181
def writeModel(model, filename, log)
Definition: jointcal.py:571