lsst.jointcal  21.0.0-18-g7027a3f+48871924a0
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import dataclasses
23 import collections
24 import os
25 
26 import astropy.time
27 import numpy as np
28 import astropy.units as u
29 
30 import lsst.geom
31 import lsst.utils
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 from lsst.afw.image import fluxErrFromABMagErr
35 import lsst.pex.exceptions as pexExceptions
37 import lsst.afw.table
38 import lsst.log
39 from lsst.obs.base import Instrument
40 from lsst.pipe.tasks.colorterms import ColortermLibrary
41 from lsst.verify import Job, Measurement
42 
43 from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44  ReferenceObjectLoader)
45 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46 
47 from .dataIds import PerTractCcdDataIdContainer
48 
49 import lsst.jointcal
50 from lsst.jointcal import MinimizeResult
51 
52 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
53 
54 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
56 
57 
58 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59 def add_measurement(job, name, value):
60  meas = Measurement(job.metrics[name], value)
61  job.measurements.insert(meas)
62 
63 
64 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65  """Subclass of TaskRunner for jointcalTask (gen2)
66 
67  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69  single dataRef, are are called repeatedly). This class transforms the processed
70  arguments generated by the ArgumentParser into the arguments expected by
71  Jointcal.runDataRef().
72 
73  See pipeBase.TaskRunner for more information.
74  """
75 
76  @staticmethod
77  def getTargetList(parsedCmd, **kwargs):
78  """
79  Return a list of tuples per tract, each containing (dataRefs, kwargs).
80 
81  Jointcal operates on lists of dataRefs simultaneously.
82  """
83  kwargs['butler'] = parsedCmd.butler
84 
85  # organize data IDs by tract
86  refListDict = {}
87  for ref in parsedCmd.id.refList:
88  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89  # we call runDataRef() once with each tract
90  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91  return result
92 
93  def __call__(self, args):
94  """
95  Parameters
96  ----------
97  args
98  Arguments for Task.runDataRef()
99 
100  Returns
101  -------
102  pipe.base.Struct
103  if self.doReturnResults is False:
104 
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106 
107  if self.doReturnResults is True:
108 
109  - ``result``: the result of calling jointcal.runDataRef()
110  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111  """
112  exitStatus = 0 # exit status for shell
113 
114  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115  dataRefList, kwargs = args
116  butler = kwargs.pop('butler')
117  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118  result = None
119  try:
120  result = task.runDataRef(dataRefList, **kwargs)
121  exitStatus = result.exitStatus
122  job_path = butler.get('verify_job_filename')
123  result.job.write(job_path[0])
124  except Exception as e: # catch everything, sort it out later.
125  if self.doRaise:
126  raise e
127  else:
128  exitStatus = 1
129  eName = type(e).__name__
130  tract = dataRefList[0].dataId['tract']
131  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
132 
133  # Put the butler back into kwargs for the other Tasks.
134  kwargs['butler'] = butler
135  if self.doReturnResults:
136  return pipeBase.Struct(result=result, exitStatus=exitStatus)
137  else:
138  return pipeBase.Struct(exitStatus=exitStatus)
139 
140 
141 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142  """Lookup function that asserts/hopes that a static calibration dataset
143  exists in a particular collection, since this task can't provide a single
144  date/time to use to search for one properly.
145 
146  This is mostly useful for the ``camera`` dataset, in cases where the task's
147  quantum dimensions do *not* include something temporal, like ``exposure``
148  or ``visit``.
149 
150  Parameters
151  ----------
152  datasetType : `lsst.daf.butler.DatasetType`
153  Type of dataset being searched for.
154  registry : `lsst.daf.butler.Registry`
155  Data repository registry to search.
156  quantumDataId : `lsst.daf.butler.DataCoordinate`
157  Data ID of the quantum this camera should match.
158  collections : `Iterable` [ `str` ]
159  Collections that should be searched - but this lookup function works
160  by ignoring this in favor of a more-or-less hard-coded value.
161 
162  Returns
163  -------
164  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165  Iterator over dataset references; should have only one element.
166 
167  Notes
168  -----
169  This implementation duplicates one in fgcmcal, and is at least quite
170  similar to another in cp_pipe. This duplicate has the most documentation.
171  Fixing this is DM-29661.
172  """
173  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175  return registry.queryDatasets(datasetType,
176  dataId=quantumDataId,
177  collections=[unboundedCollection],
178  findFirst=True)
179 
180 
181 def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182  """Lookup function that finds all refcats for all visits that overlap a
183  tract, rather than just the refcats that directly overlap the tract.
184 
185  Parameters
186  ----------
187  datasetType : `lsst.daf.butler.DatasetType`
188  Type of dataset being searched for.
189  registry : `lsst.daf.butler.Registry`
190  Data repository registry to search.
191  quantumDataId : `lsst.daf.butler.DataCoordinate`
192  Data ID of the quantum; expected to be something we can use as a
193  constraint to query for overlapping visits.
194  collections : `Iterable` [ `str` ]
195  Collections to search.
196 
197  Returns
198  -------
199  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200  Iterator over refcat references.
201  """
202  refs = set()
203  # Use .expanded() on the query methods below because we need data IDs with
204  # regions, both in the outer loop over visits (queryDatasets will expand
205  # any data ID we give it, but doing it up-front in bulk is much more
206  # efficient) and in the data IDs of the DatasetRefs this function yields
207  # (because the RefCatLoader relies on them to do some of its own
208  # filtering).
209  for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210  refs.update(
211  registry.queryDatasets(
212  datasetType,
213  collections=collections,
214  dataId=visit_data_id,
215  findFirst=True,
216  ).expanded()
217  )
218  yield from refs
219 
220 
221 class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222  dimensions=("skymap", "tract", "instrument", "physical_filter")):
223  """Middleware input/output connections for jointcal data."""
224  inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225  doc="The camera instrument that took these observations.",
226  name="camera",
227  storageClass="Camera",
228  dimensions=("instrument",),
229  isCalibration=True,
230  lookupFunction=lookupStaticCalibrations,
231  )
232  inputSourceTableVisit = pipeBase.connectionTypes.Input(
233  doc="Source table in parquet format, per visit",
234  name="sourceTable_visit",
235  storageClass="DataFrame",
236  dimensions=("instrument", "visit"),
237  deferLoad=True,
238  multiple=True,
239  )
240  inputVisitSummary = pipeBase.connectionTypes.Input(
241  doc=("Per-visit consolidated exposure metadata built from calexps. "
242  "These catalogs use detector id for the id and must be sorted for "
243  "fast lookups of a detector."),
244  name="visitSummary",
245  storageClass="ExposureCatalog",
246  dimensions=("instrument", "visit"),
247  deferLoad=True,
248  multiple=True,
249  )
250  astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251  doc="The astrometry reference catalog to match to loaded input catalog sources.",
252  name="gaia_dr2_20200414",
253  storageClass="SimpleCatalog",
254  dimensions=("skypix",),
255  deferLoad=True,
256  multiple=True,
257  lookupFunction=lookupVisitRefCats,
258  )
259  photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260  doc="The photometry reference catalog to match to loaded input catalog sources.",
261  name="ps1_pv3_3pi_20170110",
262  storageClass="SimpleCatalog",
263  dimensions=("skypix",),
264  deferLoad=True,
265  multiple=True,
266  lookupFunction=lookupVisitRefCats,
267  )
268 
269  outputWcs = pipeBase.connectionTypes.Output(
270  doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271  " These catalogs only contain entries for detectors with an output, and use"
272  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273  name="jointcalSkyWcsCatalog",
274  storageClass="ExposureCatalog",
275  dimensions=("instrument", "visit", "skymap", "tract"),
276  multiple=True
277  )
278  outputPhotoCalib = pipeBase.connectionTypes.Output(
279  doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280  " These catalogs only contain entries for detectors with an output, and use"
281  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282  name="jointcalPhotoCalibCatalog",
283  storageClass="ExposureCatalog",
284  dimensions=("instrument", "visit", "skymap", "tract"),
285  multiple=True
286  )
287 
288  def __init__(self, *, config=None):
289  super().__init__(config=config)
290  # When we are only doing one of astrometry or photometry, we don't
291  # need the reference catalog or produce the outputs for the other.
292  # This informs the middleware of that when the QuantumGraph is
293  # generated, so we don't block on getting something we won't need or
294  # create an expectation that downstream tasks will be able to consume
295  # something we won't produce.
296  if not config.doAstrometry:
297  self.prerequisiteInputs.remove("astrometryRefCat")
298  self.outputs.remove("outputWcs")
299  if not config.doPhotometry:
300  self.prerequisiteInputs.remove("photometryRefCat")
301  self.outputs.remove("outputPhotoCalib")
302 
303 
304 class JointcalConfig(pipeBase.PipelineTaskConfig,
305  pipelineConnections=JointcalTaskConnections):
306  """Configuration for JointcalTask"""
307 
308  doAstrometry = pexConfig.Field(
309  doc="Fit astrometry and write the fitted result.",
310  dtype=bool,
311  default=True
312  )
313  doPhotometry = pexConfig.Field(
314  doc="Fit photometry and write the fitted result.",
315  dtype=bool,
316  default=True
317  )
318  coaddName = pexConfig.Field(
319  doc="Type of coadd, typically deep or goodSeeing",
320  dtype=str,
321  default="deep"
322  )
323  # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
324  sourceFluxType = pexConfig.Field(
325  dtype=str,
326  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
327  default='Calib'
328  )
329  positionErrorPedestal = pexConfig.Field(
330  doc="Systematic term to apply to the measured position error (pixels)",
331  dtype=float,
332  default=0.02,
333  )
334  photometryErrorPedestal = pexConfig.Field(
335  doc="Systematic term to apply to the measured error on flux or magnitude as a "
336  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
337  dtype=float,
338  default=0.0,
339  )
340  # TODO: DM-6885 matchCut should be an geom.Angle
341  matchCut = pexConfig.Field(
342  doc="Matching radius between fitted and reference stars (arcseconds)",
343  dtype=float,
344  default=3.0,
345  )
346  minMeasurements = pexConfig.Field(
347  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
348  dtype=int,
349  default=2,
350  )
351  minMeasuredStarsPerCcd = pexConfig.Field(
352  doc="Minimum number of measuredStars per ccdImage before printing warnings",
353  dtype=int,
354  default=100,
355  )
356  minRefStarsPerCcd = pexConfig.Field(
357  doc="Minimum number of measuredStars per ccdImage before printing warnings",
358  dtype=int,
359  default=30,
360  )
361  allowLineSearch = pexConfig.Field(
362  doc="Allow a line search during minimization, if it is reasonable for the model"
363  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
364  dtype=bool,
365  default=False
366  )
367  astrometrySimpleOrder = pexConfig.Field(
368  doc="Polynomial order for fitting the simple astrometry model.",
369  dtype=int,
370  default=3,
371  )
372  astrometryChipOrder = pexConfig.Field(
373  doc="Order of the per-chip transform for the constrained astrometry model.",
374  dtype=int,
375  default=1,
376  )
377  astrometryVisitOrder = pexConfig.Field(
378  doc="Order of the per-visit transform for the constrained astrometry model.",
379  dtype=int,
380  default=5,
381  )
382  useInputWcs = pexConfig.Field(
383  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
384  dtype=bool,
385  default=True,
386  )
387  astrometryModel = pexConfig.ChoiceField(
388  doc="Type of model to fit to astrometry",
389  dtype=str,
390  default="constrained",
391  allowed={"simple": "One polynomial per ccd",
392  "constrained": "One polynomial per ccd, and one polynomial per visit"}
393  )
394  photometryModel = pexConfig.ChoiceField(
395  doc="Type of model to fit to photometry",
396  dtype=str,
397  default="constrainedMagnitude",
398  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
399  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
400  " fitting in flux space.",
401  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
402  " fitting in magnitude space.",
403  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
404  " fitting in magnitude space.",
405  }
406  )
407  applyColorTerms = pexConfig.Field(
408  doc="Apply photometric color terms to reference stars?"
409  "Requires that colorterms be set to a ColortermLibrary",
410  dtype=bool,
411  default=False
412  )
413  colorterms = pexConfig.ConfigField(
414  doc="Library of photometric reference catalog name to color term dict.",
415  dtype=ColortermLibrary,
416  )
417  photometryVisitOrder = pexConfig.Field(
418  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
419  dtype=int,
420  default=7,
421  )
422  photometryDoRankUpdate = pexConfig.Field(
423  doc=("Do the rank update step during minimization. "
424  "Skipping this can help deal with models that are too non-linear."),
425  dtype=bool,
426  default=True,
427  )
428  astrometryDoRankUpdate = pexConfig.Field(
429  doc=("Do the rank update step during minimization (should not change the astrometry fit). "
430  "Skipping this can help deal with models that are too non-linear."),
431  dtype=bool,
432  default=True,
433  )
434  outlierRejectSigma = pexConfig.Field(
435  doc="How many sigma to reject outliers at during minimization.",
436  dtype=float,
437  default=5.0,
438  )
439  maxPhotometrySteps = pexConfig.Field(
440  doc="Maximum number of minimize iterations to take when fitting photometry.",
441  dtype=int,
442  default=20,
443  )
444  maxAstrometrySteps = pexConfig.Field(
445  doc="Maximum number of minimize iterations to take when fitting photometry.",
446  dtype=int,
447  default=20,
448  )
449  astrometryRefObjLoader = pexConfig.ConfigurableField(
450  target=LoadIndexedReferenceObjectsTask,
451  doc="Reference object loader for astrometric fit",
452  )
453  photometryRefObjLoader = pexConfig.ConfigurableField(
454  target=LoadIndexedReferenceObjectsTask,
455  doc="Reference object loader for photometric fit",
456  )
457  sourceSelector = sourceSelectorRegistry.makeField(
458  doc="How to select sources for cross-matching",
459  default="astrometry"
460  )
461  astrometryReferenceSelector = pexConfig.ConfigurableField(
462  target=ReferenceSourceSelectorTask,
463  doc="How to down-select the loaded astrometry reference catalog.",
464  )
465  photometryReferenceSelector = pexConfig.ConfigurableField(
466  target=ReferenceSourceSelectorTask,
467  doc="How to down-select the loaded photometry reference catalog.",
468  )
469  astrometryReferenceErr = pexConfig.Field(
470  doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
471  "If None, then raise an exception if the reference catalog is missing coordinate errors. "
472  "If specified, overrides any existing `coord_*Err` values."),
473  dtype=float,
474  default=None,
475  optional=True
476  )
477 
478  # configs for outputting debug information
479  writeInitMatrix = pexConfig.Field(
480  dtype=bool,
481  doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
482  "Output files will be written to `config.debugOutputPath` and will "
483  "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
484  "Note that these files are the dense versions of the matrix, and so may be very large."),
485  default=False
486  )
487  writeChi2FilesInitialFinal = pexConfig.Field(
488  dtype=bool,
489  doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
490  "Output files will be written to `config.debugOutputPath` and will "
491  "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER1."),
492  default=False
493  )
494  writeChi2FilesOuterLoop = pexConfig.Field(
495  dtype=bool,
496  doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
497  "Output files will be written to `config.debugOutputPath` and will "
498  "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
499  default=False
500  )
501  writeInitialModel = pexConfig.Field(
502  dtype=bool,
503  doc=("Write the pre-initialization model to text files, for debugging. "
504  "Output files will be written to `config.debugOutputPath` and will be "
505  "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
506  default=False
507  )
508  debugOutputPath = pexConfig.Field(
509  dtype=str,
510  default=".",
511  doc=("Path to write debug output files to. Used by "
512  "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
513  )
514  detailedProfile = pexConfig.Field(
515  dtype=bool,
516  default=False,
517  doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
518  )
519 
520  def validate(self):
521  super().validate()
522  if self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms and len(self.colortermscolorterms.data) == 0:
523  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
524  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
525  if self.doAstrometrydoAstrometry and not self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms:
526  msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
527  "applyColorTerms=True will be ignored.")
528  lsst.log.warn(msg)
529 
530  def setDefaults(self):
531  # Use science source selector which can filter on extendedness, SNR, and whether blended
532  self.sourceSelectorsourceSelector.name = 'science'
533  # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
534  self.sourceSelectorsourceSelector['science'].doUnresolved = True
535  # with dependable signal to noise ratio.
536  self.sourceSelectorsourceSelector['science'].doSignalToNoise = True
537  # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
538  # and S/N > 10 to use sources that are not too faint, and thus better measured.
539  self.sourceSelectorsourceSelector['science'].signalToNoise.minimum = 10.
540  # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
541  fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
542  self.sourceSelectorsourceSelector['science'].signalToNoise.fluxField = fluxField
543  self.sourceSelectorsourceSelector['science'].signalToNoise.errField = fluxField + "Err"
544  # Do not trust blended sources' aperture fluxes which also depend on seeing
545  self.sourceSelectorsourceSelector['science'].doIsolated = True
546  # Do not trust either flux or centroid measurements with flags,
547  # chosen from the usual QA flags for stars)
548  self.sourceSelectorsourceSelector['science'].doFlags = True
549  badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
550  'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
551  'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
552  self.sourceSelectorsourceSelector['science'].flags.bad = badFlags
553 
554  # Default to Gaia-DR2 (with proper motions) for astrometry and
555  # PS1-DR1 for photometry, with a reasonable initial filterMap.
556  self.astrometryRefObjLoaderastrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
557  self.astrometryRefObjLoaderastrometryRefObjLoader.requireProperMotion = True
558  self.astrometryRefObjLoaderastrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
559  self.photometryRefObjLoaderphotometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
560 
561 
562 def writeModel(model, filename, log):
563  """Write model to outfile."""
564  with open(filename, "w") as file:
565  file.write(repr(model))
566  log.info("Wrote %s to file: %s", model, filename)
567 
568 
569 @dataclasses.dataclass
571  """The input data jointcal needs for each detector/visit."""
572  visit: int
573  """The visit identifier of this exposure."""
575  """The catalog derived from this exposure."""
576  visitInfo: lsst.afw.image.VisitInfo
577  """The VisitInfo of this exposure."""
579  """The detector of this exposure."""
580  photoCalib: lsst.afw.image.PhotoCalib
581  """The photometric calibration of this exposure."""
583  """The WCS of this exposure."""
584  bbox: lsst.geom.Box2I
585  """The bounding box of this exposure."""
587  """The filter of this exposure."""
588 
589 
590 class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
591  """Astrometricly and photometricly calibrate across multiple visits of the
592  same field.
593 
594  Parameters
595  ----------
596  butler : `lsst.daf.persistence.Butler`
597  The butler is passed to the refObjLoader constructor in case it is
598  needed. Ignored if the refObjLoader argument provides a loader directly.
599  Used to initialize the astrometry and photometry refObjLoaders.
600  initInputs : `dict`, optional
601  Dictionary used to initialize PipelineTasks (empty for jointcal).
602  """
603 
604  ConfigClass = JointcalConfig
605  RunnerClass = JointcalRunner
606  _DefaultName = "jointcal"
607 
608  def __init__(self, butler=None, initInputs=None, **kwargs):
609  super().__init__(**kwargs)
610  self.makeSubtask("sourceSelector")
611  if self.config.doAstrometry:
612  if initInputs is None:
613  # gen3 middleware does refcat things internally (and will not have a butler here)
614  self.makeSubtask('astrometryRefObjLoader', butler=butler)
615  self.makeSubtask("astrometryReferenceSelector")
616  else:
617  self.astrometryRefObjLoaderastrometryRefObjLoader = None
618  if self.config.doPhotometry:
619  if initInputs is None:
620  # gen3 middleware does refcat things internally (and will not have a butler here)
621  self.makeSubtask('photometryRefObjLoader', butler=butler)
622  self.makeSubtask("photometryReferenceSelector")
623  else:
624  self.photometryRefObjLoaderphotometryRefObjLoader = None
625 
626  # To hold various computed metrics for use by tests
627  self.jobjob = Job.load_metrics_package(subset='jointcal')
628 
629  def runQuantum(self, butlerQC, inputRefs, outputRefs):
630  # We override runQuantum to set up the refObjLoaders and write the
631  # outputs to the correct refs.
632  inputs = butlerQC.get(inputRefs)
633  # We want the tract number for writing debug files
634  tract = butlerQC.quantum.dataId['tract']
635  if self.config.doAstrometry:
636  self.astrometryRefObjLoaderastrometryRefObjLoader = ReferenceObjectLoader(
637  dataIds=[ref.datasetRef.dataId
638  for ref in inputRefs.astrometryRefCat],
639  refCats=inputs.pop('astrometryRefCat'),
640  config=self.config.astrometryRefObjLoader,
641  log=self.log)
642  if self.config.doPhotometry:
643  self.photometryRefObjLoaderphotometryRefObjLoader = ReferenceObjectLoader(
644  dataIds=[ref.datasetRef.dataId
645  for ref in inputRefs.photometryRefCat],
646  refCats=inputs.pop('photometryRefCat'),
647  config=self.config.photometryRefObjLoader,
648  log=self.log)
649  outputs = self.runrun(**inputs, tract=tract)
650  if self.config.doAstrometry:
651  self._put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
652  inputs['inputCamera'], "setWcs")
653  if self.config.doPhotometry:
654  self._put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
655  inputs['inputCamera'], "setPhotoCalib")
656 
657  def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
658  """Persist the output datasets to their appropriate datarefs.
659 
660  Parameters
661  ----------
662  butlerQC : `ButlerQuantumContext`
663  A butler which is specialized to operate in the context of a
664  `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
665  outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
666  `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
667  The fitted objects to persist.
668  outputRefs : `list` [`OutputQuantizedConnection`]
669  The DatasetRefs to persist the data to.
670  camera : `lsst.afw.cameraGeom.Camera`
671  The camera for this instrument, to get detector ids from.
672  setter : `str`
673  The method to call on the ExposureCatalog to set each output.
674  """
676  schema.addField('visit', type='I', doc='Visit number')
677 
678  def new_catalog(visit, size):
679  """Return an catalog ready to be filled with appropriate output."""
680  catalog = lsst.afw.table.ExposureCatalog(schema)
681  catalog.resize(size)
682  catalog['visit'] = visit
683  metadata = lsst.daf.base.PropertyList()
684  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
685  metadata.add("COMMENT", "Only detectors with data have entries.")
686  return catalog
687 
688  # count how many detectors have output for each visit
689  detectors_per_visit = collections.defaultdict(int)
690  for key in outputs:
691  # key is (visit, detector_id), and we only need visit here
692  detectors_per_visit[key[0]] += 1
693 
694  for ref in outputRefs:
695  visit = ref.dataId['visit']
696  catalog = new_catalog(visit, detectors_per_visit[visit])
697  # Iterate over every detector and skip the ones we don't have output for.
698  i = 0
699  for detector in camera:
700  detectorId = detector.getId()
701  key = (ref.dataId['visit'], detectorId)
702  if key not in outputs:
703  # skip detectors we don't have output for
704  self.log.debug("No %s output for detector %s in visit %s",
705  setter[3:], detectorId, visit)
706  continue
707 
708  catalog[i].setId(detectorId)
709  getattr(catalog[i], setter)(outputs[key])
710  i += 1
711 
712  catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
713  butlerQC.put(catalog, ref)
714  self.log.info("Wrote %s detectors to %s", i, ref)
715 
716  def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
717  # Docstring inherited.
718 
719  # We take values out of the Parquet table, and put them in "flux_",
720  # and the config.sourceFluxType field is used during that extraction,
721  # so just use "flux" here.
722  sourceFluxField = "flux"
723  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
724  associations = lsst.jointcal.Associations()
725  self.focalPlaneBBoxfocalPlaneBBox = inputCamera.getFpBBox()
726  oldWcsList, bands = self._load_data_load_data(inputSourceTableVisit,
727  inputVisitSummary,
728  associations,
729  jointcalControl,
730  inputCamera)
731 
732  boundingCircle, center, radius, defaultFilter = self._prep_sky_prep_sky(associations, bands)
733  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
734 
735  if self.config.doAstrometry:
736  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
737  name="astrometry",
738  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
739  referenceSelector=self.astrometryReferenceSelector,
740  fit_function=self._fit_astrometry_fit_astrometry,
741  tract=tract,
742  epoch=epoch)
743  astrometry_output = self._make_output_make_output(associations.getCcdImageList(),
744  astrometry.model,
745  "makeSkyWcs")
746  else:
747  astrometry_output = None
748 
749  if self.config.doPhotometry:
750  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
751  name="photometry",
752  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
753  referenceSelector=self.photometryReferenceSelector,
754  fit_function=self._fit_photometry_fit_photometry,
755  tract=tract,
756  epoch=epoch,
757  reject_bad_fluxes=True)
758  photometry_output = self._make_output_make_output(associations.getCcdImageList(),
759  photometry.model,
760  "toPhotoCalib")
761  else:
762  photometry_output = None
763 
764  return pipeBase.Struct(outputWcs=astrometry_output,
765  outputPhotoCalib=photometry_output,
766  job=self.jobjob,
767  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
768  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader)
769 
770  def _make_schema_table(self):
771  """Return an afw SourceTable to use as a base for creating the
772  SourceCatalog to insert values from the dataFrame into.
773 
774  Returns
775  -------
776  table : `lsst.afw.table.SourceTable`
777  Table with schema and slots to use to make SourceCatalogs.
778  """
780  schema.addField("centroid_x", "D")
781  schema.addField("centroid_y", "D")
782  schema.addField("centroid_xErr", "F")
783  schema.addField("centroid_yErr", "F")
784  schema.addField("shape_xx", "D")
785  schema.addField("shape_yy", "D")
786  schema.addField("shape_xy", "D")
787  schema.addField("flux_instFlux", "D")
788  schema.addField("flux_instFluxErr", "D")
789  table = lsst.afw.table.SourceTable.make(schema)
790  table.defineCentroid("centroid")
791  table.defineShape("shape")
792  return table
793 
794  def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
795  """Return an afw SourceCatalog extracted from a visit-level dataframe,
796  limited to just one detector.
797 
798  Parameters
799  ----------
800  table : `lsst.afw.table.SourceTable`
801  Table factory to use to make the SourceCatalog that will be
802  populated with data from ``visitCatalog``.
803  visitCatalog : `pandas.DataFrame`
804  DataFrame to extract a detector catalog from.
805  detectorId : `int`
806  Numeric id of the detector to extract from ``visitCatalog``.
807 
808  Returns
809  -------
810  catalog : `lsst.afw.table.SourceCatalog`
811  Detector-level catalog extracted from ``visitCatalog``.
812  """
813  # map from dataFrame column to afw table column
814  mapping = {'sourceId': 'id',
815  'x': 'centroid_x',
816  'y': 'centroid_y',
817  'xErr': 'centroid_xErr',
818  'yErr': 'centroid_yErr',
819  'Ixx': 'shape_xx',
820  'Iyy': 'shape_yy',
821  'Ixy': 'shape_xy',
822  f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
823  f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
824  }
825  # If the DataFrame we're reading was generated by a task running with
826  # Gen2 middleware, the column for the detector will be "ccd" for at
827  # least HSC (who knows what it might be in general!); that would be
828  # true even if the data repo is later converted to Gen3, because that
829  # doesn't touch the files themselves. In Gen3, the column for the
830  # detector will always be "detector".
831  detector_column = "detector" if "detector" in visitCatalog.columns else "ccd"
832  catalog = lsst.afw.table.SourceCatalog(table)
833  matched = visitCatalog[detector_column] == detectorId
834  catalog.resize(sum(matched))
835  view = visitCatalog.loc[matched]
836  for dfCol, afwCol in mapping.items():
837  catalog[afwCol] = view[dfCol]
838 
839  self.log.debug("%d sources selected in visit %d detector %d",
840  len(catalog),
841  view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
842  detectorId)
843  return catalog
844 
845  def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
846  jointcalControl, camera):
847  """Read the data that jointcal needs to run. (Gen3 version)
848 
849  Modifies ``associations`` in-place with the loaded data.
850 
851  Parameters
852  ----------
853  inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
854  References to visit-level DataFrames to load the catalog data from.
855  inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
856  Visit-level exposure summary catalog with metadata.
857  associations : `lsst.jointcal.Associations`
858  Object to add the loaded data to by constructing new CcdImages.
859  jointcalControl : `jointcal.JointcalControl`
860  Control object for C++ associations management.
861  camera : `lsst.afw.cameraGeom.Camera`
862  Camera object for detector geometry.
863 
864  Returns
865  -------
866  oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
867  The original WCS of the input data, to aid in writing tests.
868  bands : `list` [`str`]
869  The filter bands of each input dataset.
870  """
871  oldWcsList = []
872  filters = []
873  load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
874  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
875  table = self._make_schema_table_make_schema_table() # every detector catalog has the same layout
876  # No guarantee that the input is in the same order of visits, so we have to map one of them.
877  catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
878  detectorDict = {detector.getId(): detector for detector in camera}
879 
880  for visitSummaryRef in inputVisitSummary:
881  visitSummary = visitSummaryRef.get()
882  visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
883  selected = self.sourceSelector.run(visitCatalog)
884 
885  # Build a CcdImage for each detector in this visit.
886  detectors = {id: index for index, id in enumerate(visitSummary['id'])}
887  for id, index in detectors.items():
888  catalog = self._extract_detector_catalog_from_visit_catalog_extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
889  data = self._make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
890  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
891  if result is not None:
892  oldWcsList.append(result.wcs)
893  # A visit has only one band, so we can just use the first.
894  filters.append(data.filter)
895  if len(filters) == 0:
896  raise RuntimeError("No data to process: did source selector remove all sources?")
897  filters = collections.Counter(filters)
898 
899  return oldWcsList, filters
900 
901  def _make_one_input_data(self, visitRecord, catalog, detectorDict):
902  """Return a data structure for this detector+visit."""
903  return JointcalInputData(visit=visitRecord['visit'],
904  catalog=catalog,
905  visitInfo=visitRecord.getVisitInfo(),
906  detector=detectorDict[visitRecord.getId()],
907  photoCalib=visitRecord.getPhotoCalib(),
908  wcs=visitRecord.getWcs(),
909  bbox=visitRecord.getBBox(),
910  # ExposureRecord doesn't have a FilterLabel yet,
911  # so we have to make one.
912  filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
913  physical=visitRecord['physical_filter']))
914 
915  # We don't currently need to persist the metadata.
916  # If we do in the future, we will have to add appropriate dataset templates
917  # to each obs package (the metadata template should look like `jointcal_wcs`).
918  def _getMetadataName(self):
919  return None
920 
921  @classmethod
922  def _makeArgumentParser(cls):
923  """Create an argument parser"""
924  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
925  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
926  ContainerClass=PerTractCcdDataIdContainer)
927  return parser
928 
929  def _build_ccdImage(self, data, associations, jointcalControl):
930  """
931  Extract the necessary things from this catalog+metadata to add a new
932  ccdImage.
933 
934  Parameters
935  ----------
936  data : `JointcalInputData`
937  The loaded input data.
938  associations : `lsst.jointcal.Associations`
939  Object to add the info to, to construct a new CcdImage
940  jointcalControl : `jointcal.JointcalControl`
941  Control object for associations management
942 
943  Returns
944  ------
945  namedtuple or `None`
946  ``wcs``
947  The TAN WCS of this image, read from the calexp
948  (`lsst.afw.geom.SkyWcs`).
949  ``key``
950  A key to identify this dataRef by its visit and ccd ids
951  (`namedtuple`).
952  `None`
953  if there are no sources in the loaded catalog.
954  """
955  if len(data.catalog) == 0:
956  self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
957  return None
958 
959  associations.createCcdImage(data.catalog,
960  data.wcs,
961  data.visitInfo,
962  data.bbox,
963  data.filter.physicalLabel,
964  data.photoCalib,
965  data.detector,
966  data.visit,
967  data.detector.getId(),
968  jointcalControl)
969 
970  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
971  Key = collections.namedtuple('Key', ('visit', 'ccd'))
972  return Result(data.wcs, Key(data.visit, data.detector.getId()))
973 
974  def _readDataId(self, butler, dataId):
975  """Read all of the data for one dataId from the butler. (gen2 version)"""
976  # Not all instruments have `visit` in their dataIds.
977  if "visit" in dataId.keys():
978  visit = dataId["visit"]
979  else:
980  visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
981  detector = butler.get('calexp_detector', dataId=dataId)
982 
983  catalog = butler.get('src',
984  flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
985  dataId=dataId)
986  goodSrc = self.sourceSelector.run(catalog)
987  self.log.debug("%d sources selected in visit %d detector %d",
988  len(goodSrc.sourceCat),
989  visit,
990  detector.getId())
991  return JointcalInputData(visit=visit,
992  catalog=goodSrc.sourceCat,
993  visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
994  detector=detector,
995  photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
996  wcs=butler.get('calexp_wcs', dataId=dataId),
997  bbox=butler.get('calexp_bbox', dataId=dataId),
998  filter=butler.get('calexp_filterLabel', dataId=dataId))
999 
1000  def loadData(self, dataRefs, associations, jointcalControl):
1001  """Read the data that jointcal needs to run. (Gen2 version)"""
1002  visit_ccd_to_dataRef = {}
1003  oldWcsList = []
1004  filters = []
1005  load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1006  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1007  # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1008  camera = dataRefs[0].get('camera', immediate=True)
1009  self.focalPlaneBBoxfocalPlaneBBox = camera.getFpBBox()
1010  for dataRef in dataRefs:
1011  data = self._readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1012  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1013  if result is None:
1014  continue
1015  oldWcsList.append(result.wcs)
1016  visit_ccd_to_dataRef[result.key] = dataRef
1017  filters.append(data.filter)
1018  if len(filters) == 0:
1019  raise RuntimeError("No data to process: did source selector remove all sources?")
1020  filters = collections.Counter(filters)
1021 
1022  return oldWcsList, filters, visit_ccd_to_dataRef
1023 
1024  def _getDebugPath(self, filename):
1025  """Constructs a path to filename using the configured debug path.
1026  """
1027  return os.path.join(self.config.debugOutputPath, filename)
1028 
1029  def _prep_sky(self, associations, filters):
1030  """Prepare on-sky and other data that must be computed after data has
1031  been read.
1032  """
1033  associations.computeCommonTangentPoint()
1034 
1035  boundingCircle = associations.computeBoundingCircle()
1036  center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1037  radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1038 
1039  self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1040 
1041  # Determine a default filter band associated with the catalog. See DM-9093
1042  defaultFilter = filters.most_common(1)[0][0]
1043  self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1044 
1045  return boundingCircle, center, radius, defaultFilter
1046 
1047  @pipeBase.timeMethod
1048  def runDataRef(self, dataRefs):
1049  """
1050  Jointly calibrate the astrometry and photometry across a set of images.
1051 
1052  NOTE: this is for gen2 middleware only.
1053 
1054  Parameters
1055  ----------
1056  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1057  List of data references to the exposures to be fit.
1058 
1059  Returns
1060  -------
1061  result : `lsst.pipe.base.Struct`
1062  Struct of metadata from the fit, containing:
1063 
1064  ``dataRefs``
1065  The provided data references that were fit (with updated WCSs)
1066  ``oldWcsList``
1067  The original WCS from each dataRef
1068  ``metrics``
1069  Dictionary of internally-computed metrics for testing/validation.
1070  """
1071  if len(dataRefs) == 0:
1072  raise ValueError('Need a non-empty list of data references!')
1073 
1074  exitStatus = 0 # exit status for shell
1075 
1076  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1077  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1078  associations = lsst.jointcal.Associations()
1079 
1080  oldWcsList, filters, visit_ccd_to_dataRef = self.loadDataloadData(dataRefs,
1081  associations,
1082  jointcalControl)
1083 
1084  boundingCircle, center, radius, defaultFilter = self._prep_sky_prep_sky(associations, filters)
1085  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
1086 
1087  tract = dataRefs[0].dataId['tract']
1088 
1089  if self.config.doAstrometry:
1090  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1091  name="astrometry",
1092  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1093  referenceSelector=self.astrometryReferenceSelector,
1094  fit_function=self._fit_astrometry_fit_astrometry,
1095  tract=tract,
1096  epoch=epoch)
1097  self._write_astrometry_results_write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1098  else:
1099  astrometry = Astrometry(None, None, None)
1100 
1101  if self.config.doPhotometry:
1102  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1103  name="photometry",
1104  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1105  referenceSelector=self.photometryReferenceSelector,
1106  fit_function=self._fit_photometry_fit_photometry,
1107  tract=tract,
1108  epoch=epoch,
1109  reject_bad_fluxes=True)
1110  self._write_photometry_results_write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1111  else:
1112  photometry = Photometry(None, None)
1113 
1114  return pipeBase.Struct(dataRefs=dataRefs,
1115  oldWcsList=oldWcsList,
1116  job=self.jobjob,
1117  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1118  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1119  defaultFilter=defaultFilter,
1120  epoch=epoch,
1121  exitStatus=exitStatus)
1122 
1123  def _get_refcat_coordinate_error_override(self, refCat, name):
1124  """Check whether we should override the refcat coordinate errors, and
1125  return the overridden error if necessary.
1126 
1127  Parameters
1128  ----------
1129  refCat : `lsst.afw.table.SimpleCatalog`
1130  The reference catalog to check for a ``coord_raErr`` field.
1131  name : `str`
1132  Whether we are doing "astrometry" or "photometry".
1133 
1134  Returns
1135  -------
1136  refCoordErr : `float`
1137  The refcat coordinate error to use, or NaN if we are not overriding
1138  those fields.
1139 
1140  Raises
1141  ------
1142  lsst.pex.config.FieldValidationError
1143  Raised if the refcat does not contain coordinate errors and
1144  ``config.astrometryReferenceErr`` is not set.
1145  """
1146  # This value doesn't matter for photometry, so just set something to
1147  # keep old refcats from causing problems.
1148  if name.lower() == "photometry":
1149  if 'coord_raErr' not in refCat.schema:
1150  return 100
1151  else:
1152  return float('nan')
1153 
1154  if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1155  msg = ("Reference catalog does not contain coordinate errors, "
1156  "and config.astrometryReferenceErr not supplied.")
1157  raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1158  self.config,
1159  msg)
1160 
1161  if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1162  self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1163  self.config.astrometryReferenceErr)
1164 
1165  if self.config.astrometryReferenceErr is None:
1166  return float('nan')
1167  else:
1168  return self.config.astrometryReferenceErr
1169 
1170  def _compute_proper_motion_epoch(self, ccdImageList):
1171  """Return the proper motion correction epoch of the provided images.
1172 
1173  Parameters
1174  ----------
1175  ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1176  The images to compute the appropriate epoch for.
1177 
1178  Returns
1179  -------
1180  epoch : `astropy.time.Time`
1181  The date to use for proper motion corrections.
1182  """
1183  mjds = [ccdImage.getMjd() for ccdImage in ccdImageList]
1184  return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai")
1185 
1186  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1187  tract="", match_cut=3.0,
1188  reject_bad_fluxes=False, *,
1189  name="", refObjLoader=None, referenceSelector=None,
1190  fit_function=None, epoch=None):
1191  """Load reference catalog, perform the fit, and return the result.
1192 
1193  Parameters
1194  ----------
1195  associations : `lsst.jointcal.Associations`
1196  The star/reference star associations to fit.
1197  defaultFilter : `lsst.afw.image.FilterLabel`
1198  filter to load from reference catalog.
1199  center : `lsst.geom.SpherePoint`
1200  ICRS center of field to load from reference catalog.
1201  radius : `lsst.geom.Angle`
1202  On-sky radius to load from reference catalog.
1203  name : `str`
1204  Name of thing being fit: "astrometry" or "photometry".
1205  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1206  Reference object loader to use to load a reference catalog.
1207  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1208  Selector to use to pick objects from the loaded reference catalog.
1209  fit_function : callable
1210  Function to call to perform fit (takes Associations object).
1211  tract : `str`, optional
1212  Name of tract currently being fit.
1213  match_cut : `float`, optional
1214  Radius in arcseconds to find cross-catalog matches to during
1215  associations.associateCatalogs.
1216  reject_bad_fluxes : `bool`, optional
1217  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1218  epoch : `astropy.time.Time`, optional
1219  Epoch to which to correct refcat proper motion and parallax,
1220  or `None` to not apply such corrections.
1221 
1222  Returns
1223  -------
1224  result : `Photometry` or `Astrometry`
1225  Result of `fit_function()`
1226  """
1227  self.log.info("====== Now processing %s...", name)
1228  # TODO: this should not print "trying to invert a singular transformation:"
1229  # if it does that, something's not right about the WCS...
1230  associations.associateCatalogs(match_cut)
1231  add_measurement(self.jobjob, 'jointcal.associated_%s_fittedStars' % name,
1232  associations.fittedStarListSize())
1233 
1234  applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1235  refCat, fluxField = self._load_reference_catalog_load_reference_catalog(refObjLoader, referenceSelector,
1236  center, radius, defaultFilter,
1237  applyColorterms=applyColorterms,
1238  epoch=epoch)
1239  refCoordErr = self._get_refcat_coordinate_error_override_get_refcat_coordinate_error_override(refCat, name)
1240 
1241  associations.collectRefStars(refCat,
1242  self.config.matchCut*lsst.geom.arcseconds,
1243  fluxField,
1244  refCoordinateErr=refCoordErr,
1245  rejectBadFluxes=reject_bad_fluxes)
1246  add_measurement(self.jobjob, 'jointcal.collected_%s_refStars' % name,
1247  associations.refStarListSize())
1248 
1249  associations.prepareFittedStars(self.config.minMeasurements)
1250 
1251  self._check_star_lists_check_star_lists(associations, name)
1252  add_measurement(self.jobjob, 'jointcal.selected_%s_refStars' % name,
1253  associations.nFittedStarsWithAssociatedRefStar())
1254  add_measurement(self.jobjob, 'jointcal.selected_%s_fittedStars' % name,
1255  associations.fittedStarListSize())
1256  add_measurement(self.jobjob, 'jointcal.selected_%s_ccdImages' % name,
1257  associations.nCcdImagesValidForFit())
1258 
1259  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1260  dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1261  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1262  result = fit_function(associations, dataName)
1263  # TODO DM-12446: turn this into a "butler save" somehow.
1264  # Save reference and measurement chi2 contributions for this data
1265  if self.config.writeChi2FilesInitialFinal:
1266  baseName = self._getDebugPath_getDebugPath(f"{name}_final_chi2-{dataName}")
1267  result.fit.saveChi2Contributions(baseName+"{type}")
1268  self.log.info("Wrote chi2 contributions files: %s", baseName)
1269 
1270  return result
1271 
1272  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1273  applyColorterms=False, epoch=None):
1274  """Load the necessary reference catalog sources, convert fluxes to
1275  correct units, and apply color term corrections if requested.
1276 
1277  Parameters
1278  ----------
1279  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1280  The reference catalog loader to use to get the data.
1281  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1282  Source selector to apply to loaded reference catalog.
1283  center : `lsst.geom.SpherePoint`
1284  The center around which to load sources.
1285  radius : `lsst.geom.Angle`
1286  The radius around ``center`` to load sources in.
1287  filterLabel : `lsst.afw.image.FilterLabel`
1288  The camera filter to load fluxes for.
1289  applyColorterms : `bool`
1290  Apply colorterm corrections to the refcat for ``filterName``?
1291  epoch : `astropy.time.Time`, optional
1292  Epoch to which to correct refcat proper motion and parallax,
1293  or `None` to not apply such corrections.
1294 
1295  Returns
1296  -------
1297  refCat : `lsst.afw.table.SimpleCatalog`
1298  The loaded reference catalog.
1299  fluxField : `str`
1300  The name of the reference catalog flux field appropriate for ``filterName``.
1301  """
1302  skyCircle = refObjLoader.loadSkyCircle(center,
1303  radius,
1304  filterLabel.bandLabel,
1305  epoch=epoch)
1306 
1307  selected = referenceSelector.run(skyCircle.refCat)
1308  # Need memory contiguity to get reference filters as a vector.
1309  if not selected.sourceCat.isContiguous():
1310  refCat = selected.sourceCat.copy(deep=True)
1311  else:
1312  refCat = selected.sourceCat
1313 
1314  if applyColorterms:
1315  refCatName = refObjLoader.ref_dataset_name
1316  self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1317  filterLabel.physicalLabel, refCatName)
1318  colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1319  refCatName,
1320  doRaise=True)
1321 
1322  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1323  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1324  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1325  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1326 
1327  return refCat, skyCircle.fluxField
1328 
1329  def _check_star_lists(self, associations, name):
1330  # TODO: these should be len(blah), but we need this properly wrapped first.
1331  if associations.nCcdImagesValidForFit() == 0:
1332  raise RuntimeError('No images in the ccdImageList!')
1333  if associations.fittedStarListSize() == 0:
1334  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1335  if associations.refStarListSize() == 0:
1336  raise RuntimeError('No stars in the {} reference star list!'.format(name))
1337 
1338  def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1339  """Compute chi2, log it, validate the model, and return chi2.
1340 
1341  Parameters
1342  ----------
1343  associations : `lsst.jointcal.Associations`
1344  The star/reference star associations to fit.
1345  fit : `lsst.jointcal.FitterBase`
1346  The fitter to use for minimization.
1347  model : `lsst.jointcal.Model`
1348  The model being fit.
1349  chi2Label : `str`
1350  Label to describe the chi2 (e.g. "Initialized", "Final").
1351  writeChi2Name : `str`, optional
1352  Filename prefix to write the chi2 contributions to.
1353  Do not supply an extension: an appropriate one will be added.
1354 
1355  Returns
1356  -------
1357  chi2: `lsst.jointcal.Chi2Accumulator`
1358  The chi2 object for the current fitter and model.
1359 
1360  Raises
1361  ------
1362  FloatingPointError
1363  Raised if chi2 is infinite or NaN.
1364  ValueError
1365  Raised if the model is not valid.
1366  """
1367  if writeChi2Name is not None:
1368  fullpath = self._getDebugPath_getDebugPath(writeChi2Name)
1369  fit.saveChi2Contributions(fullpath+"{type}")
1370  self.log.info("Wrote chi2 contributions files: %s", fullpath)
1371 
1372  chi2 = fit.computeChi2()
1373  self.log.info("%s %s", chi2Label, chi2)
1374  self._check_stars_check_stars(associations)
1375  if not np.isfinite(chi2.chi2):
1376  raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1377  if not model.validate(associations.getCcdImageList(), chi2.ndof):
1378  raise ValueError("Model is not valid: check log messages for warnings.")
1379  return chi2
1380 
1381  def _fit_photometry(self, associations, dataName=None):
1382  """
1383  Fit the photometric data.
1384 
1385  Parameters
1386  ----------
1387  associations : `lsst.jointcal.Associations`
1388  The star/reference star associations to fit.
1389  dataName : `str`
1390  Name of the data being processed (e.g. "1234_HSC-Y"), for
1391  identifying debugging files.
1392 
1393  Returns
1394  -------
1395  fit_result : `namedtuple`
1396  fit : `lsst.jointcal.PhotometryFit`
1397  The photometric fitter used to perform the fit.
1398  model : `lsst.jointcal.PhotometryModel`
1399  The photometric model that was fit.
1400  """
1401  self.log.info("=== Starting photometric fitting...")
1402 
1403  # TODO: should use pex.config.RegistryField here (see DM-9195)
1404  if self.config.photometryModel == "constrainedFlux":
1405  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1406  self.focalPlaneBBoxfocalPlaneBBox,
1407  visitOrder=self.config.photometryVisitOrder,
1408  errorPedestal=self.config.photometryErrorPedestal)
1409  # potentially nonlinear problem, so we may need a line search to converge.
1410  doLineSearch = self.config.allowLineSearch
1411  elif self.config.photometryModel == "constrainedMagnitude":
1412  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1413  self.focalPlaneBBoxfocalPlaneBBox,
1414  visitOrder=self.config.photometryVisitOrder,
1415  errorPedestal=self.config.photometryErrorPedestal)
1416  # potentially nonlinear problem, so we may need a line search to converge.
1417  doLineSearch = self.config.allowLineSearch
1418  elif self.config.photometryModel == "simpleFlux":
1419  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1420  errorPedestal=self.config.photometryErrorPedestal)
1421  doLineSearch = False # purely linear in model parameters, so no line search needed
1422  elif self.config.photometryModel == "simpleMagnitude":
1423  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1424  errorPedestal=self.config.photometryErrorPedestal)
1425  doLineSearch = False # purely linear in model parameters, so no line search needed
1426 
1427  fit = lsst.jointcal.PhotometryFit(associations, model)
1428  # TODO DM-12446: turn this into a "butler save" somehow.
1429  # Save reference and measurement chi2 contributions for this data
1430  if self.config.writeChi2FilesInitialFinal:
1431  baseName = f"photometry_initial_chi2-{dataName}"
1432  else:
1433  baseName = None
1434  if self.config.writeInitialModel:
1435  fullpath = self._getDebugPath_getDebugPath(f"initial_photometry_model-{dataName}.txt")
1436  writeModel(model, fullpath, self.log)
1437  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1438 
1439  def getChi2Name(whatToFit):
1440  if self.config.writeChi2FilesOuterLoop:
1441  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1442  else:
1443  return None
1444 
1445  # The constrained model needs the visit transform fit first; the chip
1446  # transform is initialized from the singleFrame PhotoCalib, so it's close.
1447  if self.config.writeInitMatrix:
1448  dumpMatrixFile = self._getDebugPath_getDebugPath(f"photometry_preinit-{dataName}")
1449  else:
1450  dumpMatrixFile = ""
1451  if self.config.photometryModel.startswith("constrained"):
1452  # no line search: should be purely (or nearly) linear,
1453  # and we want a large step size to initialize with.
1454  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1455  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1456  writeChi2Name=getChi2Name("ModelVisit"))
1457  dumpMatrixFile = "" # so we don't redo the output on the next step
1458 
1459  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1460  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Model",
1461  writeChi2Name=getChi2Name("Model"))
1462 
1463  fit.minimize("Fluxes") # no line search: always purely linear.
1464  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1465  writeChi2Name=getChi2Name("Fluxes"))
1466 
1467  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1468  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1469  writeChi2Name=getChi2Name("ModelFluxes"))
1470 
1471  model.freezeErrorTransform()
1472  self.log.debug("Photometry error scales are frozen.")
1473 
1474  chi2 = self._iterate_fit_iterate_fit(associations,
1475  fit,
1476  self.config.maxPhotometrySteps,
1477  "photometry",
1478  "Model Fluxes",
1479  doRankUpdate=self.config.photometryDoRankUpdate,
1480  doLineSearch=doLineSearch,
1481  dataName=dataName)
1482 
1483  add_measurement(self.jobjob, 'jointcal.photometry_final_chi2', chi2.chi2)
1484  add_measurement(self.jobjob, 'jointcal.photometry_final_ndof', chi2.ndof)
1485  return Photometry(fit, model)
1486 
1487  def _fit_astrometry(self, associations, dataName=None):
1488  """
1489  Fit the astrometric data.
1490 
1491  Parameters
1492  ----------
1493  associations : `lsst.jointcal.Associations`
1494  The star/reference star associations to fit.
1495  dataName : `str`
1496  Name of the data being processed (e.g. "1234_HSC-Y"), for
1497  identifying debugging files.
1498 
1499  Returns
1500  -------
1501  fit_result : `namedtuple`
1502  fit : `lsst.jointcal.AstrometryFit`
1503  The astrometric fitter used to perform the fit.
1504  model : `lsst.jointcal.AstrometryModel`
1505  The astrometric model that was fit.
1506  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1507  The model for the sky to tangent plane projection that was used in the fit.
1508  """
1509 
1510  self.log.info("=== Starting astrometric fitting...")
1511 
1512  associations.deprojectFittedStars()
1513 
1514  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1515  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1516  # them so carefully?
1517  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1518 
1519  if self.config.astrometryModel == "constrained":
1520  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1521  sky_to_tan_projection,
1522  chipOrder=self.config.astrometryChipOrder,
1523  visitOrder=self.config.astrometryVisitOrder)
1524  elif self.config.astrometryModel == "simple":
1525  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1526  sky_to_tan_projection,
1527  self.config.useInputWcs,
1528  nNotFit=0,
1529  order=self.config.astrometrySimpleOrder)
1530 
1531  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1532  # TODO DM-12446: turn this into a "butler save" somehow.
1533  # Save reference and measurement chi2 contributions for this data
1534  if self.config.writeChi2FilesInitialFinal:
1535  baseName = f"astrometry_initial_chi2-{dataName}"
1536  else:
1537  baseName = None
1538  if self.config.writeInitialModel:
1539  fullpath = self._getDebugPath_getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1540  writeModel(model, fullpath, self.log)
1541  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1542 
1543  def getChi2Name(whatToFit):
1544  if self.config.writeChi2FilesOuterLoop:
1545  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1546  else:
1547  return None
1548 
1549  if self.config.writeInitMatrix:
1550  dumpMatrixFile = self._getDebugPath_getDebugPath(f"astrometry_preinit-{dataName}")
1551  else:
1552  dumpMatrixFile = ""
1553  # The constrained model needs the visit transform fit first; the chip
1554  # transform is initialized from the detector's cameraGeom, so it's close.
1555  if self.config.astrometryModel == "constrained":
1556  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1557  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1558  writeChi2Name=getChi2Name("DistortionsVisit"))
1559  dumpMatrixFile = "" # so we don't redo the output on the next step
1560 
1561  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1562  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1563  writeChi2Name=getChi2Name("Distortions"))
1564 
1565  fit.minimize("Positions")
1566  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Positions",
1567  writeChi2Name=getChi2Name("Positions"))
1568 
1569  fit.minimize("Distortions Positions")
1570  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1571  writeChi2Name=getChi2Name("DistortionsPositions"))
1572 
1573  chi2 = self._iterate_fit_iterate_fit(associations,
1574  fit,
1575  self.config.maxAstrometrySteps,
1576  "astrometry",
1577  "Distortions Positions",
1578  doRankUpdate=self.config.astrometryDoRankUpdate,
1579  dataName=dataName)
1580 
1581  add_measurement(self.jobjob, 'jointcal.astrometry_final_chi2', chi2.chi2)
1582  add_measurement(self.jobjob, 'jointcal.astrometry_final_ndof', chi2.ndof)
1583 
1584  return Astrometry(fit, model, sky_to_tan_projection)
1585 
1586  def _check_stars(self, associations):
1587  """Count measured and reference stars per ccd and warn/log them."""
1588  for ccdImage in associations.getCcdImageList():
1589  nMeasuredStars, nRefStars = ccdImage.countStars()
1590  self.log.debug("ccdImage %s has %s measured and %s reference stars",
1591  ccdImage.getName(), nMeasuredStars, nRefStars)
1592  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1593  self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1594  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1595  if nRefStars < self.config.minRefStarsPerCcd:
1596  self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1597  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1598 
1599  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1600  dataName="",
1601  doRankUpdate=True,
1602  doLineSearch=False):
1603  """Run fitter.minimize up to max_steps times, returning the final chi2.
1604 
1605  Parameters
1606  ----------
1607  associations : `lsst.jointcal.Associations`
1608  The star/reference star associations to fit.
1609  fitter : `lsst.jointcal.FitterBase`
1610  The fitter to use for minimization.
1611  max_steps : `int`
1612  Maximum number of steps to run outlier rejection before declaring
1613  convergence failure.
1614  name : {'photometry' or 'astrometry'}
1615  What type of data are we fitting (for logs and debugging files).
1616  whatToFit : `str`
1617  Passed to ``fitter.minimize()`` to define the parameters to fit.
1618  dataName : `str`, optional
1619  Descriptive name for this dataset (e.g. tract and filter),
1620  for debugging.
1621  doRankUpdate : `bool`, optional
1622  Do an Eigen rank update during minimization, or recompute the full
1623  matrix and gradient?
1624  doLineSearch : `bool`, optional
1625  Do a line search for the optimum step during minimization?
1626 
1627  Returns
1628  -------
1629  chi2: `lsst.jointcal.Chi2Statistic`
1630  The final chi2 after the fit converges, or is forced to end.
1631 
1632  Raises
1633  ------
1634  FloatingPointError
1635  Raised if the fitter fails with a non-finite value.
1636  RuntimeError
1637  Raised if the fitter fails for some other reason;
1638  log messages will provide further details.
1639  """
1640  if self.config.writeInitMatrix:
1641  dumpMatrixFile = self._getDebugPath_getDebugPath(f"{name}_postinit-{dataName}")
1642  else:
1643  dumpMatrixFile = ""
1644  oldChi2 = lsst.jointcal.Chi2Statistic()
1645  oldChi2.chi2 = float("inf")
1646  for i in range(max_steps):
1647  if self.config.writeChi2FilesOuterLoop:
1648  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1649  else:
1650  writeChi2Name = None
1651  result = fitter.minimize(whatToFit,
1652  self.config.outlierRejectSigma,
1653  doRankUpdate=doRankUpdate,
1654  doLineSearch=doLineSearch,
1655  dumpMatrixFile=dumpMatrixFile)
1656  dumpMatrixFile = "" # clear it so we don't write the matrix again.
1657  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1658  f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1659 
1660  if result == MinimizeResult.Converged:
1661  if doRankUpdate:
1662  self.log.debug("fit has converged - no more outliers - redo minimization "
1663  "one more time in case we have lost accuracy in rank update.")
1664  # Redo minimization one more time in case we have lost accuracy in rank update
1665  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1666  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1667 
1668  # log a message for a large final chi2, TODO: DM-15247 for something better
1669  if chi2.chi2/chi2.ndof >= 4.0:
1670  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1671 
1672  break
1673  elif result == MinimizeResult.Chi2Increased:
1674  self.log.warn("Still some outliers remaining but chi2 increased - retry")
1675  # Check whether the increase was large enough to cause trouble.
1676  chi2Ratio = chi2.chi2 / oldChi2.chi2
1677  if chi2Ratio > 1.5:
1678  self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1679  chi2.chi2, oldChi2.chi2, chi2Ratio)
1680  # Based on a variety of HSC jointcal logs (see DM-25779), it
1681  # appears that chi2 increases more than a factor of ~2 always
1682  # result in the fit diverging rapidly and ending at chi2 > 1e10.
1683  # Using 10 as the "failure" threshold gives some room between
1684  # leaving a warning and bailing early.
1685  if chi2Ratio > 10:
1686  msg = ("Large chi2 increase between steps: fit likely cannot converge."
1687  " Try setting one or more of the `writeChi2*` config fields and looking"
1688  " at how individual star chi2-values evolve during the fit.")
1689  raise RuntimeError(msg)
1690  oldChi2 = chi2
1691  elif result == MinimizeResult.NonFinite:
1692  filename = self._getDebugPath_getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1693  # TODO DM-12446: turn this into a "butler save" somehow.
1694  fitter.saveChi2Contributions(filename+"{type}")
1695  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1696  raise FloatingPointError(msg.format(filename))
1697  elif result == MinimizeResult.Failed:
1698  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1699  else:
1700  raise RuntimeError("Unxepected return code from minimize().")
1701  else:
1702  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1703 
1704  return chi2
1705 
1706  def _make_output(self, ccdImageList, model, func):
1707  """Return the internal jointcal models converted to the afw
1708  structures that will be saved to disk.
1709 
1710  Parameters
1711  ----------
1712  ccdImageList : `lsst.jointcal.CcdImageList`
1713  The list of CcdImages to get the output for.
1714  model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1715  The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1716  func : `str`
1717  The name of the function to call on ``model`` to get the converted
1718  structure. Must accept an `lsst.jointcal.CcdImage`.
1719 
1720  Returns
1721  -------
1722  output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1723  `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1724  The data to be saved, keyed on (visit, detector).
1725  """
1726  output = {}
1727  for ccdImage in ccdImageList:
1728  ccd = ccdImage.ccdId
1729  visit = ccdImage.visit
1730  self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1731  output[(visit, ccd)] = getattr(model, func)(ccdImage)
1732  return output
1733 
1734  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1735  """
1736  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1737 
1738  Parameters
1739  ----------
1740  associations : `lsst.jointcal.Associations`
1741  The star/reference star associations to fit.
1742  model : `lsst.jointcal.AstrometryModel`
1743  The astrometric model that was fit.
1744  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1745  Dict of ccdImage identifiers to dataRefs that were fit.
1746  """
1747  ccdImageList = associations.getCcdImageList()
1748  output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1749  for key, skyWcs in output.items():
1750  dataRef = visit_ccd_to_dataRef[key]
1751  try:
1752  dataRef.put(skyWcs, 'jointcal_wcs')
1753  except pexExceptions.Exception as e:
1754  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1755  raise e
1756 
1757  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1758  """
1759  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1760 
1761  Parameters
1762  ----------
1763  associations : `lsst.jointcal.Associations`
1764  The star/reference star associations to fit.
1765  model : `lsst.jointcal.PhotometryModel`
1766  The photoometric model that was fit.
1767  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1768  Dict of ccdImage identifiers to dataRefs that were fit.
1769  """
1770 
1771  ccdImageList = associations.getCcdImageList()
1772  output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1773  for key, photoCalib in output.items():
1774  dataRef = visit_ccd_to_dataRef[key]
1775  try:
1776  dataRef.put(photoCalib, 'jointcal_photoCalib')
1777  except pexExceptions.Exception as e:
1778  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1779  raise e
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:54
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
Definition: jointcal.py:77
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:846
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:1170
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:1123
def _getDebugPath(self, filename)
Definition: jointcal.py:1024
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1329
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:901
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1734
def _check_stars(self, associations)
Definition: jointcal.py:1586
def _prep_sky(self, associations, filters)
Definition: jointcal.py:1029
def _readDataId(self, butler, dataId)
Definition: jointcal.py:974
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:629
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:1190
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:929
def runDataRef(self, dataRefs)
Definition: jointcal.py:1048
def __init__(self, butler=None, initInputs=None, **kwargs)
Definition: jointcal.py:608
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId)
Definition: jointcal.py:794
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1602
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:716
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1381
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1757
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1338
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1706
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:657
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1273
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1487
def loadData(self, dataRefs, associations, jointcalControl)
Definition: jointcal.py:1000
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:141
def add_measurement(job, name, value)
Definition: jointcal.py:59
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:181
def writeModel(model, filename, log)
Definition: jointcal.py:562