lsst.jointcal  22.0.1-21-gbbdf43d+21391e3ec4
jointcal.py
Go to the documentation of this file.
1 # This file is part of jointcal.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import dataclasses
23 import collections
24 import os
25 
26 import astropy.time
27 import numpy as np
28 import astropy.units as u
29 
30 import lsst.geom
31 import lsst.utils
32 import lsst.pex.config as pexConfig
33 import lsst.pipe.base as pipeBase
34 from lsst.afw.image import fluxErrFromABMagErr
35 import lsst.pex.exceptions as pexExceptions
37 import lsst.afw.table
38 import lsst.log
39 from lsst.obs.base import Instrument
40 from lsst.pipe.tasks.colorterms import ColortermLibrary
41 from lsst.verify import Job, Measurement
42 
43 from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44  ReferenceObjectLoader)
45 from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46 
47 from .dataIds import PerTractCcdDataIdContainer
48 
49 import lsst.jointcal
50 from lsst.jointcal import MinimizeResult
51 
52 __all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
53 
54 Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55 Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
56 
57 
58 # TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59 def add_measurement(job, name, value):
60  meas = Measurement(job.metrics[name], value)
61  job.measurements.insert(meas)
62 
63 
64 class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65  """Subclass of TaskRunner for jointcalTask (gen2)
66 
67  jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68  extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69  single dataRef, are are called repeatedly). This class transforms the processed
70  arguments generated by the ArgumentParser into the arguments expected by
71  Jointcal.runDataRef().
72 
73  See pipeBase.TaskRunner for more information.
74  """
75 
76  @staticmethod
77  def getTargetList(parsedCmd, **kwargs):
78  """
79  Return a list of tuples per tract, each containing (dataRefs, kwargs).
80 
81  Jointcal operates on lists of dataRefs simultaneously.
82  """
83  kwargs['butler'] = parsedCmd.butler
84 
85  # organize data IDs by tract
86  refListDict = {}
87  for ref in parsedCmd.id.refList:
88  refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89  # we call runDataRef() once with each tract
90  result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91  return result
92 
93  def __call__(self, args):
94  """
95  Parameters
96  ----------
97  args
98  Arguments for Task.runDataRef()
99 
100  Returns
101  -------
102  pipe.base.Struct
103  if self.doReturnResults is False:
104 
105  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
106 
107  if self.doReturnResults is True:
108 
109  - ``result``: the result of calling jointcal.runDataRef()
110  - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111  """
112  exitStatus = 0 # exit status for shell
113 
114  # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115  dataRefList, kwargs = args
116  butler = kwargs.pop('butler')
117  task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118  result = None
119  try:
120  result = task.runDataRef(dataRefList, **kwargs)
121  exitStatus = result.exitStatus
122  job_path = butler.get('verify_job_filename')
123  result.job.write(job_path[0])
124  except Exception as e: # catch everything, sort it out later.
125  if self.doRaise:
126  raise e
127  else:
128  exitStatus = 1
129  eName = type(e).__name__
130  tract = dataRefList[0].dataId['tract']
131  task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
132 
133  # Put the butler back into kwargs for the other Tasks.
134  kwargs['butler'] = butler
135  if self.doReturnResults:
136  return pipeBase.Struct(result=result, exitStatus=exitStatus)
137  else:
138  return pipeBase.Struct(exitStatus=exitStatus)
139 
140 
141 def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142  """Lookup function that asserts/hopes that a static calibration dataset
143  exists in a particular collection, since this task can't provide a single
144  date/time to use to search for one properly.
145 
146  This is mostly useful for the ``camera`` dataset, in cases where the task's
147  quantum dimensions do *not* include something temporal, like ``exposure``
148  or ``visit``.
149 
150  Parameters
151  ----------
152  datasetType : `lsst.daf.butler.DatasetType`
153  Type of dataset being searched for.
154  registry : `lsst.daf.butler.Registry`
155  Data repository registry to search.
156  quantumDataId : `lsst.daf.butler.DataCoordinate`
157  Data ID of the quantum this camera should match.
158  collections : `Iterable` [ `str` ]
159  Collections that should be searched - but this lookup function works
160  by ignoring this in favor of a more-or-less hard-coded value.
161 
162  Returns
163  -------
164  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165  Iterator over dataset references; should have only one element.
166 
167  Notes
168  -----
169  This implementation duplicates one in fgcmcal, and is at least quite
170  similar to another in cp_pipe. This duplicate has the most documentation.
171  Fixing this is DM-29661.
172  """
173  instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174  unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175  return registry.queryDatasets(datasetType,
176  dataId=quantumDataId,
177  collections=[unboundedCollection],
178  findFirst=True)
179 
180 
181 def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182  """Lookup function that finds all refcats for all visits that overlap a
183  tract, rather than just the refcats that directly overlap the tract.
184 
185  Parameters
186  ----------
187  datasetType : `lsst.daf.butler.DatasetType`
188  Type of dataset being searched for.
189  registry : `lsst.daf.butler.Registry`
190  Data repository registry to search.
191  quantumDataId : `lsst.daf.butler.DataCoordinate`
192  Data ID of the quantum; expected to be something we can use as a
193  constraint to query for overlapping visits.
194  collections : `Iterable` [ `str` ]
195  Collections to search.
196 
197  Returns
198  -------
199  refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200  Iterator over refcat references.
201  """
202  refs = set()
203  # Use .expanded() on the query methods below because we need data IDs with
204  # regions, both in the outer loop over visits (queryDatasets will expand
205  # any data ID we give it, but doing it up-front in bulk is much more
206  # efficient) and in the data IDs of the DatasetRefs this function yields
207  # (because the RefCatLoader relies on them to do some of its own
208  # filtering).
209  for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210  refs.update(
211  registry.queryDatasets(
212  datasetType,
213  collections=collections,
214  dataId=visit_data_id,
215  findFirst=True,
216  ).expanded()
217  )
218  yield from refs
219 
220 
221 class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222  dimensions=("skymap", "tract", "instrument", "physical_filter")):
223  """Middleware input/output connections for jointcal data."""
224  inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225  doc="The camera instrument that took these observations.",
226  name="camera",
227  storageClass="Camera",
228  dimensions=("instrument",),
229  isCalibration=True,
230  lookupFunction=lookupStaticCalibrations,
231  )
232  inputSourceTableVisit = pipeBase.connectionTypes.Input(
233  doc="Source table in parquet format, per visit",
234  name="sourceTable_visit",
235  storageClass="DataFrame",
236  dimensions=("instrument", "visit"),
237  deferLoad=True,
238  multiple=True,
239  )
240  inputVisitSummary = pipeBase.connectionTypes.Input(
241  doc=("Per-visit consolidated exposure metadata built from calexps. "
242  "These catalogs use detector id for the id and must be sorted for "
243  "fast lookups of a detector."),
244  name="visitSummary",
245  storageClass="ExposureCatalog",
246  dimensions=("instrument", "visit"),
247  deferLoad=True,
248  multiple=True,
249  )
250  astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251  doc="The astrometry reference catalog to match to loaded input catalog sources.",
252  name="gaia_dr2_20200414",
253  storageClass="SimpleCatalog",
254  dimensions=("skypix",),
255  deferLoad=True,
256  multiple=True,
257  lookupFunction=lookupVisitRefCats,
258  )
259  photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260  doc="The photometry reference catalog to match to loaded input catalog sources.",
261  name="ps1_pv3_3pi_20170110",
262  storageClass="SimpleCatalog",
263  dimensions=("skypix",),
264  deferLoad=True,
265  multiple=True,
266  lookupFunction=lookupVisitRefCats,
267  )
268 
269  outputWcs = pipeBase.connectionTypes.Output(
270  doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271  " These catalogs only contain entries for detectors with an output, and use"
272  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273  name="jointcalSkyWcsCatalog",
274  storageClass="ExposureCatalog",
275  dimensions=("instrument", "visit", "skymap", "tract"),
276  multiple=True
277  )
278  outputPhotoCalib = pipeBase.connectionTypes.Output(
279  doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280  " These catalogs only contain entries for detectors with an output, and use"
281  " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282  name="jointcalPhotoCalibCatalog",
283  storageClass="ExposureCatalog",
284  dimensions=("instrument", "visit", "skymap", "tract"),
285  multiple=True
286  )
287 
288  # measurements of metrics
289  # The vars() trick used here allows us to set class attributes
290  # programatically. Taken from:
291  # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
292  for name in ("astrometry", "photometry"):
293  vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
294  doc=f"The number of cross-matched fittedStars for {name}",
295  name=f"metricvalue_jointcal_{name}_matched_fittedStars",
296  storageClass="MetricValue",
297  dimensions=("skymap", "tract", "instrument", "physical_filter"),
298  )
299  vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
300  doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.",
301  name=f"metricvalue_jointcal_{name}_collected_refStars",
302  storageClass="MetricValue",
303  dimensions=("skymap", "tract", "instrument", "physical_filter"),
304  )
305  vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
306  doc=f"The number of {name} reference stars matched to fittedStars.",
307  name=f"metricvalue_jointcal_{name}_prepared_refStars",
308  storageClass="MetricValue",
309  dimensions=("skymap", "tract", "instrument", "physical_filter"),
310  )
311  vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
312  doc=f"The number of cross-matched fittedStars after cleanup, for {name}.",
313  name=f"metricvalue_jointcal_{name}_prepared_fittedStars",
314  storageClass="MetricValue",
315  dimensions=("skymap", "tract", "instrument", "physical_filter"),
316  )
317  vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
318  doc=f"The number of ccdImages that will be fit for {name}, after cleanup.",
319  name=f"metricvalue_jointcal_{name}_prepared_ccdImages",
320  storageClass="MetricValue",
321  dimensions=("skymap", "tract", "instrument", "physical_filter"),
322  )
323  vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
324  doc=f"The final chi2 of the {name} fit.",
325  name=f"metricvalue_jointcal_{name}_final_chi2",
326  storageClass="MetricValue",
327  dimensions=("skymap", "tract", "instrument", "physical_filter"),
328  )
329  vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
330  doc=f"The number of degrees of freedom of the fitted {name}.",
331  name=f"metricvalue_jointcal_{name}_final_ndof",
332  storageClass="MetricValue",
333  dimensions=("skymap", "tract", "instrument", "physical_filter"),
334  )
335 
336  def __init__(self, *, config=None):
337  super().__init__(config=config)
338  # When we are only doing one of astrometry or photometry, we don't
339  # need the reference catalog or produce the outputs for the other.
340  # This informs the middleware of that when the QuantumGraph is
341  # generated, so we don't block on getting something we won't need or
342  # create an expectation that downstream tasks will be able to consume
343  # something we won't produce.
344  if not config.doAstrometry:
345  self.prerequisiteInputs.remove("astrometryRefCat")
346  self.outputs.remove("outputWcs")
347  for key in list(self.outputs):
348  if "metricvalue_jointcal_astrometry" in key:
349  self.outputs.remove(key)
350  if not config.doPhotometry:
351  self.prerequisiteInputs.remove("photometryRefCat")
352  self.outputs.remove("outputPhotoCalib")
353  for key in list(self.outputs):
354  if "metricvalue_jointcal_photometry" in key:
355  self.outputs.remove(key)
356 
357 
358 class JointcalConfig(pipeBase.PipelineTaskConfig,
359  pipelineConnections=JointcalTaskConnections):
360  """Configuration for JointcalTask"""
361 
362  doAstrometry = pexConfig.Field(
363  doc="Fit astrometry and write the fitted result.",
364  dtype=bool,
365  default=True
366  )
367  doPhotometry = pexConfig.Field(
368  doc="Fit photometry and write the fitted result.",
369  dtype=bool,
370  default=True
371  )
372  coaddName = pexConfig.Field(
373  doc="Type of coadd, typically deep or goodSeeing",
374  dtype=str,
375  default="deep"
376  )
377  # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
378  sourceFluxType = pexConfig.Field(
379  dtype=str,
380  doc="Source flux field to use in source selection and to get fluxes from the catalog.",
381  default='Calib'
382  )
383  positionErrorPedestal = pexConfig.Field(
384  doc="Systematic term to apply to the measured position error (pixels)",
385  dtype=float,
386  default=0.02,
387  )
388  photometryErrorPedestal = pexConfig.Field(
389  doc="Systematic term to apply to the measured error on flux or magnitude as a "
390  "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
391  dtype=float,
392  default=0.0,
393  )
394  # TODO: DM-6885 matchCut should be an geom.Angle
395  matchCut = pexConfig.Field(
396  doc="Matching radius between fitted and reference stars (arcseconds)",
397  dtype=float,
398  default=3.0,
399  )
400  minMeasurements = pexConfig.Field(
401  doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
402  dtype=int,
403  default=2,
404  )
405  minMeasuredStarsPerCcd = pexConfig.Field(
406  doc="Minimum number of measuredStars per ccdImage before printing warnings",
407  dtype=int,
408  default=100,
409  )
410  minRefStarsPerCcd = pexConfig.Field(
411  doc="Minimum number of measuredStars per ccdImage before printing warnings",
412  dtype=int,
413  default=30,
414  )
415  allowLineSearch = pexConfig.Field(
416  doc="Allow a line search during minimization, if it is reasonable for the model"
417  " (models with a significant non-linear component, e.g. constrainedPhotometry).",
418  dtype=bool,
419  default=False
420  )
421  astrometrySimpleOrder = pexConfig.Field(
422  doc="Polynomial order for fitting the simple astrometry model.",
423  dtype=int,
424  default=3,
425  )
426  astrometryChipOrder = pexConfig.Field(
427  doc="Order of the per-chip transform for the constrained astrometry model.",
428  dtype=int,
429  default=1,
430  )
431  astrometryVisitOrder = pexConfig.Field(
432  doc="Order of the per-visit transform for the constrained astrometry model.",
433  dtype=int,
434  default=5,
435  )
436  useInputWcs = pexConfig.Field(
437  doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
438  dtype=bool,
439  default=True,
440  )
441  astrometryModel = pexConfig.ChoiceField(
442  doc="Type of model to fit to astrometry",
443  dtype=str,
444  default="constrained",
445  allowed={"simple": "One polynomial per ccd",
446  "constrained": "One polynomial per ccd, and one polynomial per visit"}
447  )
448  photometryModel = pexConfig.ChoiceField(
449  doc="Type of model to fit to photometry",
450  dtype=str,
451  default="constrainedMagnitude",
452  allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
453  "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
454  " fitting in flux space.",
455  "simpleMagnitude": "One constant zeropoint per ccd and visit,"
456  " fitting in magnitude space.",
457  "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
458  " fitting in magnitude space.",
459  }
460  )
461  applyColorTerms = pexConfig.Field(
462  doc="Apply photometric color terms to reference stars?"
463  "Requires that colorterms be set to a ColortermLibrary",
464  dtype=bool,
465  default=False
466  )
467  colorterms = pexConfig.ConfigField(
468  doc="Library of photometric reference catalog name to color term dict.",
469  dtype=ColortermLibrary,
470  )
471  photometryVisitOrder = pexConfig.Field(
472  doc="Order of the per-visit polynomial transform for the constrained photometry model.",
473  dtype=int,
474  default=7,
475  )
476  photometryDoRankUpdate = pexConfig.Field(
477  doc=("Do the rank update step during minimization. "
478  "Skipping this can help deal with models that are too non-linear."),
479  dtype=bool,
480  default=True,
481  )
482  astrometryDoRankUpdate = pexConfig.Field(
483  doc=("Do the rank update step during minimization (should not change the astrometry fit). "
484  "Skipping this can help deal with models that are too non-linear."),
485  dtype=bool,
486  default=True,
487  )
488  outlierRejectSigma = pexConfig.Field(
489  doc="How many sigma to reject outliers at during minimization.",
490  dtype=float,
491  default=5.0,
492  )
493  astrometryOutlierRelativeTolerance = pexConfig.Field(
494  doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
495  "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
496  "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
497  "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
498  "close to the solution found when removing all outliers."),
499  dtype=float,
500  default=0,
501  )
502  maxPhotometrySteps = pexConfig.Field(
503  doc="Maximum number of minimize iterations to take when fitting photometry.",
504  dtype=int,
505  default=20,
506  )
507  maxAstrometrySteps = pexConfig.Field(
508  doc="Maximum number of minimize iterations to take when fitting astrometry.",
509  dtype=int,
510  default=20,
511  )
512  astrometryRefObjLoader = pexConfig.ConfigurableField(
513  target=LoadIndexedReferenceObjectsTask,
514  doc="Reference object loader for astrometric fit",
515  )
516  photometryRefObjLoader = pexConfig.ConfigurableField(
517  target=LoadIndexedReferenceObjectsTask,
518  doc="Reference object loader for photometric fit",
519  )
520  sourceSelector = sourceSelectorRegistry.makeField(
521  doc="How to select sources for cross-matching",
522  default="astrometry"
523  )
524  astrometryReferenceSelector = pexConfig.ConfigurableField(
525  target=ReferenceSourceSelectorTask,
526  doc="How to down-select the loaded astrometry reference catalog.",
527  )
528  photometryReferenceSelector = pexConfig.ConfigurableField(
529  target=ReferenceSourceSelectorTask,
530  doc="How to down-select the loaded photometry reference catalog.",
531  )
532  astrometryReferenceErr = pexConfig.Field(
533  doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
534  "If None, then raise an exception if the reference catalog is missing coordinate errors. "
535  "If specified, overrides any existing `coord_*Err` values."),
536  dtype=float,
537  default=None,
538  optional=True
539  )
540 
541  # configs for outputting debug information
542  writeInitMatrix = pexConfig.Field(
543  dtype=bool,
544  doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
545  "Output files will be written to `config.debugOutputPath` and will "
546  "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
547  "Note that these files are the dense versions of the matrix, and so may be very large."),
548  default=False
549  )
550  writeChi2FilesInitialFinal = pexConfig.Field(
551  dtype=bool,
552  doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
553  "Output files will be written to `config.debugOutputPath` and will "
554  "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
555  default=False
556  )
557  writeChi2FilesOuterLoop = pexConfig.Field(
558  dtype=bool,
559  doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
560  "Output files will be written to `config.debugOutputPath` and will "
561  "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
562  default=False
563  )
564  writeInitialModel = pexConfig.Field(
565  dtype=bool,
566  doc=("Write the pre-initialization model to text files, for debugging. "
567  "Output files will be written to `config.debugOutputPath` and will be "
568  "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
569  default=False
570  )
571  debugOutputPath = pexConfig.Field(
572  dtype=str,
573  default=".",
574  doc=("Path to write debug output files to. Used by "
575  "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
576  )
577  detailedProfile = pexConfig.Field(
578  dtype=bool,
579  default=False,
580  doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
581  )
582 
583  def validate(self):
584  super().validate()
585  if self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms and len(self.colortermscolorterms.data) == 0:
586  msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
587  raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
588  if self.doAstrometrydoAstrometry and not self.doPhotometrydoPhotometry and self.applyColorTermsapplyColorTerms:
589  msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
590  "applyColorTerms=True will be ignored.")
591  lsst.log.warning(msg)
592 
593  def setDefaults(self):
594  # Use science source selector which can filter on extendedness, SNR, and whether blended
595  self.sourceSelectorsourceSelector.name = 'science'
596  # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
597  self.sourceSelectorsourceSelector['science'].doUnresolved = True
598  # with dependable signal to noise ratio.
599  self.sourceSelectorsourceSelector['science'].doSignalToNoise = True
600  # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
601  # and S/N > 10 to use sources that are not too faint, and thus better measured.
602  self.sourceSelectorsourceSelector['science'].signalToNoise.minimum = 10.
603  # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
604  fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
605  self.sourceSelectorsourceSelector['science'].signalToNoise.fluxField = fluxField
606  self.sourceSelectorsourceSelector['science'].signalToNoise.errField = fluxField + "Err"
607  # Do not trust blended sources' aperture fluxes which also depend on seeing
608  self.sourceSelectorsourceSelector['science'].doIsolated = True
609  # Do not trust either flux or centroid measurements with flags,
610  # chosen from the usual QA flags for stars)
611  self.sourceSelectorsourceSelector['science'].doFlags = True
612  badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
613  'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
614  'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
615  self.sourceSelectorsourceSelector['science'].flags.bad = badFlags
616 
617  # Default to Gaia-DR2 (with proper motions) for astrometry and
618  # PS1-DR1 for photometry, with a reasonable initial filterMap.
619  self.astrometryRefObjLoaderastrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
620  self.astrometryRefObjLoaderastrometryRefObjLoader.requireProperMotion = True
621  self.astrometryRefObjLoaderastrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
622  self.photometryRefObjLoaderphotometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
623 
624 
625 def writeModel(model, filename, log):
626  """Write model to outfile."""
627  with open(filename, "w") as file:
628  file.write(repr(model))
629  log.info("Wrote %s to file: %s", model, filename)
630 
631 
632 @dataclasses.dataclass
634  """The input data jointcal needs for each detector/visit."""
635  visit: int
636  """The visit identifier of this exposure."""
638  """The catalog derived from this exposure."""
639  visitInfo: lsst.afw.image.VisitInfo
640  """The VisitInfo of this exposure."""
642  """The detector of this exposure."""
643  photoCalib: lsst.afw.image.PhotoCalib
644  """The photometric calibration of this exposure."""
646  """The WCS of this exposure."""
647  bbox: lsst.geom.Box2I
648  """The bounding box of this exposure."""
650  """The filter of this exposure."""
651 
652 
653 class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
654  """Astrometricly and photometricly calibrate across multiple visits of the
655  same field.
656 
657  Parameters
658  ----------
659  butler : `lsst.daf.persistence.Butler`
660  The butler is passed to the refObjLoader constructor in case it is
661  needed. Ignored if the refObjLoader argument provides a loader directly.
662  Used to initialize the astrometry and photometry refObjLoaders.
663  initInputs : `dict`, optional
664  Dictionary used to initialize PipelineTasks (empty for jointcal).
665  """
666 
667  ConfigClass = JointcalConfig
668  RunnerClass = JointcalRunner
669  _DefaultName = "jointcal"
670 
671  def __init__(self, butler=None, initInputs=None, **kwargs):
672  super().__init__(**kwargs)
673  self.makeSubtask("sourceSelector")
674  if self.config.doAstrometry:
675  if initInputs is None:
676  # gen3 middleware does refcat things internally (and will not have a butler here)
677  self.makeSubtask('astrometryRefObjLoader', butler=butler)
678  self.makeSubtask("astrometryReferenceSelector")
679  else:
680  self.astrometryRefObjLoaderastrometryRefObjLoader = None
681  if self.config.doPhotometry:
682  if initInputs is None:
683  # gen3 middleware does refcat things internally (and will not have a butler here)
684  self.makeSubtask('photometryRefObjLoader', butler=butler)
685  self.makeSubtask("photometryReferenceSelector")
686  else:
687  self.photometryRefObjLoaderphotometryRefObjLoader = None
688 
689  # To hold various computed metrics for use by tests
690  self.jobjob = Job.load_metrics_package(subset='jointcal')
691 
692  def runQuantum(self, butlerQC, inputRefs, outputRefs):
693  # We override runQuantum to set up the refObjLoaders and write the
694  # outputs to the correct refs.
695  inputs = butlerQC.get(inputRefs)
696  # We want the tract number for writing debug files
697  tract = butlerQC.quantum.dataId['tract']
698  if self.config.doAstrometry:
699  self.astrometryRefObjLoaderastrometryRefObjLoader = ReferenceObjectLoader(
700  dataIds=[ref.datasetRef.dataId
701  for ref in inputRefs.astrometryRefCat],
702  refCats=inputs.pop('astrometryRefCat'),
703  config=self.config.astrometryRefObjLoader,
704  log=self.log)
705  if self.config.doPhotometry:
706  self.photometryRefObjLoaderphotometryRefObjLoader = ReferenceObjectLoader(
707  dataIds=[ref.datasetRef.dataId
708  for ref in inputRefs.photometryRefCat],
709  refCats=inputs.pop('photometryRefCat'),
710  config=self.config.photometryRefObjLoader,
711  log=self.log)
712  outputs = self.runrun(**inputs, tract=tract)
713  self._put_metrics_put_metrics(butlerQC, outputs.job, outputRefs)
714  if self.config.doAstrometry:
715  self._put_output_put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
716  inputs['inputCamera'], "setWcs")
717  if self.config.doPhotometry:
718  self._put_output_put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
719  inputs['inputCamera'], "setPhotoCalib")
720 
721  def _put_metrics(self, butlerQC, job, outputRefs):
722  """Persist all measured metrics stored in a job.
723 
724  Parameters
725  ----------
726  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
727  A butler which is specialized to operate in the context of a
728  `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
729  job : `lsst.verify.job.Job`
730  Measurements of metrics to persist.
731  outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
732  The DatasetRefs to persist the data to.
733  """
734  for key in job.measurements.keys():
735  butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', '')))
736 
737  def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
738  """Persist the output datasets to their appropriate datarefs.
739 
740  Parameters
741  ----------
742  butlerQC : `lsst.pipe.base.ButlerQuantumContext`
743  A butler which is specialized to operate in the context of a
744  `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
745  outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
746  `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
747  The fitted objects to persist.
748  outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
749  The DatasetRefs to persist the data to.
750  camera : `lsst.afw.cameraGeom.Camera`
751  The camera for this instrument, to get detector ids from.
752  setter : `str`
753  The method to call on the ExposureCatalog to set each output.
754  """
756  schema.addField('visit', type='I', doc='Visit number')
757 
758  def new_catalog(visit, size):
759  """Return an catalog ready to be filled with appropriate output."""
760  catalog = lsst.afw.table.ExposureCatalog(schema)
761  catalog.resize(size)
762  catalog['visit'] = visit
763  metadata = lsst.daf.base.PropertyList()
764  metadata.add("COMMENT", "Catalog id is detector id, sorted.")
765  metadata.add("COMMENT", "Only detectors with data have entries.")
766  return catalog
767 
768  # count how many detectors have output for each visit
769  detectors_per_visit = collections.defaultdict(int)
770  for key in outputs:
771  # key is (visit, detector_id), and we only need visit here
772  detectors_per_visit[key[0]] += 1
773 
774  for ref in outputRefs:
775  visit = ref.dataId['visit']
776  catalog = new_catalog(visit, detectors_per_visit[visit])
777  # Iterate over every detector and skip the ones we don't have output for.
778  i = 0
779  for detector in camera:
780  detectorId = detector.getId()
781  key = (ref.dataId['visit'], detectorId)
782  if key not in outputs:
783  # skip detectors we don't have output for
784  self.log.debug("No %s output for detector %s in visit %s",
785  setter[3:], detectorId, visit)
786  continue
787 
788  catalog[i].setId(detectorId)
789  getattr(catalog[i], setter)(outputs[key])
790  i += 1
791 
792  catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
793  butlerQC.put(catalog, ref)
794  self.log.info("Wrote %s detectors to %s", i, ref)
795 
796  def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
797  # Docstring inherited.
798 
799  # We take values out of the Parquet table, and put them in "flux_",
800  # and the config.sourceFluxType field is used during that extraction,
801  # so just use "flux" here.
802  sourceFluxField = "flux"
803  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
804  associations = lsst.jointcal.Associations()
805  self.focalPlaneBBoxfocalPlaneBBox = inputCamera.getFpBBox()
806  oldWcsList, bands = self._load_data_load_data(inputSourceTableVisit,
807  inputVisitSummary,
808  associations,
809  jointcalControl,
810  inputCamera)
811 
812  boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, bands)
813 
814  if self.config.doAstrometry:
815  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
816  name="astrometry",
817  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
818  referenceSelector=self.astrometryReferenceSelector,
819  fit_function=self._fit_astrometry_fit_astrometry,
820  tract=tract,
821  epoch=epoch)
822  astrometry_output = self._make_output_make_output(associations.getCcdImageList(),
823  astrometry.model,
824  "makeSkyWcs")
825  else:
826  astrometry_output = None
827 
828  if self.config.doPhotometry:
829  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
830  name="photometry",
831  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
832  referenceSelector=self.photometryReferenceSelector,
833  fit_function=self._fit_photometry_fit_photometry,
834  tract=tract,
835  epoch=epoch,
836  reject_bad_fluxes=True)
837  photometry_output = self._make_output_make_output(associations.getCcdImageList(),
838  photometry.model,
839  "toPhotoCalib")
840  else:
841  photometry_output = None
842 
843  return pipeBase.Struct(outputWcs=astrometry_output,
844  outputPhotoCalib=photometry_output,
845  job=self.jobjob,
846  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
847  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader)
848 
849  def _make_schema_table(self):
850  """Return an afw SourceTable to use as a base for creating the
851  SourceCatalog to insert values from the dataFrame into.
852 
853  Returns
854  -------
855  table : `lsst.afw.table.SourceTable`
856  Table with schema and slots to use to make SourceCatalogs.
857  """
859  schema.addField("centroid_x", "D")
860  schema.addField("centroid_y", "D")
861  schema.addField("centroid_xErr", "F")
862  schema.addField("centroid_yErr", "F")
863  schema.addField("shape_xx", "D")
864  schema.addField("shape_yy", "D")
865  schema.addField("shape_xy", "D")
866  schema.addField("flux_instFlux", "D")
867  schema.addField("flux_instFluxErr", "D")
868  table = lsst.afw.table.SourceTable.make(schema)
869  table.defineCentroid("centroid")
870  table.defineShape("shape")
871  return table
872 
873  def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId,
874  detectorColumn, ixxColumns):
875  """Return an afw SourceCatalog extracted from a visit-level dataframe,
876  limited to just one detector.
877 
878  Parameters
879  ----------
880  table : `lsst.afw.table.SourceTable`
881  Table factory to use to make the SourceCatalog that will be
882  populated with data from ``visitCatalog``.
883  visitCatalog : `pandas.DataFrame`
884  DataFrame to extract a detector catalog from.
885  detectorId : `int`
886  Numeric id of the detector to extract from ``visitCatalog``.
887  detectorColumn : `str`
888  Name of the detector column in the catalog.
889  ixxColumns : `list` [`str`]
890  Names of the ixx/iyy/ixy columns in the catalog.
891 
892  Returns
893  -------
894  catalog : `lsst.afw.table.SourceCatalog`
895  Detector-level catalog extracted from ``visitCatalog``.
896  """
897  # map from dataFrame column to afw table column
898  mapping = {'x': 'centroid_x',
899  'y': 'centroid_y',
900  'xErr': 'centroid_xErr',
901  'yErr': 'centroid_yErr',
902  ixxColumns[0]: 'shape_xx',
903  ixxColumns[1]: 'shape_yy',
904  ixxColumns[2]: 'shape_xy',
905  f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
906  f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
907  }
908 
909  catalog = lsst.afw.table.SourceCatalog(table)
910  matched = visitCatalog[detectorColumn] == detectorId
911  catalog.resize(sum(matched))
912  view = visitCatalog.loc[matched]
913  catalog['id'] = view.index
914  for dfCol, afwCol in mapping.items():
915  catalog[afwCol] = view[dfCol]
916 
917  self.log.debug("%d sources selected in visit %d detector %d",
918  len(catalog),
919  view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
920  detectorId)
921  return catalog
922 
923  def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
924  jointcalControl, camera):
925  """Read the data that jointcal needs to run. (Gen3 version)
926 
927  Modifies ``associations`` in-place with the loaded data.
928 
929  Parameters
930  ----------
931  inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
932  References to visit-level DataFrames to load the catalog data from.
933  inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
934  Visit-level exposure summary catalog with metadata.
935  associations : `lsst.jointcal.Associations`
936  Object to add the loaded data to by constructing new CcdImages.
937  jointcalControl : `jointcal.JointcalControl`
938  Control object for C++ associations management.
939  camera : `lsst.afw.cameraGeom.Camera`
940  Camera object for detector geometry.
941 
942  Returns
943  -------
944  oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
945  The original WCS of the input data, to aid in writing tests.
946  bands : `list` [`str`]
947  The filter bands of each input dataset.
948  """
949  oldWcsList = []
950  filters = []
951  load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
952  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
953  table = self._make_schema_table_make_schema_table() # every detector catalog has the same layout
954  # No guarantee that the input is in the same order of visits, so we have to map one of them.
955  catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
956  detectorDict = {detector.getId(): detector for detector in camera}
957 
958  columns = None
959 
960  for visitSummaryRef in inputVisitSummary:
961  visitSummary = visitSummaryRef.get()
962 
963  dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]]
964  if columns is None:
965  inColumns = dataRef.get(component='columns')
966  columns, detColumn, ixxColumns = self._get_sourceTable_visit_columns_get_sourceTable_visit_columns(inColumns)
967  visitCatalog = dataRef.get(parameters={'columns': columns})
968 
969  selected = self.sourceSelector.run(visitCatalog)
970 
971  # Build a CcdImage for each detector in this visit.
972  detectors = {id: index for index, id in enumerate(visitSummary['id'])}
973  for id, index in detectors.items():
974  catalog = self._extract_detector_catalog_from_visit_catalog_extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id,
975  detColumn, ixxColumns)
976  data = self._make_one_input_data_make_one_input_data(visitSummary[index], catalog, detectorDict)
977  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
978  if result is not None:
979  oldWcsList.append(result.wcs)
980  # A visit has only one band, so we can just use the first.
981  filters.append(data.filter)
982  if len(filters) == 0:
983  raise RuntimeError("No data to process: did source selector remove all sources?")
984  filters = collections.Counter(filters)
985 
986  return oldWcsList, filters
987 
988  def _make_one_input_data(self, visitRecord, catalog, detectorDict):
989  """Return a data structure for this detector+visit."""
990  return JointcalInputData(visit=visitRecord['visit'],
991  catalog=catalog,
992  visitInfo=visitRecord.getVisitInfo(),
993  detector=detectorDict[visitRecord.getId()],
994  photoCalib=visitRecord.getPhotoCalib(),
995  wcs=visitRecord.getWcs(),
996  bbox=visitRecord.getBBox(),
997  # ExposureRecord doesn't have a FilterLabel yet,
998  # so we have to make one.
999  filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
1000  physical=visitRecord['physical_filter']))
1001 
1002  def _get_sourceTable_visit_columns(self, inColumns):
1003  """
1004  Get the sourceTable_visit columns from the config.
1005 
1006  Parameters
1007  ----------
1008  inColumns : `list`
1009  List of columns available in the sourceTable_visit
1010 
1011  Returns
1012  -------
1013  columns : `list`
1014  List of columns to read from sourceTable_visit.
1015  detectorColumn : `str`
1016  Name of the detector column.
1017  ixxColumns : `list`
1018  Name of the ixx/iyy/ixy columns.
1019  """
1020  if 'detector' in inColumns:
1021  # Default name for Gen3.
1022  detectorColumn = 'detector'
1023  else:
1024  # Default name for Gen2 and Gen2 conversions.
1025  detectorColumn = 'ccd'
1026 
1027  columns = ['visit', detectorColumn,
1028  'sourceId', 'x', 'xErr', 'y', 'yErr',
1029  self.config.sourceFluxType + '_instFlux', self.config.sourceFluxType + '_instFluxErr']
1030 
1031  if 'ixx' in inColumns:
1032  # New columns post-DM-31825
1033  ixxColumns = ['ixx', 'iyy', 'ixy']
1034  else:
1035  # Old columns pre-DM-31825
1036  ixxColumns = ['Ixx', 'Iyy', 'Ixy']
1037  columns.extend(ixxColumns)
1038 
1039  if self.sourceSelector.config.doFlags:
1040  columns.extend(self.sourceSelector.config.flags.bad)
1041  if self.sourceSelector.config.doUnresolved:
1042  columns.append(self.sourceSelector.config.unresolved.name)
1043  if self.sourceSelector.config.doIsolated:
1044  columns.append(self.sourceSelector.config.isolated.parentName)
1045  columns.append(self.sourceSelector.config.isolated.nChildName)
1046 
1047  return columns, detectorColumn, ixxColumns
1048 
1049  # We don't currently need to persist the metadata.
1050  # If we do in the future, we will have to add appropriate dataset templates
1051  # to each obs package (the metadata template should look like `jointcal_wcs`).
1052  def _getMetadataName(self):
1053  return None
1054 
1055  @classmethod
1056  def _makeArgumentParser(cls):
1057  """Create an argument parser"""
1058  parser = pipeBase.ArgumentParser(name=cls._DefaultName_DefaultName)
1059  parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
1060  ContainerClass=PerTractCcdDataIdContainer)
1061  return parser
1062 
1063  def _build_ccdImage(self, data, associations, jointcalControl):
1064  """
1065  Extract the necessary things from this catalog+metadata to add a new
1066  ccdImage.
1067 
1068  Parameters
1069  ----------
1070  data : `JointcalInputData`
1071  The loaded input data.
1072  associations : `lsst.jointcal.Associations`
1073  Object to add the info to, to construct a new CcdImage
1074  jointcalControl : `jointcal.JointcalControl`
1075  Control object for associations management
1076 
1077  Returns
1078  ------
1079  namedtuple or `None`
1080  ``wcs``
1081  The TAN WCS of this image, read from the calexp
1082  (`lsst.afw.geom.SkyWcs`).
1083  ``key``
1084  A key to identify this dataRef by its visit and ccd ids
1085  (`namedtuple`).
1086  `None`
1087  if there are no sources in the loaded catalog.
1088  """
1089  if len(data.catalog) == 0:
1090  self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
1091  return None
1092 
1093  associations.createCcdImage(data.catalog,
1094  data.wcs,
1095  data.visitInfo,
1096  data.bbox,
1097  data.filter.physicalLabel,
1098  data.photoCalib,
1099  data.detector,
1100  data.visit,
1101  data.detector.getId(),
1102  jointcalControl)
1103 
1104  Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
1105  Key = collections.namedtuple('Key', ('visit', 'ccd'))
1106  return Result(data.wcs, Key(data.visit, data.detector.getId()))
1107 
1108  def _readDataId(self, butler, dataId):
1109  """Read all of the data for one dataId from the butler. (gen2 version)"""
1110  # Not all instruments have `visit` in their dataIds.
1111  if "visit" in dataId.keys():
1112  visit = dataId["visit"]
1113  else:
1114  visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
1115  detector = butler.get('calexp_detector', dataId=dataId)
1116 
1117  catalog = butler.get('src',
1118  flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
1119  dataId=dataId)
1120  goodSrc = self.sourceSelector.run(catalog)
1121  self.log.debug("%d sources selected in visit %d detector %d",
1122  len(goodSrc.sourceCat),
1123  visit,
1124  detector.getId())
1125  return JointcalInputData(visit=visit,
1126  catalog=goodSrc.sourceCat,
1127  visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1128  detector=detector,
1129  photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1130  wcs=butler.get('calexp_wcs', dataId=dataId),
1131  bbox=butler.get('calexp_bbox', dataId=dataId),
1132  filter=butler.get('calexp_filterLabel', dataId=dataId))
1133 
1134  def loadData(self, dataRefs, associations, jointcalControl):
1135  """Read the data that jointcal needs to run. (Gen2 version)"""
1136  visit_ccd_to_dataRef = {}
1137  oldWcsList = []
1138  filters = []
1139  load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1140  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1141  # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1142  camera = dataRefs[0].get('camera', immediate=True)
1143  self.focalPlaneBBoxfocalPlaneBBox = camera.getFpBBox()
1144  for dataRef in dataRefs:
1145  data = self._readDataId_readDataId(dataRef.getButler(), dataRef.dataId)
1146  result = self._build_ccdImage_build_ccdImage(data, associations, jointcalControl)
1147  if result is None:
1148  continue
1149  oldWcsList.append(result.wcs)
1150  visit_ccd_to_dataRef[result.key] = dataRef
1151  filters.append(data.filter)
1152  if len(filters) == 0:
1153  raise RuntimeError("No data to process: did source selector remove all sources?")
1154  filters = collections.Counter(filters)
1155 
1156  return oldWcsList, filters, visit_ccd_to_dataRef
1157 
1158  def _getDebugPath(self, filename):
1159  """Constructs a path to filename using the configured debug path.
1160  """
1161  return os.path.join(self.config.debugOutputPath, filename)
1162 
1163  def _prep_sky(self, associations, filters):
1164  """Prepare on-sky and other data that must be computed after data has
1165  been read.
1166  """
1167  associations.computeCommonTangentPoint()
1168 
1169  boundingCircle = associations.computeBoundingCircle()
1170  center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1171  radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1172 
1173  self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1174 
1175  # Determine a default filter band associated with the catalog. See DM-9093
1176  defaultFilter = filters.most_common(1)[0][0]
1177  self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1178 
1179  # compute and set the reference epoch of the observations, for proper motion corrections
1180  epoch = self._compute_proper_motion_epoch_compute_proper_motion_epoch(associations.getCcdImageList())
1181  associations.setEpoch(epoch.jyear)
1182 
1183  return boundingCircle, center, radius, defaultFilter, epoch
1184 
1185  @pipeBase.timeMethod
1186  def runDataRef(self, dataRefs):
1187  """
1188  Jointly calibrate the astrometry and photometry across a set of images.
1189 
1190  NOTE: this is for gen2 middleware only.
1191 
1192  Parameters
1193  ----------
1194  dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1195  List of data references to the exposures to be fit.
1196 
1197  Returns
1198  -------
1199  result : `lsst.pipe.base.Struct`
1200  Struct of metadata from the fit, containing:
1201 
1202  ``dataRefs``
1203  The provided data references that were fit (with updated WCSs)
1204  ``oldWcsList``
1205  The original WCS from each dataRef
1206  ``metrics``
1207  Dictionary of internally-computed metrics for testing/validation.
1208  """
1209  if len(dataRefs) == 0:
1210  raise ValueError('Need a non-empty list of data references!')
1211 
1212  exitStatus = 0 # exit status for shell
1213 
1214  sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1215  jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1216  associations = lsst.jointcal.Associations()
1217 
1218  oldWcsList, filters, visit_ccd_to_dataRef = self.loadDataloadData(dataRefs,
1219  associations,
1220  jointcalControl)
1221 
1222  boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky_prep_sky(associations, filters)
1223 
1224  tract = dataRefs[0].dataId['tract']
1225 
1226  if self.config.doAstrometry:
1227  astrometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1228  name="astrometry",
1229  refObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1230  referenceSelector=self.astrometryReferenceSelector,
1231  fit_function=self._fit_astrometry_fit_astrometry,
1232  tract=tract,
1233  epoch=epoch)
1234  self._write_astrometry_results_write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1235  else:
1236  astrometry = Astrometry(None, None, None)
1237 
1238  if self.config.doPhotometry:
1239  photometry = self._do_load_refcat_and_fit_do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1240  name="photometry",
1241  refObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1242  referenceSelector=self.photometryReferenceSelector,
1243  fit_function=self._fit_photometry_fit_photometry,
1244  tract=tract,
1245  epoch=epoch,
1246  reject_bad_fluxes=True)
1247  self._write_photometry_results_write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1248  else:
1249  photometry = Photometry(None, None)
1250 
1251  return pipeBase.Struct(dataRefs=dataRefs,
1252  oldWcsList=oldWcsList,
1253  job=self.jobjob,
1254  astrometryRefObjLoader=self.astrometryRefObjLoaderastrometryRefObjLoader,
1255  photometryRefObjLoader=self.photometryRefObjLoaderphotometryRefObjLoader,
1256  defaultFilter=defaultFilter,
1257  epoch=epoch,
1258  exitStatus=exitStatus)
1259 
1260  def _get_refcat_coordinate_error_override(self, refCat, name):
1261  """Check whether we should override the refcat coordinate errors, and
1262  return the overridden error if necessary.
1263 
1264  Parameters
1265  ----------
1266  refCat : `lsst.afw.table.SimpleCatalog`
1267  The reference catalog to check for a ``coord_raErr`` field.
1268  name : `str`
1269  Whether we are doing "astrometry" or "photometry".
1270 
1271  Returns
1272  -------
1273  refCoordErr : `float`
1274  The refcat coordinate error to use, or NaN if we are not overriding
1275  those fields.
1276 
1277  Raises
1278  ------
1279  lsst.pex.config.FieldValidationError
1280  Raised if the refcat does not contain coordinate errors and
1281  ``config.astrometryReferenceErr`` is not set.
1282  """
1283  # This value doesn't matter for photometry, so just set something to
1284  # keep old refcats from causing problems.
1285  if name.lower() == "photometry":
1286  if 'coord_raErr' not in refCat.schema:
1287  return 100
1288  else:
1289  return float('nan')
1290 
1291  if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1292  msg = ("Reference catalog does not contain coordinate errors, "
1293  "and config.astrometryReferenceErr not supplied.")
1294  raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1295  self.config,
1296  msg)
1297 
1298  if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1299  self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1300  self.config.astrometryReferenceErr)
1301 
1302  if self.config.astrometryReferenceErr is None:
1303  return float('nan')
1304  else:
1305  return self.config.astrometryReferenceErr
1306 
1307  def _compute_proper_motion_epoch(self, ccdImageList):
1308  """Return the proper motion correction epoch of the provided images.
1309 
1310  Parameters
1311  ----------
1312  ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1313  The images to compute the appropriate epoch for.
1314 
1315  Returns
1316  -------
1317  epoch : `astropy.time.Time`
1318  The date to use for proper motion corrections.
1319  """
1320  return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
1321  format="jyear",
1322  scale="tai")
1323 
1324  def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1325  tract="", match_cut=3.0,
1326  reject_bad_fluxes=False, *,
1327  name="", refObjLoader=None, referenceSelector=None,
1328  fit_function=None, epoch=None):
1329  """Load reference catalog, perform the fit, and return the result.
1330 
1331  Parameters
1332  ----------
1333  associations : `lsst.jointcal.Associations`
1334  The star/reference star associations to fit.
1335  defaultFilter : `lsst.afw.image.FilterLabel`
1336  filter to load from reference catalog.
1337  center : `lsst.geom.SpherePoint`
1338  ICRS center of field to load from reference catalog.
1339  radius : `lsst.geom.Angle`
1340  On-sky radius to load from reference catalog.
1341  name : `str`
1342  Name of thing being fit: "astrometry" or "photometry".
1343  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1344  Reference object loader to use to load a reference catalog.
1345  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1346  Selector to use to pick objects from the loaded reference catalog.
1347  fit_function : callable
1348  Function to call to perform fit (takes Associations object).
1349  tract : `str`, optional
1350  Name of tract currently being fit.
1351  match_cut : `float`, optional
1352  Radius in arcseconds to find cross-catalog matches to during
1353  associations.associateCatalogs.
1354  reject_bad_fluxes : `bool`, optional
1355  Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1356  epoch : `astropy.time.Time`, optional
1357  Epoch to which to correct refcat proper motion and parallax,
1358  or `None` to not apply such corrections.
1359 
1360  Returns
1361  -------
1362  result : `Photometry` or `Astrometry`
1363  Result of `fit_function()`
1364  """
1365  self.log.info("====== Now processing %s...", name)
1366  # TODO: this should not print "trying to invert a singular transformation:"
1367  # if it does that, something's not right about the WCS...
1368  associations.associateCatalogs(match_cut)
1369  add_measurement(self.jobjob, 'jointcal.%s_matched_fittedStars' % name,
1370  associations.fittedStarListSize())
1371 
1372  applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1373  refCat, fluxField = self._load_reference_catalog_load_reference_catalog(refObjLoader, referenceSelector,
1374  center, radius, defaultFilter,
1375  applyColorterms=applyColorterms,
1376  epoch=epoch)
1377  refCoordErr = self._get_refcat_coordinate_error_override_get_refcat_coordinate_error_override(refCat, name)
1378 
1379  associations.collectRefStars(refCat,
1380  self.config.matchCut*lsst.geom.arcseconds,
1381  fluxField,
1382  refCoordinateErr=refCoordErr,
1383  rejectBadFluxes=reject_bad_fluxes)
1384  add_measurement(self.jobjob, 'jointcal.%s_collected_refStars' % name,
1385  associations.refStarListSize())
1386 
1387  associations.prepareFittedStars(self.config.minMeasurements)
1388 
1389  self._check_star_lists_check_star_lists(associations, name)
1390  add_measurement(self.jobjob, 'jointcal.%s_prepared_refStars' % name,
1391  associations.nFittedStarsWithAssociatedRefStar())
1392  add_measurement(self.jobjob, 'jointcal.%s_prepared_fittedStars' % name,
1393  associations.fittedStarListSize())
1394  add_measurement(self.jobjob, 'jointcal.%s_prepared_ccdImages' % name,
1395  associations.nCcdImagesValidForFit())
1396 
1397  load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1398  dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1399  with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1400  result = fit_function(associations, dataName)
1401  # TODO DM-12446: turn this into a "butler save" somehow.
1402  # Save reference and measurement chi2 contributions for this data
1403  if self.config.writeChi2FilesInitialFinal:
1404  baseName = self._getDebugPath_getDebugPath(f"{name}_final_chi2-{dataName}")
1405  result.fit.saveChi2Contributions(baseName+"{type}")
1406  self.log.info("Wrote chi2 contributions files: %s", baseName)
1407 
1408  return result
1409 
1410  def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1411  applyColorterms=False, epoch=None):
1412  """Load the necessary reference catalog sources, convert fluxes to
1413  correct units, and apply color term corrections if requested.
1414 
1415  Parameters
1416  ----------
1417  refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1418  The reference catalog loader to use to get the data.
1419  referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1420  Source selector to apply to loaded reference catalog.
1421  center : `lsst.geom.SpherePoint`
1422  The center around which to load sources.
1423  radius : `lsst.geom.Angle`
1424  The radius around ``center`` to load sources in.
1425  filterLabel : `lsst.afw.image.FilterLabel`
1426  The camera filter to load fluxes for.
1427  applyColorterms : `bool`
1428  Apply colorterm corrections to the refcat for ``filterName``?
1429  epoch : `astropy.time.Time`, optional
1430  Epoch to which to correct refcat proper motion and parallax,
1431  or `None` to not apply such corrections.
1432 
1433  Returns
1434  -------
1435  refCat : `lsst.afw.table.SimpleCatalog`
1436  The loaded reference catalog.
1437  fluxField : `str`
1438  The name of the reference catalog flux field appropriate for ``filterName``.
1439  """
1440  skyCircle = refObjLoader.loadSkyCircle(center,
1441  radius,
1442  filterLabel.bandLabel,
1443  epoch=epoch)
1444 
1445  selected = referenceSelector.run(skyCircle.refCat)
1446  # Need memory contiguity to get reference filters as a vector.
1447  if not selected.sourceCat.isContiguous():
1448  refCat = selected.sourceCat.copy(deep=True)
1449  else:
1450  refCat = selected.sourceCat
1451 
1452  if applyColorterms:
1453  refCatName = refObjLoader.ref_dataset_name
1454  self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1455  filterLabel.physicalLabel, refCatName)
1456  colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1457  refCatName,
1458  doRaise=True)
1459 
1460  refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1461  refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1462  # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1463  refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1464 
1465  return refCat, skyCircle.fluxField
1466 
1467  def _check_star_lists(self, associations, name):
1468  # TODO: these should be len(blah), but we need this properly wrapped first.
1469  if associations.nCcdImagesValidForFit() == 0:
1470  raise RuntimeError('No images in the ccdImageList!')
1471  if associations.fittedStarListSize() == 0:
1472  raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1473  if associations.refStarListSize() == 0:
1474  raise RuntimeError('No stars in the {} reference star list!'.format(name))
1475 
1476  def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1477  """Compute chi2, log it, validate the model, and return chi2.
1478 
1479  Parameters
1480  ----------
1481  associations : `lsst.jointcal.Associations`
1482  The star/reference star associations to fit.
1483  fit : `lsst.jointcal.FitterBase`
1484  The fitter to use for minimization.
1485  model : `lsst.jointcal.Model`
1486  The model being fit.
1487  chi2Label : `str`
1488  Label to describe the chi2 (e.g. "Initialized", "Final").
1489  writeChi2Name : `str`, optional
1490  Filename prefix to write the chi2 contributions to.
1491  Do not supply an extension: an appropriate one will be added.
1492 
1493  Returns
1494  -------
1495  chi2: `lsst.jointcal.Chi2Accumulator`
1496  The chi2 object for the current fitter and model.
1497 
1498  Raises
1499  ------
1500  FloatingPointError
1501  Raised if chi2 is infinite or NaN.
1502  ValueError
1503  Raised if the model is not valid.
1504  """
1505  if writeChi2Name is not None:
1506  fullpath = self._getDebugPath_getDebugPath(writeChi2Name)
1507  fit.saveChi2Contributions(fullpath+"{type}")
1508  self.log.info("Wrote chi2 contributions files: %s", fullpath)
1509 
1510  chi2 = fit.computeChi2()
1511  self.log.info("%s %s", chi2Label, chi2)
1512  self._check_stars_check_stars(associations)
1513  if not np.isfinite(chi2.chi2):
1514  raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1515  if not model.validate(associations.getCcdImageList(), chi2.ndof):
1516  raise ValueError("Model is not valid: check log messages for warnings.")
1517  return chi2
1518 
1519  def _fit_photometry(self, associations, dataName=None):
1520  """
1521  Fit the photometric data.
1522 
1523  Parameters
1524  ----------
1525  associations : `lsst.jointcal.Associations`
1526  The star/reference star associations to fit.
1527  dataName : `str`
1528  Name of the data being processed (e.g. "1234_HSC-Y"), for
1529  identifying debugging files.
1530 
1531  Returns
1532  -------
1533  fit_result : `namedtuple`
1534  fit : `lsst.jointcal.PhotometryFit`
1535  The photometric fitter used to perform the fit.
1536  model : `lsst.jointcal.PhotometryModel`
1537  The photometric model that was fit.
1538  """
1539  self.log.info("=== Starting photometric fitting...")
1540 
1541  # TODO: should use pex.config.RegistryField here (see DM-9195)
1542  if self.config.photometryModel == "constrainedFlux":
1543  model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1544  self.focalPlaneBBoxfocalPlaneBBox,
1545  visitOrder=self.config.photometryVisitOrder,
1546  errorPedestal=self.config.photometryErrorPedestal)
1547  # potentially nonlinear problem, so we may need a line search to converge.
1548  doLineSearch = self.config.allowLineSearch
1549  elif self.config.photometryModel == "constrainedMagnitude":
1550  model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1551  self.focalPlaneBBoxfocalPlaneBBox,
1552  visitOrder=self.config.photometryVisitOrder,
1553  errorPedestal=self.config.photometryErrorPedestal)
1554  # potentially nonlinear problem, so we may need a line search to converge.
1555  doLineSearch = self.config.allowLineSearch
1556  elif self.config.photometryModel == "simpleFlux":
1557  model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1558  errorPedestal=self.config.photometryErrorPedestal)
1559  doLineSearch = False # purely linear in model parameters, so no line search needed
1560  elif self.config.photometryModel == "simpleMagnitude":
1561  model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1562  errorPedestal=self.config.photometryErrorPedestal)
1563  doLineSearch = False # purely linear in model parameters, so no line search needed
1564 
1565  fit = lsst.jointcal.PhotometryFit(associations, model)
1566  # TODO DM-12446: turn this into a "butler save" somehow.
1567  # Save reference and measurement chi2 contributions for this data
1568  if self.config.writeChi2FilesInitialFinal:
1569  baseName = f"photometry_initial_chi2-{dataName}"
1570  else:
1571  baseName = None
1572  if self.config.writeInitialModel:
1573  fullpath = self._getDebugPath_getDebugPath(f"initial_photometry_model-{dataName}.txt")
1574  writeModel(model, fullpath, self.log)
1575  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1576 
1577  def getChi2Name(whatToFit):
1578  if self.config.writeChi2FilesOuterLoop:
1579  return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1580  else:
1581  return None
1582 
1583  # The constrained model needs the visit transform fit first; the chip
1584  # transform is initialized from the singleFrame PhotoCalib, so it's close.
1585  if self.config.writeInitMatrix:
1586  dumpMatrixFile = self._getDebugPath_getDebugPath(f"photometry_preinit-{dataName}")
1587  else:
1588  dumpMatrixFile = ""
1589  if self.config.photometryModel.startswith("constrained"):
1590  # no line search: should be purely (or nearly) linear,
1591  # and we want a large step size to initialize with.
1592  fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1593  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1594  writeChi2Name=getChi2Name("ModelVisit"))
1595  dumpMatrixFile = "" # so we don't redo the output on the next step
1596 
1597  fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1598  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Model",
1599  writeChi2Name=getChi2Name("Model"))
1600 
1601  fit.minimize("Fluxes") # no line search: always purely linear.
1602  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1603  writeChi2Name=getChi2Name("Fluxes"))
1604 
1605  fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1606  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1607  writeChi2Name=getChi2Name("ModelFluxes"))
1608 
1609  model.freezeErrorTransform()
1610  self.log.debug("Photometry error scales are frozen.")
1611 
1612  chi2 = self._iterate_fit_iterate_fit(associations,
1613  fit,
1614  self.config.maxPhotometrySteps,
1615  "photometry",
1616  "Model Fluxes",
1617  doRankUpdate=self.config.photometryDoRankUpdate,
1618  doLineSearch=doLineSearch,
1619  dataName=dataName)
1620 
1621  add_measurement(self.jobjob, 'jointcal.photometry_final_chi2', chi2.chi2)
1622  add_measurement(self.jobjob, 'jointcal.photometry_final_ndof', chi2.ndof)
1623  return Photometry(fit, model)
1624 
1625  def _fit_astrometry(self, associations, dataName=None):
1626  """
1627  Fit the astrometric data.
1628 
1629  Parameters
1630  ----------
1631  associations : `lsst.jointcal.Associations`
1632  The star/reference star associations to fit.
1633  dataName : `str`
1634  Name of the data being processed (e.g. "1234_HSC-Y"), for
1635  identifying debugging files.
1636 
1637  Returns
1638  -------
1639  fit_result : `namedtuple`
1640  fit : `lsst.jointcal.AstrometryFit`
1641  The astrometric fitter used to perform the fit.
1642  model : `lsst.jointcal.AstrometryModel`
1643  The astrometric model that was fit.
1644  sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1645  The model for the sky to tangent plane projection that was used in the fit.
1646  """
1647 
1648  self.log.info("=== Starting astrometric fitting...")
1649 
1650  associations.deprojectFittedStars()
1651 
1652  # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1653  # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1654  # them so carefully?
1655  sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1656 
1657  if self.config.astrometryModel == "constrained":
1658  model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1659  sky_to_tan_projection,
1660  chipOrder=self.config.astrometryChipOrder,
1661  visitOrder=self.config.astrometryVisitOrder)
1662  elif self.config.astrometryModel == "simple":
1663  model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1664  sky_to_tan_projection,
1665  self.config.useInputWcs,
1666  nNotFit=0,
1667  order=self.config.astrometrySimpleOrder)
1668 
1669  fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1670  # TODO DM-12446: turn this into a "butler save" somehow.
1671  # Save reference and measurement chi2 contributions for this data
1672  if self.config.writeChi2FilesInitialFinal:
1673  baseName = f"astrometry_initial_chi2-{dataName}"
1674  else:
1675  baseName = None
1676  if self.config.writeInitialModel:
1677  fullpath = self._getDebugPath_getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1678  writeModel(model, fullpath, self.log)
1679  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1680 
1681  def getChi2Name(whatToFit):
1682  if self.config.writeChi2FilesOuterLoop:
1683  return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1684  else:
1685  return None
1686 
1687  if self.config.writeInitMatrix:
1688  dumpMatrixFile = self._getDebugPath_getDebugPath(f"astrometry_preinit-{dataName}")
1689  else:
1690  dumpMatrixFile = ""
1691  # The constrained model needs the visit transform fit first; the chip
1692  # transform is initialized from the detector's cameraGeom, so it's close.
1693  if self.config.astrometryModel == "constrained":
1694  fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1695  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1696  writeChi2Name=getChi2Name("DistortionsVisit"))
1697  dumpMatrixFile = "" # so we don't redo the output on the next step
1698 
1699  fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1700  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1701  writeChi2Name=getChi2Name("Distortions"))
1702 
1703  fit.minimize("Positions")
1704  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize Positions",
1705  writeChi2Name=getChi2Name("Positions"))
1706 
1707  fit.minimize("Distortions Positions")
1708  self._logChi2AndValidate_logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1709  writeChi2Name=getChi2Name("DistortionsPositions"))
1710 
1711  chi2 = self._iterate_fit_iterate_fit(associations,
1712  fit,
1713  self.config.maxAstrometrySteps,
1714  "astrometry",
1715  "Distortions Positions",
1716  sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1717  doRankUpdate=self.config.astrometryDoRankUpdate,
1718  dataName=dataName)
1719 
1720  add_measurement(self.jobjob, 'jointcal.astrometry_final_chi2', chi2.chi2)
1721  add_measurement(self.jobjob, 'jointcal.astrometry_final_ndof', chi2.ndof)
1722 
1723  return Astrometry(fit, model, sky_to_tan_projection)
1724 
1725  def _check_stars(self, associations):
1726  """Count measured and reference stars per ccd and warn/log them."""
1727  for ccdImage in associations.getCcdImageList():
1728  nMeasuredStars, nRefStars = ccdImage.countStars()
1729  self.log.debug("ccdImage %s has %s measured and %s reference stars",
1730  ccdImage.getName(), nMeasuredStars, nRefStars)
1731  if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1732  self.log.warning("ccdImage %s has only %s measuredStars (desired %s)",
1733  ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1734  if nRefStars < self.config.minRefStarsPerCcd:
1735  self.log.warning("ccdImage %s has only %s RefStars (desired %s)",
1736  ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1737 
1738  def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1739  dataName="",
1740  sigmaRelativeTolerance=0,
1741  doRankUpdate=True,
1742  doLineSearch=False):
1743  """Run fitter.minimize up to max_steps times, returning the final chi2.
1744 
1745  Parameters
1746  ----------
1747  associations : `lsst.jointcal.Associations`
1748  The star/reference star associations to fit.
1749  fitter : `lsst.jointcal.FitterBase`
1750  The fitter to use for minimization.
1751  max_steps : `int`
1752  Maximum number of steps to run outlier rejection before declaring
1753  convergence failure.
1754  name : {'photometry' or 'astrometry'}
1755  What type of data are we fitting (for logs and debugging files).
1756  whatToFit : `str`
1757  Passed to ``fitter.minimize()`` to define the parameters to fit.
1758  dataName : `str`, optional
1759  Descriptive name for this dataset (e.g. tract and filter),
1760  for debugging.
1761  sigmaRelativeTolerance : `float`, optional
1762  Convergence tolerance for the fractional change in the chi2 cut
1763  level for determining outliers. If set to zero, iterations will
1764  continue until there are no outliers.
1765  doRankUpdate : `bool`, optional
1766  Do an Eigen rank update during minimization, or recompute the full
1767  matrix and gradient?
1768  doLineSearch : `bool`, optional
1769  Do a line search for the optimum step during minimization?
1770 
1771  Returns
1772  -------
1773  chi2: `lsst.jointcal.Chi2Statistic`
1774  The final chi2 after the fit converges, or is forced to end.
1775 
1776  Raises
1777  ------
1778  FloatingPointError
1779  Raised if the fitter fails with a non-finite value.
1780  RuntimeError
1781  Raised if the fitter fails for some other reason;
1782  log messages will provide further details.
1783  """
1784  if self.config.writeInitMatrix:
1785  dumpMatrixFile = self._getDebugPath_getDebugPath(f"{name}_postinit-{dataName}")
1786  else:
1787  dumpMatrixFile = ""
1788  oldChi2 = lsst.jointcal.Chi2Statistic()
1789  oldChi2.chi2 = float("inf")
1790  for i in range(max_steps):
1791  if self.config.writeChi2FilesOuterLoop:
1792  writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1793  else:
1794  writeChi2Name = None
1795  result = fitter.minimize(whatToFit,
1796  self.config.outlierRejectSigma,
1797  sigmaRelativeTolerance=sigmaRelativeTolerance,
1798  doRankUpdate=doRankUpdate,
1799  doLineSearch=doLineSearch,
1800  dumpMatrixFile=dumpMatrixFile)
1801  dumpMatrixFile = "" # clear it so we don't write the matrix again.
1802  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(),
1803  f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1804 
1805  if result == MinimizeResult.Converged:
1806  if doRankUpdate:
1807  self.log.debug("fit has converged - no more outliers - redo minimization "
1808  "one more time in case we have lost accuracy in rank update.")
1809  # Redo minimization one more time in case we have lost accuracy in rank update
1810  result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1811  sigmaRelativeTolerance=sigmaRelativeTolerance)
1812  chi2 = self._logChi2AndValidate_logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1813 
1814  # log a message for a large final chi2, TODO: DM-15247 for something better
1815  if chi2.chi2/chi2.ndof >= 4.0:
1816  self.log.error("Potentially bad fit: High chi-squared/ndof.")
1817 
1818  break
1819  elif result == MinimizeResult.Chi2Increased:
1820  self.log.warning("Still some outliers remaining but chi2 increased - retry")
1821  # Check whether the increase was large enough to cause trouble.
1822  chi2Ratio = chi2.chi2 / oldChi2.chi2
1823  if chi2Ratio > 1.5:
1824  self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1825  chi2.chi2, oldChi2.chi2, chi2Ratio)
1826  # Based on a variety of HSC jointcal logs (see DM-25779), it
1827  # appears that chi2 increases more than a factor of ~2 always
1828  # result in the fit diverging rapidly and ending at chi2 > 1e10.
1829  # Using 10 as the "failure" threshold gives some room between
1830  # leaving a warning and bailing early.
1831  if chi2Ratio > 10:
1832  msg = ("Large chi2 increase between steps: fit likely cannot converge."
1833  " Try setting one or more of the `writeChi2*` config fields and looking"
1834  " at how individual star chi2-values evolve during the fit.")
1835  raise RuntimeError(msg)
1836  oldChi2 = chi2
1837  elif result == MinimizeResult.NonFinite:
1838  filename = self._getDebugPath_getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1839  # TODO DM-12446: turn this into a "butler save" somehow.
1840  fitter.saveChi2Contributions(filename+"{type}")
1841  msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1842  raise FloatingPointError(msg.format(filename))
1843  elif result == MinimizeResult.Failed:
1844  raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1845  else:
1846  raise RuntimeError("Unxepected return code from minimize().")
1847  else:
1848  self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1849 
1850  return chi2
1851 
1852  def _make_output(self, ccdImageList, model, func):
1853  """Return the internal jointcal models converted to the afw
1854  structures that will be saved to disk.
1855 
1856  Parameters
1857  ----------
1858  ccdImageList : `lsst.jointcal.CcdImageList`
1859  The list of CcdImages to get the output for.
1860  model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1861  The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1862  func : `str`
1863  The name of the function to call on ``model`` to get the converted
1864  structure. Must accept an `lsst.jointcal.CcdImage`.
1865 
1866  Returns
1867  -------
1868  output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1869  `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1870  The data to be saved, keyed on (visit, detector).
1871  """
1872  output = {}
1873  for ccdImage in ccdImageList:
1874  ccd = ccdImage.ccdId
1875  visit = ccdImage.visit
1876  self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1877  output[(visit, ccd)] = getattr(model, func)(ccdImage)
1878  return output
1879 
1880  def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1881  """
1882  Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1883 
1884  Parameters
1885  ----------
1886  associations : `lsst.jointcal.Associations`
1887  The star/reference star associations to fit.
1888  model : `lsst.jointcal.AstrometryModel`
1889  The astrometric model that was fit.
1890  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1891  Dict of ccdImage identifiers to dataRefs that were fit.
1892  """
1893  ccdImageList = associations.getCcdImageList()
1894  output = self._make_output_make_output(ccdImageList, model, "makeSkyWcs")
1895  for key, skyWcs in output.items():
1896  dataRef = visit_ccd_to_dataRef[key]
1897  try:
1898  dataRef.put(skyWcs, 'jointcal_wcs')
1899  except pexExceptions.Exception as e:
1900  self.log.fatal('Failed to write updated Wcs: %s', str(e))
1901  raise e
1902 
1903  def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1904  """
1905  Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1906 
1907  Parameters
1908  ----------
1909  associations : `lsst.jointcal.Associations`
1910  The star/reference star associations to fit.
1911  model : `lsst.jointcal.PhotometryModel`
1912  The photoometric model that was fit.
1913  visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1914  Dict of ccdImage identifiers to dataRefs that were fit.
1915  """
1916 
1917  ccdImageList = associations.getCcdImageList()
1918  output = self._make_output_make_output(ccdImageList, model, "toPhotoCalib")
1919  for key, photoCalib in output.items():
1920  dataRef = visit_ccd_to_dataRef[key]
1921  try:
1922  dataRef.put(photoCalib, 'jointcal_photoCalib')
1923  except pexExceptions.Exception as e:
1924  self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1925  raise e
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:54
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:45
A model where there is one independent transform per CcdImage.
def getTargetList(parsedCmd, **kwargs)
Definition: jointcal.py:77
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:924
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:1307
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:1260
def _getDebugPath(self, filename)
Definition: jointcal.py:1158
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1467
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:988
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1742
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1880
def _check_stars(self, associations)
Definition: jointcal.py:1725
def _prep_sky(self, associations, filters)
Definition: jointcal.py:1163
def _readDataId(self, butler, dataId)
Definition: jointcal.py:1108
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:692
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:1328
def _put_metrics(self, butlerQC, job, outputRefs)
Definition: jointcal.py:721
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:1063
def runDataRef(self, dataRefs)
Definition: jointcal.py:1186
def __init__(self, butler=None, initInputs=None, **kwargs)
Definition: jointcal.py:671
def _get_sourceTable_visit_columns(self, inColumns)
Definition: jointcal.py:1002
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:796
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1519
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Definition: jointcal.py:1903
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1476
def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId, detectorColumn, ixxColumns)
Definition: jointcal.py:874
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1852
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:737
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1411
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1625
def loadData(self, dataRefs, associations, jointcalControl)
Definition: jointcal.py:1134
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:141
def add_measurement(job, name, value)
Definition: jointcal.py:59
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:181
def writeModel(model, filename, log)
Definition: jointcal.py:625