Coverage for python/lsst/jointcal/jointcal.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import dataclasses
23import collections
24import os
26import astropy.time
27import numpy as np
28import astropy.units as u
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
36import lsst.afw.cameraGeom
37import lsst.afw.table
38import lsst.log
39from lsst.obs.base import Instrument
40from lsst.pipe.tasks.colorterms import ColortermLibrary
41from lsst.verify import Job, Measurement
43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44 ReferenceObjectLoader)
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
47from .dataIds import PerTractCcdDataIdContainer
49import lsst.jointcal
50from lsst.jointcal import MinimizeResult
52__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
54Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
58# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59def add_measurement(job, name, value):
60 meas = Measurement(job.metrics[name], value)
61 job.measurements.insert(meas)
64class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65 """Subclass of TaskRunner for jointcalTask (gen2)
67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69 single dataRef, are are called repeatedly). This class transforms the processed
70 arguments generated by the ArgumentParser into the arguments expected by
71 Jointcal.runDataRef().
73 See pipeBase.TaskRunner for more information.
74 """
76 @staticmethod
77 def getTargetList(parsedCmd, **kwargs):
78 """
79 Return a list of tuples per tract, each containing (dataRefs, kwargs).
81 Jointcal operates on lists of dataRefs simultaneously.
82 """
83 kwargs['butler'] = parsedCmd.butler
85 # organize data IDs by tract
86 refListDict = {}
87 for ref in parsedCmd.id.refList:
88 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89 # we call runDataRef() once with each tract
90 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91 return result
93 def __call__(self, args):
94 """
95 Parameters
96 ----------
97 args
98 Arguments for Task.runDataRef()
100 Returns
101 -------
102 pipe.base.Struct
103 if self.doReturnResults is False:
105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
107 if self.doReturnResults is True:
109 - ``result``: the result of calling jointcal.runDataRef()
110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111 """
112 exitStatus = 0 # exit status for shell
114 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115 dataRefList, kwargs = args
116 butler = kwargs.pop('butler')
117 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118 result = None
119 try:
120 result = task.runDataRef(dataRefList, **kwargs)
121 exitStatus = result.exitStatus
122 job_path = butler.get('verify_job_filename')
123 result.job.write(job_path[0])
124 except Exception as e: # catch everything, sort it out later.
125 if self.doRaise:
126 raise e
127 else:
128 exitStatus = 1
129 eName = type(e).__name__
130 tract = dataRefList[0].dataId['tract']
131 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
133 # Put the butler back into kwargs for the other Tasks.
134 kwargs['butler'] = butler
135 if self.doReturnResults:
136 return pipeBase.Struct(result=result, exitStatus=exitStatus)
137 else:
138 return pipeBase.Struct(exitStatus=exitStatus)
141def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142 """Lookup function that asserts/hopes that a static calibration dataset
143 exists in a particular collection, since this task can't provide a single
144 date/time to use to search for one properly.
146 This is mostly useful for the ``camera`` dataset, in cases where the task's
147 quantum dimensions do *not* include something temporal, like ``exposure``
148 or ``visit``.
150 Parameters
151 ----------
152 datasetType : `lsst.daf.butler.DatasetType`
153 Type of dataset being searched for.
154 registry : `lsst.daf.butler.Registry`
155 Data repository registry to search.
156 quantumDataId : `lsst.daf.butler.DataCoordinate`
157 Data ID of the quantum this camera should match.
158 collections : `Iterable` [ `str` ]
159 Collections that should be searched - but this lookup function works
160 by ignoring this in favor of a more-or-less hard-coded value.
162 Returns
163 -------
164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165 Iterator over dataset references; should have only one element.
167 Notes
168 -----
169 This implementation duplicates one in fgcmcal, and is at least quite
170 similar to another in cp_pipe. This duplicate has the most documentation.
171 Fixing this is DM-29661.
172 """
173 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175 return registry.queryDatasets(datasetType,
176 dataId=quantumDataId,
177 collections=[unboundedCollection],
178 findFirst=True)
181def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182 """Lookup function that finds all refcats for all visits that overlap a
183 tract, rather than just the refcats that directly overlap the tract.
185 Parameters
186 ----------
187 datasetType : `lsst.daf.butler.DatasetType`
188 Type of dataset being searched for.
189 registry : `lsst.daf.butler.Registry`
190 Data repository registry to search.
191 quantumDataId : `lsst.daf.butler.DataCoordinate`
192 Data ID of the quantum; expected to be something we can use as a
193 constraint to query for overlapping visits.
194 collections : `Iterable` [ `str` ]
195 Collections to search.
197 Returns
198 -------
199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200 Iterator over refcat references.
201 """
202 refs = set()
203 # Use .expanded() on the query methods below because we need data IDs with
204 # regions, both in the outer loop over visits (queryDatasets will expand
205 # any data ID we give it, but doing it up-front in bulk is much more
206 # efficient) and in the data IDs of the DatasetRefs this function yields
207 # (because the RefCatLoader relies on them to do some of its own
208 # filtering).
209 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210 refs.update(
211 registry.queryDatasets(
212 datasetType,
213 collections=collections,
214 dataId=visit_data_id,
215 findFirst=True,
216 ).expanded()
217 )
218 yield from refs
221class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222 dimensions=("skymap", "tract", "instrument", "physical_filter")):
223 """Middleware input/output connections for jointcal data."""
224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225 doc="The camera instrument that took these observations.",
226 name="camera",
227 storageClass="Camera",
228 dimensions=("instrument",),
229 isCalibration=True,
230 lookupFunction=lookupStaticCalibrations,
231 )
232 inputSourceTableVisit = pipeBase.connectionTypes.Input(
233 doc="Source table in parquet format, per visit",
234 name="sourceTable_visit",
235 storageClass="DataFrame",
236 dimensions=("instrument", "visit"),
237 deferLoad=True,
238 multiple=True,
239 )
240 inputVisitSummary = pipeBase.connectionTypes.Input(
241 doc=("Per-visit consolidated exposure metadata built from calexps. "
242 "These catalogs use detector id for the id and must be sorted for "
243 "fast lookups of a detector."),
244 name="visitSummary",
245 storageClass="ExposureCatalog",
246 dimensions=("instrument", "visit"),
247 deferLoad=True,
248 multiple=True,
249 )
250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251 doc="The astrometry reference catalog to match to loaded input catalog sources.",
252 name="gaia_dr2_20200414",
253 storageClass="SimpleCatalog",
254 dimensions=("skypix",),
255 deferLoad=True,
256 multiple=True,
257 lookupFunction=lookupVisitRefCats,
258 )
259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260 doc="The photometry reference catalog to match to loaded input catalog sources.",
261 name="ps1_pv3_3pi_20170110",
262 storageClass="SimpleCatalog",
263 dimensions=("skypix",),
264 deferLoad=True,
265 multiple=True,
266 lookupFunction=lookupVisitRefCats,
267 )
269 outputWcs = pipeBase.connectionTypes.Output(
270 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271 " These catalogs only contain entries for detectors with an output, and use"
272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273 name="jointcalSkyWcsCatalog",
274 storageClass="ExposureCatalog",
275 dimensions=("instrument", "visit", "skymap", "tract"),
276 multiple=True
277 )
278 outputPhotoCalib = pipeBase.connectionTypes.Output(
279 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280 " These catalogs only contain entries for detectors with an output, and use"
281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282 name="jointcalPhotoCalibCatalog",
283 storageClass="ExposureCatalog",
284 dimensions=("instrument", "visit", "skymap", "tract"),
285 multiple=True
286 )
288 def __init__(self, *, config=None):
289 super().__init__(config=config)
290 # When we are only doing one of astrometry or photometry, we don't
291 # need the reference catalog or produce the outputs for the other.
292 # This informs the middleware of that when the QuantumGraph is
293 # generated, so we don't block on getting something we won't need or
294 # create an expectation that downstream tasks will be able to consume
295 # something we won't produce.
296 if not config.doAstrometry:
297 self.prerequisiteInputs.remove("astrometryRefCat")
298 self.outputs.remove("outputWcs")
299 if not config.doPhotometry:
300 self.prerequisiteInputs.remove("photometryRefCat")
301 self.outputs.remove("outputPhotoCalib")
304class JointcalConfig(pipeBase.PipelineTaskConfig,
305 pipelineConnections=JointcalTaskConnections):
306 """Configuration for JointcalTask"""
308 doAstrometry = pexConfig.Field(
309 doc="Fit astrometry and write the fitted result.",
310 dtype=bool,
311 default=True
312 )
313 doPhotometry = pexConfig.Field(
314 doc="Fit photometry and write the fitted result.",
315 dtype=bool,
316 default=True
317 )
318 coaddName = pexConfig.Field(
319 doc="Type of coadd, typically deep or goodSeeing",
320 dtype=str,
321 default="deep"
322 )
323 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
324 sourceFluxType = pexConfig.Field(
325 dtype=str,
326 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
327 default='Calib'
328 )
329 positionErrorPedestal = pexConfig.Field(
330 doc="Systematic term to apply to the measured position error (pixels)",
331 dtype=float,
332 default=0.02,
333 )
334 photometryErrorPedestal = pexConfig.Field(
335 doc="Systematic term to apply to the measured error on flux or magnitude as a "
336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
337 dtype=float,
338 default=0.0,
339 )
340 # TODO: DM-6885 matchCut should be an geom.Angle
341 matchCut = pexConfig.Field(
342 doc="Matching radius between fitted and reference stars (arcseconds)",
343 dtype=float,
344 default=3.0,
345 )
346 minMeasurements = pexConfig.Field(
347 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
348 dtype=int,
349 default=2,
350 )
351 minMeasuredStarsPerCcd = pexConfig.Field(
352 doc="Minimum number of measuredStars per ccdImage before printing warnings",
353 dtype=int,
354 default=100,
355 )
356 minRefStarsPerCcd = pexConfig.Field(
357 doc="Minimum number of measuredStars per ccdImage before printing warnings",
358 dtype=int,
359 default=30,
360 )
361 allowLineSearch = pexConfig.Field(
362 doc="Allow a line search during minimization, if it is reasonable for the model"
363 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
364 dtype=bool,
365 default=False
366 )
367 astrometrySimpleOrder = pexConfig.Field(
368 doc="Polynomial order for fitting the simple astrometry model.",
369 dtype=int,
370 default=3,
371 )
372 astrometryChipOrder = pexConfig.Field(
373 doc="Order of the per-chip transform for the constrained astrometry model.",
374 dtype=int,
375 default=1,
376 )
377 astrometryVisitOrder = pexConfig.Field(
378 doc="Order of the per-visit transform for the constrained astrometry model.",
379 dtype=int,
380 default=5,
381 )
382 useInputWcs = pexConfig.Field(
383 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
384 dtype=bool,
385 default=True,
386 )
387 astrometryModel = pexConfig.ChoiceField(
388 doc="Type of model to fit to astrometry",
389 dtype=str,
390 default="constrained",
391 allowed={"simple": "One polynomial per ccd",
392 "constrained": "One polynomial per ccd, and one polynomial per visit"}
393 )
394 photometryModel = pexConfig.ChoiceField(
395 doc="Type of model to fit to photometry",
396 dtype=str,
397 default="constrainedMagnitude",
398 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
399 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
400 " fitting in flux space.",
401 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
402 " fitting in magnitude space.",
403 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
404 " fitting in magnitude space.",
405 }
406 )
407 applyColorTerms = pexConfig.Field(
408 doc="Apply photometric color terms to reference stars?"
409 "Requires that colorterms be set to a ColortermLibrary",
410 dtype=bool,
411 default=False
412 )
413 colorterms = pexConfig.ConfigField(
414 doc="Library of photometric reference catalog name to color term dict.",
415 dtype=ColortermLibrary,
416 )
417 photometryVisitOrder = pexConfig.Field(
418 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
419 dtype=int,
420 default=7,
421 )
422 photometryDoRankUpdate = pexConfig.Field(
423 doc=("Do the rank update step during minimization. "
424 "Skipping this can help deal with models that are too non-linear."),
425 dtype=bool,
426 default=True,
427 )
428 astrometryDoRankUpdate = pexConfig.Field(
429 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
430 "Skipping this can help deal with models that are too non-linear."),
431 dtype=bool,
432 default=True,
433 )
434 outlierRejectSigma = pexConfig.Field(
435 doc="How many sigma to reject outliers at during minimization.",
436 dtype=float,
437 default=5.0,
438 )
439 astrometryOutlierRelativeTolerance = pexConfig.Field(
440 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
441 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
442 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
443 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
444 "close to the solution found when removing all outliers."),
445 dtype=float,
446 default=0,
447 )
448 maxPhotometrySteps = pexConfig.Field(
449 doc="Maximum number of minimize iterations to take when fitting photometry.",
450 dtype=int,
451 default=20,
452 )
453 maxAstrometrySteps = pexConfig.Field(
454 doc="Maximum number of minimize iterations to take when fitting astrometry.",
455 dtype=int,
456 default=20,
457 )
458 astrometryRefObjLoader = pexConfig.ConfigurableField(
459 target=LoadIndexedReferenceObjectsTask,
460 doc="Reference object loader for astrometric fit",
461 )
462 photometryRefObjLoader = pexConfig.ConfigurableField(
463 target=LoadIndexedReferenceObjectsTask,
464 doc="Reference object loader for photometric fit",
465 )
466 sourceSelector = sourceSelectorRegistry.makeField(
467 doc="How to select sources for cross-matching",
468 default="astrometry"
469 )
470 astrometryReferenceSelector = pexConfig.ConfigurableField(
471 target=ReferenceSourceSelectorTask,
472 doc="How to down-select the loaded astrometry reference catalog.",
473 )
474 photometryReferenceSelector = pexConfig.ConfigurableField(
475 target=ReferenceSourceSelectorTask,
476 doc="How to down-select the loaded photometry reference catalog.",
477 )
478 astrometryReferenceErr = pexConfig.Field(
479 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
480 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
481 "If specified, overrides any existing `coord_*Err` values."),
482 dtype=float,
483 default=None,
484 optional=True
485 )
487 # configs for outputting debug information
488 writeInitMatrix = pexConfig.Field(
489 dtype=bool,
490 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
491 "Output files will be written to `config.debugOutputPath` and will "
492 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
493 "Note that these files are the dense versions of the matrix, and so may be very large."),
494 default=False
495 )
496 writeChi2FilesInitialFinal = pexConfig.Field(
497 dtype=bool,
498 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
499 "Output files will be written to `config.debugOutputPath` and will "
500 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
501 default=False
502 )
503 writeChi2FilesOuterLoop = pexConfig.Field(
504 dtype=bool,
505 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
506 "Output files will be written to `config.debugOutputPath` and will "
507 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
508 default=False
509 )
510 writeInitialModel = pexConfig.Field(
511 dtype=bool,
512 doc=("Write the pre-initialization model to text files, for debugging. "
513 "Output files will be written to `config.debugOutputPath` and will be "
514 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
515 default=False
516 )
517 debugOutputPath = pexConfig.Field(
518 dtype=str,
519 default=".",
520 doc=("Path to write debug output files to. Used by "
521 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
522 )
523 detailedProfile = pexConfig.Field(
524 dtype=bool,
525 default=False,
526 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
527 )
529 def validate(self):
530 super().validate()
531 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
532 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
533 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
534 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
535 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
536 "applyColorTerms=True will be ignored.")
537 lsst.log.warn(msg)
539 def setDefaults(self):
540 # Use science source selector which can filter on extendedness, SNR, and whether blended
541 self.sourceSelector.name = 'science'
542 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
543 self.sourceSelector['science'].doUnresolved = True
544 # with dependable signal to noise ratio.
545 self.sourceSelector['science'].doSignalToNoise = True
546 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
547 # and S/N > 10 to use sources that are not too faint, and thus better measured.
548 self.sourceSelector['science'].signalToNoise.minimum = 10.
549 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
550 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
551 self.sourceSelector['science'].signalToNoise.fluxField = fluxField
552 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err"
553 # Do not trust blended sources' aperture fluxes which also depend on seeing
554 self.sourceSelector['science'].doIsolated = True
555 # Do not trust either flux or centroid measurements with flags,
556 # chosen from the usual QA flags for stars)
557 self.sourceSelector['science'].doFlags = True
558 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
559 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
560 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
561 self.sourceSelector['science'].flags.bad = badFlags
563 # Default to Gaia-DR2 (with proper motions) for astrometry and
564 # PS1-DR1 for photometry, with a reasonable initial filterMap.
565 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
566 self.astrometryRefObjLoader.requireProperMotion = True
567 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
568 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
571def writeModel(model, filename, log):
572 """Write model to outfile."""
573 with open(filename, "w") as file:
574 file.write(repr(model))
575 log.info("Wrote %s to file: %s", model, filename)
578@dataclasses.dataclass
579class JointcalInputData:
580 """The input data jointcal needs for each detector/visit."""
581 visit: int
582 """The visit identifier of this exposure."""
583 catalog: lsst.afw.table.SourceCatalog
584 """The catalog derived from this exposure."""
585 visitInfo: lsst.afw.image.VisitInfo
586 """The VisitInfo of this exposure."""
587 detector: lsst.afw.cameraGeom.Detector
588 """The detector of this exposure."""
589 photoCalib: lsst.afw.image.PhotoCalib
590 """The photometric calibration of this exposure."""
591 wcs: lsst.afw.geom.skyWcs
592 """The WCS of this exposure."""
593 bbox: lsst.geom.Box2I
594 """The bounding box of this exposure."""
595 filter: lsst.afw.image.FilterLabel
596 """The filter of this exposure."""
599class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
600 """Astrometricly and photometricly calibrate across multiple visits of the
601 same field.
603 Parameters
604 ----------
605 butler : `lsst.daf.persistence.Butler`
606 The butler is passed to the refObjLoader constructor in case it is
607 needed. Ignored if the refObjLoader argument provides a loader directly.
608 Used to initialize the astrometry and photometry refObjLoaders.
609 initInputs : `dict`, optional
610 Dictionary used to initialize PipelineTasks (empty for jointcal).
611 """
613 ConfigClass = JointcalConfig
614 RunnerClass = JointcalRunner
615 _DefaultName = "jointcal"
617 def __init__(self, butler=None, initInputs=None, **kwargs):
618 super().__init__(**kwargs)
619 self.makeSubtask("sourceSelector")
620 if self.config.doAstrometry:
621 if initInputs is None:
622 # gen3 middleware does refcat things internally (and will not have a butler here)
623 self.makeSubtask('astrometryRefObjLoader', butler=butler)
624 self.makeSubtask("astrometryReferenceSelector")
625 else:
626 self.astrometryRefObjLoader = None
627 if self.config.doPhotometry:
628 if initInputs is None:
629 # gen3 middleware does refcat things internally (and will not have a butler here)
630 self.makeSubtask('photometryRefObjLoader', butler=butler)
631 self.makeSubtask("photometryReferenceSelector")
632 else:
633 self.photometryRefObjLoader = None
635 # To hold various computed metrics for use by tests
636 self.job = Job.load_metrics_package(subset='jointcal')
638 def runQuantum(self, butlerQC, inputRefs, outputRefs):
639 # We override runQuantum to set up the refObjLoaders and write the
640 # outputs to the correct refs.
641 inputs = butlerQC.get(inputRefs)
642 # We want the tract number for writing debug files
643 tract = butlerQC.quantum.dataId['tract']
644 if self.config.doAstrometry:
645 self.astrometryRefObjLoader = ReferenceObjectLoader(
646 dataIds=[ref.datasetRef.dataId
647 for ref in inputRefs.astrometryRefCat],
648 refCats=inputs.pop('astrometryRefCat'),
649 config=self.config.astrometryRefObjLoader,
650 log=self.log)
651 if self.config.doPhotometry:
652 self.photometryRefObjLoader = ReferenceObjectLoader(
653 dataIds=[ref.datasetRef.dataId
654 for ref in inputRefs.photometryRefCat],
655 refCats=inputs.pop('photometryRefCat'),
656 config=self.config.photometryRefObjLoader,
657 log=self.log)
658 outputs = self.run(**inputs, tract=tract)
659 if self.config.doAstrometry:
660 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
661 inputs['inputCamera'], "setWcs")
662 if self.config.doPhotometry:
663 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
664 inputs['inputCamera'], "setPhotoCalib")
666 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
667 """Persist the output datasets to their appropriate datarefs.
669 Parameters
670 ----------
671 butlerQC : `ButlerQuantumContext`
672 A butler which is specialized to operate in the context of a
673 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
674 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
675 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
676 The fitted objects to persist.
677 outputRefs : `list` [`OutputQuantizedConnection`]
678 The DatasetRefs to persist the data to.
679 camera : `lsst.afw.cameraGeom.Camera`
680 The camera for this instrument, to get detector ids from.
681 setter : `str`
682 The method to call on the ExposureCatalog to set each output.
683 """
684 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
685 schema.addField('visit', type='I', doc='Visit number')
687 def new_catalog(visit, size):
688 """Return an catalog ready to be filled with appropriate output."""
689 catalog = lsst.afw.table.ExposureCatalog(schema)
690 catalog.resize(size)
691 catalog['visit'] = visit
692 metadata = lsst.daf.base.PropertyList()
693 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
694 metadata.add("COMMENT", "Only detectors with data have entries.")
695 return catalog
697 # count how many detectors have output for each visit
698 detectors_per_visit = collections.defaultdict(int)
699 for key in outputs:
700 # key is (visit, detector_id), and we only need visit here
701 detectors_per_visit[key[0]] += 1
703 for ref in outputRefs:
704 visit = ref.dataId['visit']
705 catalog = new_catalog(visit, detectors_per_visit[visit])
706 # Iterate over every detector and skip the ones we don't have output for.
707 i = 0
708 for detector in camera:
709 detectorId = detector.getId()
710 key = (ref.dataId['visit'], detectorId)
711 if key not in outputs:
712 # skip detectors we don't have output for
713 self.log.debug("No %s output for detector %s in visit %s",
714 setter[3:], detectorId, visit)
715 continue
717 catalog[i].setId(detectorId)
718 getattr(catalog[i], setter)(outputs[key])
719 i += 1
721 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
722 butlerQC.put(catalog, ref)
723 self.log.info("Wrote %s detectors to %s", i, ref)
725 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
726 # Docstring inherited.
728 # We take values out of the Parquet table, and put them in "flux_",
729 # and the config.sourceFluxType field is used during that extraction,
730 # so just use "flux" here.
731 sourceFluxField = "flux"
732 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
733 associations = lsst.jointcal.Associations()
734 self.focalPlaneBBox = inputCamera.getFpBBox()
735 oldWcsList, bands = self._load_data(inputSourceTableVisit,
736 inputVisitSummary,
737 associations,
738 jointcalControl,
739 inputCamera)
741 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands)
743 if self.config.doAstrometry:
744 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
745 name="astrometry",
746 refObjLoader=self.astrometryRefObjLoader,
747 referenceSelector=self.astrometryReferenceSelector,
748 fit_function=self._fit_astrometry,
749 tract=tract,
750 epoch=epoch)
751 astrometry_output = self._make_output(associations.getCcdImageList(),
752 astrometry.model,
753 "makeSkyWcs")
754 else:
755 astrometry_output = None
757 if self.config.doPhotometry:
758 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
759 name="photometry",
760 refObjLoader=self.photometryRefObjLoader,
761 referenceSelector=self.photometryReferenceSelector,
762 fit_function=self._fit_photometry,
763 tract=tract,
764 epoch=epoch,
765 reject_bad_fluxes=True)
766 photometry_output = self._make_output(associations.getCcdImageList(),
767 photometry.model,
768 "toPhotoCalib")
769 else:
770 photometry_output = None
772 return pipeBase.Struct(outputWcs=astrometry_output,
773 outputPhotoCalib=photometry_output,
774 job=self.job,
775 astrometryRefObjLoader=self.astrometryRefObjLoader,
776 photometryRefObjLoader=self.photometryRefObjLoader)
778 def _make_schema_table(self):
779 """Return an afw SourceTable to use as a base for creating the
780 SourceCatalog to insert values from the dataFrame into.
782 Returns
783 -------
784 table : `lsst.afw.table.SourceTable`
785 Table with schema and slots to use to make SourceCatalogs.
786 """
787 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
788 schema.addField("centroid_x", "D")
789 schema.addField("centroid_y", "D")
790 schema.addField("centroid_xErr", "F")
791 schema.addField("centroid_yErr", "F")
792 schema.addField("shape_xx", "D")
793 schema.addField("shape_yy", "D")
794 schema.addField("shape_xy", "D")
795 schema.addField("flux_instFlux", "D")
796 schema.addField("flux_instFluxErr", "D")
797 table = lsst.afw.table.SourceTable.make(schema)
798 table.defineCentroid("centroid")
799 table.defineShape("shape")
800 return table
802 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
803 """Return an afw SourceCatalog extracted from a visit-level dataframe,
804 limited to just one detector.
806 Parameters
807 ----------
808 table : `lsst.afw.table.SourceTable`
809 Table factory to use to make the SourceCatalog that will be
810 populated with data from ``visitCatalog``.
811 visitCatalog : `pandas.DataFrame`
812 DataFrame to extract a detector catalog from.
813 detectorId : `int`
814 Numeric id of the detector to extract from ``visitCatalog``.
816 Returns
817 -------
818 catalog : `lsst.afw.table.SourceCatalog`
819 Detector-level catalog extracted from ``visitCatalog``.
820 """
821 # map from dataFrame column to afw table column
822 mapping = {'sourceId': 'id',
823 'x': 'centroid_x',
824 'y': 'centroid_y',
825 'xErr': 'centroid_xErr',
826 'yErr': 'centroid_yErr',
827 'Ixx': 'shape_xx',
828 'Iyy': 'shape_yy',
829 'Ixy': 'shape_xy',
830 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
831 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
832 }
833 # If the DataFrame we're reading was generated by a task running with
834 # Gen2 middleware, the column for the detector will be "ccd" for at
835 # least HSC (who knows what it might be in general!); that would be
836 # true even if the data repo is later converted to Gen3, because that
837 # doesn't touch the files themselves. In Gen3, the column for the
838 # detector will always be "detector".
839 detector_column = "detector" if "detector" in visitCatalog.columns else "ccd"
840 catalog = lsst.afw.table.SourceCatalog(table)
841 matched = visitCatalog[detector_column] == detectorId
842 catalog.resize(sum(matched))
843 view = visitCatalog.loc[matched]
844 for dfCol, afwCol in mapping.items():
845 catalog[afwCol] = view[dfCol]
847 self.log.debug("%d sources selected in visit %d detector %d",
848 len(catalog),
849 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
850 detectorId)
851 return catalog
853 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
854 jointcalControl, camera):
855 """Read the data that jointcal needs to run. (Gen3 version)
857 Modifies ``associations`` in-place with the loaded data.
859 Parameters
860 ----------
861 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
862 References to visit-level DataFrames to load the catalog data from.
863 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
864 Visit-level exposure summary catalog with metadata.
865 associations : `lsst.jointcal.Associations`
866 Object to add the loaded data to by constructing new CcdImages.
867 jointcalControl : `jointcal.JointcalControl`
868 Control object for C++ associations management.
869 camera : `lsst.afw.cameraGeom.Camera`
870 Camera object for detector geometry.
872 Returns
873 -------
874 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
875 The original WCS of the input data, to aid in writing tests.
876 bands : `list` [`str`]
877 The filter bands of each input dataset.
878 """
879 oldWcsList = []
880 filters = []
881 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
882 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
883 table = self._make_schema_table() # every detector catalog has the same layout
884 # No guarantee that the input is in the same order of visits, so we have to map one of them.
885 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
886 detectorDict = {detector.getId(): detector for detector in camera}
888 for visitSummaryRef in inputVisitSummary:
889 visitSummary = visitSummaryRef.get()
890 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
891 selected = self.sourceSelector.run(visitCatalog)
893 # Build a CcdImage for each detector in this visit.
894 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
895 for id, index in detectors.items():
896 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
897 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
898 result = self._build_ccdImage(data, associations, jointcalControl)
899 if result is not None:
900 oldWcsList.append(result.wcs)
901 # A visit has only one band, so we can just use the first.
902 filters.append(data.filter)
903 if len(filters) == 0:
904 raise RuntimeError("No data to process: did source selector remove all sources?")
905 filters = collections.Counter(filters)
907 return oldWcsList, filters
909 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
910 """Return a data structure for this detector+visit."""
911 return JointcalInputData(visit=visitRecord['visit'],
912 catalog=catalog,
913 visitInfo=visitRecord.getVisitInfo(),
914 detector=detectorDict[visitRecord.getId()],
915 photoCalib=visitRecord.getPhotoCalib(),
916 wcs=visitRecord.getWcs(),
917 bbox=visitRecord.getBBox(),
918 # ExposureRecord doesn't have a FilterLabel yet,
919 # so we have to make one.
920 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
921 physical=visitRecord['physical_filter']))
923 # We don't currently need to persist the metadata.
924 # If we do in the future, we will have to add appropriate dataset templates
925 # to each obs package (the metadata template should look like `jointcal_wcs`).
926 def _getMetadataName(self):
927 return None
929 @classmethod
930 def _makeArgumentParser(cls):
931 """Create an argument parser"""
932 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
933 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
934 ContainerClass=PerTractCcdDataIdContainer)
935 return parser
937 def _build_ccdImage(self, data, associations, jointcalControl):
938 """
939 Extract the necessary things from this catalog+metadata to add a new
940 ccdImage.
942 Parameters
943 ----------
944 data : `JointcalInputData`
945 The loaded input data.
946 associations : `lsst.jointcal.Associations`
947 Object to add the info to, to construct a new CcdImage
948 jointcalControl : `jointcal.JointcalControl`
949 Control object for associations management
951 Returns
952 ------
953 namedtuple or `None`
954 ``wcs``
955 The TAN WCS of this image, read from the calexp
956 (`lsst.afw.geom.SkyWcs`).
957 ``key``
958 A key to identify this dataRef by its visit and ccd ids
959 (`namedtuple`).
960 `None`
961 if there are no sources in the loaded catalog.
962 """
963 if len(data.catalog) == 0:
964 self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
965 return None
967 associations.createCcdImage(data.catalog,
968 data.wcs,
969 data.visitInfo,
970 data.bbox,
971 data.filter.physicalLabel,
972 data.photoCalib,
973 data.detector,
974 data.visit,
975 data.detector.getId(),
976 jointcalControl)
978 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
979 Key = collections.namedtuple('Key', ('visit', 'ccd'))
980 return Result(data.wcs, Key(data.visit, data.detector.getId()))
982 def _readDataId(self, butler, dataId):
983 """Read all of the data for one dataId from the butler. (gen2 version)"""
984 # Not all instruments have `visit` in their dataIds.
985 if "visit" in dataId.keys():
986 visit = dataId["visit"]
987 else:
988 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
989 detector = butler.get('calexp_detector', dataId=dataId)
991 catalog = butler.get('src',
992 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
993 dataId=dataId)
994 goodSrc = self.sourceSelector.run(catalog)
995 self.log.debug("%d sources selected in visit %d detector %d",
996 len(goodSrc.sourceCat),
997 visit,
998 detector.getId())
999 return JointcalInputData(visit=visit,
1000 catalog=goodSrc.sourceCat,
1001 visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1002 detector=detector,
1003 photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1004 wcs=butler.get('calexp_wcs', dataId=dataId),
1005 bbox=butler.get('calexp_bbox', dataId=dataId),
1006 filter=butler.get('calexp_filterLabel', dataId=dataId))
1008 def loadData(self, dataRefs, associations, jointcalControl):
1009 """Read the data that jointcal needs to run. (Gen2 version)"""
1010 visit_ccd_to_dataRef = {}
1011 oldWcsList = []
1012 filters = []
1013 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1014 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1015 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1016 camera = dataRefs[0].get('camera', immediate=True)
1017 self.focalPlaneBBox = camera.getFpBBox()
1018 for dataRef in dataRefs:
1019 data = self._readDataId(dataRef.getButler(), dataRef.dataId)
1020 result = self._build_ccdImage(data, associations, jointcalControl)
1021 if result is None:
1022 continue
1023 oldWcsList.append(result.wcs)
1024 visit_ccd_to_dataRef[result.key] = dataRef
1025 filters.append(data.filter)
1026 if len(filters) == 0:
1027 raise RuntimeError("No data to process: did source selector remove all sources?")
1028 filters = collections.Counter(filters)
1030 return oldWcsList, filters, visit_ccd_to_dataRef
1032 def _getDebugPath(self, filename):
1033 """Constructs a path to filename using the configured debug path.
1034 """
1035 return os.path.join(self.config.debugOutputPath, filename)
1037 def _prep_sky(self, associations, filters):
1038 """Prepare on-sky and other data that must be computed after data has
1039 been read.
1040 """
1041 associations.computeCommonTangentPoint()
1043 boundingCircle = associations.computeBoundingCircle()
1044 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1045 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1047 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1049 # Determine a default filter band associated with the catalog. See DM-9093
1050 defaultFilter = filters.most_common(1)[0][0]
1051 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1053 # compute and set the reference epoch of the observations, for proper motion corrections
1054 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
1055 associations.setEpoch(epoch.jyear)
1057 return boundingCircle, center, radius, defaultFilter, epoch
1059 @pipeBase.timeMethod
1060 def runDataRef(self, dataRefs):
1061 """
1062 Jointly calibrate the astrometry and photometry across a set of images.
1064 NOTE: this is for gen2 middleware only.
1066 Parameters
1067 ----------
1068 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1069 List of data references to the exposures to be fit.
1071 Returns
1072 -------
1073 result : `lsst.pipe.base.Struct`
1074 Struct of metadata from the fit, containing:
1076 ``dataRefs``
1077 The provided data references that were fit (with updated WCSs)
1078 ``oldWcsList``
1079 The original WCS from each dataRef
1080 ``metrics``
1081 Dictionary of internally-computed metrics for testing/validation.
1082 """
1083 if len(dataRefs) == 0:
1084 raise ValueError('Need a non-empty list of data references!')
1086 exitStatus = 0 # exit status for shell
1088 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1089 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1090 associations = lsst.jointcal.Associations()
1092 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs,
1093 associations,
1094 jointcalControl)
1096 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, filters)
1098 tract = dataRefs[0].dataId['tract']
1100 if self.config.doAstrometry:
1101 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1102 name="astrometry",
1103 refObjLoader=self.astrometryRefObjLoader,
1104 referenceSelector=self.astrometryReferenceSelector,
1105 fit_function=self._fit_astrometry,
1106 tract=tract,
1107 epoch=epoch)
1108 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1109 else:
1110 astrometry = Astrometry(None, None, None)
1112 if self.config.doPhotometry:
1113 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1114 name="photometry",
1115 refObjLoader=self.photometryRefObjLoader,
1116 referenceSelector=self.photometryReferenceSelector,
1117 fit_function=self._fit_photometry,
1118 tract=tract,
1119 epoch=epoch,
1120 reject_bad_fluxes=True)
1121 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1122 else:
1123 photometry = Photometry(None, None)
1125 return pipeBase.Struct(dataRefs=dataRefs,
1126 oldWcsList=oldWcsList,
1127 job=self.job,
1128 astrometryRefObjLoader=self.astrometryRefObjLoader,
1129 photometryRefObjLoader=self.photometryRefObjLoader,
1130 defaultFilter=defaultFilter,
1131 epoch=epoch,
1132 exitStatus=exitStatus)
1134 def _get_refcat_coordinate_error_override(self, refCat, name):
1135 """Check whether we should override the refcat coordinate errors, and
1136 return the overridden error if necessary.
1138 Parameters
1139 ----------
1140 refCat : `lsst.afw.table.SimpleCatalog`
1141 The reference catalog to check for a ``coord_raErr`` field.
1142 name : `str`
1143 Whether we are doing "astrometry" or "photometry".
1145 Returns
1146 -------
1147 refCoordErr : `float`
1148 The refcat coordinate error to use, or NaN if we are not overriding
1149 those fields.
1151 Raises
1152 ------
1153 lsst.pex.config.FieldValidationError
1154 Raised if the refcat does not contain coordinate errors and
1155 ``config.astrometryReferenceErr`` is not set.
1156 """
1157 # This value doesn't matter for photometry, so just set something to
1158 # keep old refcats from causing problems.
1159 if name.lower() == "photometry":
1160 if 'coord_raErr' not in refCat.schema:
1161 return 100
1162 else:
1163 return float('nan')
1165 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1166 msg = ("Reference catalog does not contain coordinate errors, "
1167 "and config.astrometryReferenceErr not supplied.")
1168 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1169 self.config,
1170 msg)
1172 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1173 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1174 self.config.astrometryReferenceErr)
1176 if self.config.astrometryReferenceErr is None:
1177 return float('nan')
1178 else:
1179 return self.config.astrometryReferenceErr
1181 def _compute_proper_motion_epoch(self, ccdImageList):
1182 """Return the proper motion correction epoch of the provided images.
1184 Parameters
1185 ----------
1186 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1187 The images to compute the appropriate epoch for.
1189 Returns
1190 -------
1191 epoch : `astropy.time.Time`
1192 The date to use for proper motion corrections.
1193 """
1194 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
1195 format="jyear",
1196 scale="tai")
1198 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1199 tract="", match_cut=3.0,
1200 reject_bad_fluxes=False, *,
1201 name="", refObjLoader=None, referenceSelector=None,
1202 fit_function=None, epoch=None):
1203 """Load reference catalog, perform the fit, and return the result.
1205 Parameters
1206 ----------
1207 associations : `lsst.jointcal.Associations`
1208 The star/reference star associations to fit.
1209 defaultFilter : `lsst.afw.image.FilterLabel`
1210 filter to load from reference catalog.
1211 center : `lsst.geom.SpherePoint`
1212 ICRS center of field to load from reference catalog.
1213 radius : `lsst.geom.Angle`
1214 On-sky radius to load from reference catalog.
1215 name : `str`
1216 Name of thing being fit: "astrometry" or "photometry".
1217 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1218 Reference object loader to use to load a reference catalog.
1219 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1220 Selector to use to pick objects from the loaded reference catalog.
1221 fit_function : callable
1222 Function to call to perform fit (takes Associations object).
1223 tract : `str`, optional
1224 Name of tract currently being fit.
1225 match_cut : `float`, optional
1226 Radius in arcseconds to find cross-catalog matches to during
1227 associations.associateCatalogs.
1228 reject_bad_fluxes : `bool`, optional
1229 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1230 epoch : `astropy.time.Time`, optional
1231 Epoch to which to correct refcat proper motion and parallax,
1232 or `None` to not apply such corrections.
1234 Returns
1235 -------
1236 result : `Photometry` or `Astrometry`
1237 Result of `fit_function()`
1238 """
1239 self.log.info("====== Now processing %s...", name)
1240 # TODO: this should not print "trying to invert a singular transformation:"
1241 # if it does that, something's not right about the WCS...
1242 associations.associateCatalogs(match_cut)
1243 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
1244 associations.fittedStarListSize())
1246 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1247 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1248 center, radius, defaultFilter,
1249 applyColorterms=applyColorterms,
1250 epoch=epoch)
1251 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1253 associations.collectRefStars(refCat,
1254 self.config.matchCut*lsst.geom.arcseconds,
1255 fluxField,
1256 refCoordinateErr=refCoordErr,
1257 rejectBadFluxes=reject_bad_fluxes)
1258 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
1259 associations.refStarListSize())
1261 associations.prepareFittedStars(self.config.minMeasurements)
1263 self._check_star_lists(associations, name)
1264 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
1265 associations.nFittedStarsWithAssociatedRefStar())
1266 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
1267 associations.fittedStarListSize())
1268 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
1269 associations.nCcdImagesValidForFit())
1271 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1272 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1273 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1274 result = fit_function(associations, dataName)
1275 # TODO DM-12446: turn this into a "butler save" somehow.
1276 # Save reference and measurement chi2 contributions for this data
1277 if self.config.writeChi2FilesInitialFinal:
1278 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1279 result.fit.saveChi2Contributions(baseName+"{type}")
1280 self.log.info("Wrote chi2 contributions files: %s", baseName)
1282 return result
1284 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1285 applyColorterms=False, epoch=None):
1286 """Load the necessary reference catalog sources, convert fluxes to
1287 correct units, and apply color term corrections if requested.
1289 Parameters
1290 ----------
1291 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1292 The reference catalog loader to use to get the data.
1293 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1294 Source selector to apply to loaded reference catalog.
1295 center : `lsst.geom.SpherePoint`
1296 The center around which to load sources.
1297 radius : `lsst.geom.Angle`
1298 The radius around ``center`` to load sources in.
1299 filterLabel : `lsst.afw.image.FilterLabel`
1300 The camera filter to load fluxes for.
1301 applyColorterms : `bool`
1302 Apply colorterm corrections to the refcat for ``filterName``?
1303 epoch : `astropy.time.Time`, optional
1304 Epoch to which to correct refcat proper motion and parallax,
1305 or `None` to not apply such corrections.
1307 Returns
1308 -------
1309 refCat : `lsst.afw.table.SimpleCatalog`
1310 The loaded reference catalog.
1311 fluxField : `str`
1312 The name of the reference catalog flux field appropriate for ``filterName``.
1313 """
1314 skyCircle = refObjLoader.loadSkyCircle(center,
1315 radius,
1316 filterLabel.bandLabel,
1317 epoch=epoch)
1319 selected = referenceSelector.run(skyCircle.refCat)
1320 # Need memory contiguity to get reference filters as a vector.
1321 if not selected.sourceCat.isContiguous():
1322 refCat = selected.sourceCat.copy(deep=True)
1323 else:
1324 refCat = selected.sourceCat
1326 if applyColorterms:
1327 refCatName = refObjLoader.ref_dataset_name
1328 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1329 filterLabel.physicalLabel, refCatName)
1330 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1331 refCatName,
1332 doRaise=True)
1334 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1335 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1336 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1337 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1339 return refCat, skyCircle.fluxField
1341 def _check_star_lists(self, associations, name):
1342 # TODO: these should be len(blah), but we need this properly wrapped first.
1343 if associations.nCcdImagesValidForFit() == 0:
1344 raise RuntimeError('No images in the ccdImageList!')
1345 if associations.fittedStarListSize() == 0:
1346 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1347 if associations.refStarListSize() == 0:
1348 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1350 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1351 """Compute chi2, log it, validate the model, and return chi2.
1353 Parameters
1354 ----------
1355 associations : `lsst.jointcal.Associations`
1356 The star/reference star associations to fit.
1357 fit : `lsst.jointcal.FitterBase`
1358 The fitter to use for minimization.
1359 model : `lsst.jointcal.Model`
1360 The model being fit.
1361 chi2Label : `str`
1362 Label to describe the chi2 (e.g. "Initialized", "Final").
1363 writeChi2Name : `str`, optional
1364 Filename prefix to write the chi2 contributions to.
1365 Do not supply an extension: an appropriate one will be added.
1367 Returns
1368 -------
1369 chi2: `lsst.jointcal.Chi2Accumulator`
1370 The chi2 object for the current fitter and model.
1372 Raises
1373 ------
1374 FloatingPointError
1375 Raised if chi2 is infinite or NaN.
1376 ValueError
1377 Raised if the model is not valid.
1378 """
1379 if writeChi2Name is not None:
1380 fullpath = self._getDebugPath(writeChi2Name)
1381 fit.saveChi2Contributions(fullpath+"{type}")
1382 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1384 chi2 = fit.computeChi2()
1385 self.log.info("%s %s", chi2Label, chi2)
1386 self._check_stars(associations)
1387 if not np.isfinite(chi2.chi2):
1388 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1389 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1390 raise ValueError("Model is not valid: check log messages for warnings.")
1391 return chi2
1393 def _fit_photometry(self, associations, dataName=None):
1394 """
1395 Fit the photometric data.
1397 Parameters
1398 ----------
1399 associations : `lsst.jointcal.Associations`
1400 The star/reference star associations to fit.
1401 dataName : `str`
1402 Name of the data being processed (e.g. "1234_HSC-Y"), for
1403 identifying debugging files.
1405 Returns
1406 -------
1407 fit_result : `namedtuple`
1408 fit : `lsst.jointcal.PhotometryFit`
1409 The photometric fitter used to perform the fit.
1410 model : `lsst.jointcal.PhotometryModel`
1411 The photometric model that was fit.
1412 """
1413 self.log.info("=== Starting photometric fitting...")
1415 # TODO: should use pex.config.RegistryField here (see DM-9195)
1416 if self.config.photometryModel == "constrainedFlux":
1417 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1418 self.focalPlaneBBox,
1419 visitOrder=self.config.photometryVisitOrder,
1420 errorPedestal=self.config.photometryErrorPedestal)
1421 # potentially nonlinear problem, so we may need a line search to converge.
1422 doLineSearch = self.config.allowLineSearch
1423 elif self.config.photometryModel == "constrainedMagnitude":
1424 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1425 self.focalPlaneBBox,
1426 visitOrder=self.config.photometryVisitOrder,
1427 errorPedestal=self.config.photometryErrorPedestal)
1428 # potentially nonlinear problem, so we may need a line search to converge.
1429 doLineSearch = self.config.allowLineSearch
1430 elif self.config.photometryModel == "simpleFlux":
1431 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1432 errorPedestal=self.config.photometryErrorPedestal)
1433 doLineSearch = False # purely linear in model parameters, so no line search needed
1434 elif self.config.photometryModel == "simpleMagnitude":
1435 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1436 errorPedestal=self.config.photometryErrorPedestal)
1437 doLineSearch = False # purely linear in model parameters, so no line search needed
1439 fit = lsst.jointcal.PhotometryFit(associations, model)
1440 # TODO DM-12446: turn this into a "butler save" somehow.
1441 # Save reference and measurement chi2 contributions for this data
1442 if self.config.writeChi2FilesInitialFinal:
1443 baseName = f"photometry_initial_chi2-{dataName}"
1444 else:
1445 baseName = None
1446 if self.config.writeInitialModel:
1447 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt")
1448 writeModel(model, fullpath, self.log)
1449 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1451 def getChi2Name(whatToFit):
1452 if self.config.writeChi2FilesOuterLoop:
1453 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1454 else:
1455 return None
1457 # The constrained model needs the visit transform fit first; the chip
1458 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1459 if self.config.writeInitMatrix:
1460 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}")
1461 else:
1462 dumpMatrixFile = ""
1463 if self.config.photometryModel.startswith("constrained"):
1464 # no line search: should be purely (or nearly) linear,
1465 # and we want a large step size to initialize with.
1466 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1467 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1468 writeChi2Name=getChi2Name("ModelVisit"))
1469 dumpMatrixFile = "" # so we don't redo the output on the next step
1471 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1472 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1473 writeChi2Name=getChi2Name("Model"))
1475 fit.minimize("Fluxes") # no line search: always purely linear.
1476 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1477 writeChi2Name=getChi2Name("Fluxes"))
1479 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1480 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1481 writeChi2Name=getChi2Name("ModelFluxes"))
1483 model.freezeErrorTransform()
1484 self.log.debug("Photometry error scales are frozen.")
1486 chi2 = self._iterate_fit(associations,
1487 fit,
1488 self.config.maxPhotometrySteps,
1489 "photometry",
1490 "Model Fluxes",
1491 doRankUpdate=self.config.photometryDoRankUpdate,
1492 doLineSearch=doLineSearch,
1493 dataName=dataName)
1495 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1496 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1497 return Photometry(fit, model)
1499 def _fit_astrometry(self, associations, dataName=None):
1500 """
1501 Fit the astrometric data.
1503 Parameters
1504 ----------
1505 associations : `lsst.jointcal.Associations`
1506 The star/reference star associations to fit.
1507 dataName : `str`
1508 Name of the data being processed (e.g. "1234_HSC-Y"), for
1509 identifying debugging files.
1511 Returns
1512 -------
1513 fit_result : `namedtuple`
1514 fit : `lsst.jointcal.AstrometryFit`
1515 The astrometric fitter used to perform the fit.
1516 model : `lsst.jointcal.AstrometryModel`
1517 The astrometric model that was fit.
1518 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1519 The model for the sky to tangent plane projection that was used in the fit.
1520 """
1522 self.log.info("=== Starting astrometric fitting...")
1524 associations.deprojectFittedStars()
1526 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1527 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1528 # them so carefully?
1529 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1531 if self.config.astrometryModel == "constrained":
1532 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1533 sky_to_tan_projection,
1534 chipOrder=self.config.astrometryChipOrder,
1535 visitOrder=self.config.astrometryVisitOrder)
1536 elif self.config.astrometryModel == "simple":
1537 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1538 sky_to_tan_projection,
1539 self.config.useInputWcs,
1540 nNotFit=0,
1541 order=self.config.astrometrySimpleOrder)
1543 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1544 # TODO DM-12446: turn this into a "butler save" somehow.
1545 # Save reference and measurement chi2 contributions for this data
1546 if self.config.writeChi2FilesInitialFinal:
1547 baseName = f"astrometry_initial_chi2-{dataName}"
1548 else:
1549 baseName = None
1550 if self.config.writeInitialModel:
1551 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1552 writeModel(model, fullpath, self.log)
1553 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1555 def getChi2Name(whatToFit):
1556 if self.config.writeChi2FilesOuterLoop:
1557 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1558 else:
1559 return None
1561 if self.config.writeInitMatrix:
1562 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}")
1563 else:
1564 dumpMatrixFile = ""
1565 # The constrained model needs the visit transform fit first; the chip
1566 # transform is initialized from the detector's cameraGeom, so it's close.
1567 if self.config.astrometryModel == "constrained":
1568 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1569 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1570 writeChi2Name=getChi2Name("DistortionsVisit"))
1571 dumpMatrixFile = "" # so we don't redo the output on the next step
1573 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1574 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1575 writeChi2Name=getChi2Name("Distortions"))
1577 fit.minimize("Positions")
1578 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1579 writeChi2Name=getChi2Name("Positions"))
1581 fit.minimize("Distortions Positions")
1582 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1583 writeChi2Name=getChi2Name("DistortionsPositions"))
1585 chi2 = self._iterate_fit(associations,
1586 fit,
1587 self.config.maxAstrometrySteps,
1588 "astrometry",
1589 "Distortions Positions",
1590 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1591 doRankUpdate=self.config.astrometryDoRankUpdate,
1592 dataName=dataName)
1594 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1595 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1597 return Astrometry(fit, model, sky_to_tan_projection)
1599 def _check_stars(self, associations):
1600 """Count measured and reference stars per ccd and warn/log them."""
1601 for ccdImage in associations.getCcdImageList():
1602 nMeasuredStars, nRefStars = ccdImage.countStars()
1603 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1604 ccdImage.getName(), nMeasuredStars, nRefStars)
1605 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1606 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1607 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1608 if nRefStars < self.config.minRefStarsPerCcd:
1609 self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1610 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1612 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1613 dataName="",
1614 sigmaRelativeTolerance=0,
1615 doRankUpdate=True,
1616 doLineSearch=False):
1617 """Run fitter.minimize up to max_steps times, returning the final chi2.
1619 Parameters
1620 ----------
1621 associations : `lsst.jointcal.Associations`
1622 The star/reference star associations to fit.
1623 fitter : `lsst.jointcal.FitterBase`
1624 The fitter to use for minimization.
1625 max_steps : `int`
1626 Maximum number of steps to run outlier rejection before declaring
1627 convergence failure.
1628 name : {'photometry' or 'astrometry'}
1629 What type of data are we fitting (for logs and debugging files).
1630 whatToFit : `str`
1631 Passed to ``fitter.minimize()`` to define the parameters to fit.
1632 dataName : `str`, optional
1633 Descriptive name for this dataset (e.g. tract and filter),
1634 for debugging.
1635 sigmaRelativeTolerance : `float`, optional
1636 Convergence tolerance for the fractional change in the chi2 cut
1637 level for determining outliers. If set to zero, iterations will
1638 continue until there are no outliers.
1639 doRankUpdate : `bool`, optional
1640 Do an Eigen rank update during minimization, or recompute the full
1641 matrix and gradient?
1642 doLineSearch : `bool`, optional
1643 Do a line search for the optimum step during minimization?
1645 Returns
1646 -------
1647 chi2: `lsst.jointcal.Chi2Statistic`
1648 The final chi2 after the fit converges, or is forced to end.
1650 Raises
1651 ------
1652 FloatingPointError
1653 Raised if the fitter fails with a non-finite value.
1654 RuntimeError
1655 Raised if the fitter fails for some other reason;
1656 log messages will provide further details.
1657 """
1658 if self.config.writeInitMatrix:
1659 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}")
1660 else:
1661 dumpMatrixFile = ""
1662 oldChi2 = lsst.jointcal.Chi2Statistic()
1663 oldChi2.chi2 = float("inf")
1664 for i in range(max_steps):
1665 if self.config.writeChi2FilesOuterLoop:
1666 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1667 else:
1668 writeChi2Name = None
1669 result = fitter.minimize(whatToFit,
1670 self.config.outlierRejectSigma,
1671 sigmaRelativeTolerance=sigmaRelativeTolerance,
1672 doRankUpdate=doRankUpdate,
1673 doLineSearch=doLineSearch,
1674 dumpMatrixFile=dumpMatrixFile)
1675 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1676 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1677 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1679 if result == MinimizeResult.Converged:
1680 if doRankUpdate:
1681 self.log.debug("fit has converged - no more outliers - redo minimization "
1682 "one more time in case we have lost accuracy in rank update.")
1683 # Redo minimization one more time in case we have lost accuracy in rank update
1684 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1685 sigmaRelativeTolerance=sigmaRelativeTolerance)
1686 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1688 # log a message for a large final chi2, TODO: DM-15247 for something better
1689 if chi2.chi2/chi2.ndof >= 4.0:
1690 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1692 break
1693 elif result == MinimizeResult.Chi2Increased:
1694 self.log.warn("Still some outliers remaining but chi2 increased - retry")
1695 # Check whether the increase was large enough to cause trouble.
1696 chi2Ratio = chi2.chi2 / oldChi2.chi2
1697 if chi2Ratio > 1.5:
1698 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1699 chi2.chi2, oldChi2.chi2, chi2Ratio)
1700 # Based on a variety of HSC jointcal logs (see DM-25779), it
1701 # appears that chi2 increases more than a factor of ~2 always
1702 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1703 # Using 10 as the "failure" threshold gives some room between
1704 # leaving a warning and bailing early.
1705 if chi2Ratio > 10:
1706 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1707 " Try setting one or more of the `writeChi2*` config fields and looking"
1708 " at how individual star chi2-values evolve during the fit.")
1709 raise RuntimeError(msg)
1710 oldChi2 = chi2
1711 elif result == MinimizeResult.NonFinite:
1712 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1713 # TODO DM-12446: turn this into a "butler save" somehow.
1714 fitter.saveChi2Contributions(filename+"{type}")
1715 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1716 raise FloatingPointError(msg.format(filename))
1717 elif result == MinimizeResult.Failed:
1718 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1719 else:
1720 raise RuntimeError("Unxepected return code from minimize().")
1721 else:
1722 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1724 return chi2
1726 def _make_output(self, ccdImageList, model, func):
1727 """Return the internal jointcal models converted to the afw
1728 structures that will be saved to disk.
1730 Parameters
1731 ----------
1732 ccdImageList : `lsst.jointcal.CcdImageList`
1733 The list of CcdImages to get the output for.
1734 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1735 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1736 func : `str`
1737 The name of the function to call on ``model`` to get the converted
1738 structure. Must accept an `lsst.jointcal.CcdImage`.
1740 Returns
1741 -------
1742 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1743 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1744 The data to be saved, keyed on (visit, detector).
1745 """
1746 output = {}
1747 for ccdImage in ccdImageList:
1748 ccd = ccdImage.ccdId
1749 visit = ccdImage.visit
1750 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1751 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1752 return output
1754 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1755 """
1756 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1758 Parameters
1759 ----------
1760 associations : `lsst.jointcal.Associations`
1761 The star/reference star associations to fit.
1762 model : `lsst.jointcal.AstrometryModel`
1763 The astrometric model that was fit.
1764 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1765 Dict of ccdImage identifiers to dataRefs that were fit.
1766 """
1767 ccdImageList = associations.getCcdImageList()
1768 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1769 for key, skyWcs in output.items():
1770 dataRef = visit_ccd_to_dataRef[key]
1771 try:
1772 dataRef.put(skyWcs, 'jointcal_wcs')
1773 except pexExceptions.Exception as e:
1774 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1775 raise e
1777 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1778 """
1779 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1781 Parameters
1782 ----------
1783 associations : `lsst.jointcal.Associations`
1784 The star/reference star associations to fit.
1785 model : `lsst.jointcal.PhotometryModel`
1786 The photoometric model that was fit.
1787 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1788 Dict of ccdImage identifiers to dataRefs that were fit.
1789 """
1791 ccdImageList = associations.getCcdImageList()
1792 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1793 for key, photoCalib in output.items():
1794 dataRef = visit_ccd_to_dataRef[key]
1795 try:
1796 dataRef.put(photoCalib, 'jointcal_photoCalib')
1797 except pexExceptions.Exception as e:
1798 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1799 raise e