Coverage for python/lsst/jointcal/jointcal.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import dataclasses
23import collections
24import os
26import astropy.time
27import numpy as np
28import astropy.units as u
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
36import lsst.afw.cameraGeom
37import lsst.afw.table
38import lsst.log
39from lsst.obs.base import Instrument
40from lsst.pipe.tasks.colorterms import ColortermLibrary
41from lsst.verify import Job, Measurement
43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44 ReferenceObjectLoader)
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
47from .dataIds import PerTractCcdDataIdContainer
49import lsst.jointcal
50from lsst.jointcal import MinimizeResult
52__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
54Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
55Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
58# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
59def add_measurement(job, name, value):
60 meas = Measurement(job.metrics[name], value)
61 job.measurements.insert(meas)
64class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
65 """Subclass of TaskRunner for jointcalTask (gen2)
67 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
68 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
69 single dataRef, are are called repeatedly). This class transforms the processed
70 arguments generated by the ArgumentParser into the arguments expected by
71 Jointcal.runDataRef().
73 See pipeBase.TaskRunner for more information.
74 """
76 @staticmethod
77 def getTargetList(parsedCmd, **kwargs):
78 """
79 Return a list of tuples per tract, each containing (dataRefs, kwargs).
81 Jointcal operates on lists of dataRefs simultaneously.
82 """
83 kwargs['butler'] = parsedCmd.butler
85 # organize data IDs by tract
86 refListDict = {}
87 for ref in parsedCmd.id.refList:
88 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
89 # we call runDataRef() once with each tract
90 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
91 return result
93 def __call__(self, args):
94 """
95 Parameters
96 ----------
97 args
98 Arguments for Task.runDataRef()
100 Returns
101 -------
102 pipe.base.Struct
103 if self.doReturnResults is False:
105 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
107 if self.doReturnResults is True:
109 - ``result``: the result of calling jointcal.runDataRef()
110 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
111 """
112 exitStatus = 0 # exit status for shell
114 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
115 dataRefList, kwargs = args
116 butler = kwargs.pop('butler')
117 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118 result = None
119 try:
120 result = task.runDataRef(dataRefList, **kwargs)
121 exitStatus = result.exitStatus
122 job_path = butler.get('verify_job_filename')
123 result.job.write(job_path[0])
124 except Exception as e: # catch everything, sort it out later.
125 if self.doRaise:
126 raise e
127 else:
128 exitStatus = 1
129 eName = type(e).__name__
130 tract = dataRefList[0].dataId['tract']
131 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
133 # Put the butler back into kwargs for the other Tasks.
134 kwargs['butler'] = butler
135 if self.doReturnResults:
136 return pipeBase.Struct(result=result, exitStatus=exitStatus)
137 else:
138 return pipeBase.Struct(exitStatus=exitStatus)
141def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
142 """Lookup function that asserts/hopes that a static calibration dataset
143 exists in a particular collection, since this task can't provide a single
144 date/time to use to search for one properly.
146 This is mostly useful for the ``camera`` dataset, in cases where the task's
147 quantum dimensions do *not* include something temporal, like ``exposure``
148 or ``visit``.
150 Parameters
151 ----------
152 datasetType : `lsst.daf.butler.DatasetType`
153 Type of dataset being searched for.
154 registry : `lsst.daf.butler.Registry`
155 Data repository registry to search.
156 quantumDataId : `lsst.daf.butler.DataCoordinate`
157 Data ID of the quantum this camera should match.
158 collections : `Iterable` [ `str` ]
159 Collections that should be searched - but this lookup function works
160 by ignoring this in favor of a more-or-less hard-coded value.
162 Returns
163 -------
164 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
165 Iterator over dataset references; should have only one element.
167 Notes
168 -----
169 This implementation duplicates one in fgcmcal, and is at least quite
170 similar to another in cp_pipe. This duplicate has the most documentation.
171 Fixing this is DM-29661.
172 """
173 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
174 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
175 return registry.queryDatasets(datasetType,
176 dataId=quantumDataId,
177 collections=[unboundedCollection],
178 findFirst=True)
181def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
182 """Lookup function that finds all refcats for all visits that overlap a
183 tract, rather than just the refcats that directly overlap the tract.
185 Parameters
186 ----------
187 datasetType : `lsst.daf.butler.DatasetType`
188 Type of dataset being searched for.
189 registry : `lsst.daf.butler.Registry`
190 Data repository registry to search.
191 quantumDataId : `lsst.daf.butler.DataCoordinate`
192 Data ID of the quantum; expected to be something we can use as a
193 constraint to query for overlapping visits.
194 collections : `Iterable` [ `str` ]
195 Collections to search.
197 Returns
198 -------
199 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
200 Iterator over refcat references.
201 """
202 refs = set()
203 # Use .expanded() on the query methods below because we need data IDs with
204 # regions, both in the outer loop over visits (queryDatasets will expand
205 # any data ID we give it, but doing it up-front in bulk is much more
206 # efficient) and in the data IDs of the DatasetRefs this function yields
207 # (because the RefCatLoader relies on them to do some of its own
208 # filtering).
209 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
210 refs.update(
211 registry.queryDatasets(
212 datasetType,
213 collections=collections,
214 dataId=visit_data_id,
215 findFirst=True,
216 ).expanded()
217 )
218 yield from refs
221class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
222 dimensions=("skymap", "tract", "instrument", "physical_filter")):
223 """Middleware input/output connections for jointcal data."""
224 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
225 doc="The camera instrument that took these observations.",
226 name="camera",
227 storageClass="Camera",
228 dimensions=("instrument",),
229 isCalibration=True,
230 lookupFunction=lookupStaticCalibrations,
231 )
232 inputSourceTableVisit = pipeBase.connectionTypes.Input(
233 doc="Source table in parquet format, per visit",
234 name="sourceTable_visit",
235 storageClass="DataFrame",
236 dimensions=("instrument", "visit"),
237 deferLoad=True,
238 multiple=True,
239 )
240 inputVisitSummary = pipeBase.connectionTypes.Input(
241 doc=("Per-visit consolidated exposure metadata built from calexps. "
242 "These catalogs use detector id for the id and must be sorted for "
243 "fast lookups of a detector."),
244 name="visitSummary",
245 storageClass="ExposureCatalog",
246 dimensions=("instrument", "visit"),
247 deferLoad=True,
248 multiple=True,
249 )
250 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
251 doc="The astrometry reference catalog to match to loaded input catalog sources.",
252 name="gaia_dr2_20200414",
253 storageClass="SimpleCatalog",
254 dimensions=("skypix",),
255 deferLoad=True,
256 multiple=True,
257 lookupFunction=lookupVisitRefCats,
258 )
259 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
260 doc="The photometry reference catalog to match to loaded input catalog sources.",
261 name="ps1_pv3_3pi_20170110",
262 storageClass="SimpleCatalog",
263 dimensions=("skypix",),
264 deferLoad=True,
265 multiple=True,
266 lookupFunction=lookupVisitRefCats,
267 )
269 outputWcs = pipeBase.connectionTypes.Output(
270 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
271 " These catalogs only contain entries for detectors with an output, and use"
272 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
273 name="jointcalSkyWcsCatalog",
274 storageClass="ExposureCatalog",
275 dimensions=("instrument", "visit", "skymap", "tract"),
276 multiple=True
277 )
278 outputPhotoCalib = pipeBase.connectionTypes.Output(
279 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
280 " These catalogs only contain entries for detectors with an output, and use"
281 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
282 name="jointcalPhotoCalibCatalog",
283 storageClass="ExposureCatalog",
284 dimensions=("instrument", "visit", "skymap", "tract"),
285 multiple=True
286 )
288 def __init__(self, *, config=None):
289 super().__init__(config=config)
290 # When we are only doing one of astrometry or photometry, we don't
291 # need the reference catalog or produce the outputs for the other.
292 # This informs the middleware of that when the QuantumGraph is
293 # generated, so we don't block on getting something we won't need or
294 # create an expectation that downstream tasks will be able to consume
295 # something we won't produce.
296 if not config.doAstrometry:
297 self.prerequisiteInputs.remove("astrometryRefCat")
298 self.outputs.remove("outputWcs")
299 if not config.doPhotometry:
300 self.prerequisiteInputs.remove("photometryRefCat")
301 self.outputs.remove("outputPhotoCalib")
304class JointcalConfig(pipeBase.PipelineTaskConfig,
305 pipelineConnections=JointcalTaskConnections):
306 """Configuration for JointcalTask"""
308 doAstrometry = pexConfig.Field(
309 doc="Fit astrometry and write the fitted result.",
310 dtype=bool,
311 default=True
312 )
313 doPhotometry = pexConfig.Field(
314 doc="Fit photometry and write the fitted result.",
315 dtype=bool,
316 default=True
317 )
318 coaddName = pexConfig.Field(
319 doc="Type of coadd, typically deep or goodSeeing",
320 dtype=str,
321 default="deep"
322 )
323 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
324 sourceFluxType = pexConfig.Field(
325 dtype=str,
326 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
327 default='Calib'
328 )
329 positionErrorPedestal = pexConfig.Field(
330 doc="Systematic term to apply to the measured position error (pixels)",
331 dtype=float,
332 default=0.02,
333 )
334 photometryErrorPedestal = pexConfig.Field(
335 doc="Systematic term to apply to the measured error on flux or magnitude as a "
336 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
337 dtype=float,
338 default=0.0,
339 )
340 # TODO: DM-6885 matchCut should be an geom.Angle
341 matchCut = pexConfig.Field(
342 doc="Matching radius between fitted and reference stars (arcseconds)",
343 dtype=float,
344 default=3.0,
345 )
346 minMeasurements = pexConfig.Field(
347 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
348 dtype=int,
349 default=2,
350 )
351 minMeasuredStarsPerCcd = pexConfig.Field(
352 doc="Minimum number of measuredStars per ccdImage before printing warnings",
353 dtype=int,
354 default=100,
355 )
356 minRefStarsPerCcd = pexConfig.Field(
357 doc="Minimum number of measuredStars per ccdImage before printing warnings",
358 dtype=int,
359 default=30,
360 )
361 allowLineSearch = pexConfig.Field(
362 doc="Allow a line search during minimization, if it is reasonable for the model"
363 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
364 dtype=bool,
365 default=False
366 )
367 astrometrySimpleOrder = pexConfig.Field(
368 doc="Polynomial order for fitting the simple astrometry model.",
369 dtype=int,
370 default=3,
371 )
372 astrometryChipOrder = pexConfig.Field(
373 doc="Order of the per-chip transform for the constrained astrometry model.",
374 dtype=int,
375 default=1,
376 )
377 astrometryVisitOrder = pexConfig.Field(
378 doc="Order of the per-visit transform for the constrained astrometry model.",
379 dtype=int,
380 default=5,
381 )
382 useInputWcs = pexConfig.Field(
383 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
384 dtype=bool,
385 default=True,
386 )
387 astrometryModel = pexConfig.ChoiceField(
388 doc="Type of model to fit to astrometry",
389 dtype=str,
390 default="constrained",
391 allowed={"simple": "One polynomial per ccd",
392 "constrained": "One polynomial per ccd, and one polynomial per visit"}
393 )
394 photometryModel = pexConfig.ChoiceField(
395 doc="Type of model to fit to photometry",
396 dtype=str,
397 default="constrainedMagnitude",
398 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
399 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
400 " fitting in flux space.",
401 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
402 " fitting in magnitude space.",
403 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
404 " fitting in magnitude space.",
405 }
406 )
407 applyColorTerms = pexConfig.Field(
408 doc="Apply photometric color terms to reference stars?"
409 "Requires that colorterms be set to a ColortermLibrary",
410 dtype=bool,
411 default=False
412 )
413 colorterms = pexConfig.ConfigField(
414 doc="Library of photometric reference catalog name to color term dict.",
415 dtype=ColortermLibrary,
416 )
417 photometryVisitOrder = pexConfig.Field(
418 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
419 dtype=int,
420 default=7,
421 )
422 photometryDoRankUpdate = pexConfig.Field(
423 doc=("Do the rank update step during minimization. "
424 "Skipping this can help deal with models that are too non-linear."),
425 dtype=bool,
426 default=True,
427 )
428 astrometryDoRankUpdate = pexConfig.Field(
429 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
430 "Skipping this can help deal with models that are too non-linear."),
431 dtype=bool,
432 default=True,
433 )
434 outlierRejectSigma = pexConfig.Field(
435 doc="How many sigma to reject outliers at during minimization.",
436 dtype=float,
437 default=5.0,
438 )
439 astrometryOutlierRelativeTolerance = pexConfig.Field(
440 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
441 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
442 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
443 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
444 "close to the solution found when removing all outliers."),
445 dtype=float,
446 default=0,
447 )
448 maxPhotometrySteps = pexConfig.Field(
449 doc="Maximum number of minimize iterations to take when fitting photometry.",
450 dtype=int,
451 default=20,
452 )
453 maxAstrometrySteps = pexConfig.Field(
454 doc="Maximum number of minimize iterations to take when fitting photometry.",
455 dtype=int,
456 default=20,
457 )
458 astrometryRefObjLoader = pexConfig.ConfigurableField(
459 target=LoadIndexedReferenceObjectsTask,
460 doc="Reference object loader for astrometric fit",
461 )
462 photometryRefObjLoader = pexConfig.ConfigurableField(
463 target=LoadIndexedReferenceObjectsTask,
464 doc="Reference object loader for photometric fit",
465 )
466 sourceSelector = sourceSelectorRegistry.makeField(
467 doc="How to select sources for cross-matching",
468 default="astrometry"
469 )
470 astrometryReferenceSelector = pexConfig.ConfigurableField(
471 target=ReferenceSourceSelectorTask,
472 doc="How to down-select the loaded astrometry reference catalog.",
473 )
474 photometryReferenceSelector = pexConfig.ConfigurableField(
475 target=ReferenceSourceSelectorTask,
476 doc="How to down-select the loaded photometry reference catalog.",
477 )
478 astrometryReferenceErr = pexConfig.Field(
479 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
480 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
481 "If specified, overrides any existing `coord_*Err` values."),
482 dtype=float,
483 default=None,
484 optional=True
485 )
487 # configs for outputting debug information
488 writeInitMatrix = pexConfig.Field(
489 dtype=bool,
490 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
491 "Output files will be written to `config.debugOutputPath` and will "
492 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
493 "Note that these files are the dense versions of the matrix, and so may be very large."),
494 default=False
495 )
496 writeChi2FilesInitialFinal = pexConfig.Field(
497 dtype=bool,
498 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
499 "Output files will be written to `config.debugOutputPath` and will "
500 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER1."),
501 default=False
502 )
503 writeChi2FilesOuterLoop = pexConfig.Field(
504 dtype=bool,
505 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
506 "Output files will be written to `config.debugOutputPath` and will "
507 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
508 default=False
509 )
510 writeInitialModel = pexConfig.Field(
511 dtype=bool,
512 doc=("Write the pre-initialization model to text files, for debugging. "
513 "Output files will be written to `config.debugOutputPath` and will be "
514 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
515 default=False
516 )
517 debugOutputPath = pexConfig.Field(
518 dtype=str,
519 default=".",
520 doc=("Path to write debug output files to. Used by "
521 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
522 )
523 detailedProfile = pexConfig.Field(
524 dtype=bool,
525 default=False,
526 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
527 )
529 def validate(self):
530 super().validate()
531 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
532 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
533 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
534 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
535 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
536 "applyColorTerms=True will be ignored.")
537 lsst.log.warn(msg)
539 def setDefaults(self):
540 # Use science source selector which can filter on extendedness, SNR, and whether blended
541 self.sourceSelector.name = 'science'
542 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
543 self.sourceSelector['science'].doUnresolved = True
544 # with dependable signal to noise ratio.
545 self.sourceSelector['science'].doSignalToNoise = True
546 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
547 # and S/N > 10 to use sources that are not too faint, and thus better measured.
548 self.sourceSelector['science'].signalToNoise.minimum = 10.
549 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
550 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
551 self.sourceSelector['science'].signalToNoise.fluxField = fluxField
552 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err"
553 # Do not trust blended sources' aperture fluxes which also depend on seeing
554 self.sourceSelector['science'].doIsolated = True
555 # Do not trust either flux or centroid measurements with flags,
556 # chosen from the usual QA flags for stars)
557 self.sourceSelector['science'].doFlags = True
558 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
559 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
560 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
561 self.sourceSelector['science'].flags.bad = badFlags
563 # Default to Gaia-DR2 (with proper motions) for astrometry and
564 # PS1-DR1 for photometry, with a reasonable initial filterMap.
565 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
566 self.astrometryRefObjLoader.requireProperMotion = True
567 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
568 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
571def writeModel(model, filename, log):
572 """Write model to outfile."""
573 with open(filename, "w") as file:
574 file.write(repr(model))
575 log.info("Wrote %s to file: %s", model, filename)
578@dataclasses.dataclass
579class JointcalInputData:
580 """The input data jointcal needs for each detector/visit."""
581 visit: int
582 """The visit identifier of this exposure."""
583 catalog: lsst.afw.table.SourceCatalog
584 """The catalog derived from this exposure."""
585 visitInfo: lsst.afw.image.VisitInfo
586 """The VisitInfo of this exposure."""
587 detector: lsst.afw.cameraGeom.Detector
588 """The detector of this exposure."""
589 photoCalib: lsst.afw.image.PhotoCalib
590 """The photometric calibration of this exposure."""
591 wcs: lsst.afw.geom.skyWcs
592 """The WCS of this exposure."""
593 bbox: lsst.geom.Box2I
594 """The bounding box of this exposure."""
595 filter: lsst.afw.image.FilterLabel
596 """The filter of this exposure."""
599class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
600 """Astrometricly and photometricly calibrate across multiple visits of the
601 same field.
603 Parameters
604 ----------
605 butler : `lsst.daf.persistence.Butler`
606 The butler is passed to the refObjLoader constructor in case it is
607 needed. Ignored if the refObjLoader argument provides a loader directly.
608 Used to initialize the astrometry and photometry refObjLoaders.
609 initInputs : `dict`, optional
610 Dictionary used to initialize PipelineTasks (empty for jointcal).
611 """
613 ConfigClass = JointcalConfig
614 RunnerClass = JointcalRunner
615 _DefaultName = "jointcal"
617 def __init__(self, butler=None, initInputs=None, **kwargs):
618 super().__init__(**kwargs)
619 self.makeSubtask("sourceSelector")
620 if self.config.doAstrometry:
621 if initInputs is None:
622 # gen3 middleware does refcat things internally (and will not have a butler here)
623 self.makeSubtask('astrometryRefObjLoader', butler=butler)
624 self.makeSubtask("astrometryReferenceSelector")
625 else:
626 self.astrometryRefObjLoader = None
627 if self.config.doPhotometry:
628 if initInputs is None:
629 # gen3 middleware does refcat things internally (and will not have a butler here)
630 self.makeSubtask('photometryRefObjLoader', butler=butler)
631 self.makeSubtask("photometryReferenceSelector")
632 else:
633 self.photometryRefObjLoader = None
635 # To hold various computed metrics for use by tests
636 self.job = Job.load_metrics_package(subset='jointcal')
638 def runQuantum(self, butlerQC, inputRefs, outputRefs):
639 # We override runQuantum to set up the refObjLoaders and write the
640 # outputs to the correct refs.
641 inputs = butlerQC.get(inputRefs)
642 # We want the tract number for writing debug files
643 tract = butlerQC.quantum.dataId['tract']
644 if self.config.doAstrometry:
645 self.astrometryRefObjLoader = ReferenceObjectLoader(
646 dataIds=[ref.datasetRef.dataId
647 for ref in inputRefs.astrometryRefCat],
648 refCats=inputs.pop('astrometryRefCat'),
649 config=self.config.astrometryRefObjLoader,
650 log=self.log)
651 if self.config.doPhotometry:
652 self.photometryRefObjLoader = ReferenceObjectLoader(
653 dataIds=[ref.datasetRef.dataId
654 for ref in inputRefs.photometryRefCat],
655 refCats=inputs.pop('photometryRefCat'),
656 config=self.config.photometryRefObjLoader,
657 log=self.log)
658 outputs = self.run(**inputs, tract=tract)
659 if self.config.doAstrometry:
660 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
661 inputs['inputCamera'], "setWcs")
662 if self.config.doPhotometry:
663 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
664 inputs['inputCamera'], "setPhotoCalib")
666 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
667 """Persist the output datasets to their appropriate datarefs.
669 Parameters
670 ----------
671 butlerQC : `ButlerQuantumContext`
672 A butler which is specialized to operate in the context of a
673 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
674 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
675 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
676 The fitted objects to persist.
677 outputRefs : `list` [`OutputQuantizedConnection`]
678 The DatasetRefs to persist the data to.
679 camera : `lsst.afw.cameraGeom.Camera`
680 The camera for this instrument, to get detector ids from.
681 setter : `str`
682 The method to call on the ExposureCatalog to set each output.
683 """
684 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
685 schema.addField('visit', type='I', doc='Visit number')
687 def new_catalog(visit, size):
688 """Return an catalog ready to be filled with appropriate output."""
689 catalog = lsst.afw.table.ExposureCatalog(schema)
690 catalog.resize(size)
691 catalog['visit'] = visit
692 metadata = lsst.daf.base.PropertyList()
693 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
694 metadata.add("COMMENT", "Only detectors with data have entries.")
695 return catalog
697 # count how many detectors have output for each visit
698 detectors_per_visit = collections.defaultdict(int)
699 for key in outputs:
700 # key is (visit, detector_id), and we only need visit here
701 detectors_per_visit[key[0]] += 1
703 for ref in outputRefs:
704 visit = ref.dataId['visit']
705 catalog = new_catalog(visit, detectors_per_visit[visit])
706 # Iterate over every detector and skip the ones we don't have output for.
707 i = 0
708 for detector in camera:
709 detectorId = detector.getId()
710 key = (ref.dataId['visit'], detectorId)
711 if key not in outputs:
712 # skip detectors we don't have output for
713 self.log.debug("No %s output for detector %s in visit %s",
714 setter[3:], detectorId, visit)
715 continue
717 catalog[i].setId(detectorId)
718 getattr(catalog[i], setter)(outputs[key])
719 i += 1
721 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
722 butlerQC.put(catalog, ref)
723 self.log.info("Wrote %s detectors to %s", i, ref)
725 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
726 # Docstring inherited.
728 # We take values out of the Parquet table, and put them in "flux_",
729 # and the config.sourceFluxType field is used during that extraction,
730 # so just use "flux" here.
731 sourceFluxField = "flux"
732 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
733 associations = lsst.jointcal.Associations()
734 self.focalPlaneBBox = inputCamera.getFpBBox()
735 oldWcsList, bands = self._load_data(inputSourceTableVisit,
736 inputVisitSummary,
737 associations,
738 jointcalControl,
739 inputCamera)
741 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, bands)
742 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
744 if self.config.doAstrometry:
745 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
746 name="astrometry",
747 refObjLoader=self.astrometryRefObjLoader,
748 referenceSelector=self.astrometryReferenceSelector,
749 fit_function=self._fit_astrometry,
750 tract=tract,
751 epoch=epoch)
752 astrometry_output = self._make_output(associations.getCcdImageList(),
753 astrometry.model,
754 "makeSkyWcs")
755 else:
756 astrometry_output = None
758 if self.config.doPhotometry:
759 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
760 name="photometry",
761 refObjLoader=self.photometryRefObjLoader,
762 referenceSelector=self.photometryReferenceSelector,
763 fit_function=self._fit_photometry,
764 tract=tract,
765 epoch=epoch,
766 reject_bad_fluxes=True)
767 photometry_output = self._make_output(associations.getCcdImageList(),
768 photometry.model,
769 "toPhotoCalib")
770 else:
771 photometry_output = None
773 return pipeBase.Struct(outputWcs=astrometry_output,
774 outputPhotoCalib=photometry_output,
775 job=self.job,
776 astrometryRefObjLoader=self.astrometryRefObjLoader,
777 photometryRefObjLoader=self.photometryRefObjLoader)
779 def _make_schema_table(self):
780 """Return an afw SourceTable to use as a base for creating the
781 SourceCatalog to insert values from the dataFrame into.
783 Returns
784 -------
785 table : `lsst.afw.table.SourceTable`
786 Table with schema and slots to use to make SourceCatalogs.
787 """
788 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
789 schema.addField("centroid_x", "D")
790 schema.addField("centroid_y", "D")
791 schema.addField("centroid_xErr", "F")
792 schema.addField("centroid_yErr", "F")
793 schema.addField("shape_xx", "D")
794 schema.addField("shape_yy", "D")
795 schema.addField("shape_xy", "D")
796 schema.addField("flux_instFlux", "D")
797 schema.addField("flux_instFluxErr", "D")
798 table = lsst.afw.table.SourceTable.make(schema)
799 table.defineCentroid("centroid")
800 table.defineShape("shape")
801 return table
803 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId):
804 """Return an afw SourceCatalog extracted from a visit-level dataframe,
805 limited to just one detector.
807 Parameters
808 ----------
809 table : `lsst.afw.table.SourceTable`
810 Table factory to use to make the SourceCatalog that will be
811 populated with data from ``visitCatalog``.
812 visitCatalog : `pandas.DataFrame`
813 DataFrame to extract a detector catalog from.
814 detectorId : `int`
815 Numeric id of the detector to extract from ``visitCatalog``.
817 Returns
818 -------
819 catalog : `lsst.afw.table.SourceCatalog`
820 Detector-level catalog extracted from ``visitCatalog``.
821 """
822 # map from dataFrame column to afw table column
823 mapping = {'sourceId': 'id',
824 'x': 'centroid_x',
825 'y': 'centroid_y',
826 'xErr': 'centroid_xErr',
827 'yErr': 'centroid_yErr',
828 'Ixx': 'shape_xx',
829 'Iyy': 'shape_yy',
830 'Ixy': 'shape_xy',
831 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
832 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
833 }
834 # If the DataFrame we're reading was generated by a task running with
835 # Gen2 middleware, the column for the detector will be "ccd" for at
836 # least HSC (who knows what it might be in general!); that would be
837 # true even if the data repo is later converted to Gen3, because that
838 # doesn't touch the files themselves. In Gen3, the column for the
839 # detector will always be "detector".
840 detector_column = "detector" if "detector" in visitCatalog.columns else "ccd"
841 catalog = lsst.afw.table.SourceCatalog(table)
842 matched = visitCatalog[detector_column] == detectorId
843 catalog.resize(sum(matched))
844 view = visitCatalog.loc[matched]
845 for dfCol, afwCol in mapping.items():
846 catalog[afwCol] = view[dfCol]
848 self.log.debug("%d sources selected in visit %d detector %d",
849 len(catalog),
850 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
851 detectorId)
852 return catalog
854 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
855 jointcalControl, camera):
856 """Read the data that jointcal needs to run. (Gen3 version)
858 Modifies ``associations`` in-place with the loaded data.
860 Parameters
861 ----------
862 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
863 References to visit-level DataFrames to load the catalog data from.
864 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
865 Visit-level exposure summary catalog with metadata.
866 associations : `lsst.jointcal.Associations`
867 Object to add the loaded data to by constructing new CcdImages.
868 jointcalControl : `jointcal.JointcalControl`
869 Control object for C++ associations management.
870 camera : `lsst.afw.cameraGeom.Camera`
871 Camera object for detector geometry.
873 Returns
874 -------
875 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
876 The original WCS of the input data, to aid in writing tests.
877 bands : `list` [`str`]
878 The filter bands of each input dataset.
879 """
880 oldWcsList = []
881 filters = []
882 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
883 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
884 table = self._make_schema_table() # every detector catalog has the same layout
885 # No guarantee that the input is in the same order of visits, so we have to map one of them.
886 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
887 detectorDict = {detector.getId(): detector for detector in camera}
889 for visitSummaryRef in inputVisitSummary:
890 visitSummary = visitSummaryRef.get()
891 visitCatalog = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]].get()
892 selected = self.sourceSelector.run(visitCatalog)
894 # Build a CcdImage for each detector in this visit.
895 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
896 for id, index in detectors.items():
897 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id)
898 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
899 result = self._build_ccdImage(data, associations, jointcalControl)
900 if result is not None:
901 oldWcsList.append(result.wcs)
902 # A visit has only one band, so we can just use the first.
903 filters.append(data.filter)
904 if len(filters) == 0:
905 raise RuntimeError("No data to process: did source selector remove all sources?")
906 filters = collections.Counter(filters)
908 return oldWcsList, filters
910 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
911 """Return a data structure for this detector+visit."""
912 return JointcalInputData(visit=visitRecord['visit'],
913 catalog=catalog,
914 visitInfo=visitRecord.getVisitInfo(),
915 detector=detectorDict[visitRecord.getId()],
916 photoCalib=visitRecord.getPhotoCalib(),
917 wcs=visitRecord.getWcs(),
918 bbox=visitRecord.getBBox(),
919 # ExposureRecord doesn't have a FilterLabel yet,
920 # so we have to make one.
921 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
922 physical=visitRecord['physical_filter']))
924 # We don't currently need to persist the metadata.
925 # If we do in the future, we will have to add appropriate dataset templates
926 # to each obs package (the metadata template should look like `jointcal_wcs`).
927 def _getMetadataName(self):
928 return None
930 @classmethod
931 def _makeArgumentParser(cls):
932 """Create an argument parser"""
933 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
934 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
935 ContainerClass=PerTractCcdDataIdContainer)
936 return parser
938 def _build_ccdImage(self, data, associations, jointcalControl):
939 """
940 Extract the necessary things from this catalog+metadata to add a new
941 ccdImage.
943 Parameters
944 ----------
945 data : `JointcalInputData`
946 The loaded input data.
947 associations : `lsst.jointcal.Associations`
948 Object to add the info to, to construct a new CcdImage
949 jointcalControl : `jointcal.JointcalControl`
950 Control object for associations management
952 Returns
953 ------
954 namedtuple or `None`
955 ``wcs``
956 The TAN WCS of this image, read from the calexp
957 (`lsst.afw.geom.SkyWcs`).
958 ``key``
959 A key to identify this dataRef by its visit and ccd ids
960 (`namedtuple`).
961 `None`
962 if there are no sources in the loaded catalog.
963 """
964 if len(data.catalog) == 0:
965 self.log.warn("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
966 return None
968 associations.createCcdImage(data.catalog,
969 data.wcs,
970 data.visitInfo,
971 data.bbox,
972 data.filter.physicalLabel,
973 data.photoCalib,
974 data.detector,
975 data.visit,
976 data.detector.getId(),
977 jointcalControl)
979 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
980 Key = collections.namedtuple('Key', ('visit', 'ccd'))
981 return Result(data.wcs, Key(data.visit, data.detector.getId()))
983 def _readDataId(self, butler, dataId):
984 """Read all of the data for one dataId from the butler. (gen2 version)"""
985 # Not all instruments have `visit` in their dataIds.
986 if "visit" in dataId.keys():
987 visit = dataId["visit"]
988 else:
989 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
990 detector = butler.get('calexp_detector', dataId=dataId)
992 catalog = butler.get('src',
993 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
994 dataId=dataId)
995 goodSrc = self.sourceSelector.run(catalog)
996 self.log.debug("%d sources selected in visit %d detector %d",
997 len(goodSrc.sourceCat),
998 visit,
999 detector.getId())
1000 return JointcalInputData(visit=visit,
1001 catalog=goodSrc.sourceCat,
1002 visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1003 detector=detector,
1004 photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1005 wcs=butler.get('calexp_wcs', dataId=dataId),
1006 bbox=butler.get('calexp_bbox', dataId=dataId),
1007 filter=butler.get('calexp_filterLabel', dataId=dataId))
1009 def loadData(self, dataRefs, associations, jointcalControl):
1010 """Read the data that jointcal needs to run. (Gen2 version)"""
1011 visit_ccd_to_dataRef = {}
1012 oldWcsList = []
1013 filters = []
1014 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1015 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1016 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1017 camera = dataRefs[0].get('camera', immediate=True)
1018 self.focalPlaneBBox = camera.getFpBBox()
1019 for dataRef in dataRefs:
1020 data = self._readDataId(dataRef.getButler(), dataRef.dataId)
1021 result = self._build_ccdImage(data, associations, jointcalControl)
1022 if result is None:
1023 continue
1024 oldWcsList.append(result.wcs)
1025 visit_ccd_to_dataRef[result.key] = dataRef
1026 filters.append(data.filter)
1027 if len(filters) == 0:
1028 raise RuntimeError("No data to process: did source selector remove all sources?")
1029 filters = collections.Counter(filters)
1031 return oldWcsList, filters, visit_ccd_to_dataRef
1033 def _getDebugPath(self, filename):
1034 """Constructs a path to filename using the configured debug path.
1035 """
1036 return os.path.join(self.config.debugOutputPath, filename)
1038 def _prep_sky(self, associations, filters):
1039 """Prepare on-sky and other data that must be computed after data has
1040 been read.
1041 """
1042 associations.computeCommonTangentPoint()
1044 boundingCircle = associations.computeBoundingCircle()
1045 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1046 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1048 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1050 # Determine a default filter band associated with the catalog. See DM-9093
1051 defaultFilter = filters.most_common(1)[0][0]
1052 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1054 return boundingCircle, center, radius, defaultFilter
1056 @pipeBase.timeMethod
1057 def runDataRef(self, dataRefs):
1058 """
1059 Jointly calibrate the astrometry and photometry across a set of images.
1061 NOTE: this is for gen2 middleware only.
1063 Parameters
1064 ----------
1065 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1066 List of data references to the exposures to be fit.
1068 Returns
1069 -------
1070 result : `lsst.pipe.base.Struct`
1071 Struct of metadata from the fit, containing:
1073 ``dataRefs``
1074 The provided data references that were fit (with updated WCSs)
1075 ``oldWcsList``
1076 The original WCS from each dataRef
1077 ``metrics``
1078 Dictionary of internally-computed metrics for testing/validation.
1079 """
1080 if len(dataRefs) == 0:
1081 raise ValueError('Need a non-empty list of data references!')
1083 exitStatus = 0 # exit status for shell
1085 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1086 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1087 associations = lsst.jointcal.Associations()
1089 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs,
1090 associations,
1091 jointcalControl)
1093 boundingCircle, center, radius, defaultFilter = self._prep_sky(associations, filters)
1094 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
1096 tract = dataRefs[0].dataId['tract']
1098 if self.config.doAstrometry:
1099 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1100 name="astrometry",
1101 refObjLoader=self.astrometryRefObjLoader,
1102 referenceSelector=self.astrometryReferenceSelector,
1103 fit_function=self._fit_astrometry,
1104 tract=tract,
1105 epoch=epoch)
1106 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1107 else:
1108 astrometry = Astrometry(None, None, None)
1110 if self.config.doPhotometry:
1111 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1112 name="photometry",
1113 refObjLoader=self.photometryRefObjLoader,
1114 referenceSelector=self.photometryReferenceSelector,
1115 fit_function=self._fit_photometry,
1116 tract=tract,
1117 epoch=epoch,
1118 reject_bad_fluxes=True)
1119 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1120 else:
1121 photometry = Photometry(None, None)
1123 return pipeBase.Struct(dataRefs=dataRefs,
1124 oldWcsList=oldWcsList,
1125 job=self.job,
1126 astrometryRefObjLoader=self.astrometryRefObjLoader,
1127 photometryRefObjLoader=self.photometryRefObjLoader,
1128 defaultFilter=defaultFilter,
1129 epoch=epoch,
1130 exitStatus=exitStatus)
1132 def _get_refcat_coordinate_error_override(self, refCat, name):
1133 """Check whether we should override the refcat coordinate errors, and
1134 return the overridden error if necessary.
1136 Parameters
1137 ----------
1138 refCat : `lsst.afw.table.SimpleCatalog`
1139 The reference catalog to check for a ``coord_raErr`` field.
1140 name : `str`
1141 Whether we are doing "astrometry" or "photometry".
1143 Returns
1144 -------
1145 refCoordErr : `float`
1146 The refcat coordinate error to use, or NaN if we are not overriding
1147 those fields.
1149 Raises
1150 ------
1151 lsst.pex.config.FieldValidationError
1152 Raised if the refcat does not contain coordinate errors and
1153 ``config.astrometryReferenceErr`` is not set.
1154 """
1155 # This value doesn't matter for photometry, so just set something to
1156 # keep old refcats from causing problems.
1157 if name.lower() == "photometry":
1158 if 'coord_raErr' not in refCat.schema:
1159 return 100
1160 else:
1161 return float('nan')
1163 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1164 msg = ("Reference catalog does not contain coordinate errors, "
1165 "and config.astrometryReferenceErr not supplied.")
1166 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1167 self.config,
1168 msg)
1170 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1171 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1172 self.config.astrometryReferenceErr)
1174 if self.config.astrometryReferenceErr is None:
1175 return float('nan')
1176 else:
1177 return self.config.astrometryReferenceErr
1179 def _compute_proper_motion_epoch(self, ccdImageList):
1180 """Return the proper motion correction epoch of the provided images.
1182 Parameters
1183 ----------
1184 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1185 The images to compute the appropriate epoch for.
1187 Returns
1188 -------
1189 epoch : `astropy.time.Time`
1190 The date to use for proper motion corrections.
1191 """
1192 mjds = [ccdImage.getMjd() for ccdImage in ccdImageList]
1193 return astropy.time.Time(np.mean(mjds), format='mjd', scale="tai")
1195 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1196 tract="", match_cut=3.0,
1197 reject_bad_fluxes=False, *,
1198 name="", refObjLoader=None, referenceSelector=None,
1199 fit_function=None, epoch=None):
1200 """Load reference catalog, perform the fit, and return the result.
1202 Parameters
1203 ----------
1204 associations : `lsst.jointcal.Associations`
1205 The star/reference star associations to fit.
1206 defaultFilter : `lsst.afw.image.FilterLabel`
1207 filter to load from reference catalog.
1208 center : `lsst.geom.SpherePoint`
1209 ICRS center of field to load from reference catalog.
1210 radius : `lsst.geom.Angle`
1211 On-sky radius to load from reference catalog.
1212 name : `str`
1213 Name of thing being fit: "astrometry" or "photometry".
1214 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1215 Reference object loader to use to load a reference catalog.
1216 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1217 Selector to use to pick objects from the loaded reference catalog.
1218 fit_function : callable
1219 Function to call to perform fit (takes Associations object).
1220 tract : `str`, optional
1221 Name of tract currently being fit.
1222 match_cut : `float`, optional
1223 Radius in arcseconds to find cross-catalog matches to during
1224 associations.associateCatalogs.
1225 reject_bad_fluxes : `bool`, optional
1226 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1227 epoch : `astropy.time.Time`, optional
1228 Epoch to which to correct refcat proper motion and parallax,
1229 or `None` to not apply such corrections.
1231 Returns
1232 -------
1233 result : `Photometry` or `Astrometry`
1234 Result of `fit_function()`
1235 """
1236 self.log.info("====== Now processing %s...", name)
1237 # TODO: this should not print "trying to invert a singular transformation:"
1238 # if it does that, something's not right about the WCS...
1239 associations.associateCatalogs(match_cut)
1240 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
1241 associations.fittedStarListSize())
1243 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1244 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1245 center, radius, defaultFilter,
1246 applyColorterms=applyColorterms,
1247 epoch=epoch)
1248 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1250 associations.collectRefStars(refCat,
1251 self.config.matchCut*lsst.geom.arcseconds,
1252 fluxField,
1253 refCoordinateErr=refCoordErr,
1254 rejectBadFluxes=reject_bad_fluxes)
1255 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
1256 associations.refStarListSize())
1258 associations.prepareFittedStars(self.config.minMeasurements)
1260 self._check_star_lists(associations, name)
1261 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
1262 associations.nFittedStarsWithAssociatedRefStar())
1263 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
1264 associations.fittedStarListSize())
1265 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
1266 associations.nCcdImagesValidForFit())
1268 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1269 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1270 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1271 result = fit_function(associations, dataName)
1272 # TODO DM-12446: turn this into a "butler save" somehow.
1273 # Save reference and measurement chi2 contributions for this data
1274 if self.config.writeChi2FilesInitialFinal:
1275 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1276 result.fit.saveChi2Contributions(baseName+"{type}")
1277 self.log.info("Wrote chi2 contributions files: %s", baseName)
1279 return result
1281 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1282 applyColorterms=False, epoch=None):
1283 """Load the necessary reference catalog sources, convert fluxes to
1284 correct units, and apply color term corrections if requested.
1286 Parameters
1287 ----------
1288 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1289 The reference catalog loader to use to get the data.
1290 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1291 Source selector to apply to loaded reference catalog.
1292 center : `lsst.geom.SpherePoint`
1293 The center around which to load sources.
1294 radius : `lsst.geom.Angle`
1295 The radius around ``center`` to load sources in.
1296 filterLabel : `lsst.afw.image.FilterLabel`
1297 The camera filter to load fluxes for.
1298 applyColorterms : `bool`
1299 Apply colorterm corrections to the refcat for ``filterName``?
1300 epoch : `astropy.time.Time`, optional
1301 Epoch to which to correct refcat proper motion and parallax,
1302 or `None` to not apply such corrections.
1304 Returns
1305 -------
1306 refCat : `lsst.afw.table.SimpleCatalog`
1307 The loaded reference catalog.
1308 fluxField : `str`
1309 The name of the reference catalog flux field appropriate for ``filterName``.
1310 """
1311 skyCircle = refObjLoader.loadSkyCircle(center,
1312 radius,
1313 filterLabel.bandLabel,
1314 epoch=epoch)
1316 selected = referenceSelector.run(skyCircle.refCat)
1317 # Need memory contiguity to get reference filters as a vector.
1318 if not selected.sourceCat.isContiguous():
1319 refCat = selected.sourceCat.copy(deep=True)
1320 else:
1321 refCat = selected.sourceCat
1323 if applyColorterms:
1324 refCatName = refObjLoader.ref_dataset_name
1325 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1326 filterLabel.physicalLabel, refCatName)
1327 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1328 refCatName,
1329 doRaise=True)
1331 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1332 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1333 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1334 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1336 return refCat, skyCircle.fluxField
1338 def _check_star_lists(self, associations, name):
1339 # TODO: these should be len(blah), but we need this properly wrapped first.
1340 if associations.nCcdImagesValidForFit() == 0:
1341 raise RuntimeError('No images in the ccdImageList!')
1342 if associations.fittedStarListSize() == 0:
1343 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1344 if associations.refStarListSize() == 0:
1345 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1347 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1348 """Compute chi2, log it, validate the model, and return chi2.
1350 Parameters
1351 ----------
1352 associations : `lsst.jointcal.Associations`
1353 The star/reference star associations to fit.
1354 fit : `lsst.jointcal.FitterBase`
1355 The fitter to use for minimization.
1356 model : `lsst.jointcal.Model`
1357 The model being fit.
1358 chi2Label : `str`
1359 Label to describe the chi2 (e.g. "Initialized", "Final").
1360 writeChi2Name : `str`, optional
1361 Filename prefix to write the chi2 contributions to.
1362 Do not supply an extension: an appropriate one will be added.
1364 Returns
1365 -------
1366 chi2: `lsst.jointcal.Chi2Accumulator`
1367 The chi2 object for the current fitter and model.
1369 Raises
1370 ------
1371 FloatingPointError
1372 Raised if chi2 is infinite or NaN.
1373 ValueError
1374 Raised if the model is not valid.
1375 """
1376 if writeChi2Name is not None:
1377 fullpath = self._getDebugPath(writeChi2Name)
1378 fit.saveChi2Contributions(fullpath+"{type}")
1379 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1381 chi2 = fit.computeChi2()
1382 self.log.info("%s %s", chi2Label, chi2)
1383 self._check_stars(associations)
1384 if not np.isfinite(chi2.chi2):
1385 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1386 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1387 raise ValueError("Model is not valid: check log messages for warnings.")
1388 return chi2
1390 def _fit_photometry(self, associations, dataName=None):
1391 """
1392 Fit the photometric data.
1394 Parameters
1395 ----------
1396 associations : `lsst.jointcal.Associations`
1397 The star/reference star associations to fit.
1398 dataName : `str`
1399 Name of the data being processed (e.g. "1234_HSC-Y"), for
1400 identifying debugging files.
1402 Returns
1403 -------
1404 fit_result : `namedtuple`
1405 fit : `lsst.jointcal.PhotometryFit`
1406 The photometric fitter used to perform the fit.
1407 model : `lsst.jointcal.PhotometryModel`
1408 The photometric model that was fit.
1409 """
1410 self.log.info("=== Starting photometric fitting...")
1412 # TODO: should use pex.config.RegistryField here (see DM-9195)
1413 if self.config.photometryModel == "constrainedFlux":
1414 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1415 self.focalPlaneBBox,
1416 visitOrder=self.config.photometryVisitOrder,
1417 errorPedestal=self.config.photometryErrorPedestal)
1418 # potentially nonlinear problem, so we may need a line search to converge.
1419 doLineSearch = self.config.allowLineSearch
1420 elif self.config.photometryModel == "constrainedMagnitude":
1421 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1422 self.focalPlaneBBox,
1423 visitOrder=self.config.photometryVisitOrder,
1424 errorPedestal=self.config.photometryErrorPedestal)
1425 # potentially nonlinear problem, so we may need a line search to converge.
1426 doLineSearch = self.config.allowLineSearch
1427 elif self.config.photometryModel == "simpleFlux":
1428 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1429 errorPedestal=self.config.photometryErrorPedestal)
1430 doLineSearch = False # purely linear in model parameters, so no line search needed
1431 elif self.config.photometryModel == "simpleMagnitude":
1432 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1433 errorPedestal=self.config.photometryErrorPedestal)
1434 doLineSearch = False # purely linear in model parameters, so no line search needed
1436 fit = lsst.jointcal.PhotometryFit(associations, model)
1437 # TODO DM-12446: turn this into a "butler save" somehow.
1438 # Save reference and measurement chi2 contributions for this data
1439 if self.config.writeChi2FilesInitialFinal:
1440 baseName = f"photometry_initial_chi2-{dataName}"
1441 else:
1442 baseName = None
1443 if self.config.writeInitialModel:
1444 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt")
1445 writeModel(model, fullpath, self.log)
1446 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1448 def getChi2Name(whatToFit):
1449 if self.config.writeChi2FilesOuterLoop:
1450 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1451 else:
1452 return None
1454 # The constrained model needs the visit transform fit first; the chip
1455 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1456 if self.config.writeInitMatrix:
1457 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}")
1458 else:
1459 dumpMatrixFile = ""
1460 if self.config.photometryModel.startswith("constrained"):
1461 # no line search: should be purely (or nearly) linear,
1462 # and we want a large step size to initialize with.
1463 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1464 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1465 writeChi2Name=getChi2Name("ModelVisit"))
1466 dumpMatrixFile = "" # so we don't redo the output on the next step
1468 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1469 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1470 writeChi2Name=getChi2Name("Model"))
1472 fit.minimize("Fluxes") # no line search: always purely linear.
1473 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1474 writeChi2Name=getChi2Name("Fluxes"))
1476 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1477 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1478 writeChi2Name=getChi2Name("ModelFluxes"))
1480 model.freezeErrorTransform()
1481 self.log.debug("Photometry error scales are frozen.")
1483 chi2 = self._iterate_fit(associations,
1484 fit,
1485 self.config.maxPhotometrySteps,
1486 "photometry",
1487 "Model Fluxes",
1488 doRankUpdate=self.config.photometryDoRankUpdate,
1489 doLineSearch=doLineSearch,
1490 dataName=dataName)
1492 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1493 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1494 return Photometry(fit, model)
1496 def _fit_astrometry(self, associations, dataName=None):
1497 """
1498 Fit the astrometric data.
1500 Parameters
1501 ----------
1502 associations : `lsst.jointcal.Associations`
1503 The star/reference star associations to fit.
1504 dataName : `str`
1505 Name of the data being processed (e.g. "1234_HSC-Y"), for
1506 identifying debugging files.
1508 Returns
1509 -------
1510 fit_result : `namedtuple`
1511 fit : `lsst.jointcal.AstrometryFit`
1512 The astrometric fitter used to perform the fit.
1513 model : `lsst.jointcal.AstrometryModel`
1514 The astrometric model that was fit.
1515 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1516 The model for the sky to tangent plane projection that was used in the fit.
1517 """
1519 self.log.info("=== Starting astrometric fitting...")
1521 associations.deprojectFittedStars()
1523 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1524 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1525 # them so carefully?
1526 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1528 if self.config.astrometryModel == "constrained":
1529 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1530 sky_to_tan_projection,
1531 chipOrder=self.config.astrometryChipOrder,
1532 visitOrder=self.config.astrometryVisitOrder)
1533 elif self.config.astrometryModel == "simple":
1534 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1535 sky_to_tan_projection,
1536 self.config.useInputWcs,
1537 nNotFit=0,
1538 order=self.config.astrometrySimpleOrder)
1540 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1541 # TODO DM-12446: turn this into a "butler save" somehow.
1542 # Save reference and measurement chi2 contributions for this data
1543 if self.config.writeChi2FilesInitialFinal:
1544 baseName = f"astrometry_initial_chi2-{dataName}"
1545 else:
1546 baseName = None
1547 if self.config.writeInitialModel:
1548 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1549 writeModel(model, fullpath, self.log)
1550 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1552 def getChi2Name(whatToFit):
1553 if self.config.writeChi2FilesOuterLoop:
1554 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1555 else:
1556 return None
1558 if self.config.writeInitMatrix:
1559 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}")
1560 else:
1561 dumpMatrixFile = ""
1562 # The constrained model needs the visit transform fit first; the chip
1563 # transform is initialized from the detector's cameraGeom, so it's close.
1564 if self.config.astrometryModel == "constrained":
1565 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1566 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1567 writeChi2Name=getChi2Name("DistortionsVisit"))
1568 dumpMatrixFile = "" # so we don't redo the output on the next step
1570 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1571 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1572 writeChi2Name=getChi2Name("Distortions"))
1574 fit.minimize("Positions")
1575 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1576 writeChi2Name=getChi2Name("Positions"))
1578 fit.minimize("Distortions Positions")
1579 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1580 writeChi2Name=getChi2Name("DistortionsPositions"))
1582 chi2 = self._iterate_fit(associations,
1583 fit,
1584 self.config.maxAstrometrySteps,
1585 "astrometry",
1586 "Distortions Positions",
1587 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1588 doRankUpdate=self.config.astrometryDoRankUpdate,
1589 dataName=dataName)
1591 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1592 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1594 return Astrometry(fit, model, sky_to_tan_projection)
1596 def _check_stars(self, associations):
1597 """Count measured and reference stars per ccd and warn/log them."""
1598 for ccdImage in associations.getCcdImageList():
1599 nMeasuredStars, nRefStars = ccdImage.countStars()
1600 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1601 ccdImage.getName(), nMeasuredStars, nRefStars)
1602 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1603 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1604 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1605 if nRefStars < self.config.minRefStarsPerCcd:
1606 self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1607 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1609 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1610 dataName="",
1611 sigmaRelativeTolerance=0,
1612 doRankUpdate=True,
1613 doLineSearch=False):
1614 """Run fitter.minimize up to max_steps times, returning the final chi2.
1616 Parameters
1617 ----------
1618 associations : `lsst.jointcal.Associations`
1619 The star/reference star associations to fit.
1620 fitter : `lsst.jointcal.FitterBase`
1621 The fitter to use for minimization.
1622 max_steps : `int`
1623 Maximum number of steps to run outlier rejection before declaring
1624 convergence failure.
1625 name : {'photometry' or 'astrometry'}
1626 What type of data are we fitting (for logs and debugging files).
1627 whatToFit : `str`
1628 Passed to ``fitter.minimize()`` to define the parameters to fit.
1629 dataName : `str`, optional
1630 Descriptive name for this dataset (e.g. tract and filter),
1631 for debugging.
1632 sigmaRelativeTolerance : `float`, optional
1633 Convergence tolerance for the fractional change in the chi2 cut
1634 level for determining outliers. If set to zero, iterations will
1635 continue until there are no outliers.
1636 doRankUpdate : `bool`, optional
1637 Do an Eigen rank update during minimization, or recompute the full
1638 matrix and gradient?
1639 doLineSearch : `bool`, optional
1640 Do a line search for the optimum step during minimization?
1642 Returns
1643 -------
1644 chi2: `lsst.jointcal.Chi2Statistic`
1645 The final chi2 after the fit converges, or is forced to end.
1647 Raises
1648 ------
1649 FloatingPointError
1650 Raised if the fitter fails with a non-finite value.
1651 RuntimeError
1652 Raised if the fitter fails for some other reason;
1653 log messages will provide further details.
1654 """
1655 if self.config.writeInitMatrix:
1656 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}")
1657 else:
1658 dumpMatrixFile = ""
1659 oldChi2 = lsst.jointcal.Chi2Statistic()
1660 oldChi2.chi2 = float("inf")
1661 for i in range(max_steps):
1662 if self.config.writeChi2FilesOuterLoop:
1663 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1664 else:
1665 writeChi2Name = None
1666 result = fitter.minimize(whatToFit,
1667 self.config.outlierRejectSigma,
1668 sigmaRelativeTolerance=sigmaRelativeTolerance,
1669 doRankUpdate=doRankUpdate,
1670 doLineSearch=doLineSearch,
1671 dumpMatrixFile=dumpMatrixFile)
1672 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1673 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1674 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1676 if result == MinimizeResult.Converged:
1677 if doRankUpdate:
1678 self.log.debug("fit has converged - no more outliers - redo minimization "
1679 "one more time in case we have lost accuracy in rank update.")
1680 # Redo minimization one more time in case we have lost accuracy in rank update
1681 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1682 sigmaRelativeTolerance=sigmaRelativeTolerance)
1683 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1685 # log a message for a large final chi2, TODO: DM-15247 for something better
1686 if chi2.chi2/chi2.ndof >= 4.0:
1687 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1689 break
1690 elif result == MinimizeResult.Chi2Increased:
1691 self.log.warn("Still some outliers remaining but chi2 increased - retry")
1692 # Check whether the increase was large enough to cause trouble.
1693 chi2Ratio = chi2.chi2 / oldChi2.chi2
1694 if chi2Ratio > 1.5:
1695 self.log.warn('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1696 chi2.chi2, oldChi2.chi2, chi2Ratio)
1697 # Based on a variety of HSC jointcal logs (see DM-25779), it
1698 # appears that chi2 increases more than a factor of ~2 always
1699 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1700 # Using 10 as the "failure" threshold gives some room between
1701 # leaving a warning and bailing early.
1702 if chi2Ratio > 10:
1703 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1704 " Try setting one or more of the `writeChi2*` config fields and looking"
1705 " at how individual star chi2-values evolve during the fit.")
1706 raise RuntimeError(msg)
1707 oldChi2 = chi2
1708 elif result == MinimizeResult.NonFinite:
1709 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1710 # TODO DM-12446: turn this into a "butler save" somehow.
1711 fitter.saveChi2Contributions(filename+"{type}")
1712 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1713 raise FloatingPointError(msg.format(filename))
1714 elif result == MinimizeResult.Failed:
1715 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1716 else:
1717 raise RuntimeError("Unxepected return code from minimize().")
1718 else:
1719 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1721 return chi2
1723 def _make_output(self, ccdImageList, model, func):
1724 """Return the internal jointcal models converted to the afw
1725 structures that will be saved to disk.
1727 Parameters
1728 ----------
1729 ccdImageList : `lsst.jointcal.CcdImageList`
1730 The list of CcdImages to get the output for.
1731 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1732 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1733 func : `str`
1734 The name of the function to call on ``model`` to get the converted
1735 structure. Must accept an `lsst.jointcal.CcdImage`.
1737 Returns
1738 -------
1739 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1740 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1741 The data to be saved, keyed on (visit, detector).
1742 """
1743 output = {}
1744 for ccdImage in ccdImageList:
1745 ccd = ccdImage.ccdId
1746 visit = ccdImage.visit
1747 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1748 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1749 return output
1751 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1752 """
1753 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1755 Parameters
1756 ----------
1757 associations : `lsst.jointcal.Associations`
1758 The star/reference star associations to fit.
1759 model : `lsst.jointcal.AstrometryModel`
1760 The astrometric model that was fit.
1761 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1762 Dict of ccdImage identifiers to dataRefs that were fit.
1763 """
1764 ccdImageList = associations.getCcdImageList()
1765 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1766 for key, skyWcs in output.items():
1767 dataRef = visit_ccd_to_dataRef[key]
1768 try:
1769 dataRef.put(skyWcs, 'jointcal_wcs')
1770 except pexExceptions.Exception as e:
1771 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1772 raise e
1774 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1775 """
1776 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1778 Parameters
1779 ----------
1780 associations : `lsst.jointcal.Associations`
1781 The star/reference star associations to fit.
1782 model : `lsst.jointcal.PhotometryModel`
1783 The photoometric model that was fit.
1784 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1785 Dict of ccdImage identifiers to dataRefs that were fit.
1786 """
1788 ccdImageList = associations.getCcdImageList()
1789 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1790 for key, photoCalib in output.items():
1791 dataRef = visit_ccd_to_dataRef[key]
1792 try:
1793 dataRef.put(photoCalib, 'jointcal_photoCalib')
1794 except pexExceptions.Exception as e:
1795 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1796 raise e