lsst.jointcal g0eae3de20c+d7f9f0e7f4
jointcal.py
Go to the documentation of this file.
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import dataclasses
23import collections
24import os
25
26import astropy.time
27import numpy as np
28import astropy.units as u
29
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
37import lsst.afw.table
38import lsst.log
39from lsst.pipe.base import Instrument
40from lsst.pipe.tasks.colorterms import ColortermLibrary
41from lsst.verify import Job, Measurement
42
43from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask,
44 LoadIndexedReferenceObjectsConfig)
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46
47import lsst.jointcal
48from lsst.jointcal import MinimizeResult
49
50__all__ = ["JointcalConfig", "JointcalTask"]
51
52Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
53Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
54
55
56# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
57def add_measurement(job, name, value):
58 meas = Measurement(job.metrics[name], value)
59 job.measurements.insert(meas)
60
61
62def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
63 """Lookup function that asserts/hopes that a static calibration dataset
64 exists in a particular collection, since this task can't provide a single
65 date/time to use to search for one properly.
66
67 This is mostly useful for the ``camera`` dataset, in cases where the task's
68 quantum dimensions do *not* include something temporal, like ``exposure``
69 or ``visit``.
70
71 Parameters
72 ----------
73 datasetType : `lsst.daf.butler.DatasetType`
74 Type of dataset being searched for.
75 registry : `lsst.daf.butler.Registry`
76 Data repository registry to search.
77 quantumDataId : `lsst.daf.butler.DataCoordinate`
78 Data ID of the quantum this camera should match.
79 collections : `Iterable` [ `str` ]
80 Collections that should be searched - but this lookup function works
81 by ignoring this in favor of a more-or-less hard-coded value.
82
83 Returns
84 -------
85 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
86 Iterator over dataset references; should have only one element.
87
88 Notes
89 -----
90 This implementation duplicates one in fgcmcal, and is at least quite
91 similar to another in cp_pipe. This duplicate has the most documentation.
92 Fixing this is DM-29661.
93 """
94 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
95 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
96 return registry.queryDatasets(datasetType,
97 dataId=quantumDataId,
98 collections=[unboundedCollection],
99 findFirst=True)
100
101
102def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
103 """Lookup function that finds all refcats for all visits that overlap a
104 tract, rather than just the refcats that directly overlap the tract.
105
106 Parameters
107 ----------
108 datasetType : `lsst.daf.butler.DatasetType`
109 Type of dataset being searched for.
110 registry : `lsst.daf.butler.Registry`
111 Data repository registry to search.
112 quantumDataId : `lsst.daf.butler.DataCoordinate`
113 Data ID of the quantum; expected to be something we can use as a
114 constraint to query for overlapping visits.
115 collections : `Iterable` [ `str` ]
116 Collections to search.
117
118 Returns
119 -------
120 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
121 Iterator over refcat references.
122 """
123 refs = set()
124 # Use .expanded() on the query methods below because we need data IDs with
125 # regions, both in the outer loop over visits (queryDatasets will expand
126 # any data ID we give it, but doing it up-front in bulk is much more
127 # efficient) and in the data IDs of the DatasetRefs this function yields
128 # (because the RefCatLoader relies on them to do some of its own
129 # filtering).
130 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
131 refs.update(
132 registry.queryDatasets(
133 datasetType,
134 collections=collections,
135 dataId=visit_data_id,
136 findFirst=True,
137 ).expanded()
138 )
139 yield from refs
140
141
142class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
143 dimensions=("skymap", "tract", "instrument", "physical_filter")):
144 """Middleware input/output connections for jointcal data."""
145 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
146 doc="The camera instrument that took these observations.",
147 name="camera",
148 storageClass="Camera",
149 dimensions=("instrument",),
150 isCalibration=True,
151 lookupFunction=lookupStaticCalibrations,
152 )
153 inputSourceTableVisit = pipeBase.connectionTypes.Input(
154 doc="Source table in parquet format, per visit",
155 name="sourceTable_visit",
156 storageClass="DataFrame",
157 dimensions=("instrument", "visit"),
158 deferLoad=True,
159 multiple=True,
160 )
161 inputVisitSummary = pipeBase.connectionTypes.Input(
162 doc=("Per-visit consolidated exposure metadata built from calexps. "
163 "These catalogs use detector id for the id and must be sorted for "
164 "fast lookups of a detector."),
165 name="visitSummary",
166 storageClass="ExposureCatalog",
167 dimensions=("instrument", "visit"),
168 deferLoad=True,
169 multiple=True,
170 )
171 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
172 doc="The astrometry reference catalog to match to loaded input catalog sources.",
173 name="gaia_dr2_20200414",
174 storageClass="SimpleCatalog",
175 dimensions=("skypix",),
176 deferLoad=True,
177 multiple=True,
178 lookupFunction=lookupVisitRefCats,
179 )
180 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
181 doc="The photometry reference catalog to match to loaded input catalog sources.",
182 name="ps1_pv3_3pi_20170110",
183 storageClass="SimpleCatalog",
184 dimensions=("skypix",),
185 deferLoad=True,
186 multiple=True,
187 lookupFunction=lookupVisitRefCats,
188 )
189
190 outputWcs = pipeBase.connectionTypes.Output(
191 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
192 " These catalogs only contain entries for detectors with an output, and use"
193 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
194 name="jointcalSkyWcsCatalog",
195 storageClass="ExposureCatalog",
196 dimensions=("instrument", "visit", "skymap", "tract"),
197 multiple=True
198 )
199 outputPhotoCalib = pipeBase.connectionTypes.Output(
200 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
201 " These catalogs only contain entries for detectors with an output, and use"
202 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
203 name="jointcalPhotoCalibCatalog",
204 storageClass="ExposureCatalog",
205 dimensions=("instrument", "visit", "skymap", "tract"),
206 multiple=True
207 )
208
209 # measurements of metrics
210 # The vars() trick used here allows us to set class attributes
211 # programatically. Taken from:
212 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
213 for name in ("astrometry", "photometry"):
214 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
215 doc=f"The number of cross-matched fittedStars for {name}",
216 name=f"metricvalue_jointcal_{name}_matched_fittedStars",
217 storageClass="MetricValue",
218 dimensions=("skymap", "tract", "instrument", "physical_filter"),
219 )
220 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
221 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.",
222 name=f"metricvalue_jointcal_{name}_collected_refStars",
223 storageClass="MetricValue",
224 dimensions=("skymap", "tract", "instrument", "physical_filter"),
225 )
226 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
227 doc=f"The number of {name} reference stars matched to fittedStars.",
228 name=f"metricvalue_jointcal_{name}_prepared_refStars",
229 storageClass="MetricValue",
230 dimensions=("skymap", "tract", "instrument", "physical_filter"),
231 )
232 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
233 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.",
234 name=f"metricvalue_jointcal_{name}_prepared_fittedStars",
235 storageClass="MetricValue",
236 dimensions=("skymap", "tract", "instrument", "physical_filter"),
237 )
238 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
239 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.",
240 name=f"metricvalue_jointcal_{name}_prepared_ccdImages",
241 storageClass="MetricValue",
242 dimensions=("skymap", "tract", "instrument", "physical_filter"),
243 )
244 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
245 doc=f"The final chi2 of the {name} fit.",
246 name=f"metricvalue_jointcal_{name}_final_chi2",
247 storageClass="MetricValue",
248 dimensions=("skymap", "tract", "instrument", "physical_filter"),
249 )
250 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
251 doc=f"The number of degrees of freedom of the fitted {name}.",
252 name=f"metricvalue_jointcal_{name}_final_ndof",
253 storageClass="MetricValue",
254 dimensions=("skymap", "tract", "instrument", "physical_filter"),
255 )
256
257 def __init__(self, *, config=None):
258 super().__init__(config=config)
259 # When we are only doing one of astrometry or photometry, we don't
260 # need the reference catalog or produce the outputs for the other.
261 # This informs the middleware of that when the QuantumGraph is
262 # generated, so we don't block on getting something we won't need or
263 # create an expectation that downstream tasks will be able to consume
264 # something we won't produce.
265 if not config.doAstrometry:
266 self.prerequisiteInputs.remove("astrometryRefCat")
267 self.outputs.remove("outputWcs")
268 for key in list(self.outputs):
269 if "metricvalue_jointcal_astrometry" in key:
270 self.outputs.remove(key)
271 if not config.doPhotometry:
272 self.prerequisiteInputs.remove("photometryRefCat")
273 self.outputs.remove("outputPhotoCalib")
274 for key in list(self.outputs):
275 if "metricvalue_jointcal_photometry" in key:
276 self.outputs.remove(key)
277
278
279class JointcalConfig(pipeBase.PipelineTaskConfig,
280 pipelineConnections=JointcalTaskConnections):
281 """Configuration for JointcalTask"""
282
283 doAstrometry = pexConfig.Field(
284 doc="Fit astrometry and write the fitted result.",
285 dtype=bool,
286 default=True
287 )
288 doPhotometry = pexConfig.Field(
289 doc="Fit photometry and write the fitted result.",
290 dtype=bool,
291 default=True
292 )
293 sourceFluxType = pexConfig.Field(
294 dtype=str,
295 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
296 default='apFlux_12_0'
297 )
298 positionErrorPedestal = pexConfig.Field(
299 doc="Systematic term to apply to the measured position error (pixels)",
300 dtype=float,
301 default=0.02,
302 )
303 photometryErrorPedestal = pexConfig.Field(
304 doc="Systematic term to apply to the measured error on flux or magnitude as a "
305 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
306 dtype=float,
307 default=0.0,
308 )
309 # TODO: DM-6885 matchCut should be an geom.Angle
310 matchCut = pexConfig.Field(
311 doc="Matching radius between fitted and reference stars (arcseconds)",
312 dtype=float,
313 default=3.0,
314 )
315 minMeasurements = pexConfig.Field(
316 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
317 dtype=int,
318 default=2,
319 )
320 minMeasuredStarsPerCcd = pexConfig.Field(
321 doc="Minimum number of measuredStars per ccdImage before printing warnings",
322 dtype=int,
323 default=100,
324 )
325 minRefStarsPerCcd = pexConfig.Field(
326 doc="Minimum number of measuredStars per ccdImage before printing warnings",
327 dtype=int,
328 default=30,
329 )
330 allowLineSearch = pexConfig.Field(
331 doc="Allow a line search during minimization, if it is reasonable for the model"
332 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
333 dtype=bool,
334 default=False
335 )
336 astrometrySimpleOrder = pexConfig.Field(
337 doc="Polynomial order for fitting the simple astrometry model.",
338 dtype=int,
339 default=3,
340 )
341 astrometryChipOrder = pexConfig.Field(
342 doc="Order of the per-chip transform for the constrained astrometry model.",
343 dtype=int,
344 default=1,
345 )
346 astrometryVisitOrder = pexConfig.Field(
347 doc="Order of the per-visit transform for the constrained astrometry model.",
348 dtype=int,
349 default=5,
350 )
351 useInputWcs = pexConfig.Field(
352 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
353 dtype=bool,
354 default=True,
355 )
356 astrometryModel = pexConfig.ChoiceField(
357 doc="Type of model to fit to astrometry",
358 dtype=str,
359 default="constrained",
360 allowed={"simple": "One polynomial per ccd",
361 "constrained": "One polynomial per ccd, and one polynomial per visit"}
362 )
363 photometryModel = pexConfig.ChoiceField(
364 doc="Type of model to fit to photometry",
365 dtype=str,
366 default="constrainedMagnitude",
367 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
368 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
369 " fitting in flux space.",
370 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
371 " fitting in magnitude space.",
372 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
373 " fitting in magnitude space.",
374 }
375 )
376 applyColorTerms = pexConfig.Field(
377 doc="Apply photometric color terms to reference stars?"
378 "Requires that colorterms be set to a ColortermLibrary",
379 dtype=bool,
380 default=False
381 )
382 colorterms = pexConfig.ConfigField(
383 doc="Library of photometric reference catalog name to color term dict.",
384 dtype=ColortermLibrary,
385 )
386 photometryVisitOrder = pexConfig.Field(
387 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
388 dtype=int,
389 default=7,
390 )
391 photometryDoRankUpdate = pexConfig.Field(
392 doc=("Do the rank update step during minimization. "
393 "Skipping this can help deal with models that are too non-linear."),
394 dtype=bool,
395 default=True,
396 )
397 astrometryDoRankUpdate = pexConfig.Field(
398 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
399 "Skipping this can help deal with models that are too non-linear."),
400 dtype=bool,
401 default=True,
402 )
403 outlierRejectSigma = pexConfig.Field(
404 doc="How many sigma to reject outliers at during minimization.",
405 dtype=float,
406 default=5.0,
407 )
408 astrometryOutlierRelativeTolerance = pexConfig.Field(
409 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
410 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
411 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
412 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
413 "close to the solution found when removing all outliers."),
414 dtype=float,
415 default=0,
416 )
417 maxPhotometrySteps = pexConfig.Field(
418 doc="Maximum number of minimize iterations to take when fitting photometry.",
419 dtype=int,
420 default=20,
421 )
422 maxAstrometrySteps = pexConfig.Field(
423 doc="Maximum number of minimize iterations to take when fitting astrometry.",
424 dtype=int,
425 default=20,
426 )
427 astrometryRefObjLoader = pexConfig.ConfigField(
428 dtype=LoadIndexedReferenceObjectsConfig,
429 doc="Reference object loader for astrometric fit",
430 )
431 photometryRefObjLoader = pexConfig.ConfigField(
432 dtype=LoadIndexedReferenceObjectsConfig,
433 doc="Reference object loader for photometric fit",
434 )
435 sourceSelector = sourceSelectorRegistry.makeField(
436 doc="How to select sources for cross-matching",
437 default="science"
438 )
439 astrometryReferenceSelector = pexConfig.ConfigurableField(
440 target=ReferenceSourceSelectorTask,
441 doc="How to down-select the loaded astrometry reference catalog.",
442 )
443 photometryReferenceSelector = pexConfig.ConfigurableField(
444 target=ReferenceSourceSelectorTask,
445 doc="How to down-select the loaded photometry reference catalog.",
446 )
447 astrometryReferenceErr = pexConfig.Field(
448 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
449 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
450 "If specified, overrides any existing `coord_*Err` values."),
451 dtype=float,
452 default=None,
453 optional=True
454 )
455
456 # configs for outputting debug information
457 writeInitMatrix = pexConfig.Field(
458 dtype=bool,
459 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
460 "Output files will be written to `config.debugOutputPath` and will "
461 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
462 "Note that these files are the dense versions of the matrix, and so may be very large."),
463 default=False
464 )
465 writeChi2FilesInitialFinal = pexConfig.Field(
466 dtype=bool,
467 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
468 "Output files will be written to `config.debugOutputPath` and will "
469 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
470 default=False
471 )
472 writeChi2FilesOuterLoop = pexConfig.Field(
473 dtype=bool,
474 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
475 "Output files will be written to `config.debugOutputPath` and will "
476 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
477 default=False
478 )
479 writeInitialModel = pexConfig.Field(
480 dtype=bool,
481 doc=("Write the pre-initialization model to text files, for debugging. "
482 "Output files will be written to `config.debugOutputPath` and will be "
483 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
484 default=False
485 )
486 debugOutputPath = pexConfig.Field(
487 dtype=str,
488 default=".",
489 doc=("Path to write debug output files to. Used by "
490 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
491 )
492 detailedProfile = pexConfig.Field(
493 dtype=bool,
494 default=False,
495 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
496 )
497
498 def validate(self):
499 super().validate()
500 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
501 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
502 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
503 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
504 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
505 "applyColorTerms=True will be ignored.")
506 lsst.log.warning(msg)
507
508 def setDefaults(self):
509 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing.
510 self.sourceSelector["science"].doUnresolved = True
511 self.sourceSelector["science"].unresolved.name = "extendedness"
512 # with dependable signal to noise ratio.
513 self.sourceSelector["science"].doSignalToNoise = True
514 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
515 # and S/N > 10 to use sources that are not too faint, and thus better measured.
516 self.sourceSelector["science"].signalToNoise.minimum = 10.
517 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits.
518 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux"
519 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr"
520 # Do not trust blended sources" aperture fluxes which also depend on seeing.
521 self.sourceSelector["science"].doIsolated = True
522 self.sourceSelector["science"].isolated.parentName = "parentSourceId"
523 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild"
524 # Do not trust either flux or centroid measurements with flags,
525 # chosen from the usual QA flags for stars)
526 self.sourceSelector["science"].doFlags = True
527 badFlags = ["pixelFlags_edge",
528 "pixelFlags_saturated",
529 "pixelFlags_interpolatedCenter",
530 "pixelFlags_interpolated",
531 "pixelFlags_crCenter",
532 "pixelFlags_bad",
533 "hsmPsfMoments_flag",
534 f"{self.sourceFluxType}_flag",
535 ]
536 self.sourceSelector["science"].flags.bad = badFlags
537
538 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the
539 # primary Gaia band, but is not like any normal photometric band.
540 self.astrometryRefObjLoader.requireProperMotion = True
541 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean"
542
543
544def writeModel(model, filename, log):
545 """Write model to outfile."""
546 with open(filename, "w") as file:
547 file.write(repr(model))
548 log.info("Wrote %s to file: %s", model, filename)
549
550
551@dataclasses.dataclass
553 """The input data jointcal needs for each detector/visit."""
554 visit: int
555 """The visit identifier of this exposure."""
557 """The catalog derived from this exposure."""
558 visitInfo: lsst.afw.image.VisitInfo
559 """The VisitInfo of this exposure."""
561 """The detector of this exposure."""
562 photoCalib: lsst.afw.image.PhotoCalib
563 """The photometric calibration of this exposure."""
565 """The WCS of this exposure."""
566 bbox: lsst.geom.Box2I
567 """The bounding box of this exposure."""
569 """The filter of this exposure."""
570
571
572class JointcalTask(pipeBase.PipelineTask):
573 """Astrometricly and photometricly calibrate across multiple visits of the
574 same field.
575 """
576
577 ConfigClass = JointcalConfig
578 _DefaultName = "jointcal"
579
580 def __init__(self, **kwargs):
581 super().__init__(**kwargs)
582 self.makeSubtask("sourceSelector")
583 if self.config.doAstrometry:
584 self.makeSubtask("astrometryReferenceSelector")
585 else:
587 if self.config.doPhotometry:
588 self.makeSubtask("photometryReferenceSelector")
589 else:
591
592 # To hold various computed metrics for use by tests
593 self.job = Job.load_metrics_package(subset='jointcal')
594
595 def runQuantum(self, butlerQC, inputRefs, outputRefs):
596 # We override runQuantum to set up the refObjLoaders and write the
597 # outputs to the correct refs.
598 inputs = butlerQC.get(inputRefs)
599 # We want the tract number for writing debug files
600 tract = butlerQC.quantum.dataId['tract']
601 if self.config.doAstrometry:
602 self.astrometryRefObjLoader = ReferenceObjectLoader(
603 dataIds=[ref.datasetRef.dataId
604 for ref in inputRefs.astrometryRefCat],
605 refCats=inputs.pop('astrometryRefCat'),
606 config=self.config.astrometryRefObjLoader,
607 log=self.log)
608 if self.config.doPhotometry:
609 self.photometryRefObjLoader = ReferenceObjectLoader(
610 dataIds=[ref.datasetRef.dataId
611 for ref in inputRefs.photometryRefCat],
612 refCats=inputs.pop('photometryRefCat'),
613 config=self.config.photometryRefObjLoader,
614 log=self.log)
615 outputs = self.run(**inputs, tract=tract)
616 self._put_metrics(butlerQC, outputs.job, outputRefs)
617 if self.config.doAstrometry:
618 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
619 inputs['inputCamera'], "setWcs")
620 if self.config.doPhotometry:
621 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
622 inputs['inputCamera'], "setPhotoCalib")
623
624 def _put_metrics(self, butlerQC, job, outputRefs):
625 """Persist all measured metrics stored in a job.
626
627 Parameters
628 ----------
629 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
630 A butler which is specialized to operate in the context of a
631 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
632 job : `lsst.verify.job.Job`
633 Measurements of metrics to persist.
634 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
635 The DatasetRefs to persist the data to.
636 """
637 for key in job.measurements.keys():
638 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', '')))
639
640 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
641 """Persist the output datasets to their appropriate datarefs.
642
643 Parameters
644 ----------
645 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
646 A butler which is specialized to operate in the context of a
647 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
648 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
649 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
650 The fitted objects to persist.
651 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
652 The DatasetRefs to persist the data to.
654 The camera for this instrument, to get detector ids from.
655 setter : `str`
656 The method to call on the ExposureCatalog to set each output.
657 """
659 schema.addField('visit', type='L', doc='Visit number')
660
661 def new_catalog(visit, size):
662 """Return an catalog ready to be filled with appropriate output."""
663 catalog = lsst.afw.table.ExposureCatalog(schema)
664 catalog.resize(size)
665 catalog['visit'] = visit
666 metadata = lsst.daf.base.PropertyList()
667 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
668 metadata.add("COMMENT", "Only detectors with data have entries.")
669 return catalog
670
671 # count how many detectors have output for each visit
672 detectors_per_visit = collections.defaultdict(int)
673 for key in outputs:
674 # key is (visit, detector_id), and we only need visit here
675 detectors_per_visit[key[0]] += 1
676
677 for ref in outputRefs:
678 visit = ref.dataId['visit']
679 catalog = new_catalog(visit, detectors_per_visit[visit])
680 # Iterate over every detector and skip the ones we don't have output for.
681 i = 0
682 for detector in camera:
683 detectorId = detector.getId()
684 key = (ref.dataId['visit'], detectorId)
685 if key not in outputs:
686 # skip detectors we don't have output for
687 self.log.debug("No %s output for detector %s in visit %s",
688 setter[3:], detectorId, visit)
689 continue
690
691 catalog[i].setId(detectorId)
692 getattr(catalog[i], setter)(outputs[key])
693 i += 1
694
695 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
696 butlerQC.put(catalog, ref)
697 self.log.info("Wrote %s detectors to %s", i, ref)
698
699 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
700 # Docstring inherited.
701
702 # We take values out of the Parquet table, and put them in "flux_",
703 # and the config.sourceFluxType field is used during that extraction,
704 # so just use "flux" here.
705 sourceFluxField = "flux"
706 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
707 associations = lsst.jointcal.Associations()
708 self.focalPlaneBBox = inputCamera.getFpBBox()
709 oldWcsList, bands = self._load_data(inputSourceTableVisit,
710 inputVisitSummary,
711 associations,
712 jointcalControl,
713 inputCamera)
714
715 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands)
716
717 if self.config.doAstrometry:
718 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
719 name="astrometry",
720 refObjLoader=self.astrometryRefObjLoader,
721 referenceSelector=self.astrometryReferenceSelector,
722 fit_function=self._fit_astrometry,
723 tract=tract,
724 epoch=epoch)
725 astrometry_output = self._make_output(associations.getCcdImageList(),
726 astrometry.model,
727 "makeSkyWcs")
728 else:
729 astrometry_output = None
730
731 if self.config.doPhotometry:
732 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
733 name="photometry",
734 refObjLoader=self.photometryRefObjLoader,
735 referenceSelector=self.photometryReferenceSelector,
736 fit_function=self._fit_photometry,
737 tract=tract,
738 epoch=epoch,
739 reject_bad_fluxes=True)
740 photometry_output = self._make_output(associations.getCcdImageList(),
741 photometry.model,
742 "toPhotoCalib")
743 else:
744 photometry_output = None
745
746 return pipeBase.Struct(outputWcs=astrometry_output,
747 outputPhotoCalib=photometry_output,
748 job=self.job,
749 astrometryRefObjLoader=self.astrometryRefObjLoader,
750 photometryRefObjLoader=self.photometryRefObjLoader)
751
752 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
753 jointcalControl, camera):
754 """Read the data that jointcal needs to run.
755
756 Modifies ``associations`` in-place with the loaded data.
757
758 Parameters
759 ----------
760 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
761 References to visit-level DataFrames to load the catalog data from.
762 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
763 Visit-level exposure summary catalog with metadata.
764 associations : `lsst.jointcal.Associations`
765 Object to add the loaded data to by constructing new CcdImages.
766 jointcalControl : `jointcal.JointcalControl`
767 Control object for C++ associations management.
769 Camera object for detector geometry.
770
771 Returns
772 -------
773 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
774 The original WCS of the input data, to aid in writing tests.
775 bands : `list` [`str`]
776 The filter bands of each input dataset.
777 """
778 oldWcsList = []
779 filters = []
780 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
781 with pipeBase.cmdLineTask.profile(load_cat_profile_file):
782 table = make_schema_table() # every detector catalog has the same layout
783 # No guarantee that the input is in the same order of visits, so we have to map one of them.
784 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
785 detectorDict = {detector.getId(): detector for detector in camera}
786
787 columns = None
788
789 for visitSummaryRef in inputVisitSummary:
790 visitSummary = visitSummaryRef.get()
791
792 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]]
793 if columns is None:
794 inColumns = dataRef.get(component='columns')
795 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns,
796 self.config,
797 self.sourceSelector)
798 visitCatalog = dataRef.get(parameters={'columns': columns})
799
800 selected = self.sourceSelector.run(visitCatalog)
801
802 # Build a CcdImage for each detector in this visit.
803 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
804 for id, index in detectors.items():
806 selected.sourceCat,
807 id,
808 detColumn,
809 ixxColumns,
810 self.config.sourceFluxType,
811 self.log)
812 if catalog is None:
813 continue
814 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
815 result = self._build_ccdImage(data, associations, jointcalControl)
816 if result is not None:
817 oldWcsList.append(result.wcs)
818 # A visit has only one band, so we can just use the first.
819 filters.append(data.filter)
820 if len(filters) == 0:
821 raise RuntimeError("No data to process: did source selector remove all sources?")
822 filters = collections.Counter(filters)
823
824 return oldWcsList, filters
825
826 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
827 """Return a data structure for this detector+visit."""
828 return JointcalInputData(visit=visitRecord['visit'],
829 catalog=catalog,
830 visitInfo=visitRecord.getVisitInfo(),
831 detector=detectorDict[visitRecord.getId()],
832 photoCalib=visitRecord.getPhotoCalib(),
833 wcs=visitRecord.getWcs(),
834 bbox=visitRecord.getBBox(),
835 # ExposureRecord doesn't have a FilterLabel yet,
836 # so we have to make one.
837 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
838 physical=visitRecord['physical_filter']))
839
840 # We don't currently need to persist the metadata.
841 # If we do in the future, we will have to add appropriate dataset templates
842 # to each obs package (the metadata template should look like `jointcal_wcs`).
843 def _getMetadataName(self):
844 return None
845
846 def _build_ccdImage(self, data, associations, jointcalControl):
847 """
848 Extract the necessary things from this catalog+metadata to add a new
849 ccdImage.
850
851 Parameters
852 ----------
853 data : `JointcalInputData`
854 The loaded input data.
855 associations : `lsst.jointcal.Associations`
856 Object to add the info to, to construct a new CcdImage
857 jointcalControl : `jointcal.JointcalControl`
858 Control object for associations management
859
860 Returns
861 ------
862 namedtuple or `None`
863 ``wcs``
864 The TAN WCS of this image, read from the calexp
866 ``key``
867 A key to identify this dataRef by its visit and ccd ids
868 (`namedtuple`).
869 `None`
870 if there are no sources in the loaded catalog.
871 """
872 if len(data.catalog) == 0:
873 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
874 return None
875
876 associations.createCcdImage(data.catalog,
877 data.wcs,
878 data.visitInfo,
879 data.bbox,
880 data.filter.physicalLabel,
881 data.photoCalib,
882 data.detector,
883 data.visit,
884 data.detector.getId(),
885 jointcalControl)
886
887 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
888 Key = collections.namedtuple('Key', ('visit', 'ccd'))
889 return Result(data.wcs, Key(data.visit, data.detector.getId()))
890
891 def _getDebugPath(self, filename):
892 """Constructs a path to filename using the configured debug path.
893 """
894 return os.path.join(self.config.debugOutputPath, filename)
895
896 def _prep_sky(self, associations, filters):
897 """Prepare on-sky and other data that must be computed after data has
898 been read.
899 """
900 associations.computeCommonTangentPoint()
901
902 boundingCircle = associations.computeBoundingCircle()
903 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
904 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
905
906 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
907
908 # Determine a default filter band associated with the catalog. See DM-9093
909 defaultFilter = filters.most_common(1)[0][0]
910 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
911
912 # compute and set the reference epoch of the observations, for proper motion corrections
913 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
914 associations.setEpoch(epoch.jyear)
915
916 return boundingCircle, center, radius, defaultFilter, epoch
917
918 def _get_refcat_coordinate_error_override(self, refCat, name):
919 """Check whether we should override the refcat coordinate errors, and
920 return the overridden error if necessary.
921
922 Parameters
923 ----------
925 The reference catalog to check for a ``coord_raErr`` field.
926 name : `str`
927 Whether we are doing "astrometry" or "photometry".
928
929 Returns
930 -------
931 refCoordErr : `float`
932 The refcat coordinate error to use, or NaN if we are not overriding
933 those fields.
934
935 Raises
936 ------
937 lsst.pex.config.FieldValidationError
938 Raised if the refcat does not contain coordinate errors and
939 ``config.astrometryReferenceErr`` is not set.
940 """
941 # This value doesn't matter for photometry, so just set something to
942 # keep old refcats from causing problems.
943 if name.lower() == "photometry":
944 if 'coord_raErr' not in refCat.schema:
945 return 100
946 else:
947 return float('nan')
948
949 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
950 msg = ("Reference catalog does not contain coordinate errors, "
951 "and config.astrometryReferenceErr not supplied.")
952 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
953 self.config,
954 msg)
955
956 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
957 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
958 self.config.astrometryReferenceErr)
959
960 if self.config.astrometryReferenceErr is None:
961 return float('nan')
962 else:
963 return self.config.astrometryReferenceErr
964
965 def _compute_proper_motion_epoch(self, ccdImageList):
966 """Return the proper motion correction epoch of the provided images.
967
968 Parameters
969 ----------
970 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
971 The images to compute the appropriate epoch for.
972
973 Returns
974 -------
975 epoch : `astropy.time.Time`
976 The date to use for proper motion corrections.
977 """
978 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
979 format="jyear",
980 scale="tai")
981
982 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
983 tract="", match_cut=3.0,
984 reject_bad_fluxes=False, *,
985 name="", refObjLoader=None, referenceSelector=None,
986 fit_function=None, epoch=None):
987 """Load reference catalog, perform the fit, and return the result.
988
989 Parameters
990 ----------
991 associations : `lsst.jointcal.Associations`
992 The star/reference star associations to fit.
993 defaultFilter : `lsst.afw.image.FilterLabel`
994 filter to load from reference catalog.
995 center : `lsst.geom.SpherePoint`
996 ICRS center of field to load from reference catalog.
997 radius : `lsst.geom.Angle`
998 On-sky radius to load from reference catalog.
999 name : `str`
1000 Name of thing being fit: "astrometry" or "photometry".
1001 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1002 Reference object loader to use to load a reference catalog.
1003 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1004 Selector to use to pick objects from the loaded reference catalog.
1005 fit_function : callable
1006 Function to call to perform fit (takes Associations object).
1007 tract : `str`, optional
1008 Name of tract currently being fit.
1009 match_cut : `float`, optional
1010 Radius in arcseconds to find cross-catalog matches to during
1011 associations.associateCatalogs.
1012 reject_bad_fluxes : `bool`, optional
1013 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1014 epoch : `astropy.time.Time`, optional
1015 Epoch to which to correct refcat proper motion and parallax,
1016 or `None` to not apply such corrections.
1017
1018 Returns
1019 -------
1020 result : `Photometry` or `Astrometry`
1021 Result of `fit_function()`
1022 """
1023 self.log.info("====== Now processing %s...", name)
1024 # TODO: this should not print "trying to invert a singular transformation:"
1025 # if it does that, something's not right about the WCS...
1026 associations.associateCatalogs(match_cut)
1027 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name,
1028 associations.fittedStarListSize())
1029
1030 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1031 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1032 center, radius, defaultFilter,
1033 applyColorterms=applyColorterms,
1034 epoch=epoch)
1035 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1036
1037 associations.collectRefStars(refCat,
1038 self.config.matchCut*lsst.geom.arcseconds,
1039 fluxField,
1040 refCoordinateErr=refCoordErr,
1041 rejectBadFluxes=reject_bad_fluxes)
1042 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name,
1043 associations.refStarListSize())
1044
1045 associations.prepareFittedStars(self.config.minMeasurements)
1046
1047 self._check_star_lists(associations, name)
1048 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name,
1049 associations.nFittedStarsWithAssociatedRefStar())
1050 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name,
1051 associations.fittedStarListSize())
1052 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name,
1053 associations.nCcdImagesValidForFit())
1054
1055 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1056 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1057 with pipeBase.cmdLineTask.profile(fit_profile_file):
1058 result = fit_function(associations, dataName)
1059 # TODO DM-12446: turn this into a "butler save" somehow.
1060 # Save reference and measurement chi2 contributions for this data
1061 if self.config.writeChi2FilesInitialFinal:
1062 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1063 result.fit.saveChi2Contributions(baseName+"{type}")
1064 self.log.info("Wrote chi2 contributions files: %s", baseName)
1065
1066 return result
1067
1068 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1069 applyColorterms=False, epoch=None):
1070 """Load the necessary reference catalog sources, convert fluxes to
1071 correct units, and apply color term corrections if requested.
1072
1073 Parameters
1074 ----------
1075 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1076 The reference catalog loader to use to get the data.
1077 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1078 Source selector to apply to loaded reference catalog.
1079 center : `lsst.geom.SpherePoint`
1080 The center around which to load sources.
1081 radius : `lsst.geom.Angle`
1082 The radius around ``center`` to load sources in.
1083 filterLabel : `lsst.afw.image.FilterLabel`
1084 The camera filter to load fluxes for.
1085 applyColorterms : `bool`
1086 Apply colorterm corrections to the refcat for ``filterName``?
1087 epoch : `astropy.time.Time`, optional
1088 Epoch to which to correct refcat proper motion and parallax,
1089 or `None` to not apply such corrections.
1090
1091 Returns
1092 -------
1094 The loaded reference catalog.
1095 fluxField : `str`
1096 The name of the reference catalog flux field appropriate for ``filterName``.
1097 """
1098 skyCircle = refObjLoader.loadSkyCircle(center,
1099 radius,
1100 filterLabel.bandLabel,
1101 epoch=epoch)
1102
1103 selected = referenceSelector.run(skyCircle.refCat)
1104 # Need memory contiguity to get reference filters as a vector.
1105 if not selected.sourceCat.isContiguous():
1106 refCat = selected.sourceCat.copy(deep=True)
1107 else:
1108 refCat = selected.sourceCat
1109
1110 if applyColorterms:
1111 refCatName = self.config.connections.photometryRefCat
1112 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1113 filterLabel.physicalLabel, refCatName)
1114 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1115 refCatName,
1116 doRaise=True)
1117
1118 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1119 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1120 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1121 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1122
1123 return refCat, skyCircle.fluxField
1124
1125 def _check_star_lists(self, associations, name):
1126 # TODO: these should be len(blah), but we need this properly wrapped first.
1127 if associations.nCcdImagesValidForFit() == 0:
1128 raise RuntimeError('No images in the ccdImageList!')
1129 if associations.fittedStarListSize() == 0:
1130 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1131 if associations.refStarListSize() == 0:
1132 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1133
1134 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1135 """Compute chi2, log it, validate the model, and return chi2.
1136
1137 Parameters
1138 ----------
1139 associations : `lsst.jointcal.Associations`
1140 The star/reference star associations to fit.
1142 The fitter to use for minimization.
1143 model : `lsst.jointcal.Model`
1144 The model being fit.
1145 chi2Label : `str`
1146 Label to describe the chi2 (e.g. "Initialized", "Final").
1147 writeChi2Name : `str`, optional
1148 Filename prefix to write the chi2 contributions to.
1149 Do not supply an extension: an appropriate one will be added.
1150
1151 Returns
1152 -------
1154 The chi2 object for the current fitter and model.
1155
1156 Raises
1157 ------
1158 FloatingPointError
1159 Raised if chi2 is infinite or NaN.
1160 ValueError
1161 Raised if the model is not valid.
1162 """
1163 if writeChi2Name is not None:
1164 fullpath = self._getDebugPath(writeChi2Name)
1165 fit.saveChi2Contributions(fullpath+"{type}")
1166 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1167
1168 chi2 = fit.computeChi2()
1169 self.log.info("%s %s", chi2Label, chi2)
1170 self._check_stars(associations)
1171 if not np.isfinite(chi2.chi2):
1172 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1173 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1174 raise ValueError("Model is not valid: check log messages for warnings.")
1175 return chi2
1176
1177 def _fit_photometry(self, associations, dataName=None):
1178 """
1179 Fit the photometric data.
1180
1181 Parameters
1182 ----------
1183 associations : `lsst.jointcal.Associations`
1184 The star/reference star associations to fit.
1185 dataName : `str`
1186 Name of the data being processed (e.g. "1234_HSC-Y"), for
1187 identifying debugging files.
1188
1189 Returns
1190 -------
1191 fit_result : `namedtuple`
1193 The photometric fitter used to perform the fit.
1195 The photometric model that was fit.
1196 """
1197 self.log.info("=== Starting photometric fitting...")
1198
1199 # TODO: should use pex.config.RegistryField here (see DM-9195)
1200 if self.config.photometryModel == "constrainedFlux":
1201 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1202 self.focalPlaneBBox,
1203 visitOrder=self.config.photometryVisitOrder,
1204 errorPedestal=self.config.photometryErrorPedestal)
1205 # potentially nonlinear problem, so we may need a line search to converge.
1206 doLineSearch = self.config.allowLineSearch
1207 elif self.config.photometryModel == "constrainedMagnitude":
1208 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1209 self.focalPlaneBBox,
1210 visitOrder=self.config.photometryVisitOrder,
1211 errorPedestal=self.config.photometryErrorPedestal)
1212 # potentially nonlinear problem, so we may need a line search to converge.
1213 doLineSearch = self.config.allowLineSearch
1214 elif self.config.photometryModel == "simpleFlux":
1215 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1216 errorPedestal=self.config.photometryErrorPedestal)
1217 doLineSearch = False # purely linear in model parameters, so no line search needed
1218 elif self.config.photometryModel == "simpleMagnitude":
1219 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1220 errorPedestal=self.config.photometryErrorPedestal)
1221 doLineSearch = False # purely linear in model parameters, so no line search needed
1222
1223 fit = lsst.jointcal.PhotometryFit(associations, model)
1224 # TODO DM-12446: turn this into a "butler save" somehow.
1225 # Save reference and measurement chi2 contributions for this data
1226 if self.config.writeChi2FilesInitialFinal:
1227 baseName = f"photometry_initial_chi2-{dataName}"
1228 else:
1229 baseName = None
1230 if self.config.writeInitialModel:
1231 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt")
1232 writeModel(model, fullpath, self.log)
1233 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1234
1235 def getChi2Name(whatToFit):
1236 if self.config.writeChi2FilesOuterLoop:
1237 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1238 else:
1239 return None
1240
1241 # The constrained model needs the visit transform fit first; the chip
1242 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1243 if self.config.writeInitMatrix:
1244 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}")
1245 else:
1246 dumpMatrixFile = ""
1247 if self.config.photometryModel.startswith("constrained"):
1248 # no line search: should be purely (or nearly) linear,
1249 # and we want a large step size to initialize with.
1250 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1251 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1252 writeChi2Name=getChi2Name("ModelVisit"))
1253 dumpMatrixFile = "" # so we don't redo the output on the next step
1254
1255 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1256 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1257 writeChi2Name=getChi2Name("Model"))
1258
1259 fit.minimize("Fluxes") # no line search: always purely linear.
1260 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1261 writeChi2Name=getChi2Name("Fluxes"))
1262
1263 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1264 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1265 writeChi2Name=getChi2Name("ModelFluxes"))
1266
1267 model.freezeErrorTransform()
1268 self.log.debug("Photometry error scales are frozen.")
1269
1270 chi2 = self._iterate_fit(associations,
1271 fit,
1272 self.config.maxPhotometrySteps,
1273 "photometry",
1274 "Model Fluxes",
1275 doRankUpdate=self.config.photometryDoRankUpdate,
1276 doLineSearch=doLineSearch,
1277 dataName=dataName)
1278
1279 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1280 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1281 return Photometry(fit, model)
1282
1283 def _fit_astrometry(self, associations, dataName=None):
1284 """
1285 Fit the astrometric data.
1286
1287 Parameters
1288 ----------
1289 associations : `lsst.jointcal.Associations`
1290 The star/reference star associations to fit.
1291 dataName : `str`
1292 Name of the data being processed (e.g. "1234_HSC-Y"), for
1293 identifying debugging files.
1294
1295 Returns
1296 -------
1297 fit_result : `namedtuple`
1299 The astrometric fitter used to perform the fit.
1301 The astrometric model that was fit.
1302 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1303 The model for the sky to tangent plane projection that was used in the fit.
1304 """
1305
1306 self.log.info("=== Starting astrometric fitting...")
1307
1308 associations.deprojectFittedStars()
1309
1310 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1311 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1312 # them so carefully?
1313 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1314
1315 if self.config.astrometryModel == "constrained":
1316 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1317 sky_to_tan_projection,
1318 chipOrder=self.config.astrometryChipOrder,
1319 visitOrder=self.config.astrometryVisitOrder)
1320 elif self.config.astrometryModel == "simple":
1321 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1322 sky_to_tan_projection,
1323 self.config.useInputWcs,
1324 nNotFit=0,
1325 order=self.config.astrometrySimpleOrder)
1326
1327 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1328 # TODO DM-12446: turn this into a "butler save" somehow.
1329 # Save reference and measurement chi2 contributions for this data
1330 if self.config.writeChi2FilesInitialFinal:
1331 baseName = f"astrometry_initial_chi2-{dataName}"
1332 else:
1333 baseName = None
1334 if self.config.writeInitialModel:
1335 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1336 writeModel(model, fullpath, self.log)
1337 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1338
1339 def getChi2Name(whatToFit):
1340 if self.config.writeChi2FilesOuterLoop:
1341 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1342 else:
1343 return None
1344
1345 if self.config.writeInitMatrix:
1346 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}")
1347 else:
1348 dumpMatrixFile = ""
1349 # The constrained model needs the visit transform fit first; the chip
1350 # transform is initialized from the detector's cameraGeom, so it's close.
1351 if self.config.astrometryModel == "constrained":
1352 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1353 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1354 writeChi2Name=getChi2Name("DistortionsVisit"))
1355 dumpMatrixFile = "" # so we don't redo the output on the next step
1356
1357 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1358 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1359 writeChi2Name=getChi2Name("Distortions"))
1360
1361 fit.minimize("Positions")
1362 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1363 writeChi2Name=getChi2Name("Positions"))
1364
1365 fit.minimize("Distortions Positions")
1366 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1367 writeChi2Name=getChi2Name("DistortionsPositions"))
1368
1369 chi2 = self._iterate_fit(associations,
1370 fit,
1371 self.config.maxAstrometrySteps,
1372 "astrometry",
1373 "Distortions Positions",
1374 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1375 doRankUpdate=self.config.astrometryDoRankUpdate,
1376 dataName=dataName)
1377
1378 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1379 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1380
1381 return Astrometry(fit, model, sky_to_tan_projection)
1382
1383 def _check_stars(self, associations):
1384 """Count measured and reference stars per ccd and warn/log them."""
1385 for ccdImage in associations.getCcdImageList():
1386 nMeasuredStars, nRefStars = ccdImage.countStars()
1387 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1388 ccdImage.getName(), nMeasuredStars, nRefStars)
1389 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1390 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)",
1391 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1392 if nRefStars < self.config.minRefStarsPerCcd:
1393 self.log.warning("ccdImage %s has only %s RefStars (desired %s)",
1394 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1395
1396 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1397 dataName="",
1398 sigmaRelativeTolerance=0,
1399 doRankUpdate=True,
1400 doLineSearch=False):
1401 """Run fitter.minimize up to max_steps times, returning the final chi2.
1402
1403 Parameters
1404 ----------
1405 associations : `lsst.jointcal.Associations`
1406 The star/reference star associations to fit.
1407 fitter : `lsst.jointcal.FitterBase`
1408 The fitter to use for minimization.
1409 max_steps : `int`
1410 Maximum number of steps to run outlier rejection before declaring
1411 convergence failure.
1412 name : {'photometry' or 'astrometry'}
1413 What type of data are we fitting (for logs and debugging files).
1414 whatToFit : `str`
1415 Passed to ``fitter.minimize()`` to define the parameters to fit.
1416 dataName : `str`, optional
1417 Descriptive name for this dataset (e.g. tract and filter),
1418 for debugging.
1419 sigmaRelativeTolerance : `float`, optional
1420 Convergence tolerance for the fractional change in the chi2 cut
1421 level for determining outliers. If set to zero, iterations will
1422 continue until there are no outliers.
1423 doRankUpdate : `bool`, optional
1424 Do an Eigen rank update during minimization, or recompute the full
1425 matrix and gradient?
1426 doLineSearch : `bool`, optional
1427 Do a line search for the optimum step during minimization?
1428
1429 Returns
1430 -------
1432 The final chi2 after the fit converges, or is forced to end.
1433
1434 Raises
1435 ------
1436 FloatingPointError
1437 Raised if the fitter fails with a non-finite value.
1438 RuntimeError
1439 Raised if the fitter fails for some other reason;
1440 log messages will provide further details.
1441 """
1442 if self.config.writeInitMatrix:
1443 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}")
1444 else:
1445 dumpMatrixFile = ""
1446 oldChi2 = lsst.jointcal.Chi2Statistic()
1447 oldChi2.chi2 = float("inf")
1448 for i in range(max_steps):
1449 if self.config.writeChi2FilesOuterLoop:
1450 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1451 else:
1452 writeChi2Name = None
1453 result = fitter.minimize(whatToFit,
1454 self.config.outlierRejectSigma,
1455 sigmaRelativeTolerance=sigmaRelativeTolerance,
1456 doRankUpdate=doRankUpdate,
1457 doLineSearch=doLineSearch,
1458 dumpMatrixFile=dumpMatrixFile)
1459 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1460 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1461 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1462
1463 if result == MinimizeResult.Converged:
1464 if doRankUpdate:
1465 self.log.debug("fit has converged - no more outliers - redo minimization "
1466 "one more time in case we have lost accuracy in rank update.")
1467 # Redo minimization one more time in case we have lost accuracy in rank update
1468 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1469 sigmaRelativeTolerance=sigmaRelativeTolerance)
1470 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1471
1472 # log a message for a large final chi2, TODO: DM-15247 for something better
1473 if chi2.chi2/chi2.ndof >= 4.0:
1474 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1475
1476 break
1477 elif result == MinimizeResult.Chi2Increased:
1478 self.log.warning("Still some outliers remaining but chi2 increased - retry")
1479 # Check whether the increase was large enough to cause trouble.
1480 chi2Ratio = chi2.chi2 / oldChi2.chi2
1481 if chi2Ratio > 1.5:
1482 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1483 chi2.chi2, oldChi2.chi2, chi2Ratio)
1484 # Based on a variety of HSC jointcal logs (see DM-25779), it
1485 # appears that chi2 increases more than a factor of ~2 always
1486 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1487 # Using 10 as the "failure" threshold gives some room between
1488 # leaving a warning and bailing early.
1489 if chi2Ratio > 10:
1490 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1491 " Try setting one or more of the `writeChi2*` config fields and looking"
1492 " at how individual star chi2-values evolve during the fit.")
1493 raise RuntimeError(msg)
1494 oldChi2 = chi2
1495 elif result == MinimizeResult.NonFinite:
1496 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1497 # TODO DM-12446: turn this into a "butler save" somehow.
1498 fitter.saveChi2Contributions(filename+"{type}")
1499 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1500 raise FloatingPointError(msg.format(filename))
1501 elif result == MinimizeResult.Failed:
1502 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1503 else:
1504 raise RuntimeError("Unxepected return code from minimize().")
1505 else:
1506 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1507
1508 return chi2
1509
1510 def _make_output(self, ccdImageList, model, func):
1511 """Return the internal jointcal models converted to the afw
1512 structures that will be saved to disk.
1513
1514 Parameters
1515 ----------
1516 ccdImageList : `lsst.jointcal.CcdImageList`
1517 The list of CcdImages to get the output for.
1519 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1520 func : `str`
1521 The name of the function to call on ``model`` to get the converted
1522 structure. Must accept an `lsst.jointcal.CcdImage`.
1523
1524 Returns
1525 -------
1526 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1527 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1528 The data to be saved, keyed on (visit, detector).
1529 """
1530 output = {}
1531 for ccdImage in ccdImageList:
1532 ccd = ccdImage.ccdId
1533 visit = ccdImage.visit
1534 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1535 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1536 return output
1537
1538 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1539 """
1540 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1541
1542 Parameters
1543 ----------
1544 associations : `lsst.jointcal.Associations`
1545 The star/reference star associations to fit.
1547 The astrometric model that was fit.
1548 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1549 Dict of ccdImage identifiers to dataRefs that were fit.
1550 """
1551 ccdImageList = associations.getCcdImageList()
1552 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1553 for key, skyWcs in output.items():
1554 dataRef = visit_ccd_to_dataRef[key]
1555 try:
1556 dataRef.put(skyWcs, 'jointcal_wcs')
1557 except pexExceptions.Exception as e:
1558 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1559 raise e
1560
1561 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1562 """
1563 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1564
1565 Parameters
1566 ----------
1567 associations : `lsst.jointcal.Associations`
1568 The star/reference star associations to fit.
1570 The photoometric model that was fit.
1571 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1572 Dict of ccdImage identifiers to dataRefs that were fit.
1573 """
1574
1575 ccdImageList = associations.getCcdImageList()
1576 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1577 for key, photoCalib in output.items():
1578 dataRef = visit_ccd_to_dataRef[key]
1579 try:
1580 dataRef.put(photoCalib, 'jointcal_photoCalib')
1581 except pexExceptions.Exception as e:
1582 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1583 raise e
1584
1585
1587 """Return an afw SourceTable to use as a base for creating the
1588 SourceCatalog to insert values from the dataFrame into.
1589
1590 Returns
1591 -------
1593 Table with schema and slots to use to make SourceCatalogs.
1594 """
1596 schema.addField("centroid_x", "D")
1597 schema.addField("centroid_y", "D")
1598 schema.addField("centroid_xErr", "F")
1599 schema.addField("centroid_yErr", "F")
1600 schema.addField("shape_xx", "D")
1601 schema.addField("shape_yy", "D")
1602 schema.addField("shape_xy", "D")
1603 schema.addField("flux_instFlux", "D")
1604 schema.addField("flux_instFluxErr", "D")
1605 table = lsst.afw.table.SourceTable.make(schema)
1606 table.defineCentroid("centroid")
1607 table.defineShape("shape")
1608 return table
1609
1610
1611def get_sourceTable_visit_columns(inColumns, config, sourceSelector):
1612 """
1613 Get the sourceTable_visit columns to load from the catalogs.
1614
1615 Parameters
1616 ----------
1617 inColumns : `list`
1618 List of columns known to be available in the sourceTable_visit.
1619 config : `JointcalConfig`
1620 A filled-in config to to help define column names.
1621 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask`
1622 A configured source selector to define column names to load.
1623
1624 Returns
1625 -------
1626 columns : `list`
1627 List of columns to read from sourceTable_visit.
1628 detectorColumn : `str`
1629 Name of the detector column.
1630 ixxColumns : `list`
1631 Name of the ixx/iyy/ixy columns.
1632 """
1633 if 'detector' in inColumns:
1634 # Default name for Gen3.
1635 detectorColumn = 'detector'
1636 else:
1637 # Default name for Gen2 conversions (still used in tests, CI, and older catalogs)
1638 detectorColumn = 'ccd'
1639
1640 columns = ['visit', detectorColumn,
1641 'sourceId', 'x', 'xErr', 'y', 'yErr',
1642 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr']
1643
1644 if 'ixx' in inColumns:
1645 # New columns post-DM-31825
1646 ixxColumns = ['ixx', 'iyy', 'ixy']
1647 else:
1648 # Old columns pre-DM-31825
1649 ixxColumns = ['Ixx', 'Iyy', 'Ixy']
1650 columns.extend(ixxColumns)
1651
1652 if sourceSelector.config.doFlags:
1653 columns.extend(sourceSelector.config.flags.bad)
1654 if sourceSelector.config.doUnresolved:
1655 columns.append(sourceSelector.config.unresolved.name)
1656 if sourceSelector.config.doIsolated:
1657 columns.append(sourceSelector.config.isolated.parentName)
1658 columns.append(sourceSelector.config.isolated.nChildName)
1659
1660 return columns, detectorColumn, ixxColumns
1661
1662
1663def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId,
1664 detectorColumn, ixxColumns, sourceFluxType, log):
1665 """Return an afw SourceCatalog extracted from a visit-level dataframe,
1666 limited to just one detector.
1667
1668 Parameters
1669 ----------
1671 Table factory to use to make the SourceCatalog that will be
1672 populated with data from ``visitCatalog``.
1673 visitCatalog : `pandas.DataFrame`
1674 DataFrame to extract a detector catalog from.
1675 detectorId : `int`
1676 Numeric id of the detector to extract from ``visitCatalog``.
1677 detectorColumn : `str`
1678 Name of the detector column in the catalog.
1679 ixxColumns : `list` [`str`]
1680 Names of the ixx/iyy/ixy columns in the catalog.
1681 sourceFluxType : `str`
1682 Name of the catalog field to load instFluxes from.
1683 log : `lsst.log.Log`
1684 Logging instance to log to.
1685
1686 Returns
1687 -------
1688 catalog : `lsst.afw.table.SourceCatalog`, or `None`
1689 Detector-level catalog extracted from ``visitCatalog``, or `None`
1690 if there was no data to load.
1691 """
1692 # map from dataFrame column to afw table column
1693 mapping = {'x': 'centroid_x',
1694 'y': 'centroid_y',
1695 'xErr': 'centroid_xErr',
1696 'yErr': 'centroid_yErr',
1697 ixxColumns[0]: 'shape_xx',
1698 ixxColumns[1]: 'shape_yy',
1699 ixxColumns[2]: 'shape_xy',
1700 f'{sourceFluxType}_instFlux': 'flux_instFlux',
1701 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr',
1702 }
1703
1704 catalog = lsst.afw.table.SourceCatalog(table)
1705 matched = visitCatalog[detectorColumn] == detectorId
1706 n = sum(matched)
1707 if n == 0:
1708 return None
1709 catalog.resize(sum(matched))
1710 view = visitCatalog.loc[matched]
1711 catalog['id'] = view.index
1712 for dfCol, afwCol in mapping.items():
1713 catalog[afwCol] = view[dfCol]
1714
1715 log.debug("%d sources selected in visit %d detector %d",
1716 len(catalog),
1717 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
1718 detectorId)
1719 return catalog
static std::shared_ptr< SourceTable > make(Schema const &schema, std::shared_ptr< IdFactory > const &idFactory)
static Schema makeMinimalSchema()
The class that implements the relations between MeasuredStar and FittedStar.
Definition: Associations.h:55
Class that handles the astrometric least squares problem.
Definition: AstrometryFit.h:78
Interface between AstrometryFit and the combinations of Mappings from pixels to some tangent plane (a...
Handler of an actual image from a single CCD.
Definition: CcdImage.h:64
Base class for Chi2Statistic and Chi2List, to allow addEntry inside Fitter for either class.
Definition: Chi2.h:44
Simple structure to accumulate chi2 and ndof.
Definition: Chi2.h:52
A multi-component model, fitting mappings for sensors and visits simultaneously.
Base class for fitters.
Definition: FitterBase.h:55
A projection handler in which all CCDs from the same visit have the same tangent point.
Class that handles the photometric least squares problem.
Definition: PhotometryFit.h:46
A model where there is one independent transform per CcdImage.
def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations, jointcalControl, camera)
Definition: jointcal.py:753
def _compute_proper_motion_epoch(self, ccdImageList)
Definition: jointcal.py:965
def _get_refcat_coordinate_error_override(self, refCat, name)
Definition: jointcal.py:918
def _getDebugPath(self, filename)
Definition: jointcal.py:891
def _check_star_lists(self, associations, name)
Definition: jointcal.py:1125
def _make_one_input_data(self, visitRecord, catalog, detectorDict)
Definition: jointcal.py:826
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", sigmaRelativeTolerance=0, doRankUpdate=True, doLineSearch=False)
Definition: jointcal.py:1400
def _check_stars(self, associations)
Definition: jointcal.py:1383
def _prep_sky(self, associations, filters)
Definition: jointcal.py:896
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: jointcal.py:595
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, tract="", match_cut=3.0, reject_bad_fluxes=False, *name="", refObjLoader=None, referenceSelector=None, fit_function=None, epoch=None)
Definition: jointcal.py:986
def _put_metrics(self, butlerQC, job, outputRefs)
Definition: jointcal.py:624
def _build_ccdImage(self, data, associations, jointcalControl)
Definition: jointcal.py:846
def __init__(self, **kwargs)
Definition: jointcal.py:580
def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None)
Definition: jointcal.py:699
def _fit_photometry(self, associations, dataName=None)
Definition: jointcal.py:1177
def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None)
Definition: jointcal.py:1134
def _make_output(self, ccdImageList, model, func)
Definition: jointcal.py:1510
def _put_output(self, butlerQC, outputs, outputRefs, camera, setter)
Definition: jointcal.py:640
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel, applyColorterms=False, epoch=None)
Definition: jointcal.py:1069
def _fit_astrometry(self, associations, dataName=None)
Definition: jointcal.py:1283
def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:62
def add_measurement(job, name, value)
Definition: jointcal.py:57
def get_sourceTable_visit_columns(inColumns, config, sourceSelector)
Definition: jointcal.py:1611
def lookupVisitRefCats(datasetType, registry, quantumDataId, collections)
Definition: jointcal.py:102
def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId, detectorColumn, ixxColumns, sourceFluxType, log)
Definition: jointcal.py:1664
def writeModel(model, filename, log)
Definition: jointcal.py:544
This is a virtual class that allows a lot of freedom in the choice of the projection from "Sky" (wher...