Coverage for python/lsst/jointcal/jointcal.py: 18%
543 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-14 02:57 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2022-10-14 02:57 -0700
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import dataclasses
23import collections
24import os
25import logging
27import astropy.time
28import numpy as np
29import astropy.units as u
31import lsst.geom
32import lsst.utils
33import lsst.pex.config as pexConfig
34import lsst.pipe.base as pipeBase
35from lsst.afw.image import fluxErrFromABMagErr
36import lsst.afw.cameraGeom
37import lsst.afw.table
38from lsst.pipe.base import Instrument
39from lsst.pipe.tasks.colorterms import ColortermLibrary
40from lsst.verify import Job, Measurement
42from lsst.meas.algorithms import (ReferenceObjectLoader, ReferenceSourceSelectorTask,
43 LoadIndexedReferenceObjectsConfig)
44from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46import lsst.jointcal
47from lsst.jointcal import MinimizeResult
49__all__ = ["JointcalConfig", "JointcalTask"]
51Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
52Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
55# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
56def add_measurement(job, name, value):
57 meas = Measurement(job.metrics[name], value)
58 job.measurements.insert(meas)
61def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
62 """Lookup function that asserts/hopes that a static calibration dataset
63 exists in a particular collection, since this task can't provide a single
64 date/time to use to search for one properly.
66 This is mostly useful for the ``camera`` dataset, in cases where the task's
67 quantum dimensions do *not* include something temporal, like ``exposure``
68 or ``visit``.
70 Parameters
71 ----------
72 datasetType : `lsst.daf.butler.DatasetType`
73 Type of dataset being searched for.
74 registry : `lsst.daf.butler.Registry`
75 Data repository registry to search.
76 quantumDataId : `lsst.daf.butler.DataCoordinate`
77 Data ID of the quantum this camera should match.
78 collections : `Iterable` [ `str` ]
79 Collections that should be searched - but this lookup function works
80 by ignoring this in favor of a more-or-less hard-coded value.
82 Returns
83 -------
84 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
85 Iterator over dataset references; should have only one element.
87 Notes
88 -----
89 This implementation duplicates one in fgcmcal, and is at least quite
90 similar to another in cp_pipe. This duplicate has the most documentation.
91 Fixing this is DM-29661.
92 """
93 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
94 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
95 return registry.queryDatasets(datasetType,
96 dataId=quantumDataId,
97 collections=[unboundedCollection],
98 findFirst=True)
101def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
102 """Lookup function that finds all refcats for all visits that overlap a
103 tract, rather than just the refcats that directly overlap the tract.
105 Parameters
106 ----------
107 datasetType : `lsst.daf.butler.DatasetType`
108 Type of dataset being searched for.
109 registry : `lsst.daf.butler.Registry`
110 Data repository registry to search.
111 quantumDataId : `lsst.daf.butler.DataCoordinate`
112 Data ID of the quantum; expected to be something we can use as a
113 constraint to query for overlapping visits.
114 collections : `Iterable` [ `str` ]
115 Collections to search.
117 Returns
118 -------
119 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
120 Iterator over refcat references.
121 """
122 refs = set()
123 # Use .expanded() on the query methods below because we need data IDs with
124 # regions, both in the outer loop over visits (queryDatasets will expand
125 # any data ID we give it, but doing it up-front in bulk is much more
126 # efficient) and in the data IDs of the DatasetRefs this function yields
127 # (because the RefCatLoader relies on them to do some of its own
128 # filtering).
129 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
130 refs.update(
131 registry.queryDatasets(
132 datasetType,
133 collections=collections,
134 dataId=visit_data_id,
135 findFirst=True,
136 ).expanded()
137 )
138 yield from refs
141class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
142 dimensions=("skymap", "tract", "instrument", "physical_filter")):
143 """Middleware input/output connections for jointcal data."""
144 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
145 doc="The camera instrument that took these observations.",
146 name="camera",
147 storageClass="Camera",
148 dimensions=("instrument",),
149 isCalibration=True,
150 lookupFunction=lookupStaticCalibrations,
151 )
152 inputSourceTableVisit = pipeBase.connectionTypes.Input(
153 doc="Source table in parquet format, per visit",
154 name="sourceTable_visit",
155 storageClass="DataFrame",
156 dimensions=("instrument", "visit"),
157 deferLoad=True,
158 multiple=True,
159 )
160 inputVisitSummary = pipeBase.connectionTypes.Input(
161 doc=("Per-visit consolidated exposure metadata built from calexps. "
162 "These catalogs use detector id for the id and must be sorted for "
163 "fast lookups of a detector."),
164 name="visitSummary",
165 storageClass="ExposureCatalog",
166 dimensions=("instrument", "visit"),
167 deferLoad=True,
168 multiple=True,
169 )
170 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
171 doc="The astrometry reference catalog to match to loaded input catalog sources.",
172 name="gaia_dr2_20200414",
173 storageClass="SimpleCatalog",
174 dimensions=("skypix",),
175 deferLoad=True,
176 multiple=True,
177 lookupFunction=lookupVisitRefCats,
178 )
179 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
180 doc="The photometry reference catalog to match to loaded input catalog sources.",
181 name="ps1_pv3_3pi_20170110",
182 storageClass="SimpleCatalog",
183 dimensions=("skypix",),
184 deferLoad=True,
185 multiple=True,
186 lookupFunction=lookupVisitRefCats,
187 )
189 outputWcs = pipeBase.connectionTypes.Output(
190 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
191 " These catalogs only contain entries for detectors with an output, and use"
192 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
193 name="jointcalSkyWcsCatalog",
194 storageClass="ExposureCatalog",
195 dimensions=("instrument", "visit", "skymap", "tract"),
196 multiple=True
197 )
198 outputPhotoCalib = pipeBase.connectionTypes.Output(
199 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
200 " These catalogs only contain entries for detectors with an output, and use"
201 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
202 name="jointcalPhotoCalibCatalog",
203 storageClass="ExposureCatalog",
204 dimensions=("instrument", "visit", "skymap", "tract"),
205 multiple=True
206 )
208 # measurements of metrics
209 # The vars() trick used here allows us to set class attributes
210 # programatically. Taken from:
211 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
212 for name in ("astrometry", "photometry"):
213 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
214 doc=f"The number of cross-matched fittedStars for {name}",
215 name=f"metricvalue_jointcal_{name}_matched_fittedStars",
216 storageClass="MetricValue",
217 dimensions=("skymap", "tract", "instrument", "physical_filter"),
218 )
219 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
220 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.",
221 name=f"metricvalue_jointcal_{name}_collected_refStars",
222 storageClass="MetricValue",
223 dimensions=("skymap", "tract", "instrument", "physical_filter"),
224 )
225 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
226 doc=f"The number of {name} reference stars matched to fittedStars.",
227 name=f"metricvalue_jointcal_{name}_prepared_refStars",
228 storageClass="MetricValue",
229 dimensions=("skymap", "tract", "instrument", "physical_filter"),
230 )
231 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
232 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.",
233 name=f"metricvalue_jointcal_{name}_prepared_fittedStars",
234 storageClass="MetricValue",
235 dimensions=("skymap", "tract", "instrument", "physical_filter"),
236 )
237 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
238 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.",
239 name=f"metricvalue_jointcal_{name}_prepared_ccdImages",
240 storageClass="MetricValue",
241 dimensions=("skymap", "tract", "instrument", "physical_filter"),
242 )
243 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
244 doc=f"The final chi2 of the {name} fit.",
245 name=f"metricvalue_jointcal_{name}_final_chi2",
246 storageClass="MetricValue",
247 dimensions=("skymap", "tract", "instrument", "physical_filter"),
248 )
249 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
250 doc=f"The number of degrees of freedom of the fitted {name}.",
251 name=f"metricvalue_jointcal_{name}_final_ndof",
252 storageClass="MetricValue",
253 dimensions=("skymap", "tract", "instrument", "physical_filter"),
254 )
256 def __init__(self, *, config=None):
257 super().__init__(config=config)
258 # When we are only doing one of astrometry or photometry, we don't
259 # need the reference catalog or produce the outputs for the other.
260 # This informs the middleware of that when the QuantumGraph is
261 # generated, so we don't block on getting something we won't need or
262 # create an expectation that downstream tasks will be able to consume
263 # something we won't produce.
264 if not config.doAstrometry:
265 self.prerequisiteInputs.remove("astrometryRefCat")
266 self.outputs.remove("outputWcs")
267 for key in list(self.outputs):
268 if "metricvalue_jointcal_astrometry" in key:
269 self.outputs.remove(key)
270 if not config.doPhotometry:
271 self.prerequisiteInputs.remove("photometryRefCat")
272 self.outputs.remove("outputPhotoCalib")
273 for key in list(self.outputs):
274 if "metricvalue_jointcal_photometry" in key:
275 self.outputs.remove(key)
278class JointcalConfig(pipeBase.PipelineTaskConfig,
279 pipelineConnections=JointcalTaskConnections):
280 """Configuration for JointcalTask"""
282 doAstrometry = pexConfig.Field(
283 doc="Fit astrometry and write the fitted result.",
284 dtype=bool,
285 default=True
286 )
287 doPhotometry = pexConfig.Field(
288 doc="Fit photometry and write the fitted result.",
289 dtype=bool,
290 default=True
291 )
292 sourceFluxType = pexConfig.Field(
293 dtype=str,
294 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
295 default='apFlux_12_0'
296 )
297 positionErrorPedestal = pexConfig.Field(
298 doc="Systematic term to apply to the measured position error (pixels)",
299 dtype=float,
300 default=0.02,
301 )
302 photometryErrorPedestal = pexConfig.Field(
303 doc="Systematic term to apply to the measured error on flux or magnitude as a "
304 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
305 dtype=float,
306 default=0.0,
307 )
308 # TODO: DM-6885 matchCut should be an geom.Angle
309 matchCut = pexConfig.Field(
310 doc="Matching radius between fitted and reference stars (arcseconds)",
311 dtype=float,
312 default=3.0,
313 )
314 minMeasurements = pexConfig.Field(
315 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
316 dtype=int,
317 default=2,
318 )
319 minMeasuredStarsPerCcd = pexConfig.Field(
320 doc="Minimum number of measuredStars per ccdImage before printing warnings",
321 dtype=int,
322 default=100,
323 )
324 minRefStarsPerCcd = pexConfig.Field(
325 doc="Minimum number of measuredStars per ccdImage before printing warnings",
326 dtype=int,
327 default=30,
328 )
329 allowLineSearch = pexConfig.Field(
330 doc="Allow a line search during minimization, if it is reasonable for the model"
331 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
332 dtype=bool,
333 default=False
334 )
335 astrometrySimpleOrder = pexConfig.Field(
336 doc="Polynomial order for fitting the simple astrometry model.",
337 dtype=int,
338 default=3,
339 )
340 astrometryChipOrder = pexConfig.Field(
341 doc="Order of the per-chip transform for the constrained astrometry model.",
342 dtype=int,
343 default=1,
344 )
345 astrometryVisitOrder = pexConfig.Field(
346 doc="Order of the per-visit transform for the constrained astrometry model.",
347 dtype=int,
348 default=5,
349 )
350 useInputWcs = pexConfig.Field(
351 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
352 dtype=bool,
353 default=True,
354 )
355 astrometryModel = pexConfig.ChoiceField(
356 doc="Type of model to fit to astrometry",
357 dtype=str,
358 default="constrained",
359 allowed={"simple": "One polynomial per ccd",
360 "constrained": "One polynomial per ccd, and one polynomial per visit"}
361 )
362 photometryModel = pexConfig.ChoiceField(
363 doc="Type of model to fit to photometry",
364 dtype=str,
365 default="constrainedMagnitude",
366 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
367 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
368 " fitting in flux space.",
369 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
370 " fitting in magnitude space.",
371 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
372 " fitting in magnitude space.",
373 }
374 )
375 applyColorTerms = pexConfig.Field(
376 doc="Apply photometric color terms to reference stars?"
377 "Requires that colorterms be set to a ColortermLibrary",
378 dtype=bool,
379 default=False
380 )
381 colorterms = pexConfig.ConfigField(
382 doc="Library of photometric reference catalog name to color term dict.",
383 dtype=ColortermLibrary,
384 )
385 photometryVisitOrder = pexConfig.Field(
386 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
387 dtype=int,
388 default=7,
389 )
390 photometryDoRankUpdate = pexConfig.Field(
391 doc=("Do the rank update step during minimization. "
392 "Skipping this can help deal with models that are too non-linear."),
393 dtype=bool,
394 default=True,
395 )
396 astrometryDoRankUpdate = pexConfig.Field(
397 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
398 "Skipping this can help deal with models that are too non-linear."),
399 dtype=bool,
400 default=True,
401 )
402 outlierRejectSigma = pexConfig.Field(
403 doc="How many sigma to reject outliers at during minimization.",
404 dtype=float,
405 default=5.0,
406 )
407 astrometryOutlierRelativeTolerance = pexConfig.Field(
408 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
409 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
410 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
411 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
412 "close to the solution found when removing all outliers."),
413 dtype=float,
414 default=0,
415 )
416 maxPhotometrySteps = pexConfig.Field(
417 doc="Maximum number of minimize iterations to take when fitting photometry.",
418 dtype=int,
419 default=20,
420 )
421 maxAstrometrySteps = pexConfig.Field(
422 doc="Maximum number of minimize iterations to take when fitting astrometry.",
423 dtype=int,
424 default=20,
425 )
426 astrometryRefObjLoader = pexConfig.ConfigField(
427 dtype=LoadIndexedReferenceObjectsConfig,
428 doc="Reference object loader for astrometric fit",
429 )
430 photometryRefObjLoader = pexConfig.ConfigField(
431 dtype=LoadIndexedReferenceObjectsConfig,
432 doc="Reference object loader for photometric fit",
433 )
434 sourceSelector = sourceSelectorRegistry.makeField(
435 doc="How to select sources for cross-matching",
436 default="science"
437 )
438 astrometryReferenceSelector = pexConfig.ConfigurableField(
439 target=ReferenceSourceSelectorTask,
440 doc="How to down-select the loaded astrometry reference catalog.",
441 )
442 photometryReferenceSelector = pexConfig.ConfigurableField(
443 target=ReferenceSourceSelectorTask,
444 doc="How to down-select the loaded photometry reference catalog.",
445 )
446 astrometryReferenceErr = pexConfig.Field(
447 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
448 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
449 "If specified, overrides any existing `coord_*Err` values."),
450 dtype=float,
451 default=None,
452 optional=True
453 )
455 # configs for outputting debug information
456 writeInitMatrix = pexConfig.Field(
457 dtype=bool,
458 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
459 "Output files will be written to `config.debugOutputPath` and will "
460 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
461 "Note that these files are the dense versions of the matrix, and so may be very large."),
462 default=False
463 )
464 writeChi2FilesInitialFinal = pexConfig.Field(
465 dtype=bool,
466 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
467 "Output files will be written to `config.debugOutputPath` and will "
468 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
469 default=False
470 )
471 writeChi2FilesOuterLoop = pexConfig.Field(
472 dtype=bool,
473 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
474 "Output files will be written to `config.debugOutputPath` and will "
475 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
476 default=False
477 )
478 writeInitialModel = pexConfig.Field(
479 dtype=bool,
480 doc=("Write the pre-initialization model to text files, for debugging. "
481 "Output files will be written to `config.debugOutputPath` and will be "
482 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
483 default=False
484 )
485 debugOutputPath = pexConfig.Field(
486 dtype=str,
487 default=".",
488 doc=("Path to write debug output files to. Used by "
489 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
490 )
491 detailedProfile = pexConfig.Field(
492 dtype=bool,
493 default=False,
494 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
495 )
497 def validate(self):
498 super().validate()
499 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
500 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
501 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
502 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
503 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
504 "applyColorTerms=True will be ignored.")
505 logging.getLogger("lsst.jointcal").warning(msg)
507 def setDefaults(self):
508 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing.
509 self.sourceSelector["science"].doUnresolved = True
510 self.sourceSelector["science"].unresolved.name = "extendedness"
511 # with dependable signal to noise ratio.
512 self.sourceSelector["science"].doSignalToNoise = True
513 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
514 # and S/N > 10 to use sources that are not too faint, and thus better measured.
515 self.sourceSelector["science"].signalToNoise.minimum = 10.
516 # Base SNR selection on `sourceFluxType` because that is the flux that jointcal fits.
517 self.sourceSelector["science"].signalToNoise.fluxField = f"{self.sourceFluxType}_instFlux"
518 self.sourceSelector["science"].signalToNoise.errField = f"{self.sourceFluxType}_instFluxErr"
519 # Do not trust blended sources" aperture fluxes which also depend on seeing.
520 self.sourceSelector["science"].doIsolated = True
521 self.sourceSelector["science"].isolated.parentName = "parentSourceId"
522 self.sourceSelector["science"].isolated.nChildName = "deblend_nChild"
523 # Do not trust either flux or centroid measurements with flags,
524 # chosen from the usual QA flags for stars)
525 self.sourceSelector["science"].doFlags = True
526 badFlags = ["pixelFlags_edge",
527 "pixelFlags_saturated",
528 "pixelFlags_interpolatedCenter",
529 "pixelFlags_interpolated",
530 "pixelFlags_crCenter",
531 "pixelFlags_bad",
532 "hsmPsfMoments_flag",
533 f"{self.sourceFluxType}_flag",
534 ]
535 self.sourceSelector["science"].flags.bad = badFlags
536 self.sourceSelector["science"].doRequireFiniteRaDec = True
537 self.sourceSelector["science"].requireFiniteRaDec.raColName = "ra"
538 self.sourceSelector["science"].requireFiniteRaDec.decColName = "decl"
540 # Use Gaia-DR2 with proper motions for astrometry; phot_g_mean is the
541 # primary Gaia band, but is not like any normal photometric band.
542 self.astrometryRefObjLoader.requireProperMotion = True
543 self.astrometryRefObjLoader.anyFilterMapsToThis = "phot_g_mean"
546def writeModel(model, filename, log):
547 """Write model to outfile."""
548 with open(filename, "w") as file:
549 file.write(repr(model))
550 log.info("Wrote %s to file: %s", model, filename)
553@dataclasses.dataclass
554class JointcalInputData:
555 """The input data jointcal needs for each detector/visit."""
556 visit: int
557 """The visit identifier of this exposure."""
558 catalog: lsst.afw.table.SourceCatalog
559 """The catalog derived from this exposure."""
560 visitInfo: lsst.afw.image.VisitInfo
561 """The VisitInfo of this exposure."""
562 detector: lsst.afw.cameraGeom.Detector
563 """The detector of this exposure."""
564 photoCalib: lsst.afw.image.PhotoCalib
565 """The photometric calibration of this exposure."""
566 wcs: lsst.afw.geom.skyWcs
567 """The WCS of this exposure."""
568 bbox: lsst.geom.Box2I
569 """The bounding box of this exposure."""
570 filter: lsst.afw.image.FilterLabel
571 """The filter of this exposure."""
574class JointcalTask(pipeBase.PipelineTask):
575 """Astrometricly and photometricly calibrate across multiple visits of the
576 same field.
577 """
579 ConfigClass = JointcalConfig
580 _DefaultName = "jointcal"
582 def __init__(self, **kwargs):
583 super().__init__(**kwargs)
584 self.makeSubtask("sourceSelector")
585 if self.config.doAstrometry:
586 self.makeSubtask("astrometryReferenceSelector")
587 else:
588 self.astrometryRefObjLoader = None
589 if self.config.doPhotometry:
590 self.makeSubtask("photometryReferenceSelector")
591 else:
592 self.photometryRefObjLoader = None
594 # To hold various computed metrics for use by tests
595 self.job = Job.load_metrics_package(subset='jointcal')
597 def runQuantum(self, butlerQC, inputRefs, outputRefs):
598 # We override runQuantum to set up the refObjLoaders and write the
599 # outputs to the correct refs.
600 inputs = butlerQC.get(inputRefs)
601 # We want the tract number for writing debug files
602 tract = butlerQC.quantum.dataId['tract']
603 if self.config.doAstrometry:
604 self.astrometryRefObjLoader = ReferenceObjectLoader(
605 dataIds=[ref.datasetRef.dataId
606 for ref in inputRefs.astrometryRefCat],
607 refCats=inputs.pop('astrometryRefCat'),
608 config=self.config.astrometryRefObjLoader,
609 name=self.config.connections.astrometryRefCat,
610 log=self.log)
611 if self.config.doPhotometry:
612 self.photometryRefObjLoader = ReferenceObjectLoader(
613 dataIds=[ref.datasetRef.dataId
614 for ref in inputRefs.photometryRefCat],
615 refCats=inputs.pop('photometryRefCat'),
616 config=self.config.photometryRefObjLoader,
617 name=self.config.connections.photometryRefCat,
618 log=self.log)
619 outputs = self.run(**inputs, tract=tract)
620 self._put_metrics(butlerQC, outputs.job, outputRefs)
621 if self.config.doAstrometry:
622 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
623 inputs['inputCamera'], "setWcs")
624 if self.config.doPhotometry:
625 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
626 inputs['inputCamera'], "setPhotoCalib")
628 def _put_metrics(self, butlerQC, job, outputRefs):
629 """Persist all measured metrics stored in a job.
631 Parameters
632 ----------
633 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
634 A butler which is specialized to operate in the context of a
635 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
636 job : `lsst.verify.job.Job`
637 Measurements of metrics to persist.
638 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
639 The DatasetRefs to persist the data to.
640 """
641 for key in job.measurements.keys():
642 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', '')))
644 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
645 """Persist the output datasets to their appropriate datarefs.
647 Parameters
648 ----------
649 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
650 A butler which is specialized to operate in the context of a
651 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
652 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
653 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
654 The fitted objects to persist.
655 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
656 The DatasetRefs to persist the data to.
657 camera : `lsst.afw.cameraGeom.Camera`
658 The camera for this instrument, to get detector ids from.
659 setter : `str`
660 The method to call on the ExposureCatalog to set each output.
661 """
662 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
663 schema.addField('visit', type='L', doc='Visit number')
665 def new_catalog(visit, size):
666 """Return an catalog ready to be filled with appropriate output."""
667 catalog = lsst.afw.table.ExposureCatalog(schema)
668 catalog.resize(size)
669 catalog['visit'] = visit
670 metadata = lsst.daf.base.PropertyList()
671 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
672 metadata.add("COMMENT", "Only detectors with data have entries.")
673 return catalog
675 # count how many detectors have output for each visit
676 detectors_per_visit = collections.defaultdict(int)
677 for key in outputs:
678 # key is (visit, detector_id), and we only need visit here
679 detectors_per_visit[key[0]] += 1
681 for ref in outputRefs:
682 visit = ref.dataId['visit']
683 catalog = new_catalog(visit, detectors_per_visit[visit])
684 # Iterate over every detector and skip the ones we don't have output for.
685 i = 0
686 for detector in camera:
687 detectorId = detector.getId()
688 key = (ref.dataId['visit'], detectorId)
689 if key not in outputs:
690 # skip detectors we don't have output for
691 self.log.debug("No %s output for detector %s in visit %s",
692 setter[3:], detectorId, visit)
693 continue
695 catalog[i].setId(detectorId)
696 getattr(catalog[i], setter)(outputs[key])
697 i += 1
699 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
700 butlerQC.put(catalog, ref)
701 self.log.info("Wrote %s detectors to %s", i, ref)
703 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
704 # Docstring inherited.
706 # We take values out of the Parquet table, and put them in "flux_",
707 # and the config.sourceFluxType field is used during that extraction,
708 # so just use "flux" here.
709 sourceFluxField = "flux"
710 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
711 associations = lsst.jointcal.Associations()
712 self.focalPlaneBBox = inputCamera.getFpBBox()
713 oldWcsList, bands = self._load_data(inputSourceTableVisit,
714 inputVisitSummary,
715 associations,
716 jointcalControl,
717 inputCamera)
719 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands)
721 if self.config.doAstrometry:
722 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
723 name="astrometry",
724 refObjLoader=self.astrometryRefObjLoader,
725 referenceSelector=self.astrometryReferenceSelector,
726 fit_function=self._fit_astrometry,
727 tract=tract,
728 epoch=epoch)
729 astrometry_output = self._make_output(associations.getCcdImageList(),
730 astrometry.model,
731 "makeSkyWcs")
732 else:
733 astrometry_output = None
735 if self.config.doPhotometry:
736 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
737 name="photometry",
738 refObjLoader=self.photometryRefObjLoader,
739 referenceSelector=self.photometryReferenceSelector,
740 fit_function=self._fit_photometry,
741 tract=tract,
742 epoch=epoch,
743 reject_bad_fluxes=True)
744 photometry_output = self._make_output(associations.getCcdImageList(),
745 photometry.model,
746 "toPhotoCalib")
747 else:
748 photometry_output = None
750 return pipeBase.Struct(outputWcs=astrometry_output,
751 outputPhotoCalib=photometry_output,
752 job=self.job,
753 astrometryRefObjLoader=self.astrometryRefObjLoader,
754 photometryRefObjLoader=self.photometryRefObjLoader)
756 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
757 jointcalControl, camera):
758 """Read the data that jointcal needs to run.
760 Modifies ``associations`` in-place with the loaded data.
762 Parameters
763 ----------
764 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
765 References to visit-level DataFrames to load the catalog data from.
766 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
767 Visit-level exposure summary catalog with metadata.
768 associations : `lsst.jointcal.Associations`
769 Object to add the loaded data to by constructing new CcdImages.
770 jointcalControl : `jointcal.JointcalControl`
771 Control object for C++ associations management.
772 camera : `lsst.afw.cameraGeom.Camera`
773 Camera object for detector geometry.
775 Returns
776 -------
777 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
778 The original WCS of the input data, to aid in writing tests.
779 bands : `list` [`str`]
780 The filter bands of each input dataset.
781 """
782 oldWcsList = []
783 filters = []
784 load_cat_profile_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
785 with lsst.utils.timer.profile(load_cat_profile_file):
786 table = make_schema_table() # every detector catalog has the same layout
787 # No guarantee that the input is in the same order of visits, so we have to map one of them.
788 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
789 detectorDict = {detector.getId(): detector for detector in camera}
791 columns = None
793 for visitSummaryRef in inputVisitSummary:
794 visitSummary = visitSummaryRef.get()
796 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]]
797 if columns is None:
798 inColumns = dataRef.get(component='columns')
799 columns, detColumn, ixxColumns = get_sourceTable_visit_columns(inColumns,
800 self.config,
801 self.sourceSelector)
802 visitCatalog = dataRef.get(parameters={'columns': columns})
804 selected = self.sourceSelector.run(visitCatalog)
805 if len(selected.sourceCat) == 0:
806 self.log.warning("No sources selected in visit %s. Skipping...",
807 visitSummary["visit"][0])
808 continue
810 # Build a CcdImage for each detector in this visit.
811 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
812 for id, index in detectors.items():
813 catalog = extract_detector_catalog_from_visit_catalog(table,
814 selected.sourceCat,
815 id,
816 detColumn,
817 ixxColumns,
818 self.config.sourceFluxType,
819 self.log)
820 if catalog is None:
821 continue
822 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
823 result = self._build_ccdImage(data, associations, jointcalControl)
824 if result is not None:
825 oldWcsList.append(result.wcs)
826 # A visit has only one band, so we can just use the first.
827 filters.append(data.filter)
828 if len(filters) == 0:
829 raise RuntimeError("No data to process: did source selector remove all sources?")
830 filters = collections.Counter(filters)
832 return oldWcsList, filters
834 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
835 """Return a data structure for this detector+visit."""
836 return JointcalInputData(visit=visitRecord['visit'],
837 catalog=catalog,
838 visitInfo=visitRecord.getVisitInfo(),
839 detector=detectorDict[visitRecord.getId()],
840 photoCalib=visitRecord.getPhotoCalib(),
841 wcs=visitRecord.getWcs(),
842 bbox=visitRecord.getBBox(),
843 # ExposureRecord doesn't have a FilterLabel yet,
844 # so we have to make one.
845 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
846 physical=visitRecord['physical_filter']))
848 def _build_ccdImage(self, data, associations, jointcalControl):
849 """
850 Extract the necessary things from this catalog+metadata to add a new
851 ccdImage.
853 Parameters
854 ----------
855 data : `JointcalInputData`
856 The loaded input data.
857 associations : `lsst.jointcal.Associations`
858 Object to add the info to, to construct a new CcdImage
859 jointcalControl : `jointcal.JointcalControl`
860 Control object for associations management
862 Returns
863 ------
864 namedtuple or `None`
865 ``wcs``
866 The TAN WCS of this image, read from the calexp
867 (`lsst.afw.geom.SkyWcs`).
868 ``key``
869 A key to identify this dataRef by its visit and ccd ids
870 (`namedtuple`).
871 `None`
872 if there are no sources in the loaded catalog.
873 """
874 if len(data.catalog) == 0:
875 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
876 return None
878 associations.createCcdImage(data.catalog,
879 data.wcs,
880 data.visitInfo,
881 data.bbox,
882 data.filter.physicalLabel,
883 data.photoCalib,
884 data.detector,
885 data.visit,
886 data.detector.getId(),
887 jointcalControl)
889 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
890 Key = collections.namedtuple('Key', ('visit', 'ccd'))
891 return Result(data.wcs, Key(data.visit, data.detector.getId()))
893 def _getDebugPath(self, filename):
894 """Constructs a path to filename using the configured debug path.
895 """
896 return os.path.join(self.config.debugOutputPath, filename)
898 def _prep_sky(self, associations, filters):
899 """Prepare on-sky and other data that must be computed after data has
900 been read.
901 """
902 associations.computeCommonTangentPoint()
904 boundingCircle = associations.computeBoundingCircle()
905 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
906 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
908 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
910 # Determine a default filter band associated with the catalog. See DM-9093
911 defaultFilter = filters.most_common(1)[0][0]
912 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
914 # compute and set the reference epoch of the observations, for proper motion corrections
915 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
916 associations.setEpoch(epoch.jyear)
918 return boundingCircle, center, radius, defaultFilter, epoch
920 def _get_refcat_coordinate_error_override(self, refCat, name):
921 """Check whether we should override the refcat coordinate errors, and
922 return the overridden error if necessary.
924 Parameters
925 ----------
926 refCat : `lsst.afw.table.SimpleCatalog`
927 The reference catalog to check for a ``coord_raErr`` field.
928 name : `str`
929 Whether we are doing "astrometry" or "photometry".
931 Returns
932 -------
933 refCoordErr : `float`
934 The refcat coordinate error to use, or NaN if we are not overriding
935 those fields.
937 Raises
938 ------
939 lsst.pex.config.FieldValidationError
940 Raised if the refcat does not contain coordinate errors and
941 ``config.astrometryReferenceErr`` is not set.
942 """
943 # This value doesn't matter for photometry, so just set something to
944 # keep old refcats from causing problems.
945 if name.lower() == "photometry":
946 if 'coord_raErr' not in refCat.schema:
947 return 100
948 else:
949 return float('nan')
951 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
952 msg = ("Reference catalog does not contain coordinate errors, "
953 "and config.astrometryReferenceErr not supplied.")
954 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
955 self.config,
956 msg)
958 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
959 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
960 self.config.astrometryReferenceErr)
962 if self.config.astrometryReferenceErr is None:
963 return float('nan')
964 else:
965 return self.config.astrometryReferenceErr
967 def _compute_proper_motion_epoch(self, ccdImageList):
968 """Return the proper motion correction epoch of the provided images.
970 Parameters
971 ----------
972 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
973 The images to compute the appropriate epoch for.
975 Returns
976 -------
977 epoch : `astropy.time.Time`
978 The date to use for proper motion corrections.
979 """
980 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
981 format="jyear",
982 scale="tai")
984 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
985 tract="", match_cut=3.0,
986 reject_bad_fluxes=False, *,
987 name="", refObjLoader=None, referenceSelector=None,
988 fit_function=None, epoch=None):
989 """Load reference catalog, perform the fit, and return the result.
991 Parameters
992 ----------
993 associations : `lsst.jointcal.Associations`
994 The star/reference star associations to fit.
995 defaultFilter : `lsst.afw.image.FilterLabel`
996 filter to load from reference catalog.
997 center : `lsst.geom.SpherePoint`
998 ICRS center of field to load from reference catalog.
999 radius : `lsst.geom.Angle`
1000 On-sky radius to load from reference catalog.
1001 name : `str`
1002 Name of thing being fit: "astrometry" or "photometry".
1003 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1004 Reference object loader to use to load a reference catalog.
1005 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1006 Selector to use to pick objects from the loaded reference catalog.
1007 fit_function : callable
1008 Function to call to perform fit (takes Associations object).
1009 tract : `str`, optional
1010 Name of tract currently being fit.
1011 match_cut : `float`, optional
1012 Radius in arcseconds to find cross-catalog matches to during
1013 associations.associateCatalogs.
1014 reject_bad_fluxes : `bool`, optional
1015 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1016 epoch : `astropy.time.Time`, optional
1017 Epoch to which to correct refcat proper motion and parallax,
1018 or `None` to not apply such corrections.
1020 Returns
1021 -------
1022 result : `Photometry` or `Astrometry`
1023 Result of `fit_function()`
1024 """
1025 self.log.info("====== Now processing %s...", name)
1026 # TODO: this should not print "trying to invert a singular transformation:"
1027 # if it does that, something's not right about the WCS...
1028 associations.associateCatalogs(match_cut)
1029 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name,
1030 associations.fittedStarListSize())
1032 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1033 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1034 center, radius, defaultFilter,
1035 applyColorterms=applyColorterms,
1036 epoch=epoch)
1037 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1039 associations.collectRefStars(refCat,
1040 self.config.matchCut*lsst.geom.arcseconds,
1041 fluxField,
1042 refCoordinateErr=refCoordErr,
1043 rejectBadFluxes=reject_bad_fluxes)
1044 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name,
1045 associations.refStarListSize())
1047 associations.prepareFittedStars(self.config.minMeasurements)
1049 self._check_star_lists(associations, name)
1050 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name,
1051 associations.nFittedStarsWithAssociatedRefStar())
1052 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name,
1053 associations.fittedStarListSize())
1054 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name,
1055 associations.nCcdImagesValidForFit())
1057 fit_profile_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1058 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1059 with lsst.utils.timer.profile(fit_profile_file):
1060 result = fit_function(associations, dataName)
1061 # TODO DM-12446: turn this into a "butler save" somehow.
1062 # Save reference and measurement chi2 contributions for this data
1063 if self.config.writeChi2FilesInitialFinal:
1064 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1065 result.fit.saveChi2Contributions(baseName+"{type}")
1066 self.log.info("Wrote chi2 contributions files: %s", baseName)
1068 return result
1070 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1071 applyColorterms=False, epoch=None):
1072 """Load the necessary reference catalog sources, convert fluxes to
1073 correct units, and apply color term corrections if requested.
1075 Parameters
1076 ----------
1077 refObjLoader : `lsst.meas.algorithms.ReferenceObjectLoader`
1078 The reference catalog loader to use to get the data.
1079 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1080 Source selector to apply to loaded reference catalog.
1081 center : `lsst.geom.SpherePoint`
1082 The center around which to load sources.
1083 radius : `lsst.geom.Angle`
1084 The radius around ``center`` to load sources in.
1085 filterLabel : `lsst.afw.image.FilterLabel`
1086 The camera filter to load fluxes for.
1087 applyColorterms : `bool`
1088 Apply colorterm corrections to the refcat for ``filterName``?
1089 epoch : `astropy.time.Time`, optional
1090 Epoch to which to correct refcat proper motion and parallax,
1091 or `None` to not apply such corrections.
1093 Returns
1094 -------
1095 refCat : `lsst.afw.table.SimpleCatalog`
1096 The loaded reference catalog.
1097 fluxField : `str`
1098 The name of the reference catalog flux field appropriate for ``filterName``.
1099 """
1100 skyCircle = refObjLoader.loadSkyCircle(center,
1101 radius,
1102 filterLabel.bandLabel,
1103 epoch=epoch)
1105 selected = referenceSelector.run(skyCircle.refCat)
1106 # Need memory contiguity to get reference filters as a vector.
1107 if not selected.sourceCat.isContiguous():
1108 refCat = selected.sourceCat.copy(deep=True)
1109 else:
1110 refCat = selected.sourceCat
1112 if applyColorterms:
1113 refCatName = refObjLoader.name
1114 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1115 filterLabel.physicalLabel, refCatName)
1116 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1117 refCatName,
1118 doRaise=True)
1120 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1121 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1122 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1123 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1125 return refCat, skyCircle.fluxField
1127 def _check_star_lists(self, associations, name):
1128 # TODO: these should be len(blah), but we need this properly wrapped first.
1129 if associations.nCcdImagesValidForFit() == 0:
1130 raise RuntimeError('No images in the ccdImageList!')
1131 if associations.fittedStarListSize() == 0:
1132 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1133 if associations.refStarListSize() == 0:
1134 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1136 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1137 """Compute chi2, log it, validate the model, and return chi2.
1139 Parameters
1140 ----------
1141 associations : `lsst.jointcal.Associations`
1142 The star/reference star associations to fit.
1143 fit : `lsst.jointcal.FitterBase`
1144 The fitter to use for minimization.
1145 model : `lsst.jointcal.Model`
1146 The model being fit.
1147 chi2Label : `str`
1148 Label to describe the chi2 (e.g. "Initialized", "Final").
1149 writeChi2Name : `str`, optional
1150 Filename prefix to write the chi2 contributions to.
1151 Do not supply an extension: an appropriate one will be added.
1153 Returns
1154 -------
1155 chi2: `lsst.jointcal.Chi2Accumulator`
1156 The chi2 object for the current fitter and model.
1158 Raises
1159 ------
1160 FloatingPointError
1161 Raised if chi2 is infinite or NaN.
1162 ValueError
1163 Raised if the model is not valid.
1164 """
1165 if writeChi2Name is not None:
1166 fullpath = self._getDebugPath(writeChi2Name)
1167 fit.saveChi2Contributions(fullpath+"{type}")
1168 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1170 chi2 = fit.computeChi2()
1171 self.log.info("%s %s", chi2Label, chi2)
1172 self._check_stars(associations)
1173 if not np.isfinite(chi2.chi2):
1174 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1175 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1176 raise ValueError("Model is not valid: check log messages for warnings.")
1177 return chi2
1179 def _fit_photometry(self, associations, dataName=None):
1180 """
1181 Fit the photometric data.
1183 Parameters
1184 ----------
1185 associations : `lsst.jointcal.Associations`
1186 The star/reference star associations to fit.
1187 dataName : `str`
1188 Name of the data being processed (e.g. "1234_HSC-Y"), for
1189 identifying debugging files.
1191 Returns
1192 -------
1193 fit_result : `namedtuple`
1194 fit : `lsst.jointcal.PhotometryFit`
1195 The photometric fitter used to perform the fit.
1196 model : `lsst.jointcal.PhotometryModel`
1197 The photometric model that was fit.
1198 """
1199 self.log.info("=== Starting photometric fitting...")
1201 # TODO: should use pex.config.RegistryField here (see DM-9195)
1202 if self.config.photometryModel == "constrainedFlux":
1203 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1204 self.focalPlaneBBox,
1205 visitOrder=self.config.photometryVisitOrder,
1206 errorPedestal=self.config.photometryErrorPedestal)
1207 # potentially nonlinear problem, so we may need a line search to converge.
1208 doLineSearch = self.config.allowLineSearch
1209 elif self.config.photometryModel == "constrainedMagnitude":
1210 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1211 self.focalPlaneBBox,
1212 visitOrder=self.config.photometryVisitOrder,
1213 errorPedestal=self.config.photometryErrorPedestal)
1214 # potentially nonlinear problem, so we may need a line search to converge.
1215 doLineSearch = self.config.allowLineSearch
1216 elif self.config.photometryModel == "simpleFlux":
1217 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1218 errorPedestal=self.config.photometryErrorPedestal)
1219 doLineSearch = False # purely linear in model parameters, so no line search needed
1220 elif self.config.photometryModel == "simpleMagnitude":
1221 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1222 errorPedestal=self.config.photometryErrorPedestal)
1223 doLineSearch = False # purely linear in model parameters, so no line search needed
1225 fit = lsst.jointcal.PhotometryFit(associations, model)
1226 # TODO DM-12446: turn this into a "butler save" somehow.
1227 # Save reference and measurement chi2 contributions for this data
1228 if self.config.writeChi2FilesInitialFinal:
1229 baseName = f"photometry_initial_chi2-{dataName}"
1230 else:
1231 baseName = None
1232 if self.config.writeInitialModel:
1233 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt")
1234 writeModel(model, fullpath, self.log)
1235 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1237 def getChi2Name(whatToFit):
1238 if self.config.writeChi2FilesOuterLoop:
1239 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1240 else:
1241 return None
1243 # The constrained model needs the visit transform fit first; the chip
1244 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1245 if self.config.writeInitMatrix:
1246 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}")
1247 else:
1248 dumpMatrixFile = ""
1249 if self.config.photometryModel.startswith("constrained"):
1250 # no line search: should be purely (or nearly) linear,
1251 # and we want a large step size to initialize with.
1252 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1253 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1254 writeChi2Name=getChi2Name("ModelVisit"))
1255 dumpMatrixFile = "" # so we don't redo the output on the next step
1257 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1258 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1259 writeChi2Name=getChi2Name("Model"))
1261 fit.minimize("Fluxes") # no line search: always purely linear.
1262 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1263 writeChi2Name=getChi2Name("Fluxes"))
1265 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1266 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1267 writeChi2Name=getChi2Name("ModelFluxes"))
1269 model.freezeErrorTransform()
1270 self.log.debug("Photometry error scales are frozen.")
1272 chi2 = self._iterate_fit(associations,
1273 fit,
1274 self.config.maxPhotometrySteps,
1275 "photometry",
1276 "Model Fluxes",
1277 doRankUpdate=self.config.photometryDoRankUpdate,
1278 doLineSearch=doLineSearch,
1279 dataName=dataName)
1281 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1282 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1283 return Photometry(fit, model)
1285 def _fit_astrometry(self, associations, dataName=None):
1286 """
1287 Fit the astrometric data.
1289 Parameters
1290 ----------
1291 associations : `lsst.jointcal.Associations`
1292 The star/reference star associations to fit.
1293 dataName : `str`
1294 Name of the data being processed (e.g. "1234_HSC-Y"), for
1295 identifying debugging files.
1297 Returns
1298 -------
1299 fit_result : `namedtuple`
1300 fit : `lsst.jointcal.AstrometryFit`
1301 The astrometric fitter used to perform the fit.
1302 model : `lsst.jointcal.AstrometryModel`
1303 The astrometric model that was fit.
1304 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1305 The model for the sky to tangent plane projection that was used in the fit.
1306 """
1308 self.log.info("=== Starting astrometric fitting...")
1310 associations.deprojectFittedStars()
1312 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1313 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1314 # them so carefully?
1315 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1317 if self.config.astrometryModel == "constrained":
1318 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1319 sky_to_tan_projection,
1320 chipOrder=self.config.astrometryChipOrder,
1321 visitOrder=self.config.astrometryVisitOrder)
1322 elif self.config.astrometryModel == "simple":
1323 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1324 sky_to_tan_projection,
1325 self.config.useInputWcs,
1326 nNotFit=0,
1327 order=self.config.astrometrySimpleOrder)
1329 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1330 # TODO DM-12446: turn this into a "butler save" somehow.
1331 # Save reference and measurement chi2 contributions for this data
1332 if self.config.writeChi2FilesInitialFinal:
1333 baseName = f"astrometry_initial_chi2-{dataName}"
1334 else:
1335 baseName = None
1336 if self.config.writeInitialModel:
1337 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1338 writeModel(model, fullpath, self.log)
1339 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1341 def getChi2Name(whatToFit):
1342 if self.config.writeChi2FilesOuterLoop:
1343 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1344 else:
1345 return None
1347 if self.config.writeInitMatrix:
1348 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}")
1349 else:
1350 dumpMatrixFile = ""
1351 # The constrained model needs the visit transform fit first; the chip
1352 # transform is initialized from the detector's cameraGeom, so it's close.
1353 if self.config.astrometryModel == "constrained":
1354 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1355 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1356 writeChi2Name=getChi2Name("DistortionsVisit"))
1357 dumpMatrixFile = "" # so we don't redo the output on the next step
1359 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1360 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1361 writeChi2Name=getChi2Name("Distortions"))
1363 fit.minimize("Positions")
1364 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1365 writeChi2Name=getChi2Name("Positions"))
1367 fit.minimize("Distortions Positions")
1368 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1369 writeChi2Name=getChi2Name("DistortionsPositions"))
1371 chi2 = self._iterate_fit(associations,
1372 fit,
1373 self.config.maxAstrometrySteps,
1374 "astrometry",
1375 "Distortions Positions",
1376 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1377 doRankUpdate=self.config.astrometryDoRankUpdate,
1378 dataName=dataName)
1380 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1381 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1383 return Astrometry(fit, model, sky_to_tan_projection)
1385 def _check_stars(self, associations):
1386 """Count measured and reference stars per ccd and warn/log them."""
1387 for ccdImage in associations.getCcdImageList():
1388 nMeasuredStars, nRefStars = ccdImage.countStars()
1389 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1390 ccdImage.getName(), nMeasuredStars, nRefStars)
1391 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1392 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)",
1393 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1394 if nRefStars < self.config.minRefStarsPerCcd:
1395 self.log.warning("ccdImage %s has only %s RefStars (desired %s)",
1396 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1398 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1399 dataName="",
1400 sigmaRelativeTolerance=0,
1401 doRankUpdate=True,
1402 doLineSearch=False):
1403 """Run fitter.minimize up to max_steps times, returning the final chi2.
1405 Parameters
1406 ----------
1407 associations : `lsst.jointcal.Associations`
1408 The star/reference star associations to fit.
1409 fitter : `lsst.jointcal.FitterBase`
1410 The fitter to use for minimization.
1411 max_steps : `int`
1412 Maximum number of steps to run outlier rejection before declaring
1413 convergence failure.
1414 name : {'photometry' or 'astrometry'}
1415 What type of data are we fitting (for logs and debugging files).
1416 whatToFit : `str`
1417 Passed to ``fitter.minimize()`` to define the parameters to fit.
1418 dataName : `str`, optional
1419 Descriptive name for this dataset (e.g. tract and filter),
1420 for debugging.
1421 sigmaRelativeTolerance : `float`, optional
1422 Convergence tolerance for the fractional change in the chi2 cut
1423 level for determining outliers. If set to zero, iterations will
1424 continue until there are no outliers.
1425 doRankUpdate : `bool`, optional
1426 Do an Eigen rank update during minimization, or recompute the full
1427 matrix and gradient?
1428 doLineSearch : `bool`, optional
1429 Do a line search for the optimum step during minimization?
1431 Returns
1432 -------
1433 chi2: `lsst.jointcal.Chi2Statistic`
1434 The final chi2 after the fit converges, or is forced to end.
1436 Raises
1437 ------
1438 FloatingPointError
1439 Raised if the fitter fails with a non-finite value.
1440 RuntimeError
1441 Raised if the fitter fails for some other reason;
1442 log messages will provide further details.
1443 """
1444 if self.config.writeInitMatrix:
1445 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}")
1446 else:
1447 dumpMatrixFile = ""
1448 oldChi2 = lsst.jointcal.Chi2Statistic()
1449 oldChi2.chi2 = float("inf")
1450 for i in range(max_steps):
1451 if self.config.writeChi2FilesOuterLoop:
1452 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1453 else:
1454 writeChi2Name = None
1455 result = fitter.minimize(whatToFit,
1456 self.config.outlierRejectSigma,
1457 sigmaRelativeTolerance=sigmaRelativeTolerance,
1458 doRankUpdate=doRankUpdate,
1459 doLineSearch=doLineSearch,
1460 dumpMatrixFile=dumpMatrixFile)
1461 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1462 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1463 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1465 if result == MinimizeResult.Converged:
1466 if doRankUpdate:
1467 self.log.debug("fit has converged - no more outliers - redo minimization "
1468 "one more time in case we have lost accuracy in rank update.")
1469 # Redo minimization one more time in case we have lost accuracy in rank update
1470 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1471 sigmaRelativeTolerance=sigmaRelativeTolerance)
1472 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1474 # log a message for a large final chi2, TODO: DM-15247 for something better
1475 if chi2.chi2/chi2.ndof >= 4.0:
1476 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1478 break
1479 elif result == MinimizeResult.Chi2Increased:
1480 self.log.warning("Still some outliers remaining but chi2 increased - retry")
1481 # Check whether the increase was large enough to cause trouble.
1482 chi2Ratio = chi2.chi2 / oldChi2.chi2
1483 if chi2Ratio > 1.5:
1484 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1485 chi2.chi2, oldChi2.chi2, chi2Ratio)
1486 # Based on a variety of HSC jointcal logs (see DM-25779), it
1487 # appears that chi2 increases more than a factor of ~2 always
1488 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1489 # Using 10 as the "failure" threshold gives some room between
1490 # leaving a warning and bailing early.
1491 if chi2Ratio > 10:
1492 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1493 " Try setting one or more of the `writeChi2*` config fields and looking"
1494 " at how individual star chi2-values evolve during the fit.")
1495 raise RuntimeError(msg)
1496 oldChi2 = chi2
1497 elif result == MinimizeResult.NonFinite:
1498 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1499 # TODO DM-12446: turn this into a "butler save" somehow.
1500 fitter.saveChi2Contributions(filename+"{type}")
1501 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1502 raise FloatingPointError(msg.format(filename))
1503 elif result == MinimizeResult.Failed:
1504 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1505 else:
1506 raise RuntimeError("Unxepected return code from minimize().")
1507 else:
1508 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1510 return chi2
1512 def _make_output(self, ccdImageList, model, func):
1513 """Return the internal jointcal models converted to the afw
1514 structures that will be saved to disk.
1516 Parameters
1517 ----------
1518 ccdImageList : `lsst.jointcal.CcdImageList`
1519 The list of CcdImages to get the output for.
1520 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1521 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1522 func : `str`
1523 The name of the function to call on ``model`` to get the converted
1524 structure. Must accept an `lsst.jointcal.CcdImage`.
1526 Returns
1527 -------
1528 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1529 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1530 The data to be saved, keyed on (visit, detector).
1531 """
1532 output = {}
1533 for ccdImage in ccdImageList:
1534 ccd = ccdImage.ccdId
1535 visit = ccdImage.visit
1536 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1537 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1538 return output
1541def make_schema_table():
1542 """Return an afw SourceTable to use as a base for creating the
1543 SourceCatalog to insert values from the dataFrame into.
1545 Returns
1546 -------
1547 table : `lsst.afw.table.SourceTable`
1548 Table with schema and slots to use to make SourceCatalogs.
1549 """
1550 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
1551 schema.addField("centroid_x", "D")
1552 schema.addField("centroid_y", "D")
1553 schema.addField("centroid_xErr", "F")
1554 schema.addField("centroid_yErr", "F")
1555 schema.addField("shape_xx", "D")
1556 schema.addField("shape_yy", "D")
1557 schema.addField("shape_xy", "D")
1558 schema.addField("flux_instFlux", "D")
1559 schema.addField("flux_instFluxErr", "D")
1560 table = lsst.afw.table.SourceTable.make(schema)
1561 table.defineCentroid("centroid")
1562 table.defineShape("shape")
1563 return table
1566def get_sourceTable_visit_columns(inColumns, config, sourceSelector):
1567 """
1568 Get the sourceTable_visit columns to load from the catalogs.
1570 Parameters
1571 ----------
1572 inColumns : `list`
1573 List of columns known to be available in the sourceTable_visit.
1574 config : `JointcalConfig`
1575 A filled-in config to to help define column names.
1576 sourceSelector : `lsst.meas.algorithms.BaseSourceSelectorTask`
1577 A configured source selector to define column names to load.
1579 Returns
1580 -------
1581 columns : `list`
1582 List of columns to read from sourceTable_visit.
1583 detectorColumn : `str`
1584 Name of the detector column.
1585 ixxColumns : `list`
1586 Name of the ixx/iyy/ixy columns.
1587 """
1588 if 'detector' in inColumns:
1589 # Default name for Gen3.
1590 detectorColumn = 'detector'
1591 else:
1592 # Default name for Gen2 conversions (still used in tests, CI, and older catalogs)
1593 detectorColumn = 'ccd'
1595 columns = ['visit', detectorColumn,
1596 'sourceId', 'x', 'xErr', 'y', 'yErr',
1597 config.sourceFluxType + '_instFlux', config.sourceFluxType + '_instFluxErr']
1599 if 'ixx' in inColumns:
1600 # New columns post-DM-31825
1601 ixxColumns = ['ixx', 'iyy', 'ixy']
1602 else:
1603 # Old columns pre-DM-31825
1604 ixxColumns = ['Ixx', 'Iyy', 'Ixy']
1605 columns.extend(ixxColumns)
1607 if sourceSelector.config.doFlags:
1608 columns.extend(sourceSelector.config.flags.bad)
1609 if sourceSelector.config.doUnresolved:
1610 columns.append(sourceSelector.config.unresolved.name)
1611 if sourceSelector.config.doIsolated:
1612 columns.append(sourceSelector.config.isolated.parentName)
1613 columns.append(sourceSelector.config.isolated.nChildName)
1614 if sourceSelector.config.doRequireFiniteRaDec:
1615 columns.append(sourceSelector.config.requireFiniteRaDec.raColName)
1616 columns.append(sourceSelector.config.requireFiniteRaDec.decColName)
1618 return columns, detectorColumn, ixxColumns
1621def extract_detector_catalog_from_visit_catalog(table, visitCatalog, detectorId,
1622 detectorColumn, ixxColumns, sourceFluxType, log):
1623 """Return an afw SourceCatalog extracted from a visit-level dataframe,
1624 limited to just one detector.
1626 Parameters
1627 ----------
1628 table : `lsst.afw.table.SourceTable`
1629 Table factory to use to make the SourceCatalog that will be
1630 populated with data from ``visitCatalog``.
1631 visitCatalog : `pandas.DataFrame`
1632 DataFrame to extract a detector catalog from.
1633 detectorId : `int`
1634 Numeric id of the detector to extract from ``visitCatalog``.
1635 detectorColumn : `str`
1636 Name of the detector column in the catalog.
1637 ixxColumns : `list` [`str`]
1638 Names of the ixx/iyy/ixy columns in the catalog.
1639 sourceFluxType : `str`
1640 Name of the catalog field to load instFluxes from.
1641 log : `logging.Logger`
1642 Logging instance to log to.
1644 Returns
1645 -------
1646 catalog : `lsst.afw.table.SourceCatalog`, or `None`
1647 Detector-level catalog extracted from ``visitCatalog``, or `None`
1648 if there was no data to load.
1649 """
1650 # map from dataFrame column to afw table column
1651 mapping = {'x': 'centroid_x',
1652 'y': 'centroid_y',
1653 'xErr': 'centroid_xErr',
1654 'yErr': 'centroid_yErr',
1655 ixxColumns[0]: 'shape_xx',
1656 ixxColumns[1]: 'shape_yy',
1657 ixxColumns[2]: 'shape_xy',
1658 f'{sourceFluxType}_instFlux': 'flux_instFlux',
1659 f'{sourceFluxType}_instFluxErr': 'flux_instFluxErr',
1660 }
1662 catalog = lsst.afw.table.SourceCatalog(table)
1663 matched = visitCatalog[detectorColumn] == detectorId
1664 n = sum(matched)
1665 if n == 0:
1666 return None
1667 catalog.resize(sum(matched))
1668 view = visitCatalog.loc[matched]
1669 catalog['id'] = view.index
1670 for dfCol, afwCol in mapping.items():
1671 catalog[afwCol] = view[dfCol]
1673 log.debug("%d sources selected in visit %d detector %d",
1674 len(catalog),
1675 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
1676 detectorId)
1677 return catalog