Coverage for python/lsst/jointcal/jointcal.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import dataclasses
23import collections
24import os
26import astropy.time
27import numpy as np
28import astropy.units as u
30import lsst.geom
31import lsst.utils
32import lsst.pex.config as pexConfig
33import lsst.pipe.base as pipeBase
34from lsst.afw.image import fluxErrFromABMagErr
35import lsst.pex.exceptions as pexExceptions
36import lsst.afw.cameraGeom
37import lsst.afw.table
38import lsst.log
39from lsst.obs.base import Instrument
40from lsst.pipe.tasks.colorterms import ColortermLibrary
41from lsst.verify import Job, Measurement
43from lsst.meas.algorithms import (LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask,
44 ReferenceObjectLoader)
45from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
46from lsst.utils.timer import timeMethod
48from .dataIds import PerTractCcdDataIdContainer
50import lsst.jointcal
51from lsst.jointcal import MinimizeResult
53__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
55Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
56Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
59# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
60def add_measurement(job, name, value):
61 meas = Measurement(job.metrics[name], value)
62 job.measurements.insert(meas)
65class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
66 """Subclass of TaskRunner for jointcalTask (gen2)
68 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
69 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
70 single dataRef, are are called repeatedly). This class transforms the processed
71 arguments generated by the ArgumentParser into the arguments expected by
72 Jointcal.runDataRef().
74 See pipeBase.TaskRunner for more information.
75 """
77 @staticmethod
78 def getTargetList(parsedCmd, **kwargs):
79 """
80 Return a list of tuples per tract, each containing (dataRefs, kwargs).
82 Jointcal operates on lists of dataRefs simultaneously.
83 """
84 kwargs['butler'] = parsedCmd.butler
86 # organize data IDs by tract
87 refListDict = {}
88 for ref in parsedCmd.id.refList:
89 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
90 # we call runDataRef() once with each tract
91 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
92 return result
94 def __call__(self, args):
95 """
96 Parameters
97 ----------
98 args
99 Arguments for Task.runDataRef()
101 Returns
102 -------
103 pipe.base.Struct
104 if self.doReturnResults is False:
106 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
108 if self.doReturnResults is True:
110 - ``result``: the result of calling jointcal.runDataRef()
111 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
112 """
113 exitStatus = 0 # exit status for shell
115 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
116 dataRefList, kwargs = args
117 butler = kwargs.pop('butler')
118 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
119 result = None
120 try:
121 result = task.runDataRef(dataRefList, **kwargs)
122 exitStatus = result.exitStatus
123 job_path = butler.get('verify_job_filename')
124 result.job.write(job_path[0])
125 except Exception as e: # catch everything, sort it out later.
126 if self.doRaise:
127 raise e
128 else:
129 exitStatus = 1
130 eName = type(e).__name__
131 tract = dataRefList[0].dataId['tract']
132 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
134 # Put the butler back into kwargs for the other Tasks.
135 kwargs['butler'] = butler
136 if self.doReturnResults:
137 return pipeBase.Struct(result=result, exitStatus=exitStatus)
138 else:
139 return pipeBase.Struct(exitStatus=exitStatus)
142def lookupStaticCalibrations(datasetType, registry, quantumDataId, collections):
143 """Lookup function that asserts/hopes that a static calibration dataset
144 exists in a particular collection, since this task can't provide a single
145 date/time to use to search for one properly.
147 This is mostly useful for the ``camera`` dataset, in cases where the task's
148 quantum dimensions do *not* include something temporal, like ``exposure``
149 or ``visit``.
151 Parameters
152 ----------
153 datasetType : `lsst.daf.butler.DatasetType`
154 Type of dataset being searched for.
155 registry : `lsst.daf.butler.Registry`
156 Data repository registry to search.
157 quantumDataId : `lsst.daf.butler.DataCoordinate`
158 Data ID of the quantum this camera should match.
159 collections : `Iterable` [ `str` ]
160 Collections that should be searched - but this lookup function works
161 by ignoring this in favor of a more-or-less hard-coded value.
163 Returns
164 -------
165 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
166 Iterator over dataset references; should have only one element.
168 Notes
169 -----
170 This implementation duplicates one in fgcmcal, and is at least quite
171 similar to another in cp_pipe. This duplicate has the most documentation.
172 Fixing this is DM-29661.
173 """
174 instrument = Instrument.fromName(quantumDataId["instrument"], registry)
175 unboundedCollection = instrument.makeUnboundedCalibrationRunName()
176 return registry.queryDatasets(datasetType,
177 dataId=quantumDataId,
178 collections=[unboundedCollection],
179 findFirst=True)
182def lookupVisitRefCats(datasetType, registry, quantumDataId, collections):
183 """Lookup function that finds all refcats for all visits that overlap a
184 tract, rather than just the refcats that directly overlap the tract.
186 Parameters
187 ----------
188 datasetType : `lsst.daf.butler.DatasetType`
189 Type of dataset being searched for.
190 registry : `lsst.daf.butler.Registry`
191 Data repository registry to search.
192 quantumDataId : `lsst.daf.butler.DataCoordinate`
193 Data ID of the quantum; expected to be something we can use as a
194 constraint to query for overlapping visits.
195 collections : `Iterable` [ `str` ]
196 Collections to search.
198 Returns
199 -------
200 refs : `Iterator` [ `lsst.daf.butler.DatasetRef` ]
201 Iterator over refcat references.
202 """
203 refs = set()
204 # Use .expanded() on the query methods below because we need data IDs with
205 # regions, both in the outer loop over visits (queryDatasets will expand
206 # any data ID we give it, but doing it up-front in bulk is much more
207 # efficient) and in the data IDs of the DatasetRefs this function yields
208 # (because the RefCatLoader relies on them to do some of its own
209 # filtering).
210 for visit_data_id in set(registry.queryDataIds("visit", dataId=quantumDataId).expanded()):
211 refs.update(
212 registry.queryDatasets(
213 datasetType,
214 collections=collections,
215 dataId=visit_data_id,
216 findFirst=True,
217 ).expanded()
218 )
219 yield from refs
222class JointcalTaskConnections(pipeBase.PipelineTaskConnections,
223 dimensions=("skymap", "tract", "instrument", "physical_filter")):
224 """Middleware input/output connections for jointcal data."""
225 inputCamera = pipeBase.connectionTypes.PrerequisiteInput(
226 doc="The camera instrument that took these observations.",
227 name="camera",
228 storageClass="Camera",
229 dimensions=("instrument",),
230 isCalibration=True,
231 lookupFunction=lookupStaticCalibrations,
232 )
233 inputSourceTableVisit = pipeBase.connectionTypes.Input(
234 doc="Source table in parquet format, per visit",
235 name="sourceTable_visit",
236 storageClass="DataFrame",
237 dimensions=("instrument", "visit"),
238 deferLoad=True,
239 multiple=True,
240 )
241 inputVisitSummary = pipeBase.connectionTypes.Input(
242 doc=("Per-visit consolidated exposure metadata built from calexps. "
243 "These catalogs use detector id for the id and must be sorted for "
244 "fast lookups of a detector."),
245 name="visitSummary",
246 storageClass="ExposureCatalog",
247 dimensions=("instrument", "visit"),
248 deferLoad=True,
249 multiple=True,
250 )
251 astrometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
252 doc="The astrometry reference catalog to match to loaded input catalog sources.",
253 name="gaia_dr2_20200414",
254 storageClass="SimpleCatalog",
255 dimensions=("skypix",),
256 deferLoad=True,
257 multiple=True,
258 lookupFunction=lookupVisitRefCats,
259 )
260 photometryRefCat = pipeBase.connectionTypes.PrerequisiteInput(
261 doc="The photometry reference catalog to match to loaded input catalog sources.",
262 name="ps1_pv3_3pi_20170110",
263 storageClass="SimpleCatalog",
264 dimensions=("skypix",),
265 deferLoad=True,
266 multiple=True,
267 lookupFunction=lookupVisitRefCats,
268 )
270 outputWcs = pipeBase.connectionTypes.Output(
271 doc=("Per-tract, per-visit world coordinate systems derived from the fitted model."
272 " These catalogs only contain entries for detectors with an output, and use"
273 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
274 name="jointcalSkyWcsCatalog",
275 storageClass="ExposureCatalog",
276 dimensions=("instrument", "visit", "skymap", "tract"),
277 multiple=True
278 )
279 outputPhotoCalib = pipeBase.connectionTypes.Output(
280 doc=("Per-tract, per-visit photometric calibrations derived from the fitted model."
281 " These catalogs only contain entries for detectors with an output, and use"
282 " the detector id for the catalog id, sorted on id for fast lookups of a detector."),
283 name="jointcalPhotoCalibCatalog",
284 storageClass="ExposureCatalog",
285 dimensions=("instrument", "visit", "skymap", "tract"),
286 multiple=True
287 )
289 # measurements of metrics
290 # The vars() trick used here allows us to set class attributes
291 # programatically. Taken from:
292 # https://stackoverflow.com/questions/2519807/setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
293 for name in ("astrometry", "photometry"):
294 vars()[f"{name}_matched_fittedStars"] = pipeBase.connectionTypes.Output(
295 doc=f"The number of cross-matched fittedStars for {name}",
296 name=f"metricvalue_jointcal_{name}_matched_fittedStars",
297 storageClass="MetricValue",
298 dimensions=("skymap", "tract", "instrument", "physical_filter"),
299 )
300 vars()[f"{name}_collected_refStars"] = pipeBase.connectionTypes.Output(
301 doc=f"The number of {name} reference stars drawn from the reference catalog, before matching.",
302 name=f"metricvalue_jointcal_{name}_collected_refStars",
303 storageClass="MetricValue",
304 dimensions=("skymap", "tract", "instrument", "physical_filter"),
305 )
306 vars()[f"{name}_prepared_refStars"] = pipeBase.connectionTypes.Output(
307 doc=f"The number of {name} reference stars matched to fittedStars.",
308 name=f"metricvalue_jointcal_{name}_prepared_refStars",
309 storageClass="MetricValue",
310 dimensions=("skymap", "tract", "instrument", "physical_filter"),
311 )
312 vars()[f"{name}_prepared_fittedStars"] = pipeBase.connectionTypes.Output(
313 doc=f"The number of cross-matched fittedStars after cleanup, for {name}.",
314 name=f"metricvalue_jointcal_{name}_prepared_fittedStars",
315 storageClass="MetricValue",
316 dimensions=("skymap", "tract", "instrument", "physical_filter"),
317 )
318 vars()[f"{name}_prepared_ccdImages"] = pipeBase.connectionTypes.Output(
319 doc=f"The number of ccdImages that will be fit for {name}, after cleanup.",
320 name=f"metricvalue_jointcal_{name}_prepared_ccdImages",
321 storageClass="MetricValue",
322 dimensions=("skymap", "tract", "instrument", "physical_filter"),
323 )
324 vars()[f"{name}_final_chi2"] = pipeBase.connectionTypes.Output(
325 doc=f"The final chi2 of the {name} fit.",
326 name=f"metricvalue_jointcal_{name}_final_chi2",
327 storageClass="MetricValue",
328 dimensions=("skymap", "tract", "instrument", "physical_filter"),
329 )
330 vars()[f"{name}_final_ndof"] = pipeBase.connectionTypes.Output(
331 doc=f"The number of degrees of freedom of the fitted {name}.",
332 name=f"metricvalue_jointcal_{name}_final_ndof",
333 storageClass="MetricValue",
334 dimensions=("skymap", "tract", "instrument", "physical_filter"),
335 )
337 def __init__(self, *, config=None):
338 super().__init__(config=config)
339 # When we are only doing one of astrometry or photometry, we don't
340 # need the reference catalog or produce the outputs for the other.
341 # This informs the middleware of that when the QuantumGraph is
342 # generated, so we don't block on getting something we won't need or
343 # create an expectation that downstream tasks will be able to consume
344 # something we won't produce.
345 if not config.doAstrometry:
346 self.prerequisiteInputs.remove("astrometryRefCat")
347 self.outputs.remove("outputWcs")
348 for key in list(self.outputs):
349 if "metricvalue_jointcal_astrometry" in key:
350 self.outputs.remove(key)
351 if not config.doPhotometry:
352 self.prerequisiteInputs.remove("photometryRefCat")
353 self.outputs.remove("outputPhotoCalib")
354 for key in list(self.outputs):
355 if "metricvalue_jointcal_photometry" in key:
356 self.outputs.remove(key)
359class JointcalConfig(pipeBase.PipelineTaskConfig,
360 pipelineConnections=JointcalTaskConnections):
361 """Configuration for JointcalTask"""
363 doAstrometry = pexConfig.Field(
364 doc="Fit astrometry and write the fitted result.",
365 dtype=bool,
366 default=True
367 )
368 doPhotometry = pexConfig.Field(
369 doc="Fit photometry and write the fitted result.",
370 dtype=bool,
371 default=True
372 )
373 coaddName = pexConfig.Field(
374 doc="Type of coadd, typically deep or goodSeeing",
375 dtype=str,
376 default="deep"
377 )
378 # TODO DM-29008: Change this to "ApFlux_12_0" before gen2 removal.
379 sourceFluxType = pexConfig.Field(
380 dtype=str,
381 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
382 default='Calib'
383 )
384 positionErrorPedestal = pexConfig.Field(
385 doc="Systematic term to apply to the measured position error (pixels)",
386 dtype=float,
387 default=0.02,
388 )
389 photometryErrorPedestal = pexConfig.Field(
390 doc="Systematic term to apply to the measured error on flux or magnitude as a "
391 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
392 dtype=float,
393 default=0.0,
394 )
395 # TODO: DM-6885 matchCut should be an geom.Angle
396 matchCut = pexConfig.Field(
397 doc="Matching radius between fitted and reference stars (arcseconds)",
398 dtype=float,
399 default=3.0,
400 )
401 minMeasurements = pexConfig.Field(
402 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
403 dtype=int,
404 default=2,
405 )
406 minMeasuredStarsPerCcd = pexConfig.Field(
407 doc="Minimum number of measuredStars per ccdImage before printing warnings",
408 dtype=int,
409 default=100,
410 )
411 minRefStarsPerCcd = pexConfig.Field(
412 doc="Minimum number of measuredStars per ccdImage before printing warnings",
413 dtype=int,
414 default=30,
415 )
416 allowLineSearch = pexConfig.Field(
417 doc="Allow a line search during minimization, if it is reasonable for the model"
418 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
419 dtype=bool,
420 default=False
421 )
422 astrometrySimpleOrder = pexConfig.Field(
423 doc="Polynomial order for fitting the simple astrometry model.",
424 dtype=int,
425 default=3,
426 )
427 astrometryChipOrder = pexConfig.Field(
428 doc="Order of the per-chip transform for the constrained astrometry model.",
429 dtype=int,
430 default=1,
431 )
432 astrometryVisitOrder = pexConfig.Field(
433 doc="Order of the per-visit transform for the constrained astrometry model.",
434 dtype=int,
435 default=5,
436 )
437 useInputWcs = pexConfig.Field(
438 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
439 dtype=bool,
440 default=True,
441 )
442 astrometryModel = pexConfig.ChoiceField(
443 doc="Type of model to fit to astrometry",
444 dtype=str,
445 default="constrained",
446 allowed={"simple": "One polynomial per ccd",
447 "constrained": "One polynomial per ccd, and one polynomial per visit"}
448 )
449 photometryModel = pexConfig.ChoiceField(
450 doc="Type of model to fit to photometry",
451 dtype=str,
452 default="constrainedMagnitude",
453 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
454 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
455 " fitting in flux space.",
456 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
457 " fitting in magnitude space.",
458 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
459 " fitting in magnitude space.",
460 }
461 )
462 applyColorTerms = pexConfig.Field(
463 doc="Apply photometric color terms to reference stars?"
464 "Requires that colorterms be set to a ColortermLibrary",
465 dtype=bool,
466 default=False
467 )
468 colorterms = pexConfig.ConfigField(
469 doc="Library of photometric reference catalog name to color term dict.",
470 dtype=ColortermLibrary,
471 )
472 photometryVisitOrder = pexConfig.Field(
473 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
474 dtype=int,
475 default=7,
476 )
477 photometryDoRankUpdate = pexConfig.Field(
478 doc=("Do the rank update step during minimization. "
479 "Skipping this can help deal with models that are too non-linear."),
480 dtype=bool,
481 default=True,
482 )
483 astrometryDoRankUpdate = pexConfig.Field(
484 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
485 "Skipping this can help deal with models that are too non-linear."),
486 dtype=bool,
487 default=True,
488 )
489 outlierRejectSigma = pexConfig.Field(
490 doc="How many sigma to reject outliers at during minimization.",
491 dtype=float,
492 default=5.0,
493 )
494 astrometryOutlierRelativeTolerance = pexConfig.Field(
495 doc=("Convergence tolerance for outlier rejection threshold when fitting astrometry. Iterations will "
496 "stop when the fractional change in the chi2 cut level is below this value. If tolerance is set "
497 "to zero, iterations will continue until there are no more outliers. We suggest a value of 0.002"
498 "as a balance between a shorter minimization runtime and achieving a final fitted model that is"
499 "close to the solution found when removing all outliers."),
500 dtype=float,
501 default=0,
502 )
503 maxPhotometrySteps = pexConfig.Field(
504 doc="Maximum number of minimize iterations to take when fitting photometry.",
505 dtype=int,
506 default=20,
507 )
508 maxAstrometrySteps = pexConfig.Field(
509 doc="Maximum number of minimize iterations to take when fitting astrometry.",
510 dtype=int,
511 default=20,
512 )
513 astrometryRefObjLoader = pexConfig.ConfigurableField(
514 target=LoadIndexedReferenceObjectsTask,
515 doc="Reference object loader for astrometric fit",
516 )
517 photometryRefObjLoader = pexConfig.ConfigurableField(
518 target=LoadIndexedReferenceObjectsTask,
519 doc="Reference object loader for photometric fit",
520 )
521 sourceSelector = sourceSelectorRegistry.makeField(
522 doc="How to select sources for cross-matching",
523 default="astrometry"
524 )
525 astrometryReferenceSelector = pexConfig.ConfigurableField(
526 target=ReferenceSourceSelectorTask,
527 doc="How to down-select the loaded astrometry reference catalog.",
528 )
529 photometryReferenceSelector = pexConfig.ConfigurableField(
530 target=ReferenceSourceSelectorTask,
531 doc="How to down-select the loaded photometry reference catalog.",
532 )
533 astrometryReferenceErr = pexConfig.Field(
534 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
535 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
536 "If specified, overrides any existing `coord_*Err` values."),
537 dtype=float,
538 default=None,
539 optional=True
540 )
542 # configs for outputting debug information
543 writeInitMatrix = pexConfig.Field(
544 dtype=bool,
545 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
546 "Output files will be written to `config.debugOutputPath` and will "
547 "be of the form 'astrometry_[pre|post]init-TRACT-FILTER-mat.txt'. "
548 "Note that these files are the dense versions of the matrix, and so may be very large."),
549 default=False
550 )
551 writeChi2FilesInitialFinal = pexConfig.Field(
552 dtype=bool,
553 doc=("Write .csv files containing the contributions to chi2 for the initialization and final fit. "
554 "Output files will be written to `config.debugOutputPath` and will "
555 "be of the form `astrometry_[initial|final]_chi2-TRACT-FILTER."),
556 default=False
557 )
558 writeChi2FilesOuterLoop = pexConfig.Field(
559 dtype=bool,
560 doc=("Write .csv files containing the contributions to chi2 for the outer fit loop. "
561 "Output files will be written to `config.debugOutputPath` and will "
562 "be of the form `astrometry_init-NN_chi2-TRACT-FILTER`."),
563 default=False
564 )
565 writeInitialModel = pexConfig.Field(
566 dtype=bool,
567 doc=("Write the pre-initialization model to text files, for debugging. "
568 "Output files will be written to `config.debugOutputPath` and will be "
569 "of the form `initial_astrometry_model-TRACT_FILTER.txt`."),
570 default=False
571 )
572 debugOutputPath = pexConfig.Field(
573 dtype=str,
574 default=".",
575 doc=("Path to write debug output files to. Used by "
576 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
577 )
578 detailedProfile = pexConfig.Field(
579 dtype=bool,
580 default=False,
581 doc="Output separate profiling information for different parts of jointcal, e.g. data read, fitting"
582 )
584 def validate(self):
585 super().validate()
586 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
587 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
588 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
589 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
590 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
591 "applyColorTerms=True will be ignored.")
592 lsst.log.warning(msg)
594 def setDefaults(self):
595 # Use science source selector which can filter on extendedness, SNR, and whether blended
596 self.sourceSelector.name = 'science'
597 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
598 self.sourceSelector['science'].doUnresolved = True
599 # with dependable signal to noise ratio.
600 self.sourceSelector['science'].doSignalToNoise = True
601 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
602 # and S/N > 10 to use sources that are not too faint, and thus better measured.
603 self.sourceSelector['science'].signalToNoise.minimum = 10.
604 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
605 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
606 self.sourceSelector['science'].signalToNoise.fluxField = fluxField
607 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err"
608 # Do not trust blended sources' aperture fluxes which also depend on seeing
609 self.sourceSelector['science'].doIsolated = True
610 # Do not trust either flux or centroid measurements with flags,
611 # chosen from the usual QA flags for stars)
612 self.sourceSelector['science'].doFlags = True
613 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
614 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
615 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
616 self.sourceSelector['science'].flags.bad = badFlags
618 # Default to Gaia-DR2 (with proper motions) for astrometry and
619 # PS1-DR1 for photometry, with a reasonable initial filterMap.
620 self.astrometryRefObjLoader.ref_dataset_name = "gaia_dr2_20200414"
621 self.astrometryRefObjLoader.requireProperMotion = True
622 self.astrometryRefObjLoader.anyFilterMapsToThis = 'phot_g_mean'
623 self.photometryRefObjLoader.ref_dataset_name = "ps1_pv3_3pi_20170110"
626def writeModel(model, filename, log):
627 """Write model to outfile."""
628 with open(filename, "w") as file:
629 file.write(repr(model))
630 log.info("Wrote %s to file: %s", model, filename)
633@dataclasses.dataclass
634class JointcalInputData:
635 """The input data jointcal needs for each detector/visit."""
636 visit: int
637 """The visit identifier of this exposure."""
638 catalog: lsst.afw.table.SourceCatalog
639 """The catalog derived from this exposure."""
640 visitInfo: lsst.afw.image.VisitInfo
641 """The VisitInfo of this exposure."""
642 detector: lsst.afw.cameraGeom.Detector
643 """The detector of this exposure."""
644 photoCalib: lsst.afw.image.PhotoCalib
645 """The photometric calibration of this exposure."""
646 wcs: lsst.afw.geom.skyWcs
647 """The WCS of this exposure."""
648 bbox: lsst.geom.Box2I
649 """The bounding box of this exposure."""
650 filter: lsst.afw.image.FilterLabel
651 """The filter of this exposure."""
654class JointcalTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
655 """Astrometricly and photometricly calibrate across multiple visits of the
656 same field.
658 Parameters
659 ----------
660 butler : `lsst.daf.persistence.Butler`
661 The butler is passed to the refObjLoader constructor in case it is
662 needed. Ignored if the refObjLoader argument provides a loader directly.
663 Used to initialize the astrometry and photometry refObjLoaders.
664 initInputs : `dict`, optional
665 Dictionary used to initialize PipelineTasks (empty for jointcal).
666 """
668 ConfigClass = JointcalConfig
669 RunnerClass = JointcalRunner
670 _DefaultName = "jointcal"
672 def __init__(self, butler=None, initInputs=None, **kwargs):
673 super().__init__(**kwargs)
674 self.makeSubtask("sourceSelector")
675 if self.config.doAstrometry:
676 if initInputs is None:
677 # gen3 middleware does refcat things internally (and will not have a butler here)
678 self.makeSubtask('astrometryRefObjLoader', butler=butler)
679 self.makeSubtask("astrometryReferenceSelector")
680 else:
681 self.astrometryRefObjLoader = None
682 if self.config.doPhotometry:
683 if initInputs is None:
684 # gen3 middleware does refcat things internally (and will not have a butler here)
685 self.makeSubtask('photometryRefObjLoader', butler=butler)
686 self.makeSubtask("photometryReferenceSelector")
687 else:
688 self.photometryRefObjLoader = None
690 # To hold various computed metrics for use by tests
691 self.job = Job.load_metrics_package(subset='jointcal')
693 def runQuantum(self, butlerQC, inputRefs, outputRefs):
694 # We override runQuantum to set up the refObjLoaders and write the
695 # outputs to the correct refs.
696 inputs = butlerQC.get(inputRefs)
697 # We want the tract number for writing debug files
698 tract = butlerQC.quantum.dataId['tract']
699 if self.config.doAstrometry:
700 self.astrometryRefObjLoader = ReferenceObjectLoader(
701 dataIds=[ref.datasetRef.dataId
702 for ref in inputRefs.astrometryRefCat],
703 refCats=inputs.pop('astrometryRefCat'),
704 config=self.config.astrometryRefObjLoader,
705 log=self.log)
706 if self.config.doPhotometry:
707 self.photometryRefObjLoader = ReferenceObjectLoader(
708 dataIds=[ref.datasetRef.dataId
709 for ref in inputRefs.photometryRefCat],
710 refCats=inputs.pop('photometryRefCat'),
711 config=self.config.photometryRefObjLoader,
712 log=self.log)
713 outputs = self.run(**inputs, tract=tract)
714 self._put_metrics(butlerQC, outputs.job, outputRefs)
715 if self.config.doAstrometry:
716 self._put_output(butlerQC, outputs.outputWcs, outputRefs.outputWcs,
717 inputs['inputCamera'], "setWcs")
718 if self.config.doPhotometry:
719 self._put_output(butlerQC, outputs.outputPhotoCalib, outputRefs.outputPhotoCalib,
720 inputs['inputCamera'], "setPhotoCalib")
722 def _put_metrics(self, butlerQC, job, outputRefs):
723 """Persist all measured metrics stored in a job.
725 Parameters
726 ----------
727 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
728 A butler which is specialized to operate in the context of a
729 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
730 job : `lsst.verify.job.Job`
731 Measurements of metrics to persist.
732 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
733 The DatasetRefs to persist the data to.
734 """
735 for key in job.measurements.keys():
736 butlerQC.put(job.measurements[key], getattr(outputRefs, key.fqn.replace('jointcal.', '')))
738 def _put_output(self, butlerQC, outputs, outputRefs, camera, setter):
739 """Persist the output datasets to their appropriate datarefs.
741 Parameters
742 ----------
743 butlerQC : `lsst.pipe.base.ButlerQuantumContext`
744 A butler which is specialized to operate in the context of a
745 `lsst.daf.butler.Quantum`; This is the input to `runQuantum`.
746 outputs : `dict` [`tuple`, `lsst.afw.geom.SkyWcs`] or
747 `dict` [`tuple, `lsst.afw.image.PhotoCalib`]
748 The fitted objects to persist.
749 outputRefs : `list` [`lsst.pipe.base.connectionTypes.OutputQuantizedConnection`]
750 The DatasetRefs to persist the data to.
751 camera : `lsst.afw.cameraGeom.Camera`
752 The camera for this instrument, to get detector ids from.
753 setter : `str`
754 The method to call on the ExposureCatalog to set each output.
755 """
756 schema = lsst.afw.table.ExposureTable.makeMinimalSchema()
757 schema.addField('visit', type='I', doc='Visit number')
759 def new_catalog(visit, size):
760 """Return an catalog ready to be filled with appropriate output."""
761 catalog = lsst.afw.table.ExposureCatalog(schema)
762 catalog.resize(size)
763 catalog['visit'] = visit
764 metadata = lsst.daf.base.PropertyList()
765 metadata.add("COMMENT", "Catalog id is detector id, sorted.")
766 metadata.add("COMMENT", "Only detectors with data have entries.")
767 return catalog
769 # count how many detectors have output for each visit
770 detectors_per_visit = collections.defaultdict(int)
771 for key in outputs:
772 # key is (visit, detector_id), and we only need visit here
773 detectors_per_visit[key[0]] += 1
775 for ref in outputRefs:
776 visit = ref.dataId['visit']
777 catalog = new_catalog(visit, detectors_per_visit[visit])
778 # Iterate over every detector and skip the ones we don't have output for.
779 i = 0
780 for detector in camera:
781 detectorId = detector.getId()
782 key = (ref.dataId['visit'], detectorId)
783 if key not in outputs:
784 # skip detectors we don't have output for
785 self.log.debug("No %s output for detector %s in visit %s",
786 setter[3:], detectorId, visit)
787 continue
789 catalog[i].setId(detectorId)
790 getattr(catalog[i], setter)(outputs[key])
791 i += 1
793 catalog.sort() # ensure that the detectors are in sorted order, for fast lookups
794 butlerQC.put(catalog, ref)
795 self.log.info("Wrote %s detectors to %s", i, ref)
797 def run(self, inputSourceTableVisit, inputVisitSummary, inputCamera, tract=None):
798 # Docstring inherited.
800 # We take values out of the Parquet table, and put them in "flux_",
801 # and the config.sourceFluxType field is used during that extraction,
802 # so just use "flux" here.
803 sourceFluxField = "flux"
804 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
805 associations = lsst.jointcal.Associations()
806 self.focalPlaneBBox = inputCamera.getFpBBox()
807 oldWcsList, bands = self._load_data(inputSourceTableVisit,
808 inputVisitSummary,
809 associations,
810 jointcalControl,
811 inputCamera)
813 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, bands)
815 if self.config.doAstrometry:
816 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
817 name="astrometry",
818 refObjLoader=self.astrometryRefObjLoader,
819 referenceSelector=self.astrometryReferenceSelector,
820 fit_function=self._fit_astrometry,
821 tract=tract,
822 epoch=epoch)
823 astrometry_output = self._make_output(associations.getCcdImageList(),
824 astrometry.model,
825 "makeSkyWcs")
826 else:
827 astrometry_output = None
829 if self.config.doPhotometry:
830 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
831 name="photometry",
832 refObjLoader=self.photometryRefObjLoader,
833 referenceSelector=self.photometryReferenceSelector,
834 fit_function=self._fit_photometry,
835 tract=tract,
836 epoch=epoch,
837 reject_bad_fluxes=True)
838 photometry_output = self._make_output(associations.getCcdImageList(),
839 photometry.model,
840 "toPhotoCalib")
841 else:
842 photometry_output = None
844 return pipeBase.Struct(outputWcs=astrometry_output,
845 outputPhotoCalib=photometry_output,
846 job=self.job,
847 astrometryRefObjLoader=self.astrometryRefObjLoader,
848 photometryRefObjLoader=self.photometryRefObjLoader)
850 def _make_schema_table(self):
851 """Return an afw SourceTable to use as a base for creating the
852 SourceCatalog to insert values from the dataFrame into.
854 Returns
855 -------
856 table : `lsst.afw.table.SourceTable`
857 Table with schema and slots to use to make SourceCatalogs.
858 """
859 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
860 schema.addField("centroid_x", "D")
861 schema.addField("centroid_y", "D")
862 schema.addField("centroid_xErr", "F")
863 schema.addField("centroid_yErr", "F")
864 schema.addField("shape_xx", "D")
865 schema.addField("shape_yy", "D")
866 schema.addField("shape_xy", "D")
867 schema.addField("flux_instFlux", "D")
868 schema.addField("flux_instFluxErr", "D")
869 table = lsst.afw.table.SourceTable.make(schema)
870 table.defineCentroid("centroid")
871 table.defineShape("shape")
872 return table
874 def _extract_detector_catalog_from_visit_catalog(self, table, visitCatalog, detectorId,
875 detectorColumn, ixxColumns):
876 """Return an afw SourceCatalog extracted from a visit-level dataframe,
877 limited to just one detector.
879 Parameters
880 ----------
881 table : `lsst.afw.table.SourceTable`
882 Table factory to use to make the SourceCatalog that will be
883 populated with data from ``visitCatalog``.
884 visitCatalog : `pandas.DataFrame`
885 DataFrame to extract a detector catalog from.
886 detectorId : `int`
887 Numeric id of the detector to extract from ``visitCatalog``.
888 detectorColumn : `str`
889 Name of the detector column in the catalog.
890 ixxColumns : `list` [`str`]
891 Names of the ixx/iyy/ixy columns in the catalog.
893 Returns
894 -------
895 catalog : `lsst.afw.table.SourceCatalog`
896 Detector-level catalog extracted from ``visitCatalog``.
897 """
898 # map from dataFrame column to afw table column
899 mapping = {'x': 'centroid_x',
900 'y': 'centroid_y',
901 'xErr': 'centroid_xErr',
902 'yErr': 'centroid_yErr',
903 ixxColumns[0]: 'shape_xx',
904 ixxColumns[1]: 'shape_yy',
905 ixxColumns[2]: 'shape_xy',
906 f'{self.config.sourceFluxType}_instFlux': 'flux_instFlux',
907 f'{self.config.sourceFluxType}_instFluxErr': 'flux_instFluxErr',
908 }
910 catalog = lsst.afw.table.SourceCatalog(table)
911 matched = visitCatalog[detectorColumn] == detectorId
912 catalog.resize(sum(matched))
913 view = visitCatalog.loc[matched]
914 catalog['id'] = view.index
915 for dfCol, afwCol in mapping.items():
916 catalog[afwCol] = view[dfCol]
918 self.log.debug("%d sources selected in visit %d detector %d",
919 len(catalog),
920 view['visit'].iloc[0], # all visits in this catalog are the same, so take the first
921 detectorId)
922 return catalog
924 def _load_data(self, inputSourceTableVisit, inputVisitSummary, associations,
925 jointcalControl, camera):
926 """Read the data that jointcal needs to run. (Gen3 version)
928 Modifies ``associations`` in-place with the loaded data.
930 Parameters
931 ----------
932 inputSourceTableVisit : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
933 References to visit-level DataFrames to load the catalog data from.
934 inputVisitSummary : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
935 Visit-level exposure summary catalog with metadata.
936 associations : `lsst.jointcal.Associations`
937 Object to add the loaded data to by constructing new CcdImages.
938 jointcalControl : `jointcal.JointcalControl`
939 Control object for C++ associations management.
940 camera : `lsst.afw.cameraGeom.Camera`
941 Camera object for detector geometry.
943 Returns
944 -------
945 oldWcsList: `list` [`lsst.afw.geom.SkyWcs`]
946 The original WCS of the input data, to aid in writing tests.
947 bands : `list` [`str`]
948 The filter bands of each input dataset.
949 """
950 oldWcsList = []
951 filters = []
952 load_cat_prof_file = 'jointcal_load_data.prof' if self.config.detailedProfile else ''
953 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
954 table = self._make_schema_table() # every detector catalog has the same layout
955 # No guarantee that the input is in the same order of visits, so we have to map one of them.
956 catalogMap = {ref.dataId['visit']: i for i, ref in enumerate(inputSourceTableVisit)}
957 detectorDict = {detector.getId(): detector for detector in camera}
959 columns = None
961 for visitSummaryRef in inputVisitSummary:
962 visitSummary = visitSummaryRef.get()
964 dataRef = inputSourceTableVisit[catalogMap[visitSummaryRef.dataId['visit']]]
965 if columns is None:
966 inColumns = dataRef.get(component='columns')
967 columns, detColumn, ixxColumns = self._get_sourceTable_visit_columns(inColumns)
968 visitCatalog = dataRef.get(parameters={'columns': columns})
970 selected = self.sourceSelector.run(visitCatalog)
972 # Build a CcdImage for each detector in this visit.
973 detectors = {id: index for index, id in enumerate(visitSummary['id'])}
974 for id, index in detectors.items():
975 catalog = self._extract_detector_catalog_from_visit_catalog(table, selected.sourceCat, id,
976 detColumn, ixxColumns)
977 data = self._make_one_input_data(visitSummary[index], catalog, detectorDict)
978 result = self._build_ccdImage(data, associations, jointcalControl)
979 if result is not None:
980 oldWcsList.append(result.wcs)
981 # A visit has only one band, so we can just use the first.
982 filters.append(data.filter)
983 if len(filters) == 0:
984 raise RuntimeError("No data to process: did source selector remove all sources?")
985 filters = collections.Counter(filters)
987 return oldWcsList, filters
989 def _make_one_input_data(self, visitRecord, catalog, detectorDict):
990 """Return a data structure for this detector+visit."""
991 return JointcalInputData(visit=visitRecord['visit'],
992 catalog=catalog,
993 visitInfo=visitRecord.getVisitInfo(),
994 detector=detectorDict[visitRecord.getId()],
995 photoCalib=visitRecord.getPhotoCalib(),
996 wcs=visitRecord.getWcs(),
997 bbox=visitRecord.getBBox(),
998 # ExposureRecord doesn't have a FilterLabel yet,
999 # so we have to make one.
1000 filter=lsst.afw.image.FilterLabel(band=visitRecord['band'],
1001 physical=visitRecord['physical_filter']))
1003 def _get_sourceTable_visit_columns(self, inColumns):
1004 """
1005 Get the sourceTable_visit columns from the config.
1007 Parameters
1008 ----------
1009 inColumns : `list`
1010 List of columns available in the sourceTable_visit
1012 Returns
1013 -------
1014 columns : `list`
1015 List of columns to read from sourceTable_visit.
1016 detectorColumn : `str`
1017 Name of the detector column.
1018 ixxColumns : `list`
1019 Name of the ixx/iyy/ixy columns.
1020 """
1021 if 'detector' in inColumns:
1022 # Default name for Gen3.
1023 detectorColumn = 'detector'
1024 else:
1025 # Default name for Gen2 and Gen2 conversions.
1026 detectorColumn = 'ccd'
1028 columns = ['visit', detectorColumn,
1029 'sourceId', 'x', 'xErr', 'y', 'yErr',
1030 self.config.sourceFluxType + '_instFlux', self.config.sourceFluxType + '_instFluxErr']
1032 if 'ixx' in inColumns:
1033 # New columns post-DM-31825
1034 ixxColumns = ['ixx', 'iyy', 'ixy']
1035 else:
1036 # Old columns pre-DM-31825
1037 ixxColumns = ['Ixx', 'Iyy', 'Ixy']
1038 columns.extend(ixxColumns)
1040 if self.sourceSelector.config.doFlags:
1041 columns.extend(self.sourceSelector.config.flags.bad)
1042 if self.sourceSelector.config.doUnresolved:
1043 columns.append(self.sourceSelector.config.unresolved.name)
1044 if self.sourceSelector.config.doIsolated:
1045 columns.append(self.sourceSelector.config.isolated.parentName)
1046 columns.append(self.sourceSelector.config.isolated.nChildName)
1048 return columns, detectorColumn, ixxColumns
1050 # We don't currently need to persist the metadata.
1051 # If we do in the future, we will have to add appropriate dataset templates
1052 # to each obs package (the metadata template should look like `jointcal_wcs`).
1053 def _getMetadataName(self):
1054 return None
1056 @classmethod
1057 def _makeArgumentParser(cls):
1058 """Create an argument parser"""
1059 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
1060 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
1061 ContainerClass=PerTractCcdDataIdContainer)
1062 return parser
1064 def _build_ccdImage(self, data, associations, jointcalControl):
1065 """
1066 Extract the necessary things from this catalog+metadata to add a new
1067 ccdImage.
1069 Parameters
1070 ----------
1071 data : `JointcalInputData`
1072 The loaded input data.
1073 associations : `lsst.jointcal.Associations`
1074 Object to add the info to, to construct a new CcdImage
1075 jointcalControl : `jointcal.JointcalControl`
1076 Control object for associations management
1078 Returns
1079 ------
1080 namedtuple or `None`
1081 ``wcs``
1082 The TAN WCS of this image, read from the calexp
1083 (`lsst.afw.geom.SkyWcs`).
1084 ``key``
1085 A key to identify this dataRef by its visit and ccd ids
1086 (`namedtuple`).
1087 `None`
1088 if there are no sources in the loaded catalog.
1089 """
1090 if len(data.catalog) == 0:
1091 self.log.warning("No sources selected in visit %s ccd %s", data.visit, data.detector.getId())
1092 return None
1094 associations.createCcdImage(data.catalog,
1095 data.wcs,
1096 data.visitInfo,
1097 data.bbox,
1098 data.filter.physicalLabel,
1099 data.photoCalib,
1100 data.detector,
1101 data.visit,
1102 data.detector.getId(),
1103 jointcalControl)
1105 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key'))
1106 Key = collections.namedtuple('Key', ('visit', 'ccd'))
1107 return Result(data.wcs, Key(data.visit, data.detector.getId()))
1109 def _readDataId(self, butler, dataId):
1110 """Read all of the data for one dataId from the butler. (gen2 version)"""
1111 # Not all instruments have `visit` in their dataIds.
1112 if "visit" in dataId.keys():
1113 visit = dataId["visit"]
1114 else:
1115 visit = butler.getButler().queryMetadata("calexp", ("visit"), butler.dataId)[0]
1116 detector = butler.get('calexp_detector', dataId=dataId)
1118 catalog = butler.get('src',
1119 flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS,
1120 dataId=dataId)
1121 goodSrc = self.sourceSelector.run(catalog)
1122 self.log.debug("%d sources selected in visit %d detector %d",
1123 len(goodSrc.sourceCat),
1124 visit,
1125 detector.getId())
1126 return JointcalInputData(visit=visit,
1127 catalog=goodSrc.sourceCat,
1128 visitInfo=butler.get('calexp_visitInfo', dataId=dataId),
1129 detector=detector,
1130 photoCalib=butler.get('calexp_photoCalib', dataId=dataId),
1131 wcs=butler.get('calexp_wcs', dataId=dataId),
1132 bbox=butler.get('calexp_bbox', dataId=dataId),
1133 filter=butler.get('calexp_filterLabel', dataId=dataId))
1135 def loadData(self, dataRefs, associations, jointcalControl):
1136 """Read the data that jointcal needs to run. (Gen2 version)"""
1137 visit_ccd_to_dataRef = {}
1138 oldWcsList = []
1139 filters = []
1140 load_cat_prof_file = 'jointcal_loadData.prof' if self.config.detailedProfile else ''
1141 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1142 # Need the bounding-box of the focal plane (the same for all visits) for photometry visit models
1143 camera = dataRefs[0].get('camera', immediate=True)
1144 self.focalPlaneBBox = camera.getFpBBox()
1145 for dataRef in dataRefs:
1146 data = self._readDataId(dataRef.getButler(), dataRef.dataId)
1147 result = self._build_ccdImage(data, associations, jointcalControl)
1148 if result is None:
1149 continue
1150 oldWcsList.append(result.wcs)
1151 visit_ccd_to_dataRef[result.key] = dataRef
1152 filters.append(data.filter)
1153 if len(filters) == 0:
1154 raise RuntimeError("No data to process: did source selector remove all sources?")
1155 filters = collections.Counter(filters)
1157 return oldWcsList, filters, visit_ccd_to_dataRef
1159 def _getDebugPath(self, filename):
1160 """Constructs a path to filename using the configured debug path.
1161 """
1162 return os.path.join(self.config.debugOutputPath, filename)
1164 def _prep_sky(self, associations, filters):
1165 """Prepare on-sky and other data that must be computed after data has
1166 been read.
1167 """
1168 associations.computeCommonTangentPoint()
1170 boundingCircle = associations.computeBoundingCircle()
1171 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
1172 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
1174 self.log.info(f"Data has center={center} with radius={radius.asDegrees()} degrees.")
1176 # Determine a default filter band associated with the catalog. See DM-9093
1177 defaultFilter = filters.most_common(1)[0][0]
1178 self.log.debug("Using '%s' filter for reference flux", defaultFilter.physicalLabel)
1180 # compute and set the reference epoch of the observations, for proper motion corrections
1181 epoch = self._compute_proper_motion_epoch(associations.getCcdImageList())
1182 associations.setEpoch(epoch.jyear)
1184 return boundingCircle, center, radius, defaultFilter, epoch
1186 @timeMethod
1187 def runDataRef(self, dataRefs):
1188 """
1189 Jointly calibrate the astrometry and photometry across a set of images.
1191 NOTE: this is for gen2 middleware only.
1193 Parameters
1194 ----------
1195 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
1196 List of data references to the exposures to be fit.
1198 Returns
1199 -------
1200 result : `lsst.pipe.base.Struct`
1201 Struct of metadata from the fit, containing:
1203 ``dataRefs``
1204 The provided data references that were fit (with updated WCSs)
1205 ``oldWcsList``
1206 The original WCS from each dataRef
1207 ``metrics``
1208 Dictionary of internally-computed metrics for testing/validation.
1209 """
1210 if len(dataRefs) == 0:
1211 raise ValueError('Need a non-empty list of data references!')
1213 exitStatus = 0 # exit status for shell
1215 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
1216 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
1217 associations = lsst.jointcal.Associations()
1219 oldWcsList, filters, visit_ccd_to_dataRef = self.loadData(dataRefs,
1220 associations,
1221 jointcalControl)
1223 boundingCircle, center, radius, defaultFilter, epoch = self._prep_sky(associations, filters)
1225 tract = dataRefs[0].dataId['tract']
1227 if self.config.doAstrometry:
1228 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1229 name="astrometry",
1230 refObjLoader=self.astrometryRefObjLoader,
1231 referenceSelector=self.astrometryReferenceSelector,
1232 fit_function=self._fit_astrometry,
1233 tract=tract,
1234 epoch=epoch)
1235 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
1236 else:
1237 astrometry = Astrometry(None, None, None)
1239 if self.config.doPhotometry:
1240 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
1241 name="photometry",
1242 refObjLoader=self.photometryRefObjLoader,
1243 referenceSelector=self.photometryReferenceSelector,
1244 fit_function=self._fit_photometry,
1245 tract=tract,
1246 epoch=epoch,
1247 reject_bad_fluxes=True)
1248 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
1249 else:
1250 photometry = Photometry(None, None)
1252 return pipeBase.Struct(dataRefs=dataRefs,
1253 oldWcsList=oldWcsList,
1254 job=self.job,
1255 astrometryRefObjLoader=self.astrometryRefObjLoader,
1256 photometryRefObjLoader=self.photometryRefObjLoader,
1257 defaultFilter=defaultFilter,
1258 epoch=epoch,
1259 exitStatus=exitStatus)
1261 def _get_refcat_coordinate_error_override(self, refCat, name):
1262 """Check whether we should override the refcat coordinate errors, and
1263 return the overridden error if necessary.
1265 Parameters
1266 ----------
1267 refCat : `lsst.afw.table.SimpleCatalog`
1268 The reference catalog to check for a ``coord_raErr`` field.
1269 name : `str`
1270 Whether we are doing "astrometry" or "photometry".
1272 Returns
1273 -------
1274 refCoordErr : `float`
1275 The refcat coordinate error to use, or NaN if we are not overriding
1276 those fields.
1278 Raises
1279 ------
1280 lsst.pex.config.FieldValidationError
1281 Raised if the refcat does not contain coordinate errors and
1282 ``config.astrometryReferenceErr`` is not set.
1283 """
1284 # This value doesn't matter for photometry, so just set something to
1285 # keep old refcats from causing problems.
1286 if name.lower() == "photometry":
1287 if 'coord_raErr' not in refCat.schema:
1288 return 100
1289 else:
1290 return float('nan')
1292 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
1293 msg = ("Reference catalog does not contain coordinate errors, "
1294 "and config.astrometryReferenceErr not supplied.")
1295 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
1296 self.config,
1297 msg)
1299 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
1300 self.log.warning("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
1301 self.config.astrometryReferenceErr)
1303 if self.config.astrometryReferenceErr is None:
1304 return float('nan')
1305 else:
1306 return self.config.astrometryReferenceErr
1308 def _compute_proper_motion_epoch(self, ccdImageList):
1309 """Return the proper motion correction epoch of the provided images.
1311 Parameters
1312 ----------
1313 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
1314 The images to compute the appropriate epoch for.
1316 Returns
1317 -------
1318 epoch : `astropy.time.Time`
1319 The date to use for proper motion corrections.
1320 """
1321 return astropy.time.Time(np.mean([ccdImage.getEpoch() for ccdImage in ccdImageList]),
1322 format="jyear",
1323 scale="tai")
1325 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
1326 tract="", match_cut=3.0,
1327 reject_bad_fluxes=False, *,
1328 name="", refObjLoader=None, referenceSelector=None,
1329 fit_function=None, epoch=None):
1330 """Load reference catalog, perform the fit, and return the result.
1332 Parameters
1333 ----------
1334 associations : `lsst.jointcal.Associations`
1335 The star/reference star associations to fit.
1336 defaultFilter : `lsst.afw.image.FilterLabel`
1337 filter to load from reference catalog.
1338 center : `lsst.geom.SpherePoint`
1339 ICRS center of field to load from reference catalog.
1340 radius : `lsst.geom.Angle`
1341 On-sky radius to load from reference catalog.
1342 name : `str`
1343 Name of thing being fit: "astrometry" or "photometry".
1344 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1345 Reference object loader to use to load a reference catalog.
1346 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1347 Selector to use to pick objects from the loaded reference catalog.
1348 fit_function : callable
1349 Function to call to perform fit (takes Associations object).
1350 tract : `str`, optional
1351 Name of tract currently being fit.
1352 match_cut : `float`, optional
1353 Radius in arcseconds to find cross-catalog matches to during
1354 associations.associateCatalogs.
1355 reject_bad_fluxes : `bool`, optional
1356 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
1357 epoch : `astropy.time.Time`, optional
1358 Epoch to which to correct refcat proper motion and parallax,
1359 or `None` to not apply such corrections.
1361 Returns
1362 -------
1363 result : `Photometry` or `Astrometry`
1364 Result of `fit_function()`
1365 """
1366 self.log.info("====== Now processing %s...", name)
1367 # TODO: this should not print "trying to invert a singular transformation:"
1368 # if it does that, something's not right about the WCS...
1369 associations.associateCatalogs(match_cut)
1370 add_measurement(self.job, 'jointcal.%s_matched_fittedStars' % name,
1371 associations.fittedStarListSize())
1373 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
1374 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
1375 center, radius, defaultFilter,
1376 applyColorterms=applyColorterms,
1377 epoch=epoch)
1378 refCoordErr = self._get_refcat_coordinate_error_override(refCat, name)
1380 associations.collectRefStars(refCat,
1381 self.config.matchCut*lsst.geom.arcseconds,
1382 fluxField,
1383 refCoordinateErr=refCoordErr,
1384 rejectBadFluxes=reject_bad_fluxes)
1385 add_measurement(self.job, 'jointcal.%s_collected_refStars' % name,
1386 associations.refStarListSize())
1388 associations.prepareFittedStars(self.config.minMeasurements)
1390 self._check_star_lists(associations, name)
1391 add_measurement(self.job, 'jointcal.%s_prepared_refStars' % name,
1392 associations.nFittedStarsWithAssociatedRefStar())
1393 add_measurement(self.job, 'jointcal.%s_prepared_fittedStars' % name,
1394 associations.fittedStarListSize())
1395 add_measurement(self.job, 'jointcal.%s_prepared_ccdImages' % name,
1396 associations.nCcdImagesValidForFit())
1398 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if self.config.detailedProfile else ''
1399 dataName = "{}_{}".format(tract, defaultFilter.physicalLabel)
1400 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
1401 result = fit_function(associations, dataName)
1402 # TODO DM-12446: turn this into a "butler save" somehow.
1403 # Save reference and measurement chi2 contributions for this data
1404 if self.config.writeChi2FilesInitialFinal:
1405 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
1406 result.fit.saveChi2Contributions(baseName+"{type}")
1407 self.log.info("Wrote chi2 contributions files: %s", baseName)
1409 return result
1411 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterLabel,
1412 applyColorterms=False, epoch=None):
1413 """Load the necessary reference catalog sources, convert fluxes to
1414 correct units, and apply color term corrections if requested.
1416 Parameters
1417 ----------
1418 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
1419 The reference catalog loader to use to get the data.
1420 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
1421 Source selector to apply to loaded reference catalog.
1422 center : `lsst.geom.SpherePoint`
1423 The center around which to load sources.
1424 radius : `lsst.geom.Angle`
1425 The radius around ``center`` to load sources in.
1426 filterLabel : `lsst.afw.image.FilterLabel`
1427 The camera filter to load fluxes for.
1428 applyColorterms : `bool`
1429 Apply colorterm corrections to the refcat for ``filterName``?
1430 epoch : `astropy.time.Time`, optional
1431 Epoch to which to correct refcat proper motion and parallax,
1432 or `None` to not apply such corrections.
1434 Returns
1435 -------
1436 refCat : `lsst.afw.table.SimpleCatalog`
1437 The loaded reference catalog.
1438 fluxField : `str`
1439 The name of the reference catalog flux field appropriate for ``filterName``.
1440 """
1441 skyCircle = refObjLoader.loadSkyCircle(center,
1442 radius,
1443 filterLabel.bandLabel,
1444 epoch=epoch)
1446 selected = referenceSelector.run(skyCircle.refCat)
1447 # Need memory contiguity to get reference filters as a vector.
1448 if not selected.sourceCat.isContiguous():
1449 refCat = selected.sourceCat.copy(deep=True)
1450 else:
1451 refCat = selected.sourceCat
1453 if applyColorterms:
1454 refCatName = refObjLoader.ref_dataset_name
1455 self.log.info("Applying color terms for physical filter=%r reference catalog=%s",
1456 filterLabel.physicalLabel, refCatName)
1457 colorterm = self.config.colorterms.getColorterm(filterLabel.physicalLabel,
1458 refCatName,
1459 doRaise=True)
1461 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat)
1462 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
1463 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
1464 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
1466 return refCat, skyCircle.fluxField
1468 def _check_star_lists(self, associations, name):
1469 # TODO: these should be len(blah), but we need this properly wrapped first.
1470 if associations.nCcdImagesValidForFit() == 0:
1471 raise RuntimeError('No images in the ccdImageList!')
1472 if associations.fittedStarListSize() == 0:
1473 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
1474 if associations.refStarListSize() == 0:
1475 raise RuntimeError('No stars in the {} reference star list!'.format(name))
1477 def _logChi2AndValidate(self, associations, fit, model, chi2Label, writeChi2Name=None):
1478 """Compute chi2, log it, validate the model, and return chi2.
1480 Parameters
1481 ----------
1482 associations : `lsst.jointcal.Associations`
1483 The star/reference star associations to fit.
1484 fit : `lsst.jointcal.FitterBase`
1485 The fitter to use for minimization.
1486 model : `lsst.jointcal.Model`
1487 The model being fit.
1488 chi2Label : `str`
1489 Label to describe the chi2 (e.g. "Initialized", "Final").
1490 writeChi2Name : `str`, optional
1491 Filename prefix to write the chi2 contributions to.
1492 Do not supply an extension: an appropriate one will be added.
1494 Returns
1495 -------
1496 chi2: `lsst.jointcal.Chi2Accumulator`
1497 The chi2 object for the current fitter and model.
1499 Raises
1500 ------
1501 FloatingPointError
1502 Raised if chi2 is infinite or NaN.
1503 ValueError
1504 Raised if the model is not valid.
1505 """
1506 if writeChi2Name is not None:
1507 fullpath = self._getDebugPath(writeChi2Name)
1508 fit.saveChi2Contributions(fullpath+"{type}")
1509 self.log.info("Wrote chi2 contributions files: %s", fullpath)
1511 chi2 = fit.computeChi2()
1512 self.log.info("%s %s", chi2Label, chi2)
1513 self._check_stars(associations)
1514 if not np.isfinite(chi2.chi2):
1515 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
1516 if not model.validate(associations.getCcdImageList(), chi2.ndof):
1517 raise ValueError("Model is not valid: check log messages for warnings.")
1518 return chi2
1520 def _fit_photometry(self, associations, dataName=None):
1521 """
1522 Fit the photometric data.
1524 Parameters
1525 ----------
1526 associations : `lsst.jointcal.Associations`
1527 The star/reference star associations to fit.
1528 dataName : `str`
1529 Name of the data being processed (e.g. "1234_HSC-Y"), for
1530 identifying debugging files.
1532 Returns
1533 -------
1534 fit_result : `namedtuple`
1535 fit : `lsst.jointcal.PhotometryFit`
1536 The photometric fitter used to perform the fit.
1537 model : `lsst.jointcal.PhotometryModel`
1538 The photometric model that was fit.
1539 """
1540 self.log.info("=== Starting photometric fitting...")
1542 # TODO: should use pex.config.RegistryField here (see DM-9195)
1543 if self.config.photometryModel == "constrainedFlux":
1544 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
1545 self.focalPlaneBBox,
1546 visitOrder=self.config.photometryVisitOrder,
1547 errorPedestal=self.config.photometryErrorPedestal)
1548 # potentially nonlinear problem, so we may need a line search to converge.
1549 doLineSearch = self.config.allowLineSearch
1550 elif self.config.photometryModel == "constrainedMagnitude":
1551 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
1552 self.focalPlaneBBox,
1553 visitOrder=self.config.photometryVisitOrder,
1554 errorPedestal=self.config.photometryErrorPedestal)
1555 # potentially nonlinear problem, so we may need a line search to converge.
1556 doLineSearch = self.config.allowLineSearch
1557 elif self.config.photometryModel == "simpleFlux":
1558 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
1559 errorPedestal=self.config.photometryErrorPedestal)
1560 doLineSearch = False # purely linear in model parameters, so no line search needed
1561 elif self.config.photometryModel == "simpleMagnitude":
1562 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
1563 errorPedestal=self.config.photometryErrorPedestal)
1564 doLineSearch = False # purely linear in model parameters, so no line search needed
1566 fit = lsst.jointcal.PhotometryFit(associations, model)
1567 # TODO DM-12446: turn this into a "butler save" somehow.
1568 # Save reference and measurement chi2 contributions for this data
1569 if self.config.writeChi2FilesInitialFinal:
1570 baseName = f"photometry_initial_chi2-{dataName}"
1571 else:
1572 baseName = None
1573 if self.config.writeInitialModel:
1574 fullpath = self._getDebugPath(f"initial_photometry_model-{dataName}.txt")
1575 writeModel(model, fullpath, self.log)
1576 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
1578 def getChi2Name(whatToFit):
1579 if self.config.writeChi2FilesOuterLoop:
1580 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
1581 else:
1582 return None
1584 # The constrained model needs the visit transform fit first; the chip
1585 # transform is initialized from the singleFrame PhotoCalib, so it's close.
1586 if self.config.writeInitMatrix:
1587 dumpMatrixFile = self._getDebugPath(f"photometry_preinit-{dataName}")
1588 else:
1589 dumpMatrixFile = ""
1590 if self.config.photometryModel.startswith("constrained"):
1591 # no line search: should be purely (or nearly) linear,
1592 # and we want a large step size to initialize with.
1593 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
1594 self._logChi2AndValidate(associations, fit, model, "Initialize ModelVisit",
1595 writeChi2Name=getChi2Name("ModelVisit"))
1596 dumpMatrixFile = "" # so we don't redo the output on the next step
1598 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
1599 self._logChi2AndValidate(associations, fit, model, "Initialize Model",
1600 writeChi2Name=getChi2Name("Model"))
1602 fit.minimize("Fluxes") # no line search: always purely linear.
1603 self._logChi2AndValidate(associations, fit, model, "Initialize Fluxes",
1604 writeChi2Name=getChi2Name("Fluxes"))
1606 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
1607 self._logChi2AndValidate(associations, fit, model, "Initialize ModelFluxes",
1608 writeChi2Name=getChi2Name("ModelFluxes"))
1610 model.freezeErrorTransform()
1611 self.log.debug("Photometry error scales are frozen.")
1613 chi2 = self._iterate_fit(associations,
1614 fit,
1615 self.config.maxPhotometrySteps,
1616 "photometry",
1617 "Model Fluxes",
1618 doRankUpdate=self.config.photometryDoRankUpdate,
1619 doLineSearch=doLineSearch,
1620 dataName=dataName)
1622 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
1623 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
1624 return Photometry(fit, model)
1626 def _fit_astrometry(self, associations, dataName=None):
1627 """
1628 Fit the astrometric data.
1630 Parameters
1631 ----------
1632 associations : `lsst.jointcal.Associations`
1633 The star/reference star associations to fit.
1634 dataName : `str`
1635 Name of the data being processed (e.g. "1234_HSC-Y"), for
1636 identifying debugging files.
1638 Returns
1639 -------
1640 fit_result : `namedtuple`
1641 fit : `lsst.jointcal.AstrometryFit`
1642 The astrometric fitter used to perform the fit.
1643 model : `lsst.jointcal.AstrometryModel`
1644 The astrometric model that was fit.
1645 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
1646 The model for the sky to tangent plane projection that was used in the fit.
1647 """
1649 self.log.info("=== Starting astrometric fitting...")
1651 associations.deprojectFittedStars()
1653 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
1654 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
1655 # them so carefully?
1656 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
1658 if self.config.astrometryModel == "constrained":
1659 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
1660 sky_to_tan_projection,
1661 chipOrder=self.config.astrometryChipOrder,
1662 visitOrder=self.config.astrometryVisitOrder)
1663 elif self.config.astrometryModel == "simple":
1664 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
1665 sky_to_tan_projection,
1666 self.config.useInputWcs,
1667 nNotFit=0,
1668 order=self.config.astrometrySimpleOrder)
1670 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
1671 # TODO DM-12446: turn this into a "butler save" somehow.
1672 # Save reference and measurement chi2 contributions for this data
1673 if self.config.writeChi2FilesInitialFinal:
1674 baseName = f"astrometry_initial_chi2-{dataName}"
1675 else:
1676 baseName = None
1677 if self.config.writeInitialModel:
1678 fullpath = self._getDebugPath(f"initial_astrometry_model-{dataName}.txt")
1679 writeModel(model, fullpath, self.log)
1680 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
1682 def getChi2Name(whatToFit):
1683 if self.config.writeChi2FilesOuterLoop:
1684 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
1685 else:
1686 return None
1688 if self.config.writeInitMatrix:
1689 dumpMatrixFile = self._getDebugPath(f"astrometry_preinit-{dataName}")
1690 else:
1691 dumpMatrixFile = ""
1692 # The constrained model needs the visit transform fit first; the chip
1693 # transform is initialized from the detector's cameraGeom, so it's close.
1694 if self.config.astrometryModel == "constrained":
1695 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1696 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsVisit",
1697 writeChi2Name=getChi2Name("DistortionsVisit"))
1698 dumpMatrixFile = "" # so we don't redo the output on the next step
1700 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
1701 self._logChi2AndValidate(associations, fit, model, "Initialize Distortions",
1702 writeChi2Name=getChi2Name("Distortions"))
1704 fit.minimize("Positions")
1705 self._logChi2AndValidate(associations, fit, model, "Initialize Positions",
1706 writeChi2Name=getChi2Name("Positions"))
1708 fit.minimize("Distortions Positions")
1709 self._logChi2AndValidate(associations, fit, model, "Initialize DistortionsPositions",
1710 writeChi2Name=getChi2Name("DistortionsPositions"))
1712 chi2 = self._iterate_fit(associations,
1713 fit,
1714 self.config.maxAstrometrySteps,
1715 "astrometry",
1716 "Distortions Positions",
1717 sigmaRelativeTolerance=self.config.astrometryOutlierRelativeTolerance,
1718 doRankUpdate=self.config.astrometryDoRankUpdate,
1719 dataName=dataName)
1721 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
1722 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
1724 return Astrometry(fit, model, sky_to_tan_projection)
1726 def _check_stars(self, associations):
1727 """Count measured and reference stars per ccd and warn/log them."""
1728 for ccdImage in associations.getCcdImageList():
1729 nMeasuredStars, nRefStars = ccdImage.countStars()
1730 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1731 ccdImage.getName(), nMeasuredStars, nRefStars)
1732 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1733 self.log.warning("ccdImage %s has only %s measuredStars (desired %s)",
1734 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1735 if nRefStars < self.config.minRefStarsPerCcd:
1736 self.log.warning("ccdImage %s has only %s RefStars (desired %s)",
1737 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1739 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1740 dataName="",
1741 sigmaRelativeTolerance=0,
1742 doRankUpdate=True,
1743 doLineSearch=False):
1744 """Run fitter.minimize up to max_steps times, returning the final chi2.
1746 Parameters
1747 ----------
1748 associations : `lsst.jointcal.Associations`
1749 The star/reference star associations to fit.
1750 fitter : `lsst.jointcal.FitterBase`
1751 The fitter to use for minimization.
1752 max_steps : `int`
1753 Maximum number of steps to run outlier rejection before declaring
1754 convergence failure.
1755 name : {'photometry' or 'astrometry'}
1756 What type of data are we fitting (for logs and debugging files).
1757 whatToFit : `str`
1758 Passed to ``fitter.minimize()`` to define the parameters to fit.
1759 dataName : `str`, optional
1760 Descriptive name for this dataset (e.g. tract and filter),
1761 for debugging.
1762 sigmaRelativeTolerance : `float`, optional
1763 Convergence tolerance for the fractional change in the chi2 cut
1764 level for determining outliers. If set to zero, iterations will
1765 continue until there are no outliers.
1766 doRankUpdate : `bool`, optional
1767 Do an Eigen rank update during minimization, or recompute the full
1768 matrix and gradient?
1769 doLineSearch : `bool`, optional
1770 Do a line search for the optimum step during minimization?
1772 Returns
1773 -------
1774 chi2: `lsst.jointcal.Chi2Statistic`
1775 The final chi2 after the fit converges, or is forced to end.
1777 Raises
1778 ------
1779 FloatingPointError
1780 Raised if the fitter fails with a non-finite value.
1781 RuntimeError
1782 Raised if the fitter fails for some other reason;
1783 log messages will provide further details.
1784 """
1785 if self.config.writeInitMatrix:
1786 dumpMatrixFile = self._getDebugPath(f"{name}_postinit-{dataName}")
1787 else:
1788 dumpMatrixFile = ""
1789 oldChi2 = lsst.jointcal.Chi2Statistic()
1790 oldChi2.chi2 = float("inf")
1791 for i in range(max_steps):
1792 if self.config.writeChi2FilesOuterLoop:
1793 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1794 else:
1795 writeChi2Name = None
1796 result = fitter.minimize(whatToFit,
1797 self.config.outlierRejectSigma,
1798 sigmaRelativeTolerance=sigmaRelativeTolerance,
1799 doRankUpdate=doRankUpdate,
1800 doLineSearch=doLineSearch,
1801 dumpMatrixFile=dumpMatrixFile)
1802 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1803 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1804 f"Fit iteration {i}", writeChi2Name=writeChi2Name)
1806 if result == MinimizeResult.Converged:
1807 if doRankUpdate:
1808 self.log.debug("fit has converged - no more outliers - redo minimization "
1809 "one more time in case we have lost accuracy in rank update.")
1810 # Redo minimization one more time in case we have lost accuracy in rank update
1811 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma,
1812 sigmaRelativeTolerance=sigmaRelativeTolerance)
1813 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1815 # log a message for a large final chi2, TODO: DM-15247 for something better
1816 if chi2.chi2/chi2.ndof >= 4.0:
1817 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1819 break
1820 elif result == MinimizeResult.Chi2Increased:
1821 self.log.warning("Still some outliers remaining but chi2 increased - retry")
1822 # Check whether the increase was large enough to cause trouble.
1823 chi2Ratio = chi2.chi2 / oldChi2.chi2
1824 if chi2Ratio > 1.5:
1825 self.log.warning('Significant chi2 increase by a factor of %.4g / %.4g = %.4g',
1826 chi2.chi2, oldChi2.chi2, chi2Ratio)
1827 # Based on a variety of HSC jointcal logs (see DM-25779), it
1828 # appears that chi2 increases more than a factor of ~2 always
1829 # result in the fit diverging rapidly and ending at chi2 > 1e10.
1830 # Using 10 as the "failure" threshold gives some room between
1831 # leaving a warning and bailing early.
1832 if chi2Ratio > 10:
1833 msg = ("Large chi2 increase between steps: fit likely cannot converge."
1834 " Try setting one or more of the `writeChi2*` config fields and looking"
1835 " at how individual star chi2-values evolve during the fit.")
1836 raise RuntimeError(msg)
1837 oldChi2 = chi2
1838 elif result == MinimizeResult.NonFinite:
1839 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1840 # TODO DM-12446: turn this into a "butler save" somehow.
1841 fitter.saveChi2Contributions(filename+"{type}")
1842 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1843 raise FloatingPointError(msg.format(filename))
1844 elif result == MinimizeResult.Failed:
1845 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1846 else:
1847 raise RuntimeError("Unxepected return code from minimize().")
1848 else:
1849 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1851 return chi2
1853 def _make_output(self, ccdImageList, model, func):
1854 """Return the internal jointcal models converted to the afw
1855 structures that will be saved to disk.
1857 Parameters
1858 ----------
1859 ccdImageList : `lsst.jointcal.CcdImageList`
1860 The list of CcdImages to get the output for.
1861 model : `lsst.jointcal.AstrometryModel` or `lsst.jointcal.PhotometryModel`
1862 The internal jointcal model to convert for each `lsst.jointcal.CcdImage`.
1863 func : `str`
1864 The name of the function to call on ``model`` to get the converted
1865 structure. Must accept an `lsst.jointcal.CcdImage`.
1867 Returns
1868 -------
1869 output : `dict` [`tuple`, `lsst.jointcal.AstrometryModel`] or
1870 `dict` [`tuple`, `lsst.jointcal.PhotometryModel`]
1871 The data to be saved, keyed on (visit, detector).
1872 """
1873 output = {}
1874 for ccdImage in ccdImageList:
1875 ccd = ccdImage.ccdId
1876 visit = ccdImage.visit
1877 self.log.debug("%s for visit: %d, ccd: %d", func, visit, ccd)
1878 output[(visit, ccd)] = getattr(model, func)(ccdImage)
1879 return output
1881 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1882 """
1883 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1885 Parameters
1886 ----------
1887 associations : `lsst.jointcal.Associations`
1888 The star/reference star associations to fit.
1889 model : `lsst.jointcal.AstrometryModel`
1890 The astrometric model that was fit.
1891 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1892 Dict of ccdImage identifiers to dataRefs that were fit.
1893 """
1894 ccdImageList = associations.getCcdImageList()
1895 output = self._make_output(ccdImageList, model, "makeSkyWcs")
1896 for key, skyWcs in output.items():
1897 dataRef = visit_ccd_to_dataRef[key]
1898 try:
1899 dataRef.put(skyWcs, 'jointcal_wcs')
1900 except pexExceptions.Exception as e:
1901 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1902 raise e
1904 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1905 """
1906 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1908 Parameters
1909 ----------
1910 associations : `lsst.jointcal.Associations`
1911 The star/reference star associations to fit.
1912 model : `lsst.jointcal.PhotometryModel`
1913 The photoometric model that was fit.
1914 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1915 Dict of ccdImage identifiers to dataRefs that were fit.
1916 """
1918 ccdImageList = associations.getCcdImageList()
1919 output = self._make_output(ccdImageList, model, "toPhotoCalib")
1920 for key, photoCalib in output.items():
1921 dataRef = visit_ccd_to_dataRef[key]
1922 try:
1923 dataRef.put(photoCalib, 'jointcal_photoCalib')
1924 except pexExceptions.Exception as e:
1925 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1926 raise e