Coverage for python/lsst/jointcal/jointcal.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of jointcal.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import collections
23import os
25import numpy as np
26import astropy.units as u
28import lsst.geom
29import lsst.utils
30import lsst.pex.config as pexConfig
31import lsst.pipe.base as pipeBase
32from lsst.afw.image import fluxErrFromABMagErr
33import lsst.pex.exceptions as pexExceptions
34import lsst.afw.table
35import lsst.log
36import lsst.meas.algorithms
37from lsst.pipe.tasks.colorterms import ColortermLibrary
38from lsst.verify import Job, Measurement
40from lsst.meas.algorithms import LoadIndexedReferenceObjectsTask, ReferenceSourceSelectorTask
41from lsst.meas.algorithms.sourceSelector import sourceSelectorRegistry
43from .dataIds import PerTractCcdDataIdContainer
45import lsst.jointcal
46from lsst.jointcal import MinimizeResult
48__all__ = ["JointcalConfig", "JointcalRunner", "JointcalTask"]
50Photometry = collections.namedtuple('Photometry', ('fit', 'model'))
51Astrometry = collections.namedtuple('Astrometry', ('fit', 'model', 'sky_to_tan_projection'))
54# TODO: move this to MeasurementSet in lsst.verify per DM-12655.
55def add_measurement(job, name, value):
56 meas = Measurement(job.metrics[name], value)
57 job.measurements.insert(meas)
60class JointcalRunner(pipeBase.ButlerInitializedTaskRunner):
61 """Subclass of TaskRunner for jointcalTask
63 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
64 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
65 single dataRef, are are called repeatedly). This class transforms the processed
66 arguments generated by the ArgumentParser into the arguments expected by
67 Jointcal.runDataRef().
69 See pipeBase.TaskRunner for more information.
70 """
72 @staticmethod
73 def getTargetList(parsedCmd, **kwargs):
74 """
75 Return a list of tuples per tract, each containing (dataRefs, kwargs).
77 Jointcal operates on lists of dataRefs simultaneously.
78 """
79 kwargs['profile_jointcal'] = parsedCmd.profile_jointcal
80 kwargs['butler'] = parsedCmd.butler
82 # organize data IDs by tract
83 refListDict = {}
84 for ref in parsedCmd.id.refList:
85 refListDict.setdefault(ref.dataId["tract"], []).append(ref)
86 # we call runDataRef() once with each tract
87 result = [(refListDict[tract], kwargs) for tract in sorted(refListDict.keys())]
88 return result
90 def __call__(self, args):
91 """
92 Parameters
93 ----------
94 args
95 Arguments for Task.runDataRef()
97 Returns
98 -------
99 pipe.base.Struct
100 if self.doReturnResults is False:
102 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
104 if self.doReturnResults is True:
106 - ``result``: the result of calling jointcal.runDataRef()
107 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
108 """
109 exitStatus = 0 # exit status for shell
111 # NOTE: cannot call self.makeTask because that assumes args[0] is a single dataRef.
112 dataRefList, kwargs = args
113 butler = kwargs.pop('butler')
114 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
115 result = None
116 try:
117 result = task.runDataRef(dataRefList, **kwargs)
118 exitStatus = result.exitStatus
119 job_path = butler.get('verify_job_filename')
120 result.job.write(job_path[0])
121 except Exception as e: # catch everything, sort it out later.
122 if self.doRaise:
123 raise e
124 else:
125 exitStatus = 1
126 eName = type(e).__name__
127 tract = dataRefList[0].dataId['tract']
128 task.log.fatal("Failed processing tract %s, %s: %s", tract, eName, e)
130 # Put the butler back into kwargs for the other Tasks.
131 kwargs['butler'] = butler
132 if self.doReturnResults:
133 return pipeBase.Struct(result=result, exitStatus=exitStatus)
134 else:
135 return pipeBase.Struct(exitStatus=exitStatus)
138class JointcalConfig(pexConfig.Config):
139 """Configuration for JointcalTask"""
141 doAstrometry = pexConfig.Field(
142 doc="Fit astrometry and write the fitted result.",
143 dtype=bool,
144 default=True
145 )
146 doPhotometry = pexConfig.Field(
147 doc="Fit photometry and write the fitted result.",
148 dtype=bool,
149 default=True
150 )
151 coaddName = pexConfig.Field(
152 doc="Type of coadd, typically deep or goodSeeing",
153 dtype=str,
154 default="deep"
155 )
156 positionErrorPedestal = pexConfig.Field(
157 doc="Systematic term to apply to the measured position error (pixels)",
158 dtype=float,
159 default=0.02,
160 )
161 photometryErrorPedestal = pexConfig.Field(
162 doc="Systematic term to apply to the measured error on flux or magnitude as a "
163 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
164 dtype=float,
165 default=0.0,
166 )
167 # TODO: DM-6885 matchCut should be an geom.Angle
168 matchCut = pexConfig.Field(
169 doc="Matching radius between fitted and reference stars (arcseconds)",
170 dtype=float,
171 default=3.0,
172 )
173 minMeasurements = pexConfig.Field(
174 doc="Minimum number of associated measured stars for a fitted star to be included in the fit",
175 dtype=int,
176 default=2,
177 )
178 minMeasuredStarsPerCcd = pexConfig.Field(
179 doc="Minimum number of measuredStars per ccdImage before printing warnings",
180 dtype=int,
181 default=100,
182 )
183 minRefStarsPerCcd = pexConfig.Field(
184 doc="Minimum number of measuredStars per ccdImage before printing warnings",
185 dtype=int,
186 default=30,
187 )
188 allowLineSearch = pexConfig.Field(
189 doc="Allow a line search during minimization, if it is reasonable for the model"
190 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
191 dtype=bool,
192 default=False
193 )
194 astrometrySimpleOrder = pexConfig.Field(
195 doc="Polynomial order for fitting the simple astrometry model.",
196 dtype=int,
197 default=3,
198 )
199 astrometryChipOrder = pexConfig.Field(
200 doc="Order of the per-chip transform for the constrained astrometry model.",
201 dtype=int,
202 default=1,
203 )
204 astrometryVisitOrder = pexConfig.Field(
205 doc="Order of the per-visit transform for the constrained astrometry model.",
206 dtype=int,
207 default=5,
208 )
209 useInputWcs = pexConfig.Field(
210 doc="Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
211 dtype=bool,
212 default=True,
213 )
214 astrometryModel = pexConfig.ChoiceField(
215 doc="Type of model to fit to astrometry",
216 dtype=str,
217 default="constrained",
218 allowed={"simple": "One polynomial per ccd",
219 "constrained": "One polynomial per ccd, and one polynomial per visit"}
220 )
221 photometryModel = pexConfig.ChoiceField(
222 doc="Type of model to fit to photometry",
223 dtype=str,
224 default="constrainedMagnitude",
225 allowed={"simpleFlux": "One constant zeropoint per ccd and visit, fitting in flux space.",
226 "constrainedFlux": "Constrained zeropoint per ccd, and one polynomial per visit,"
227 " fitting in flux space.",
228 "simpleMagnitude": "One constant zeropoint per ccd and visit,"
229 " fitting in magnitude space.",
230 "constrainedMagnitude": "Constrained zeropoint per ccd, and one polynomial per visit,"
231 " fitting in magnitude space.",
232 }
233 )
234 applyColorTerms = pexConfig.Field(
235 doc="Apply photometric color terms to reference stars?"
236 "Requires that colorterms be set to a ColortermLibrary",
237 dtype=bool,
238 default=False
239 )
240 colorterms = pexConfig.ConfigField(
241 doc="Library of photometric reference catalog name to color term dict.",
242 dtype=ColortermLibrary,
243 )
244 photometryVisitOrder = pexConfig.Field(
245 doc="Order of the per-visit polynomial transform for the constrained photometry model.",
246 dtype=int,
247 default=7,
248 )
249 photometryDoRankUpdate = pexConfig.Field(
250 doc=("Do the rank update step during minimization. "
251 "Skipping this can help deal with models that are too non-linear."),
252 dtype=bool,
253 default=True,
254 )
255 astrometryDoRankUpdate = pexConfig.Field(
256 doc=("Do the rank update step during minimization (should not change the astrometry fit). "
257 "Skipping this can help deal with models that are too non-linear."),
258 dtype=bool,
259 default=True,
260 )
261 outlierRejectSigma = pexConfig.Field(
262 doc="How many sigma to reject outliers at during minimization.",
263 dtype=float,
264 default=5.0,
265 )
266 maxPhotometrySteps = pexConfig.Field(
267 doc="Maximum number of minimize iterations to take when fitting photometry.",
268 dtype=int,
269 default=20,
270 )
271 maxAstrometrySteps = pexConfig.Field(
272 doc="Maximum number of minimize iterations to take when fitting photometry.",
273 dtype=int,
274 default=20,
275 )
276 astrometryRefObjLoader = pexConfig.ConfigurableField(
277 target=LoadIndexedReferenceObjectsTask,
278 doc="Reference object loader for astrometric fit",
279 )
280 photometryRefObjLoader = pexConfig.ConfigurableField(
281 target=LoadIndexedReferenceObjectsTask,
282 doc="Reference object loader for photometric fit",
283 )
284 sourceSelector = sourceSelectorRegistry.makeField(
285 doc="How to select sources for cross-matching",
286 default="astrometry"
287 )
288 astrometryReferenceSelector = pexConfig.ConfigurableField(
289 target=ReferenceSourceSelectorTask,
290 doc="How to down-select the loaded astrometry reference catalog.",
291 )
292 photometryReferenceSelector = pexConfig.ConfigurableField(
293 target=ReferenceSourceSelectorTask,
294 doc="How to down-select the loaded photometry reference catalog.",
295 )
296 astrometryReferenceErr = pexConfig.Field(
297 doc=("Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
298 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
299 "If specified, overrides any existing `coord_*Err` values."),
300 dtype=float,
301 default=None,
302 optional=True
303 )
304 writeInitMatrix = pexConfig.Field(
305 dtype=bool,
306 doc=("Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
307 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. "
308 "Note that these files are the dense versions of the matrix, and so may be very large."),
309 default=False
310 )
311 writeChi2FilesInitialFinal = pexConfig.Field(
312 dtype=bool,
313 doc="Write .csv files containing the contributions to chi2 for the initialization and final fit.",
314 default=False
315 )
316 writeChi2FilesOuterLoop = pexConfig.Field(
317 dtype=bool,
318 doc="Write .csv files containing the contributions to chi2 for the outer fit loop.",
319 default=False
320 )
321 writeInitialModel = pexConfig.Field(
322 dtype=bool,
323 doc=("Write the pre-initialization model to text files, for debugging."
324 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."),
325 default=False
326 )
327 debugOutputPath = pexConfig.Field(
328 dtype=str,
329 default=".",
330 doc=("Path to write debug output files to. Used by "
331 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
332 )
333 sourceFluxType = pexConfig.Field(
334 dtype=str,
335 doc="Source flux field to use in source selection and to get fluxes from the catalog.",
336 default='Calib'
337 )
339 def validate(self):
340 super().validate()
341 if self.doPhotometry and self.applyColorTerms and len(self.colorterms.data) == 0:
342 msg = "applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
343 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
344 if self.doAstrometry and not self.doPhotometry and self.applyColorTerms:
345 msg = ("Only doing astrometry, but Colorterms are not applied for astrometry;"
346 "applyColorTerms=True will be ignored.")
347 lsst.log.warn(msg)
349 def setDefaults(self):
350 # Use science source selector which can filter on extendedness, SNR, and whether blended
351 self.sourceSelector.name = 'science'
352 # Use only stars because aperture fluxes of galaxies are biased and depend on seeing
353 self.sourceSelector['science'].doUnresolved = True
354 # with dependable signal to noise ratio.
355 self.sourceSelector['science'].doSignalToNoise = True
356 # Min SNR must be > 0 because jointcal cannot handle negative fluxes,
357 # and S/N > 10 to use sources that are not too faint, and thus better measured.
358 self.sourceSelector['science'].signalToNoise.minimum = 10.
359 # Base SNR on CalibFlux because that is the flux jointcal that fits and must be positive
360 fluxField = f"slot_{self.sourceFluxType}Flux_instFlux"
361 self.sourceSelector['science'].signalToNoise.fluxField = fluxField
362 self.sourceSelector['science'].signalToNoise.errField = fluxField + "Err"
363 # Do not trust blended sources' aperture fluxes which also depend on seeing
364 self.sourceSelector['science'].doIsolated = True
365 # Do not trust either flux or centroid measurements with flags,
366 # chosen from the usual QA flags for stars)
367 self.sourceSelector['science'].doFlags = True
368 badFlags = ['base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated',
369 'base_PixelFlags_flag_interpolatedCenter', 'base_SdssCentroid_flag',
370 'base_PsfFlux_flag', 'base_PixelFlags_flag_suspectCenter']
371 self.sourceSelector['science'].flags.bad = badFlags
374def writeModel(model, filename, log):
375 """Write model to outfile."""
376 with open(filename, "w") as file:
377 file.write(repr(model))
378 log.info("Wrote %s to file: %s", model, filename)
381class JointcalTask(pipeBase.CmdLineTask):
382 """Jointly astrometrically and photometrically calibrate a group of images."""
384 ConfigClass = JointcalConfig
385 RunnerClass = JointcalRunner
386 _DefaultName = "jointcal"
388 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
389 """
390 Instantiate a JointcalTask.
392 Parameters
393 ----------
394 butler : `lsst.daf.persistence.Butler`
395 The butler is passed to the refObjLoader constructor in case it is
396 needed. Ignored if the refObjLoader argument provides a loader directly.
397 Used to initialize the astrometry and photometry refObjLoaders.
398 profile_jointcal : `bool`
399 Set to True to profile different stages of this jointcal run.
400 """
401 pipeBase.CmdLineTask.__init__(self, **kwargs)
402 self.profile_jointcal = profile_jointcal
403 self.makeSubtask("sourceSelector")
404 if self.config.doAstrometry:
405 self.makeSubtask('astrometryRefObjLoader', butler=butler)
406 self.makeSubtask("astrometryReferenceSelector")
407 else:
408 self.astrometryRefObjLoader = None
409 if self.config.doPhotometry:
410 self.makeSubtask('photometryRefObjLoader', butler=butler)
411 self.makeSubtask("photometryReferenceSelector")
412 else:
413 self.photometryRefObjLoader = None
415 # To hold various computed metrics for use by tests
416 self.job = Job.load_metrics_package(subset='jointcal')
418 # We don't currently need to persist the metadata.
419 # If we do in the future, we will have to add appropriate dataset templates
420 # to each obs package (the metadata template should look like `jointcal_wcs`).
421 def _getMetadataName(self):
422 return None
424 @classmethod
425 def _makeArgumentParser(cls):
426 """Create an argument parser"""
427 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
428 parser.add_argument("--profile_jointcal", default=False, action="store_true",
429 help="Profile steps of jointcal separately.")
430 parser.add_id_argument("--id", "calexp", help="data ID, e.g. --id visit=6789 ccd=0..9",
431 ContainerClass=PerTractCcdDataIdContainer)
432 return parser
434 def _build_ccdImage(self, dataRef, associations, jointcalControl):
435 """
436 Extract the necessary things from this dataRef to add a new ccdImage.
438 Parameters
439 ----------
440 dataRef : `lsst.daf.persistence.ButlerDataRef`
441 DataRef to extract info from.
442 associations : `lsst.jointcal.Associations`
443 Object to add the info to, to construct a new CcdImage
444 jointcalControl : `jointcal.JointcalControl`
445 Control object for associations management
447 Returns
448 ------
449 namedtuple
450 ``wcs``
451 The TAN WCS of this image, read from the calexp
452 (`lsst.afw.geom.SkyWcs`).
453 ``key``
454 A key to identify this dataRef by its visit and ccd ids
455 (`namedtuple`).
456 ``filter``
457 This calexp's filter (`str`).
458 """
459 if "visit" in dataRef.dataId.keys():
460 visit = dataRef.dataId["visit"]
461 else:
462 visit = dataRef.getButler().queryMetadata("calexp", ("visit"), dataRef.dataId)[0]
464 src = dataRef.get("src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=True)
466 visitInfo = dataRef.get('calexp_visitInfo')
467 detector = dataRef.get('calexp_detector')
468 ccdId = detector.getId()
469 photoCalib = dataRef.get('calexp_photoCalib')
470 tanWcs = dataRef.get('calexp_wcs')
471 bbox = dataRef.get('calexp_bbox')
472 filt = dataRef.get('calexp_filter')
473 filterName = filt.getName()
475 goodSrc = self.sourceSelector.run(src)
477 if len(goodSrc.sourceCat) == 0:
478 self.log.warn("No sources selected in visit %s ccd %s", visit, ccdId)
479 else:
480 self.log.info("%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
481 associations.createCcdImage(goodSrc.sourceCat,
482 tanWcs,
483 visitInfo,
484 bbox,
485 filterName,
486 photoCalib,
487 detector,
488 visit,
489 ccdId,
490 jointcalControl)
492 Result = collections.namedtuple('Result_from_build_CcdImage', ('wcs', 'key', 'filter'))
493 Key = collections.namedtuple('Key', ('visit', 'ccd'))
494 return Result(tanWcs, Key(visit, ccdId), filterName)
496 def _getDebugPath(self, filename):
497 """Constructs a path to filename using the configured debug path.
498 """
499 return os.path.join(self.config.debugOutputPath, filename)
501 @pipeBase.timeMethod
502 def runDataRef(self, dataRefs, profile_jointcal=False):
503 """
504 Jointly calibrate the astrometry and photometry across a set of images.
506 Parameters
507 ----------
508 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
509 List of data references to the exposures to be fit.
510 profile_jointcal : `bool`
511 Profile the individual steps of jointcal.
513 Returns
514 -------
515 result : `lsst.pipe.base.Struct`
516 Struct of metadata from the fit, containing:
518 ``dataRefs``
519 The provided data references that were fit (with updated WCSs)
520 ``oldWcsList``
521 The original WCS from each dataRef
522 ``metrics``
523 Dictionary of internally-computed metrics for testing/validation.
524 """
525 if len(dataRefs) == 0:
526 raise ValueError('Need a non-empty list of data references!')
528 exitStatus = 0 # exit status for shell
530 sourceFluxField = "slot_%sFlux" % (self.config.sourceFluxType,)
531 jointcalControl = lsst.jointcal.JointcalControl(sourceFluxField)
532 associations = lsst.jointcal.Associations()
534 visit_ccd_to_dataRef = {}
535 oldWcsList = []
536 filters = []
537 load_cat_prof_file = 'jointcal_build_ccdImage.prof' if profile_jointcal else ''
538 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
539 # We need the bounding-box of the focal plane for photometry visit models.
540 # NOTE: we only need to read it once, because its the same for all exposures of a camera.
541 camera = dataRefs[0].get('camera', immediate=True)
542 self.focalPlaneBBox = camera.getFpBBox()
543 for ref in dataRefs:
544 result = self._build_ccdImage(ref, associations, jointcalControl)
545 oldWcsList.append(result.wcs)
546 visit_ccd_to_dataRef[result.key] = ref
547 filters.append(result.filter)
548 filters = collections.Counter(filters)
550 associations.computeCommonTangentPoint()
552 boundingCircle = associations.computeBoundingCircle()
553 center = lsst.geom.SpherePoint(boundingCircle.getCenter())
554 radius = lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
556 # Determine a default filter associated with the catalog. See DM-9093
557 defaultFilter = filters.most_common(1)[0][0]
558 self.log.debug("Using %s band for reference flux", defaultFilter)
560 # TODO: need a better way to get the tract.
561 tract = dataRefs[0].dataId['tract']
563 if self.config.doAstrometry:
564 astrometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
565 name="astrometry",
566 refObjLoader=self.astrometryRefObjLoader,
567 referenceSelector=self.astrometryReferenceSelector,
568 fit_function=self._fit_astrometry,
569 profile_jointcal=profile_jointcal,
570 tract=tract)
571 self._write_astrometry_results(associations, astrometry.model, visit_ccd_to_dataRef)
572 else:
573 astrometry = Astrometry(None, None, None)
575 if self.config.doPhotometry:
576 photometry = self._do_load_refcat_and_fit(associations, defaultFilter, center, radius,
577 name="photometry",
578 refObjLoader=self.photometryRefObjLoader,
579 referenceSelector=self.photometryReferenceSelector,
580 fit_function=self._fit_photometry,
581 profile_jointcal=profile_jointcal,
582 tract=tract,
583 filters=filters,
584 reject_bad_fluxes=True)
585 self._write_photometry_results(associations, photometry.model, visit_ccd_to_dataRef)
586 else:
587 photometry = Photometry(None, None)
589 return pipeBase.Struct(dataRefs=dataRefs,
590 oldWcsList=oldWcsList,
591 job=self.job,
592 astrometryRefObjLoader=self.astrometryRefObjLoader,
593 photometryRefObjLoader=self.photometryRefObjLoader,
594 defaultFilter=defaultFilter,
595 exitStatus=exitStatus)
597 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
598 filters=[],
599 tract="", profile_jointcal=False, match_cut=3.0,
600 reject_bad_fluxes=False, *,
601 name="", refObjLoader=None, referenceSelector=None,
602 fit_function=None):
603 """Load reference catalog, perform the fit, and return the result.
605 Parameters
606 ----------
607 associations : `lsst.jointcal.Associations`
608 The star/reference star associations to fit.
609 defaultFilter : `str`
610 filter to load from reference catalog.
611 center : `lsst.geom.SpherePoint`
612 ICRS center of field to load from reference catalog.
613 radius : `lsst.geom.Angle`
614 On-sky radius to load from reference catalog.
615 name : `str`
616 Name of thing being fit: "astrometry" or "photometry".
617 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
618 Reference object loader to use to load a reference catalog.
619 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
620 Selector to use to pick objects from the loaded reference catalog.
621 fit_function : callable
622 Function to call to perform fit (takes Associations object).
623 filters : `list` [`str`], optional
624 List of filters to load from the reference catalog.
625 tract : `str`, optional
626 Name of tract currently being fit.
627 profile_jointcal : `bool`, optional
628 Separately profile the fitting step.
629 match_cut : `float`, optional
630 Radius in arcseconds to find cross-catalog matches to during
631 associations.associateCatalogs.
632 reject_bad_fluxes : `bool`, optional
633 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
635 Returns
636 -------
637 result : `Photometry` or `Astrometry`
638 Result of `fit_function()`
639 """
640 self.log.info("====== Now processing %s...", name)
641 # TODO: this should not print "trying to invert a singular transformation:"
642 # if it does that, something's not right about the WCS...
643 associations.associateCatalogs(match_cut)
644 add_measurement(self.job, 'jointcal.associated_%s_fittedStars' % name,
645 associations.fittedStarListSize())
647 applyColorterms = False if name.lower() == "astrometry" else self.config.applyColorTerms
648 refCat, fluxField = self._load_reference_catalog(refObjLoader, referenceSelector,
649 center, radius, defaultFilter,
650 applyColorterms=applyColorterms)
652 if self.config.astrometryReferenceErr is None:
653 refCoordErr = float('nan')
654 else:
655 refCoordErr = self.config.astrometryReferenceErr
657 associations.collectRefStars(refCat,
658 self.config.matchCut*lsst.geom.arcseconds,
659 fluxField,
660 refCoordinateErr=refCoordErr,
661 rejectBadFluxes=reject_bad_fluxes)
662 add_measurement(self.job, 'jointcal.collected_%s_refStars' % name,
663 associations.refStarListSize())
665 associations.prepareFittedStars(self.config.minMeasurements)
667 self._check_star_lists(associations, name)
668 add_measurement(self.job, 'jointcal.selected_%s_refStars' % name,
669 associations.nFittedStarsWithAssociatedRefStar())
670 add_measurement(self.job, 'jointcal.selected_%s_fittedStars' % name,
671 associations.fittedStarListSize())
672 add_measurement(self.job, 'jointcal.selected_%s_ccdImages' % name,
673 associations.nCcdImagesValidForFit())
675 load_cat_prof_file = 'jointcal_fit_%s.prof'%name if profile_jointcal else ''
676 dataName = "{}_{}".format(tract, defaultFilter)
677 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
678 result = fit_function(associations, dataName)
679 # TODO DM-12446: turn this into a "butler save" somehow.
680 # Save reference and measurement chi2 contributions for this data
681 if self.config.writeChi2FilesInitialFinal:
682 baseName = self._getDebugPath(f"{name}_final_chi2-{dataName}")
683 result.fit.saveChi2Contributions(baseName+"{type}")
684 self.log.info("Wrote chi2 contributions files: %s", baseName)
686 return result
688 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
689 applyColorterms=False):
690 """Load the necessary reference catalog sources, convert fluxes to
691 correct units, and apply color term corrections if requested.
693 Parameters
694 ----------
695 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
696 The reference catalog loader to use to get the data.
697 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
698 Source selector to apply to loaded reference catalog.
699 center : `lsst.geom.SpherePoint`
700 The center around which to load sources.
701 radius : `lsst.geom.Angle`
702 The radius around ``center`` to load sources in.
703 filterName : `str`
704 The name of the camera filter to load fluxes for.
705 applyColorterms : `bool`
706 Apply colorterm corrections to the refcat for ``filterName``?
708 Returns
709 -------
710 refCat : `lsst.afw.table.SimpleCatalog`
711 The loaded reference catalog.
712 fluxField : `str`
713 The name of the reference catalog flux field appropriate for ``filterName``.
714 """
715 skyCircle = refObjLoader.loadSkyCircle(center,
716 radius,
717 filterName)
719 selected = referenceSelector.run(skyCircle.refCat)
720 # Need memory contiguity to get reference filters as a vector.
721 if not selected.sourceCat.isContiguous():
722 refCat = selected.sourceCat.copy(deep=True)
723 else:
724 refCat = selected.sourceCat
726 if self.config.astrometryReferenceErr is None and 'coord_raErr' not in refCat.schema:
727 msg = ("Reference catalog does not contain coordinate errors, "
728 "and config.astrometryReferenceErr not supplied.")
729 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
730 self.config,
731 msg)
733 if self.config.astrometryReferenceErr is not None and 'coord_raErr' in refCat.schema:
734 self.log.warn("Overriding reference catalog coordinate errors with %f/coordinate [mas]",
735 self.config.astrometryReferenceErr)
737 if applyColorterms:
738 try:
739 refCatName = refObjLoader.ref_dataset_name
740 except AttributeError:
741 # NOTE: we need this try:except: block in place until we've completely removed a.net support.
742 raise RuntimeError("Cannot perform colorterm corrections with a.net refcats.")
743 self.log.info("Applying color terms for filterName=%r reference catalog=%s",
744 filterName, refCatName)
745 colorterm = self.config.colorterms.getColorterm(
746 filterName=filterName, photoCatName=refCatName, doRaise=True)
748 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
749 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
750 # TODO: I didn't want to use this, but I'll deal with it in DM-16903
751 refCat[skyCircle.fluxField+'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
753 return refCat, skyCircle.fluxField
755 def _check_star_lists(self, associations, name):
756 # TODO: these should be len(blah), but we need this properly wrapped first.
757 if associations.nCcdImagesValidForFit() == 0:
758 raise RuntimeError('No images in the ccdImageList!')
759 if associations.fittedStarListSize() == 0:
760 raise RuntimeError('No stars in the {} fittedStarList!'.format(name))
761 if associations.refStarListSize() == 0:
762 raise RuntimeError('No stars in the {} reference star list!'.format(name))
764 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
765 writeChi2Name=None):
766 """Compute chi2, log it, validate the model, and return chi2.
768 Parameters
769 ----------
770 associations : `lsst.jointcal.Associations`
771 The star/reference star associations to fit.
772 fit : `lsst.jointcal.FitterBase`
773 The fitter to use for minimization.
774 model : `lsst.jointcal.Model`
775 The model being fit.
776 chi2Label : str, optional
777 Label to describe the chi2 (e.g. "Initialized", "Final").
778 writeChi2Name : `str`, optional
779 Filename prefix to write the chi2 contributions to.
780 Do not supply an extension: an appropriate one will be added.
782 Returns
783 -------
784 chi2: `lsst.jointcal.Chi2Accumulator`
785 The chi2 object for the current fitter and model.
787 Raises
788 ------
789 FloatingPointError
790 Raised if chi2 is infinite or NaN.
791 ValueError
792 Raised if the model is not valid.
793 """
794 if writeChi2Name is not None:
795 fullpath = self._getDebugPath(writeChi2Name)
796 fit.saveChi2Contributions(fullpath+"{type}")
797 self.log.info("Wrote chi2 contributions files: %s", fullpath)
799 chi2 = fit.computeChi2()
800 self.log.info("%s %s", chi2Label, chi2)
801 self._check_stars(associations)
802 if not np.isfinite(chi2.chi2):
803 raise FloatingPointError(f'{chi2Label} chi2 is invalid: {chi2}')
804 if not model.validate(associations.getCcdImageList(), chi2.ndof):
805 raise ValueError("Model is not valid: check log messages for warnings.")
806 return chi2
808 def _fit_photometry(self, associations, dataName=None):
809 """
810 Fit the photometric data.
812 Parameters
813 ----------
814 associations : `lsst.jointcal.Associations`
815 The star/reference star associations to fit.
816 dataName : `str`
817 Name of the data being processed (e.g. "1234_HSC-Y"), for
818 identifying debugging files.
820 Returns
821 -------
822 fit_result : `namedtuple`
823 fit : `lsst.jointcal.PhotometryFit`
824 The photometric fitter used to perform the fit.
825 model : `lsst.jointcal.PhotometryModel`
826 The photometric model that was fit.
827 """
828 self.log.info("=== Starting photometric fitting...")
830 # TODO: should use pex.config.RegistryField here (see DM-9195)
831 if self.config.photometryModel == "constrainedFlux":
832 model = lsst.jointcal.ConstrainedFluxModel(associations.getCcdImageList(),
833 self.focalPlaneBBox,
834 visitOrder=self.config.photometryVisitOrder,
835 errorPedestal=self.config.photometryErrorPedestal)
836 # potentially nonlinear problem, so we may need a line search to converge.
837 doLineSearch = self.config.allowLineSearch
838 elif self.config.photometryModel == "constrainedMagnitude":
839 model = lsst.jointcal.ConstrainedMagnitudeModel(associations.getCcdImageList(),
840 self.focalPlaneBBox,
841 visitOrder=self.config.photometryVisitOrder,
842 errorPedestal=self.config.photometryErrorPedestal)
843 # potentially nonlinear problem, so we may need a line search to converge.
844 doLineSearch = self.config.allowLineSearch
845 elif self.config.photometryModel == "simpleFlux":
846 model = lsst.jointcal.SimpleFluxModel(associations.getCcdImageList(),
847 errorPedestal=self.config.photometryErrorPedestal)
848 doLineSearch = False # purely linear in model parameters, so no line search needed
849 elif self.config.photometryModel == "simpleMagnitude":
850 model = lsst.jointcal.SimpleMagnitudeModel(associations.getCcdImageList(),
851 errorPedestal=self.config.photometryErrorPedestal)
852 doLineSearch = False # purely linear in model parameters, so no line search needed
854 fit = lsst.jointcal.PhotometryFit(associations, model)
855 # TODO DM-12446: turn this into a "butler save" somehow.
856 # Save reference and measurement chi2 contributions for this data
857 if self.config.writeChi2FilesInitialFinal:
858 baseName = f"photometry_initial_chi2-{dataName}"
859 else:
860 baseName = None
861 if self.config.writeInitialModel:
862 fullpath = self._getDebugPath("initialPhotometryModel.txt")
863 writeModel(model, fullpath, self.log)
864 self._logChi2AndValidate(associations, fit, model, "Initialized", writeChi2Name=baseName)
866 def getChi2Name(whatToFit):
867 if self.config.writeChi2FilesOuterLoop:
868 return f"photometry_init-%s_chi2-{dataName}" % whatToFit
869 else:
870 return None
872 # The constrained model needs the visit transform fit first; the chip
873 # transform is initialized from the singleFrame PhotoCalib, so it's close.
874 dumpMatrixFile = self._getDebugPath("photometry_preinit") if self.config.writeInitMatrix else ""
875 if self.config.photometryModel.startswith("constrained"):
876 # no line search: should be purely (or nearly) linear,
877 # and we want a large step size to initialize with.
878 fit.minimize("ModelVisit", dumpMatrixFile=dumpMatrixFile)
879 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("ModelVisit"))
880 dumpMatrixFile = "" # so we don't redo the output on the next step
882 fit.minimize("Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
883 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Model"))
885 fit.minimize("Fluxes") # no line search: always purely linear.
886 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Fluxes"))
888 fit.minimize("Model Fluxes", doLineSearch=doLineSearch)
889 self._logChi2AndValidate(associations, fit, model, "Fit prepared",
890 writeChi2Name=getChi2Name("ModelFluxes"))
892 model.freezeErrorTransform()
893 self.log.debug("Photometry error scales are frozen.")
895 chi2 = self._iterate_fit(associations,
896 fit,
897 self.config.maxPhotometrySteps,
898 "photometry",
899 "Model Fluxes",
900 doRankUpdate=self.config.photometryDoRankUpdate,
901 doLineSearch=doLineSearch,
902 dataName=dataName)
904 add_measurement(self.job, 'jointcal.photometry_final_chi2', chi2.chi2)
905 add_measurement(self.job, 'jointcal.photometry_final_ndof', chi2.ndof)
906 return Photometry(fit, model)
908 def _fit_astrometry(self, associations, dataName=None):
909 """
910 Fit the astrometric data.
912 Parameters
913 ----------
914 associations : `lsst.jointcal.Associations`
915 The star/reference star associations to fit.
916 dataName : `str`
917 Name of the data being processed (e.g. "1234_HSC-Y"), for
918 identifying debugging files.
920 Returns
921 -------
922 fit_result : `namedtuple`
923 fit : `lsst.jointcal.AstrometryFit`
924 The astrometric fitter used to perform the fit.
925 model : `lsst.jointcal.AstrometryModel`
926 The astrometric model that was fit.
927 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
928 The model for the sky to tangent plane projection that was used in the fit.
929 """
931 self.log.info("=== Starting astrometric fitting...")
933 associations.deprojectFittedStars()
935 # NOTE: need to return sky_to_tan_projection so that it doesn't get garbage collected.
936 # TODO: could we package sky_to_tan_projection and model together so we don't have to manage
937 # them so carefully?
938 sky_to_tan_projection = lsst.jointcal.OneTPPerVisitHandler(associations.getCcdImageList())
940 if self.config.astrometryModel == "constrained":
941 model = lsst.jointcal.ConstrainedAstrometryModel(associations.getCcdImageList(),
942 sky_to_tan_projection,
943 chipOrder=self.config.astrometryChipOrder,
944 visitOrder=self.config.astrometryVisitOrder)
945 elif self.config.astrometryModel == "simple":
946 model = lsst.jointcal.SimpleAstrometryModel(associations.getCcdImageList(),
947 sky_to_tan_projection,
948 self.config.useInputWcs,
949 nNotFit=0,
950 order=self.config.astrometrySimpleOrder)
952 fit = lsst.jointcal.AstrometryFit(associations, model, self.config.positionErrorPedestal)
953 # TODO DM-12446: turn this into a "butler save" somehow.
954 # Save reference and measurement chi2 contributions for this data
955 if self.config.writeChi2FilesInitialFinal:
956 baseName = f"astrometry_initial_chi2-{dataName}"
957 else:
958 baseName = None
959 if self.config.writeInitialModel:
960 fullpath = self._getDebugPath("initialAstrometryModel.txt")
961 writeModel(model, fullpath, self.log)
962 self._logChi2AndValidate(associations, fit, model, "Initial", writeChi2Name=baseName)
964 def getChi2Name(whatToFit):
965 if self.config.writeChi2FilesOuterLoop:
966 return f"astrometry_init-%s_chi2-{dataName}" % whatToFit
967 else:
968 return None
970 dumpMatrixFile = self._getDebugPath("astrometry_preinit") if self.config.writeInitMatrix else ""
971 # The constrained model needs the visit transform fit first; the chip
972 # transform is initialized from the detector's cameraGeom, so it's close.
973 if self.config.astrometryModel == "constrained":
974 fit.minimize("DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
975 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("DistortionsVisit"))
976 dumpMatrixFile = "" # so we don't redo the output on the next step
978 fit.minimize("Distortions", dumpMatrixFile=dumpMatrixFile)
979 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Distortions"))
981 fit.minimize("Positions")
982 self._logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name("Positions"))
984 fit.minimize("Distortions Positions")
985 self._logChi2AndValidate(associations, fit, model, "Fit prepared",
986 writeChi2Name=getChi2Name("DistortionsPositions"))
988 chi2 = self._iterate_fit(associations,
989 fit,
990 self.config.maxAstrometrySteps,
991 "astrometry",
992 "Distortions Positions",
993 doRankUpdate=self.config.astrometryDoRankUpdate,
994 dataName=dataName)
996 add_measurement(self.job, 'jointcal.astrometry_final_chi2', chi2.chi2)
997 add_measurement(self.job, 'jointcal.astrometry_final_ndof', chi2.ndof)
999 return Astrometry(fit, model, sky_to_tan_projection)
1001 def _check_stars(self, associations):
1002 """Count measured and reference stars per ccd and warn/log them."""
1003 for ccdImage in associations.getCcdImageList():
1004 nMeasuredStars, nRefStars = ccdImage.countStars()
1005 self.log.debug("ccdImage %s has %s measured and %s reference stars",
1006 ccdImage.getName(), nMeasuredStars, nRefStars)
1007 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1008 self.log.warn("ccdImage %s has only %s measuredStars (desired %s)",
1009 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1010 if nRefStars < self.config.minRefStarsPerCcd:
1011 self.log.warn("ccdImage %s has only %s RefStars (desired %s)",
1012 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1014 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1015 dataName="",
1016 doRankUpdate=True,
1017 doLineSearch=False):
1018 """Run fitter.minimize up to max_steps times, returning the final chi2.
1020 Parameters
1021 ----------
1022 associations : `lsst.jointcal.Associations`
1023 The star/reference star associations to fit.
1024 fitter : `lsst.jointcal.FitterBase`
1025 The fitter to use for minimization.
1026 max_steps : `int`
1027 Maximum number of steps to run outlier rejection before declaring
1028 convergence failure.
1029 name : {'photometry' or 'astrometry'}
1030 What type of data are we fitting (for logs and debugging files).
1031 whatToFit : `str`
1032 Passed to ``fitter.minimize()`` to define the parameters to fit.
1033 dataName : `str`, optional
1034 Descriptive name for this dataset (e.g. tract and filter),
1035 for debugging.
1036 doRankUpdate : `bool`, optional
1037 Do an Eigen rank update during minimization, or recompute the full
1038 matrix and gradient?
1039 doLineSearch : `bool`, optional
1040 Do a line search for the optimum step during minimization?
1042 Returns
1043 -------
1044 chi2: `lsst.jointcal.Chi2Statistic`
1045 The final chi2 after the fit converges, or is forced to end.
1047 Raises
1048 ------
1049 FloatingPointError
1050 Raised if the fitter fails with a non-finite value.
1051 RuntimeError
1052 Raised if the fitter fails for some other reason;
1053 log messages will provide further details.
1054 """
1055 dumpMatrixFile = self._getDebugPath(f"{name}_postinit") if self.config.writeInitMatrix else ""
1056 for i in range(max_steps):
1057 if self.config.writeChi2FilesOuterLoop:
1058 writeChi2Name = f"{name}_iterate_{i}_chi2-{dataName}"
1059 else:
1060 writeChi2Name = None
1061 result = fitter.minimize(whatToFit,
1062 self.config.outlierRejectSigma,
1063 doRankUpdate=doRankUpdate,
1064 doLineSearch=doLineSearch,
1065 dumpMatrixFile=dumpMatrixFile)
1066 dumpMatrixFile = "" # clear it so we don't write the matrix again.
1067 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(),
1068 writeChi2Name=writeChi2Name)
1070 if result == MinimizeResult.Converged:
1071 if doRankUpdate:
1072 self.log.debug("fit has converged - no more outliers - redo minimization "
1073 "one more time in case we have lost accuracy in rank update.")
1074 # Redo minimization one more time in case we have lost accuracy in rank update
1075 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1076 chi2 = self._logChi2AndValidate(associations, fitter, fitter.getModel(), "Fit completed")
1078 # log a message for a large final chi2, TODO: DM-15247 for something better
1079 if chi2.chi2/chi2.ndof >= 4.0:
1080 self.log.error("Potentially bad fit: High chi-squared/ndof.")
1082 break
1083 elif result == MinimizeResult.Chi2Increased:
1084 self.log.warn("still some outliers but chi2 increases - retry")
1085 elif result == MinimizeResult.NonFinite:
1086 filename = self._getDebugPath("{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1087 # TODO DM-12446: turn this into a "butler save" somehow.
1088 fitter.saveChi2Contributions(filename+"{type}")
1089 msg = "Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1090 raise FloatingPointError(msg.format(filename))
1091 elif result == MinimizeResult.Failed:
1092 raise RuntimeError("Chi2 minimization failure, cannot complete fit.")
1093 else:
1094 raise RuntimeError("Unxepected return code from minimize().")
1095 else:
1096 self.log.error("%s failed to converge after %d steps"%(name, max_steps))
1098 return chi2
1100 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1101 """
1102 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1104 Parameters
1105 ----------
1106 associations : `lsst.jointcal.Associations`
1107 The star/reference star associations to fit.
1108 model : `lsst.jointcal.AstrometryModel`
1109 The astrometric model that was fit.
1110 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1111 Dict of ccdImage identifiers to dataRefs that were fit.
1112 """
1114 ccdImageList = associations.getCcdImageList()
1115 for ccdImage in ccdImageList:
1116 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1117 ccd = ccdImage.ccdId
1118 visit = ccdImage.visit
1119 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1120 self.log.info("Updating WCS for visit: %d, ccd: %d", visit, ccd)
1121 skyWcs = model.makeSkyWcs(ccdImage)
1122 try:
1123 dataRef.put(skyWcs, 'jointcal_wcs')
1124 except pexExceptions.Exception as e:
1125 self.log.fatal('Failed to write updated Wcs: %s', str(e))
1126 raise e
1128 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1129 """
1130 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1132 Parameters
1133 ----------
1134 associations : `lsst.jointcal.Associations`
1135 The star/reference star associations to fit.
1136 model : `lsst.jointcal.PhotometryModel`
1137 The photoometric model that was fit.
1138 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1139 Dict of ccdImage identifiers to dataRefs that were fit.
1140 """
1142 ccdImageList = associations.getCcdImageList()
1143 for ccdImage in ccdImageList:
1144 # TODO: there must be a better way to identify this ccdImage than a visit,ccd pair?
1145 ccd = ccdImage.ccdId
1146 visit = ccdImage.visit
1147 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1148 self.log.info("Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1149 photoCalib = model.toPhotoCalib(ccdImage)
1150 try:
1151 dataRef.put(photoCalib, 'jointcal_photoCalib')
1152 except pexExceptions.Exception as e:
1153 self.log.fatal('Failed to write updated PhotoCalib: %s', str(e))
1154 raise e