27 import astropy.units
as u
31 import lsst.pex.config
as pexConfig
38 from lsst.pipe.tasks.colorterms
import ColortermLibrary
39 from lsst.verify
import Job, Measurement
44 from .dataIds
import PerTractCcdDataIdContainer
49 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
51 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
52 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
57 meas = Measurement(job.metrics[name], value)
58 job.measurements.insert(meas)
62 """Subclass of TaskRunner for jointcalTask
64 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs
65 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take
66 single dataRef, are are called repeatedly). This class transforms the processed
67 arguments generated by the ArgumentParser into the arguments expected by
68 Jointcal.runDataRef().
70 See pipeBase.TaskRunner for more information.
76 Return a list of tuples per tract, each containing (dataRefs, kwargs).
78 Jointcal operates on lists of dataRefs simultaneously.
80 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
81 kwargs[
'butler'] = parsedCmd.butler
85 for ref
in parsedCmd.id.refList:
86 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
88 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
96 Arguments for Task.runDataRef()
101 if self.doReturnResults is False:
103 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
105 if self.doReturnResults is True:
107 - ``result``: the result of calling jointcal.runDataRef()
108 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise.
113 dataRefList, kwargs = args
114 butler = kwargs.pop(
'butler')
115 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
118 result = task.runDataRef(dataRefList, **kwargs)
119 exitStatus = result.exitStatus
120 job_path = butler.get(
'verify_job_filename')
121 result.job.write(job_path[0])
122 except Exception
as e:
127 eName = type(e).__name__
128 tract = dataRefList[0].dataId[
'tract']
129 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
132 kwargs[
'butler'] = butler
133 if self.doReturnResults:
134 return pipeBase.Struct(result=result, exitStatus=exitStatus)
136 return pipeBase.Struct(exitStatus=exitStatus)
140 """Configuration for JointcalTask"""
142 doAstrometry = pexConfig.Field(
143 doc=
"Fit astrometry and write the fitted result.",
147 doPhotometry = pexConfig.Field(
148 doc=
"Fit photometry and write the fitted result.",
152 coaddName = pexConfig.Field(
153 doc=
"Type of coadd, typically deep or goodSeeing",
157 positionErrorPedestal = pexConfig.Field(
158 doc=
"Systematic term to apply to the measured position error (pixels)",
162 photometryErrorPedestal = pexConfig.Field(
163 doc=
"Systematic term to apply to the measured error on flux or magnitude as a "
164 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
169 matchCut = pexConfig.Field(
170 doc=
"Matching radius between fitted and reference stars (arcseconds)",
174 minMeasurements = pexConfig.Field(
175 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
179 minMeasuredStarsPerCcd = pexConfig.Field(
180 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
184 minRefStarsPerCcd = pexConfig.Field(
185 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
189 allowLineSearch = pexConfig.Field(
190 doc=
"Allow a line search during minimization, if it is reasonable for the model"
191 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
195 astrometrySimpleOrder = pexConfig.Field(
196 doc=
"Polynomial order for fitting the simple astrometry model.",
200 astrometryChipOrder = pexConfig.Field(
201 doc=
"Order of the per-chip transform for the constrained astrometry model.",
205 astrometryVisitOrder = pexConfig.Field(
206 doc=
"Order of the per-visit transform for the constrained astrometry model.",
210 useInputWcs = pexConfig.Field(
211 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
215 astrometryModel = pexConfig.ChoiceField(
216 doc=
"Type of model to fit to astrometry",
218 default=
"constrained",
219 allowed={
"simple":
"One polynomial per ccd",
220 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
222 photometryModel = pexConfig.ChoiceField(
223 doc=
"Type of model to fit to photometry",
225 default=
"constrainedMagnitude",
226 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
227 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit,"
228 " fitting in flux space.",
229 "simpleMagnitude":
"One constant zeropoint per ccd and visit,"
230 " fitting in magnitude space.",
231 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit,"
232 " fitting in magnitude space.",
235 applyColorTerms = pexConfig.Field(
236 doc=
"Apply photometric color terms to reference stars?"
237 "Requires that colorterms be set to a ColortermLibrary",
241 colorterms = pexConfig.ConfigField(
242 doc=
"Library of photometric reference catalog name to color term dict.",
243 dtype=ColortermLibrary,
245 photometryVisitOrder = pexConfig.Field(
246 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
250 photometryDoRankUpdate = pexConfig.Field(
251 doc=(
"Do the rank update step during minimization. "
252 "Skipping this can help deal with models that are too non-linear."),
256 astrometryDoRankUpdate = pexConfig.Field(
257 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). "
258 "Skipping this can help deal with models that are too non-linear."),
262 outlierRejectSigma = pexConfig.Field(
263 doc=
"How many sigma to reject outliers at during minimization.",
267 maxPhotometrySteps = pexConfig.Field(
268 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
272 maxAstrometrySteps = pexConfig.Field(
273 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
277 astrometryRefObjLoader = pexConfig.ConfigurableField(
278 target=LoadIndexedReferenceObjectsTask,
279 doc=
"Reference object loader for astrometric fit",
281 photometryRefObjLoader = pexConfig.ConfigurableField(
282 target=LoadIndexedReferenceObjectsTask,
283 doc=
"Reference object loader for photometric fit",
285 sourceSelector = sourceSelectorRegistry.makeField(
286 doc=
"How to select sources for cross-matching",
289 astrometryReferenceSelector = pexConfig.ConfigurableField(
290 target=ReferenceSourceSelectorTask,
291 doc=
"How to down-select the loaded astrometry reference catalog.",
293 photometryReferenceSelector = pexConfig.ConfigurableField(
294 target=ReferenceSourceSelectorTask,
295 doc=
"How to down-select the loaded photometry reference catalog.",
297 astrometryReferenceErr = pexConfig.Field(
298 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. "
299 "If None, then raise an exception if the reference catalog is missing coordinate errors. "
300 "If specified, overrides any existing `coord_*Err` values."),
305 writeInitMatrix = pexConfig.Field(
307 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. "
308 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. "
309 "Note that these files are the dense versions of the matrix, and so may be very large."),
312 writeChi2FilesInitialFinal = pexConfig.Field(
314 doc=
"Write .csv files containing the contributions to chi2 for the initialization and final fit.",
317 writeChi2FilesOuterLoop = pexConfig.Field(
319 doc=
"Write .csv files containing the contributions to chi2 for the outer fit loop.",
322 writeInitialModel = pexConfig.Field(
324 doc=(
"Write the pre-initialization model to text files, for debugging."
325 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."),
328 debugOutputPath = pexConfig.Field(
331 doc=(
"Path to write debug output files to. Used by "
332 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
334 sourceFluxType = pexConfig.Field(
336 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
343 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary."
344 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
346 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;"
347 "applyColorTerms=True will be ignored.")
361 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux"
362 self.
sourceSelector[
'science'].signalToNoise.fluxField = fluxField
363 self.
sourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err"
369 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
370 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
371 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
388 """Write model to outfile."""
389 with open(filename,
"w")
as file:
390 file.write(repr(model))
391 log.info(
"Wrote %s to file: %s", model, filename)
395 """Jointly astrometrically and photometrically calibrate a group of images."""
397 ConfigClass = JointcalConfig
398 RunnerClass = JointcalRunner
399 _DefaultName =
"jointcal"
401 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
403 Instantiate a JointcalTask.
407 butler : `lsst.daf.persistence.Butler`
408 The butler is passed to the refObjLoader constructor in case it is
409 needed. Ignored if the refObjLoader argument provides a loader directly.
410 Used to initialize the astrometry and photometry refObjLoaders.
411 profile_jointcal : `bool`
412 Set to True to profile different stages of this jointcal run.
414 pipeBase.CmdLineTask.__init__(self, **kwargs)
416 self.makeSubtask(
"sourceSelector")
417 if self.config.doAstrometry:
418 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
419 self.makeSubtask(
"astrometryReferenceSelector")
422 if self.config.doPhotometry:
423 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
424 self.makeSubtask(
"photometryReferenceSelector")
429 self.
job = Job.load_metrics_package(subset=
'jointcal')
434 def _getMetadataName(self):
438 def _makeArgumentParser(cls):
439 """Create an argument parser"""
441 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
442 help=
"Profile steps of jointcal separately.")
443 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
444 ContainerClass=PerTractCcdDataIdContainer)
447 def _build_ccdImage(self, dataRef, associations, jointcalControl):
449 Extract the necessary things from this dataRef to add a new ccdImage.
453 dataRef : `lsst.daf.persistence.ButlerDataRef`
454 DataRef to extract info from.
455 associations : `lsst.jointcal.Associations`
456 Object to add the info to, to construct a new CcdImage
457 jointcalControl : `jointcal.JointcalControl`
458 Control object for associations management
464 The TAN WCS of this image, read from the calexp
465 (`lsst.afw.geom.SkyWcs`).
467 A key to identify this dataRef by its visit and ccd ids
470 This calexp's filter (`str`).
472 if "visit" in dataRef.dataId.keys():
473 visit = dataRef.dataId[
"visit"]
475 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
477 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
479 visitInfo = dataRef.get(
'calexp_visitInfo')
480 detector = dataRef.get(
'calexp_detector')
481 ccdId = detector.getId()
482 photoCalib = dataRef.get(
'calexp_photoCalib')
483 tanWcs = dataRef.get(
'calexp_wcs')
484 bbox = dataRef.get(
'calexp_bbox')
485 filt = dataRef.get(
'calexp_filter')
486 filterName = filt.getName()
488 goodSrc = self.sourceSelector.run(src)
490 if len(goodSrc.sourceCat) == 0:
491 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
493 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
494 associations.createCcdImage(goodSrc.sourceCat,
505 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
506 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
507 return Result(tanWcs, Key(visit, ccdId), filterName)
509 def _getDebugPath(self, filename):
510 """Constructs a path to filename using the configured debug path.
512 return os.path.join(self.config.debugOutputPath, filename)
517 Jointly calibrate the astrometry and photometry across a set of images.
521 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef`
522 List of data references to the exposures to be fit.
523 profile_jointcal : `bool`
524 Profile the individual steps of jointcal.
528 result : `lsst.pipe.base.Struct`
529 Struct of metadata from the fit, containing:
532 The provided data references that were fit (with updated WCSs)
534 The original WCS from each dataRef
536 Dictionary of internally-computed metrics for testing/validation.
538 if len(dataRefs) == 0:
539 raise ValueError(
'Need a non-empty list of data references!')
543 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
547 visit_ccd_to_dataRef = {}
550 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else ''
551 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
554 camera = dataRefs[0].get(
'camera', immediate=
True)
558 oldWcsList.append(result.wcs)
559 visit_ccd_to_dataRef[result.key] = ref
560 filters.append(result.filter)
561 filters = collections.Counter(filters)
563 associations.computeCommonTangentPoint()
565 boundingCircle = associations.computeBoundingCircle()
567 radius =
lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
569 self.log.info(f
"Data has center={center} with radius={radius.asDegrees()} degrees.")
572 defaultFilter = filters.most_common(1)[0][0]
573 self.log.debug(
"Using %s band for reference flux", defaultFilter)
576 tract = dataRefs[0].dataId[
'tract']
578 if self.config.doAstrometry:
582 referenceSelector=self.astrometryReferenceSelector,
584 profile_jointcal=profile_jointcal,
590 if self.config.doPhotometry:
594 referenceSelector=self.photometryReferenceSelector,
596 profile_jointcal=profile_jointcal,
599 reject_bad_fluxes=
True)
604 return pipeBase.Struct(dataRefs=dataRefs,
605 oldWcsList=oldWcsList,
609 defaultFilter=defaultFilter,
610 exitStatus=exitStatus)
612 def _get_refcat_coordinate_error_override(self, refCat, name):
613 """Check whether we should override the refcat coordinate errors, and
614 return the overridden error if necessary.
618 refCat : `lsst.afw.table.SimpleCatalog`
619 The reference catalog to check for a ``coord_raErr`` field.
621 Whether we are doing "astrometry" or "photometry".
625 refCoordErr : `float`
626 The refcat coordinate error to use, or NaN if we are not overriding
631 lsst.pex.config.FieldValidationError
632 Raised if the refcat does not contain coordinate errors and
633 ``config.astrometryReferenceErr`` is not set.
637 if name.lower() ==
"photometry":
638 if 'coord_raErr' not in refCat.schema:
643 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
644 msg = (
"Reference catalog does not contain coordinate errors, "
645 "and config.astrometryReferenceErr not supplied.")
646 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
650 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
651 self.log.warn(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
652 self.config.astrometryReferenceErr)
654 if self.config.astrometryReferenceErr
is None:
657 return self.config.astrometryReferenceErr
659 def _compute_proper_motion_epoch(self, ccdImageList):
660 """Return the proper motion correction epoch of the provided images.
664 ccdImageList : `list` [`lsst.jointcal.CcdImage`]
665 The images to compute the appropriate epoch for.
669 epoch : `astropy.time.Time`
670 The date to use for proper motion corrections.
672 mjds = [ccdImage.getMjd()
for ccdImage
in ccdImageList]
673 return astropy.time.Time(np.mean(mjds), format=
'mjd', scale=
"tai")
675 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
677 tract="", profile_jointcal=False, match_cut=3.0,
678 reject_bad_fluxes=False, *,
679 name="", refObjLoader=None, referenceSelector=None,
681 """Load reference catalog, perform the fit, and return the result.
685 associations : `lsst.jointcal.Associations`
686 The star/reference star associations to fit.
687 defaultFilter : `str`
688 filter to load from reference catalog.
689 center : `lsst.geom.SpherePoint`
690 ICRS center of field to load from reference catalog.
691 radius : `lsst.geom.Angle`
692 On-sky radius to load from reference catalog.
694 Name of thing being fit: "astrometry" or "photometry".
695 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
696 Reference object loader to use to load a reference catalog.
697 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
698 Selector to use to pick objects from the loaded reference catalog.
699 fit_function : callable
700 Function to call to perform fit (takes Associations object).
701 filters : `list` [`str`], optional
702 List of filters to load from the reference catalog.
703 tract : `str`, optional
704 Name of tract currently being fit.
705 profile_jointcal : `bool`, optional
706 Separately profile the fitting step.
707 match_cut : `float`, optional
708 Radius in arcseconds to find cross-catalog matches to during
709 associations.associateCatalogs.
710 reject_bad_fluxes : `bool`, optional
711 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr.
715 result : `Photometry` or `Astrometry`
716 Result of `fit_function()`
718 self.log.info(
"====== Now processing %s...", name)
721 associations.associateCatalogs(match_cut)
723 associations.fittedStarListSize())
725 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
728 center, radius, defaultFilter,
729 applyColorterms=applyColorterms,
733 associations.collectRefStars(refCat,
734 self.config.matchCut*lsst.geom.arcseconds,
736 refCoordinateErr=refCoordErr,
737 rejectBadFluxes=reject_bad_fluxes)
739 associations.refStarListSize())
741 associations.prepareFittedStars(self.config.minMeasurements)
745 associations.nFittedStarsWithAssociatedRefStar())
747 associations.fittedStarListSize())
749 associations.nCcdImagesValidForFit())
751 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else ''
752 dataName =
"{}_{}".format(tract, defaultFilter)
753 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
754 result = fit_function(associations, dataName)
757 if self.config.writeChi2FilesInitialFinal:
758 baseName = self.
_getDebugPath(f
"{name}_final_chi2-{dataName}")
759 result.fit.saveChi2Contributions(baseName+
"{type}")
760 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
764 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
765 applyColorterms=False, epoch=None):
766 """Load the necessary reference catalog sources, convert fluxes to
767 correct units, and apply color term corrections if requested.
771 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask`
772 The reference catalog loader to use to get the data.
773 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask`
774 Source selector to apply to loaded reference catalog.
775 center : `lsst.geom.SpherePoint`
776 The center around which to load sources.
777 radius : `lsst.geom.Angle`
778 The radius around ``center`` to load sources in.
780 The name of the camera filter to load fluxes for.
781 applyColorterms : `bool`
782 Apply colorterm corrections to the refcat for ``filterName``?
783 epoch : `astropy.time.Time`, optional
784 Epoch to which to correct refcat proper motion and parallax,
785 or `None` to not apply such corrections.
789 refCat : `lsst.afw.table.SimpleCatalog`
790 The loaded reference catalog.
792 The name of the reference catalog flux field appropriate for ``filterName``.
794 skyCircle = refObjLoader.loadSkyCircle(center,
799 selected = referenceSelector.run(skyCircle.refCat)
801 if not selected.sourceCat.isContiguous():
802 refCat = selected.sourceCat.copy(deep=
True)
804 refCat = selected.sourceCat
807 refCatName = refObjLoader.ref_dataset_name
808 self.log.info(
"Applying color terms for filterName=%r reference catalog=%s",
809 filterName, refCatName)
810 colorterm = self.config.colorterms.getColorterm(
811 filterName=filterName, photoCatName=refCatName, doRaise=
True)
813 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
814 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
816 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
818 return refCat, skyCircle.fluxField
820 def _check_star_lists(self, associations, name):
822 if associations.nCcdImagesValidForFit() == 0:
823 raise RuntimeError(
'No images in the ccdImageList!')
824 if associations.fittedStarListSize() == 0:
825 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
826 if associations.refStarListSize() == 0:
827 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
829 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
831 """Compute chi2, log it, validate the model, and return chi2.
835 associations : `lsst.jointcal.Associations`
836 The star/reference star associations to fit.
837 fit : `lsst.jointcal.FitterBase`
838 The fitter to use for minimization.
839 model : `lsst.jointcal.Model`
841 chi2Label : str, optional
842 Label to describe the chi2 (e.g. "Initialized", "Final").
843 writeChi2Name : `str`, optional
844 Filename prefix to write the chi2 contributions to.
845 Do not supply an extension: an appropriate one will be added.
849 chi2: `lsst.jointcal.Chi2Accumulator`
850 The chi2 object for the current fitter and model.
855 Raised if chi2 is infinite or NaN.
857 Raised if the model is not valid.
859 if writeChi2Name
is not None:
861 fit.saveChi2Contributions(fullpath+
"{type}")
862 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
864 chi2 = fit.computeChi2()
865 self.log.info(
"%s %s", chi2Label, chi2)
867 if not np.isfinite(chi2.chi2):
868 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
869 if not model.validate(associations.getCcdImageList(), chi2.ndof):
870 raise ValueError(
"Model is not valid: check log messages for warnings.")
873 def _fit_photometry(self, associations, dataName=None):
875 Fit the photometric data.
879 associations : `lsst.jointcal.Associations`
880 The star/reference star associations to fit.
882 Name of the data being processed (e.g. "1234_HSC-Y"), for
883 identifying debugging files.
887 fit_result : `namedtuple`
888 fit : `lsst.jointcal.PhotometryFit`
889 The photometric fitter used to perform the fit.
890 model : `lsst.jointcal.PhotometryModel`
891 The photometric model that was fit.
893 self.log.info(
"=== Starting photometric fitting...")
896 if self.config.photometryModel ==
"constrainedFlux":
899 visitOrder=self.config.photometryVisitOrder,
900 errorPedestal=self.config.photometryErrorPedestal)
902 doLineSearch = self.config.allowLineSearch
903 elif self.config.photometryModel ==
"constrainedMagnitude":
906 visitOrder=self.config.photometryVisitOrder,
907 errorPedestal=self.config.photometryErrorPedestal)
909 doLineSearch = self.config.allowLineSearch
910 elif self.config.photometryModel ==
"simpleFlux":
912 errorPedestal=self.config.photometryErrorPedestal)
914 elif self.config.photometryModel ==
"simpleMagnitude":
916 errorPedestal=self.config.photometryErrorPedestal)
922 if self.config.writeChi2FilesInitialFinal:
923 baseName = f
"photometry_initial_chi2-{dataName}"
926 if self.config.writeInitialModel:
931 def getChi2Name(whatToFit):
932 if self.config.writeChi2FilesOuterLoop:
933 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
939 dumpMatrixFile = self.
_getDebugPath(
"photometry_preinit")
if self.config.writeInitMatrix
else ""
940 if self.config.photometryModel.startswith(
"constrained"):
943 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
944 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"ModelVisit"))
947 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
950 fit.minimize(
"Fluxes")
953 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
955 writeChi2Name=getChi2Name(
"ModelFluxes"))
957 model.freezeErrorTransform()
958 self.log.debug(
"Photometry error scales are frozen.")
962 self.config.maxPhotometrySteps,
965 doRankUpdate=self.config.photometryDoRankUpdate,
966 doLineSearch=doLineSearch,
973 def _fit_astrometry(self, associations, dataName=None):
975 Fit the astrometric data.
979 associations : `lsst.jointcal.Associations`
980 The star/reference star associations to fit.
982 Name of the data being processed (e.g. "1234_HSC-Y"), for
983 identifying debugging files.
987 fit_result : `namedtuple`
988 fit : `lsst.jointcal.AstrometryFit`
989 The astrometric fitter used to perform the fit.
990 model : `lsst.jointcal.AstrometryModel`
991 The astrometric model that was fit.
992 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler`
993 The model for the sky to tangent plane projection that was used in the fit.
996 self.log.info(
"=== Starting astrometric fitting...")
998 associations.deprojectFittedStars()
1005 if self.config.astrometryModel ==
"constrained":
1007 sky_to_tan_projection,
1008 chipOrder=self.config.astrometryChipOrder,
1009 visitOrder=self.config.astrometryVisitOrder)
1010 elif self.config.astrometryModel ==
"simple":
1012 sky_to_tan_projection,
1013 self.config.useInputWcs,
1015 order=self.config.astrometrySimpleOrder)
1020 if self.config.writeChi2FilesInitialFinal:
1021 baseName = f
"astrometry_initial_chi2-{dataName}"
1024 if self.config.writeInitialModel:
1029 def getChi2Name(whatToFit):
1030 if self.config.writeChi2FilesOuterLoop:
1031 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
1035 dumpMatrixFile = self.
_getDebugPath(
"astrometry_preinit")
if self.config.writeInitMatrix
else ""
1038 if self.config.astrometryModel ==
"constrained":
1039 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
1040 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"DistortionsVisit"))
1043 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
1044 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"Distortions"))
1046 fit.minimize(
"Positions")
1047 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"Positions"))
1049 fit.minimize(
"Distortions Positions")
1051 writeChi2Name=getChi2Name(
"DistortionsPositions"))
1055 self.config.maxAstrometrySteps,
1057 "Distortions Positions",
1058 doRankUpdate=self.config.astrometryDoRankUpdate,
1064 return Astrometry(fit, model, sky_to_tan_projection)
1066 def _check_stars(self, associations):
1067 """Count measured and reference stars per ccd and warn/log them."""
1068 for ccdImage
in associations.getCcdImageList():
1069 nMeasuredStars, nRefStars = ccdImage.countStars()
1070 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1071 ccdImage.getName(), nMeasuredStars, nRefStars)
1072 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1073 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
1074 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1075 if nRefStars < self.config.minRefStarsPerCcd:
1076 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
1077 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1079 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1082 doLineSearch=False):
1083 """Run fitter.minimize up to max_steps times, returning the final chi2.
1087 associations : `lsst.jointcal.Associations`
1088 The star/reference star associations to fit.
1089 fitter : `lsst.jointcal.FitterBase`
1090 The fitter to use for minimization.
1092 Maximum number of steps to run outlier rejection before declaring
1093 convergence failure.
1094 name : {'photometry' or 'astrometry'}
1095 What type of data are we fitting (for logs and debugging files).
1097 Passed to ``fitter.minimize()`` to define the parameters to fit.
1098 dataName : `str`, optional
1099 Descriptive name for this dataset (e.g. tract and filter),
1101 doRankUpdate : `bool`, optional
1102 Do an Eigen rank update during minimization, or recompute the full
1103 matrix and gradient?
1104 doLineSearch : `bool`, optional
1105 Do a line search for the optimum step during minimization?
1109 chi2: `lsst.jointcal.Chi2Statistic`
1110 The final chi2 after the fit converges, or is forced to end.
1115 Raised if the fitter fails with a non-finite value.
1117 Raised if the fitter fails for some other reason;
1118 log messages will provide further details.
1120 dumpMatrixFile = self.
_getDebugPath(f
"{name}_postinit")
if self.config.writeInitMatrix
else ""
1121 for i
in range(max_steps):
1122 if self.config.writeChi2FilesOuterLoop:
1123 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}"
1125 writeChi2Name =
None
1126 result = fitter.minimize(whatToFit,
1127 self.config.outlierRejectSigma,
1128 doRankUpdate=doRankUpdate,
1129 doLineSearch=doLineSearch,
1130 dumpMatrixFile=dumpMatrixFile)
1133 writeChi2Name=writeChi2Name)
1135 if result == MinimizeResult.Converged:
1137 self.log.debug(
"fit has converged - no more outliers - redo minimization "
1138 "one more time in case we have lost accuracy in rank update.")
1140 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1144 if chi2.chi2/chi2.ndof >= 4.0:
1145 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1148 elif result == MinimizeResult.Chi2Increased:
1149 self.log.warn(
"still some outliers but chi2 increases - retry")
1150 elif result == MinimizeResult.NonFinite:
1151 filename = self.
_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1153 fitter.saveChi2Contributions(filename+
"{type}")
1154 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}"
1155 raise FloatingPointError(msg.format(filename))
1156 elif result == MinimizeResult.Failed:
1157 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1159 raise RuntimeError(
"Unxepected return code from minimize().")
1161 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1165 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1167 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef.
1171 associations : `lsst.jointcal.Associations`
1172 The star/reference star associations to fit.
1173 model : `lsst.jointcal.AstrometryModel`
1174 The astrometric model that was fit.
1175 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1176 Dict of ccdImage identifiers to dataRefs that were fit.
1179 ccdImageList = associations.getCcdImageList()
1180 for ccdImage
in ccdImageList:
1182 ccd = ccdImage.ccdId
1183 visit = ccdImage.visit
1184 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1185 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
1186 skyWcs = model.makeSkyWcs(ccdImage)
1188 dataRef.put(skyWcs,
'jointcal_wcs')
1189 except pexExceptions.Exception
as e:
1190 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1193 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1195 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef.
1199 associations : `lsst.jointcal.Associations`
1200 The star/reference star associations to fit.
1201 model : `lsst.jointcal.PhotometryModel`
1202 The photoometric model that was fit.
1203 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef`
1204 Dict of ccdImage identifiers to dataRefs that were fit.
1207 ccdImageList = associations.getCcdImageList()
1208 for ccdImage
in ccdImageList:
1210 ccd = ccdImage.ccdId
1211 visit = ccdImage.visit
1212 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1213 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1214 photoCalib = model.toPhotoCalib(ccdImage)
1216 dataRef.put(photoCalib,
'jointcal_photoCalib')
1217 except pexExceptions.Exception
as e:
1218 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))