24 import astropy.units
as u
34 from lsst.pipe.tasks.colorterms
import ColortermLibrary
35 from lsst.verify
import Job, Measurement
40 from .dataIds
import PerTractCcdDataIdContainer
45 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
47 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
48 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
53 meas = Measurement(job.metrics[name], value)
54 job.measurements.insert(meas)
58 """Subclass of TaskRunner for jointcalTask 60 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 61 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 62 single dataRef, are are called repeatedly). This class transforms the processed 63 arguments generated by the ArgumentParser into the arguments expected by 64 Jointcal.runDataRef(). 66 See pipeBase.TaskRunner for more information. 72 Return a list of tuples per tract, each containing (dataRefs, kwargs). 74 Jointcal operates on lists of dataRefs simultaneously. 76 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
77 kwargs[
'butler'] = parsedCmd.butler
81 for ref
in parsedCmd.id.refList:
82 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
84 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
92 Arguments for Task.runDataRef() 97 if self.doReturnResults is False: 99 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 101 if self.doReturnResults is True: 103 - ``result``: the result of calling jointcal.runDataRef() 104 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 109 dataRefList, kwargs = args
110 butler = kwargs.pop(
'butler')
111 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
114 result = task.runDataRef(dataRefList, **kwargs)
115 exitStatus = result.exitStatus
116 job_path = butler.get(
'verify_job_filename')
117 result.job.write(job_path[0])
118 except Exception
as e:
123 eName = type(e).__name__
124 tract = dataRefList[0].dataId[
'tract']
125 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
128 kwargs[
'butler'] = butler
129 if self.doReturnResults:
130 return pipeBase.Struct(result=result, exitStatus=exitStatus)
132 return pipeBase.Struct(exitStatus=exitStatus)
136 """Configuration for JointcalTask""" 138 doAstrometry = pexConfig.Field(
139 doc=
"Fit astrometry and write the fitted result.",
143 doPhotometry = pexConfig.Field(
144 doc=
"Fit photometry and write the fitted result.",
148 coaddName = pexConfig.Field(
149 doc=
"Type of coadd, typically deep or goodSeeing",
153 positionErrorPedestal = pexConfig.Field(
154 doc=
"Systematic term to apply to the measured position error (pixels)",
158 photometryErrorPedestal = pexConfig.Field(
159 doc=
"Systematic term to apply to the measured error on flux or magnitude as a " 160 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
165 matchCut = pexConfig.Field(
166 doc=
"Matching radius between fitted and reference stars (arcseconds)",
170 minMeasurements = pexConfig.Field(
171 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
175 minMeasuredStarsPerCcd = pexConfig.Field(
176 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
180 minRefStarsPerCcd = pexConfig.Field(
181 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
185 allowLineSearch = pexConfig.Field(
186 doc=
"Allow a line search during minimization, if it is reasonable for the model" 187 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
191 astrometrySimpleOrder = pexConfig.Field(
192 doc=
"Polynomial order for fitting the simple astrometry model.",
196 astrometryChipOrder = pexConfig.Field(
197 doc=
"Order of the per-chip transform for the constrained astrometry model.",
201 astrometryVisitOrder = pexConfig.Field(
202 doc=
"Order of the per-visit transform for the constrained astrometry model.",
206 useInputWcs = pexConfig.Field(
207 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
211 astrometryModel = pexConfig.ChoiceField(
212 doc=
"Type of model to fit to astrometry",
214 default=
"constrained",
215 allowed={
"simple":
"One polynomial per ccd",
216 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
218 photometryModel = pexConfig.ChoiceField(
219 doc=
"Type of model to fit to photometry",
221 default=
"constrainedMagnitude",
222 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
223 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit," 224 " fitting in flux space.",
225 "simpleMagnitude":
"One constant zeropoint per ccd and visit," 226 " fitting in magnitude space.",
227 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit," 228 " fitting in magnitude space.",
231 applyColorTerms = pexConfig.Field(
232 doc=
"Apply photometric color terms to reference stars?" 233 "Requires that colorterms be set to a ColortermLibrary",
237 colorterms = pexConfig.ConfigField(
238 doc=
"Library of photometric reference catalog name to color term dict.",
239 dtype=ColortermLibrary,
241 photometryVisitOrder = pexConfig.Field(
242 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
246 photometryDoRankUpdate = pexConfig.Field(
247 doc=
"Do the rank update step during minimization. " 248 "Skipping this can help deal with models that are too non-linear.",
252 astrometryDoRankUpdate = pexConfig.Field(
253 doc=
"Do the rank update step during minimization (should not change the astrometry fit). " 254 "Skipping this can help deal with models that are too non-linear.",
258 outlierRejectSigma = pexConfig.Field(
259 doc=
"How many sigma to reject outliers at during minimization.",
263 maxPhotometrySteps = pexConfig.Field(
264 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
268 maxAstrometrySteps = pexConfig.Field(
269 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
273 astrometryRefObjLoader = pexConfig.ConfigurableField(
274 target=LoadIndexedReferenceObjectsTask,
275 doc=
"Reference object loader for astrometric fit",
277 photometryRefObjLoader = pexConfig.ConfigurableField(
278 target=LoadIndexedReferenceObjectsTask,
279 doc=
"Reference object loader for photometric fit",
281 sourceSelector = sourceSelectorRegistry.makeField(
282 doc=
"How to select sources for cross-matching",
285 astrometryReferenceSelector = pexConfig.ConfigurableField(
286 target=ReferenceSourceSelectorTask,
287 doc=
"How to down-select the loaded astrometry reference catalog.",
289 photometryReferenceSelector = pexConfig.ConfigurableField(
290 target=ReferenceSourceSelectorTask,
291 doc=
"How to down-select the loaded photometry reference catalog.",
293 astrometryReferenceErr = pexConfig.Field(
294 doc=
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*_err` fields." 295 " If None, then raise an exception if the reference catalog is missing coordinate errors." 296 " If specified, overrides any existing `coord_*_err` values.",
301 writeInitMatrix = pexConfig.Field(
303 doc=
"Write the pre/post-initialization Hessian and gradient to text files, for debugging." 304 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory." 305 "Note that these files are the dense versions of the matrix, and so may be very large.",
308 writeChi2FilesInitialFinal = pexConfig.Field(
310 doc=
"Write .csv files containing the contributions to chi2 for the initialization and final fit.",
313 writeChi2FilesOuterLoop = pexConfig.Field(
315 doc=
"Write .csv files containing the contributions to chi2 for the outer fit loop.",
318 sourceFluxType = pexConfig.Field(
320 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
327 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 328 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
341 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux" 342 self.
sourceSelector[
'science'].signalToNoise.fluxField = fluxField
343 self.
sourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err" 349 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
350 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
351 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
356 """Jointly astrometrically and photometrically calibrate a group of images.""" 358 ConfigClass = JointcalConfig
359 RunnerClass = JointcalRunner
360 _DefaultName =
"jointcal" 362 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
364 Instantiate a JointcalTask. 368 butler : `lsst.daf.persistence.Butler` 369 The butler is passed to the refObjLoader constructor in case it is 370 needed. Ignored if the refObjLoader argument provides a loader directly. 371 Used to initialize the astrometry and photometry refObjLoaders. 372 profile_jointcal : `bool` 373 Set to True to profile different stages of this jointcal run. 375 pipeBase.CmdLineTask.__init__(self, **kwargs)
377 self.makeSubtask(
"sourceSelector")
378 if self.config.doAstrometry:
379 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
380 self.makeSubtask(
"astrometryReferenceSelector")
383 if self.config.doPhotometry:
384 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
385 self.makeSubtask(
"photometryReferenceSelector")
390 self.
job = Job.load_metrics_package(subset=
'jointcal')
395 def _getMetadataName(self):
399 def _makeArgumentParser(cls):
400 """Create an argument parser""" 402 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
403 help=
"Profile steps of jointcal separately.")
404 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
405 ContainerClass=PerTractCcdDataIdContainer)
408 def _build_ccdImage(self, dataRef, associations, jointcalControl):
410 Extract the necessary things from this dataRef to add a new ccdImage. 414 dataRef : `lsst.daf.persistence.ButlerDataRef` 415 DataRef to extract info from. 416 associations : `lsst.jointcal.Associations` 417 Object to add the info to, to construct a new CcdImage 418 jointcalControl : `jointcal.JointcalControl` 419 Control object for associations management 425 The TAN WCS of this image, read from the calexp 426 (`lsst.afw.geom.SkyWcs`). 428 A key to identify this dataRef by its visit and ccd ids 431 This calexp's filter (`str`). 433 if "visit" in dataRef.dataId.keys():
434 visit = dataRef.dataId[
"visit"]
436 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
438 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
440 visitInfo = dataRef.get(
'calexp_visitInfo')
441 detector = dataRef.get(
'calexp_detector')
442 ccdId = detector.getId()
443 photoCalib = dataRef.get(
'calexp_photoCalib')
444 tanWcs = dataRef.get(
'calexp_wcs')
445 bbox = dataRef.get(
'calexp_bbox')
446 filt = dataRef.get(
'calexp_filter')
447 filterName = filt.getName()
449 goodSrc = self.sourceSelector.run(src)
451 if len(goodSrc.sourceCat) == 0:
452 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
454 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
455 associations.createCcdImage(goodSrc.sourceCat,
466 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
467 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
468 return Result(tanWcs, Key(visit, ccdId), filterName)
473 Jointly calibrate the astrometry and photometry across a set of images. 477 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 478 List of data references to the exposures to be fit. 479 profile_jointcal : `bool` 480 Profile the individual steps of jointcal. 484 result : `lsst.pipe.base.Struct` 485 Struct of metadata from the fit, containing: 488 The provided data references that were fit (with updated WCSs) 490 The original WCS from each dataRef 492 Dictionary of internally-computed metrics for testing/validation. 494 if len(dataRefs) == 0:
495 raise ValueError(
'Need a non-empty list of data references!')
499 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
503 visit_ccd_to_dataRef = {}
506 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 507 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
510 camera = dataRefs[0].get(
'camera', immediate=
True)
514 oldWcsList.append(result.wcs)
515 visit_ccd_to_dataRef[result.key] = ref
516 filters.append(result.filter)
517 filters = collections.Counter(filters)
519 associations.computeCommonTangentPoint()
524 bbox = associations.getRaDecBBox()
525 bboxCenter = bbox.getCenter()
526 center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
527 bboxMax = bbox.getMax()
528 corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
529 radius = center.separation(corner).asRadians()
534 raise RuntimeError(
"astrometry_net_data is not setup")
537 defaultFilter = filters.most_common(1)[0][0]
538 self.log.debug(
"Using %s band for reference flux", defaultFilter)
541 tract = dataRefs[0].dataId[
'tract']
543 if self.config.doAstrometry:
547 referenceSelector=self.astrometryReferenceSelector,
549 profile_jointcal=profile_jointcal,
555 if self.config.doPhotometry:
559 referenceSelector=self.photometryReferenceSelector,
561 profile_jointcal=profile_jointcal,
564 reject_bad_fluxes=
True)
569 return pipeBase.Struct(dataRefs=dataRefs,
570 oldWcsList=oldWcsList,
574 defaultFilter=defaultFilter,
575 exitStatus=exitStatus)
577 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
578 name="", refObjLoader=None, referenceSelector=None,
579 filters=[], fit_function=None,
580 tract=None, profile_jointcal=False, match_cut=3.0,
581 reject_bad_fluxes=False):
582 """Load reference catalog, perform the fit, and return the result. 586 associations : `lsst.jointcal.Associations` 587 The star/reference star associations to fit. 588 defaultFilter : `str` 589 filter to load from reference catalog. 590 center : `lsst.afw.geom.SpherePoint` 591 ICRS center of field to load from reference catalog. 592 radius : `lsst.afw.geom.Angle` 593 On-sky radius to load from reference catalog. 595 Name of thing being fit: "Astrometry" or "Photometry". 596 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 597 Reference object loader to load from for fit. 598 filters : `list` of `str`, optional 599 List of filters to load from the reference catalog. 600 fit_function : callable 601 Function to call to perform fit (takes associations object). 603 Name of tract currently being fit. 604 profile_jointcal : `bool`, optional 605 Separately profile the fitting step. 606 match_cut : `float`, optional 607 Radius in arcseconds to find cross-catalog matches to during 608 associations.associateCatalogs. 609 reject_bad_fluxes : `bool`, optional 610 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 614 result : `Photometry` or `Astrometry` 615 Result of `fit_function()` 617 self.log.info(
"====== Now processing %s...", name)
620 associations.associateCatalogs(match_cut)
622 associations.fittedStarListSize())
624 applyColorterms =
False if name ==
"Astrometry" else self.config.applyColorTerms
625 if name ==
"Astrometry":
626 referenceSelector = self.config.astrometryReferenceSelector
627 elif name ==
"Photometry":
628 referenceSelector = self.config.photometryReferenceSelector
630 center, radius, defaultFilter,
631 applyColorterms=applyColorterms)
633 if self.config.astrometryReferenceErr
is None:
634 refCoordErr = float(
'nan')
636 refCoordErr = self.config.astrometryReferenceErr
638 associations.collectRefStars(refCat,
639 self.config.matchCut*afwGeom.arcseconds,
641 refCoordinateErr=refCoordErr,
642 rejectBadFluxes=reject_bad_fluxes)
644 associations.refStarListSize())
646 associations.prepareFittedStars(self.config.minMeasurements)
650 associations.nFittedStarsWithAssociatedRefStar())
652 associations.fittedStarListSize())
654 associations.nCcdImagesValidForFit())
656 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 657 dataName =
"{}_{}".format(tract, defaultFilter)
658 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
659 result = fit_function(associations, dataName)
662 if self.config.writeChi2FilesInitialFinal:
663 baseName = f
"{name}_final_chi2-{dataName}" 664 result.fit.saveChi2Contributions(baseName+
"{type}")
668 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
669 applyColorterms=False):
670 """Load the necessary reference catalog sources, convert fluxes to 671 correct units, and apply color term corrections if requested. 675 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 676 The reference catalog loader to use to get the data. 677 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 678 Source selector to apply to loaded reference catalog. 679 center : `lsst.geom.SpherePoint` 680 The center around which to load sources. 681 radius : `lsst.geom.Angle` 682 The radius around ``center`` to load sources in. 684 The name of the camera filter to load fluxes for. 685 applyColorterms : `bool` 686 Apply colorterm corrections to the refcat for ``filterName``? 690 refCat : `lsst.afw.table.SimpleCatalog` 691 The loaded reference catalog. 693 The name of the reference catalog flux field appropriate for ``filterName``. 695 skyCircle = refObjLoader.loadSkyCircle(center,
696 afwGeom.Angle(radius, afwGeom.radians),
699 selected = referenceSelector.run(skyCircle.refCat)
701 if not selected.sourceCat.isContiguous():
702 refCat = selected.sourceCat.copy(deep=
True)
704 refCat = selected.sourceCat
706 if self.config.astrometryReferenceErr
is None and 'coord_ra_err' not in refCat.schema:
707 msg = (
"Reference catalog does not contain coordinate errors, " 708 "and config.astrometryReferenceErr not supplied.")
709 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
713 if self.config.astrometryReferenceErr
is not None and 'coord_ra_err' in refCat.schema:
714 self.log.warn(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
715 self.config.astrometryReferenceErr)
719 refCatName = refObjLoader.ref_dataset_name
720 except AttributeError:
722 raise RuntimeError(
"Cannot perform colorterm corrections with a.net refcats.")
723 self.log.info(
"Applying color terms for filterName=%r reference catalog=%s",
724 filterName, refCatName)
725 colorterm = self.config.colorterms.getColorterm(
726 filterName=filterName, photoCatName=refCatName, doRaise=
True)
728 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
729 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
731 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
733 return refCat, skyCircle.fluxField
735 def _check_star_lists(self, associations, name):
737 if associations.nCcdImagesValidForFit() == 0:
738 raise RuntimeError(
'No images in the ccdImageList!')
739 if associations.fittedStarListSize() == 0:
740 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
741 if associations.refStarListSize() == 0:
742 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
744 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
746 """Compute chi2, log it, validate the model, and return chi2. 750 associations : `lsst.jointcal.Associations` 751 The star/reference star associations to fit. 752 fit : `lsst.jointcal.FitterBase` 753 The fitter to use for minimization. 754 model : `lsst.jointcal.Model` 756 chi2Label : str, optional 757 Label to describe the chi2 (e.g. "Initialized", "Final"). 758 writeChi2Name : `str`, optional 759 Filename prefix to write the chi2 contributions to. 760 Do not supply an extension: an appropriate one will be added. 764 chi2: `lsst.jointcal.Chi2Accumulator` 765 The chi2 object for the current fitter and model. 770 Raised if chi2 is infinite or NaN. 772 Raised if the model is not valid. 774 if writeChi2Name
is not None:
775 fit.saveChi2Contributions(writeChi2Name+
"{type}")
776 self.log.info(
"Wrote chi2 contributions files: %s", writeChi2Name)
778 chi2 = fit.computeChi2()
779 self.log.info(
"%s %s", chi2Label, chi2)
781 if not np.isfinite(chi2.chi2):
782 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
783 if not model.validate(associations.getCcdImageList(), chi2.ndof):
784 raise ValueError(
"Model is not valid: check log messages for warnings.")
787 def _fit_photometry(self, associations, dataName=None):
789 Fit the photometric data. 793 associations : `lsst.jointcal.Associations` 794 The star/reference star associations to fit. 796 Name of the data being processed (e.g. "1234_HSC-Y"), for 797 identifying debugging files. 801 fit_result : `namedtuple` 802 fit : `lsst.jointcal.PhotometryFit` 803 The photometric fitter used to perform the fit. 804 model : `lsst.jointcal.PhotometryModel` 805 The photometric model that was fit. 807 self.log.info(
"=== Starting photometric fitting...")
810 if self.config.photometryModel ==
"constrainedFlux":
813 visitOrder=self.config.photometryVisitOrder,
814 errorPedestal=self.config.photometryErrorPedestal)
816 doLineSearch = self.config.allowLineSearch
817 elif self.config.photometryModel ==
"constrainedMagnitude":
820 visitOrder=self.config.photometryVisitOrder,
821 errorPedestal=self.config.photometryErrorPedestal)
823 doLineSearch = self.config.allowLineSearch
824 elif self.config.photometryModel ==
"simpleFlux":
826 errorPedestal=self.config.photometryErrorPedestal)
828 elif self.config.photometryModel ==
"simpleMagnitude":
830 errorPedestal=self.config.photometryErrorPedestal)
836 if self.config.writeChi2FilesInitialFinal:
837 baseName = f
"photometry_initial_chi2-{dataName}" 842 def getChi2Name(whatToFit):
843 if self.config.writeChi2FilesOuterLoop:
844 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
850 dumpMatrixFile =
"photometry_preinit" if self.config.writeInitMatrix
else "" 851 if self.config.photometryModel.startswith(
"constrained"):
854 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
855 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"ModelVisit"))
858 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
861 fit.minimize(
"Fluxes")
864 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
866 writeChi2Name=getChi2Name(
"ModelFluxes"))
868 model.freezeErrorTransform()
869 self.log.debug(
"Photometry error scales are frozen.")
873 self.config.maxPhotometrySteps,
876 doRankUpdate=self.config.photometryDoRankUpdate,
877 doLineSearch=doLineSearch,
884 def _fit_astrometry(self, associations, dataName=None):
886 Fit the astrometric data. 890 associations : `lsst.jointcal.Associations` 891 The star/reference star associations to fit. 893 Name of the data being processed (e.g. "1234_HSC-Y"), for 894 identifying debugging files. 898 fit_result : `namedtuple` 899 fit : `lsst.jointcal.AstrometryFit` 900 The astrometric fitter used to perform the fit. 901 model : `lsst.jointcal.AstrometryModel` 902 The astrometric model that was fit. 903 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 904 The model for the sky to tangent plane projection that was used in the fit. 907 self.log.info(
"=== Starting astrometric fitting...")
909 associations.deprojectFittedStars()
916 if self.config.astrometryModel ==
"constrained":
918 sky_to_tan_projection,
919 chipOrder=self.config.astrometryChipOrder,
920 visitOrder=self.config.astrometryVisitOrder)
921 elif self.config.astrometryModel ==
"simple":
923 sky_to_tan_projection,
924 self.config.useInputWcs,
926 order=self.config.astrometrySimpleOrder)
931 if self.config.writeChi2FilesInitialFinal:
932 baseName = f
"astrometry_initial_chi2-{dataName}" 937 def getChi2Name(whatToFit):
938 if self.config.writeChi2FilesOuterLoop:
939 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
943 dumpMatrixFile =
"astrometry_preinit" if self.config.writeInitMatrix
else "" 946 if self.config.astrometryModel ==
"constrained":
947 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
948 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"DistortionsVisit"))
951 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
952 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"Distortions"))
954 fit.minimize(
"Positions")
957 fit.minimize(
"Distortions Positions")
959 writeChi2Name=getChi2Name(
"DistortionsPositions"))
963 self.config.maxAstrometrySteps,
965 "Distortions Positions",
966 doRankUpdate=self.config.astrometryDoRankUpdate,
972 return Astrometry(fit, model, sky_to_tan_projection)
974 def _check_stars(self, associations):
975 """Count measured and reference stars per ccd and warn/log them.""" 976 for ccdImage
in associations.getCcdImageList():
977 nMeasuredStars, nRefStars = ccdImage.countStars()
978 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
979 ccdImage.getName(), nMeasuredStars, nRefStars)
980 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
981 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
982 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
983 if nRefStars < self.config.minRefStarsPerCcd:
984 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
985 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
987 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
991 """Run fitter.minimize up to max_steps times, returning the final chi2. 995 associations : `lsst.jointcal.Associations` 996 The star/reference star associations to fit. 997 fitter : `lsst.jointcal.FitterBase` 998 The fitter to use for minimization. 1000 Maximum number of steps to run outlier rejection before declaring 1001 convergence failure. 1002 name : {'photometry' or 'astrometry'} 1003 What type of data are we fitting (for logs and debugging files). 1005 Passed to ``fitter.minimize()`` to define the parameters to fit. 1006 dataName : `str`, optional 1007 Descriptive name for this dataset (e.g. tract and filter), 1009 doRankUpdate : `bool`, optional 1010 Do an Eigen rank update during minimization, or recompute the full 1011 matrix and gradient? 1012 doLineSearch : `bool`, optional 1013 Do a line search for the optimum step during minimization? 1017 chi2: `lsst.jointcal.Chi2Statistic` 1018 The final chi2 after the fit converges, or is forced to end. 1023 Raised if the fitter fails with a non-finite value. 1025 Raised if the fitter fails for some other reason; 1026 log messages will provide further details. 1028 dumpMatrixFile =
"%s_postinit" % name
if self.config.writeInitMatrix
else "" 1029 for i
in range(max_steps):
1030 if self.config.writeChi2FilesOuterLoop:
1031 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}" 1033 writeChi2Name =
None 1034 result = fitter.minimize(whatToFit,
1035 self.config.outlierRejectSigma,
1036 doRankUpdate=doRankUpdate,
1037 doLineSearch=doLineSearch,
1038 dumpMatrixFile=dumpMatrixFile)
1041 writeChi2Name=writeChi2Name)
1043 if result == MinimizeResult.Converged:
1045 self.log.debug(
"fit has converged - no more outliers - redo minimization " 1046 "one more time in case we have lost accuracy in rank update.")
1048 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1052 if chi2.chi2/chi2.ndof >= 4.0:
1053 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1056 elif result == MinimizeResult.Chi2Increased:
1057 self.log.warn(
"still some outliers but chi2 increases - retry")
1058 elif result == MinimizeResult.NonFinite:
1059 filename =
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
1061 fitter.saveChi2Contributions(filename)
1062 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 1063 raise FloatingPointError(msg.format(filename))
1064 elif result == MinimizeResult.Failed:
1065 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1067 raise RuntimeError(
"Unxepected return code from minimize().")
1069 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1073 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1075 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 1079 associations : `lsst.jointcal.Associations` 1080 The star/reference star associations to fit. 1081 model : `lsst.jointcal.AstrometryModel` 1082 The astrometric model that was fit. 1083 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1084 Dict of ccdImage identifiers to dataRefs that were fit. 1087 ccdImageList = associations.getCcdImageList()
1088 for ccdImage
in ccdImageList:
1090 ccd = ccdImage.ccdId
1091 visit = ccdImage.visit
1092 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1093 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
1094 skyWcs = model.makeSkyWcs(ccdImage)
1096 dataRef.put(skyWcs,
'jointcal_wcs')
1097 except pexExceptions.Exception
as e:
1098 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1101 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1103 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 1107 associations : `lsst.jointcal.Associations` 1108 The star/reference star associations to fit. 1109 model : `lsst.jointcal.PhotometryModel` 1110 The photoometric model that was fit. 1111 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1112 Dict of ccdImage identifiers to dataRefs that were fit. 1115 ccdImageList = associations.getCcdImageList()
1116 for ccdImage
in ccdImageList:
1118 ccd = ccdImage.ccdId
1119 visit = ccdImage.visit
1120 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1121 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1122 photoCalib = model.toPhotoCalib(ccdImage)
1124 dataRef.put(photoCalib,
'jointcal_photoCalib')
1125 except pexExceptions.Exception
as e:
1126 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def runDataRef(self, dataRefs, profile_jointcal=False)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", writeChi2Name=None)
A model where there is one independent transform per CcdImage.
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
A multi-component model, fitting mappings for sensors and visits simultaneously.
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def __init__(self, butler=None, profile_jointcal=False, kwargs)