24 import astropy.units
as u
34 from lsst.pipe.tasks.colorterms
import ColortermLibrary
35 from lsst.verify
import Job, Measurement
40 from .dataIds
import PerTractCcdDataIdContainer
45 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
47 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
48 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
53 meas = Measurement(job.metrics[name], value)
54 job.measurements.insert(meas)
58 """Subclass of TaskRunner for jointcalTask 60 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 61 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 62 single dataRef, are are called repeatedly). This class transforms the processed 63 arguments generated by the ArgumentParser into the arguments expected by 64 Jointcal.runDataRef(). 66 See pipeBase.TaskRunner for more information. 72 Return a list of tuples per tract, each containing (dataRefs, kwargs). 74 Jointcal operates on lists of dataRefs simultaneously. 76 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
77 kwargs[
'butler'] = parsedCmd.butler
81 for ref
in parsedCmd.id.refList:
82 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
84 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
92 Arguments for Task.runDataRef() 97 if self.doReturnResults is False: 99 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 101 if self.doReturnResults is True: 103 - ``result``: the result of calling jointcal.runDataRef() 104 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 109 dataRefList, kwargs = args
110 butler = kwargs.pop(
'butler')
111 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
114 result = task.runDataRef(dataRefList, **kwargs)
115 exitStatus = result.exitStatus
116 job_path = butler.get(
'verify_job_filename')
117 result.job.write(job_path[0])
118 except Exception
as e:
123 eName = type(e).__name__
124 tract = dataRefList[0].dataId[
'tract']
125 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
128 kwargs[
'butler'] = butler
129 if self.doReturnResults:
130 return pipeBase.Struct(result=result, exitStatus=exitStatus)
132 return pipeBase.Struct(exitStatus=exitStatus)
136 """Config for JointcalTask""" 138 doAstrometry = pexConfig.Field(
139 doc=
"Fit astrometry and write the fitted result.",
143 doPhotometry = pexConfig.Field(
144 doc=
"Fit photometry and write the fitted result.",
148 coaddName = pexConfig.Field(
149 doc=
"Type of coadd, typically deep or goodSeeing",
153 positionErrorPedestal = pexConfig.Field(
154 doc=
"Systematic term to apply to the measured position error (pixels)",
158 photometryErrorPedestal = pexConfig.Field(
159 doc=
"Systematic term to apply to the measured error on flux or magnitude as a " 160 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
165 matchCut = pexConfig.Field(
166 doc=
"Matching radius between fitted and reference stars (arcseconds)",
170 minMeasurements = pexConfig.Field(
171 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
175 minMeasuredStarsPerCcd = pexConfig.Field(
176 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
180 minRefStarsPerCcd = pexConfig.Field(
181 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
185 allowLineSearch = pexConfig.Field(
186 doc=
"Allow a line search during minimization, if it is reasonable for the model" 187 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
191 astrometrySimpleOrder = pexConfig.Field(
192 doc=
"Polynomial order for fitting the simple astrometry model.",
196 astrometryChipOrder = pexConfig.Field(
197 doc=
"Order of the per-chip transform for the constrained astrometry model.",
201 astrometryVisitOrder = pexConfig.Field(
202 doc=
"Order of the per-visit transform for the constrained astrometry model.",
206 useInputWcs = pexConfig.Field(
207 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
211 astrometryModel = pexConfig.ChoiceField(
212 doc=
"Type of model to fit to astrometry",
214 default=
"constrained",
215 allowed={
"simple":
"One polynomial per ccd",
216 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
218 photometryModel = pexConfig.ChoiceField(
219 doc=
"Type of model to fit to photometry",
221 default=
"constrainedMagnitude",
222 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
223 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit," 224 " fitting in flux space.",
225 "simpleMagnitude":
"One constant zeropoint per ccd and visit," 226 " fitting in magnitude space.",
227 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit," 228 " fitting in magnitude space.",
231 applyColorTerms = pexConfig.Field(
232 doc=
"Apply photometric color terms to reference stars?" 233 "Requires that colorterms be set to a ColortermLibrary",
237 colorterms = pexConfig.ConfigField(
238 doc=
"Library of photometric reference catalog name to color term dict.",
239 dtype=ColortermLibrary,
241 photometryVisitOrder = pexConfig.Field(
242 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
246 photometryDoRankUpdate = pexConfig.Field(
247 doc=
"Do the rank update step during minimization. " 248 "Skipping this can help deal with models that are too non-linear.",
252 astrometryDoRankUpdate = pexConfig.Field(
253 doc=
"Do the rank update step during minimization (should not change the astrometry fit). " 254 "Skipping this can help deal with models that are too non-linear.",
258 outlierRejectSigma = pexConfig.Field(
259 doc=
"How many sigma to reject outliers at during minimization.",
263 maxPhotometrySteps = pexConfig.Field(
264 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
268 maxAstrometrySteps = pexConfig.Field(
269 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
273 astrometryRefObjLoader = pexConfig.ConfigurableField(
274 target=LoadIndexedReferenceObjectsTask,
275 doc=
"Reference object loader for astrometric fit",
277 photometryRefObjLoader = pexConfig.ConfigurableField(
278 target=LoadIndexedReferenceObjectsTask,
279 doc=
"Reference object loader for photometric fit",
281 sourceSelector = sourceSelectorRegistry.makeField(
282 doc=
"How to select sources for cross-matching",
285 astrometryReferenceSelector = pexConfig.ConfigurableField(
286 target=ReferenceSourceSelectorTask,
287 doc=
"How to down-select the loaded astrometry reference catalog.",
289 photometryReferenceSelector = pexConfig.ConfigurableField(
290 target=ReferenceSourceSelectorTask,
291 doc=
"How to down-select the loaded photometry reference catalog.",
293 writeInitMatrix = pexConfig.Field(
295 doc=
"Write the pre/post-initialization Hessian and gradient to text files, for debugging." 296 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory." 297 "Note that these files are the dense versions of the matrix, and so may be very large.",
300 writeChi2FilesInitialFinal = pexConfig.Field(
302 doc=
"Write .csv files containing the contributions to chi2 for the initialization and final fit.",
305 writeChi2FilesOuterLoop = pexConfig.Field(
307 doc=
"Write .csv files containing the contributions to chi2 for the outer fit loop.",
310 sourceFluxType = pexConfig.Field(
312 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
319 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 320 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
324 sourceSelector.setDefaults()
326 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
332 """Jointly astrometrically and photometrically calibrate a group of images.""" 334 ConfigClass = JointcalConfig
335 RunnerClass = JointcalRunner
336 _DefaultName =
"jointcal" 338 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
340 Instantiate a JointcalTask. 344 butler : `lsst.daf.persistence.Butler` 345 The butler is passed to the refObjLoader constructor in case it is 346 needed. Ignored if the refObjLoader argument provides a loader directly. 347 Used to initialize the astrometry and photometry refObjLoaders. 348 profile_jointcal : `bool` 349 Set to True to profile different stages of this jointcal run. 351 pipeBase.CmdLineTask.__init__(self, **kwargs)
353 self.makeSubtask(
"sourceSelector")
354 if self.config.doAstrometry:
355 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
356 self.makeSubtask(
"astrometryReferenceSelector")
359 if self.config.doPhotometry:
360 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
361 self.makeSubtask(
"photometryReferenceSelector")
366 self.
job = Job.load_metrics_package(subset=
'jointcal')
371 def _getMetadataName(self):
375 def _makeArgumentParser(cls):
376 """Create an argument parser""" 378 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
379 help=
"Profile steps of jointcal separately.")
380 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
381 ContainerClass=PerTractCcdDataIdContainer)
384 def _build_ccdImage(self, dataRef, associations, jointcalControl):
386 Extract the necessary things from this dataRef to add a new ccdImage. 390 dataRef : `lsst.daf.persistence.ButlerDataRef` 391 DataRef to extract info from. 392 associations : `lsst.jointcal.Associations` 393 Object to add the info to, to construct a new CcdImage 394 jointcalControl : `jointcal.JointcalControl` 395 Control object for associations management 401 The TAN WCS of this image, read from the calexp 402 (`lsst.afw.geom.SkyWcs`). 404 A key to identify this dataRef by its visit and ccd ids 407 This calexp's filter (`str`). 409 if "visit" in dataRef.dataId.keys():
410 visit = dataRef.dataId[
"visit"]
412 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
414 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
416 visitInfo = dataRef.get(
'calexp_visitInfo')
417 detector = dataRef.get(
'calexp_detector')
418 ccdId = detector.getId()
419 photoCalib = dataRef.get(
'calexp_photoCalib')
420 tanWcs = dataRef.get(
'calexp_wcs')
421 bbox = dataRef.get(
'calexp_bbox')
422 filt = dataRef.get(
'calexp_filter')
423 filterName = filt.getName()
425 goodSrc = self.sourceSelector.run(src)
427 if len(goodSrc.sourceCat) == 0:
428 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
430 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
431 associations.createCcdImage(goodSrc.sourceCat,
442 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
443 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
444 return Result(tanWcs, Key(visit, ccdId), filterName)
449 Jointly calibrate the astrometry and photometry across a set of images. 453 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 454 List of data references to the exposures to be fit. 455 profile_jointcal : `bool` 456 Profile the individual steps of jointcal. 460 result : `lsst.pipe.base.Struct` 461 Struct of metadata from the fit, containing: 464 The provided data references that were fit (with updated WCSs) 466 The original WCS from each dataRef 468 Dictionary of internally-computed metrics for testing/validation. 470 if len(dataRefs) == 0:
471 raise ValueError(
'Need a non-empty list of data references!')
475 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
479 visit_ccd_to_dataRef = {}
482 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 483 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
486 camera = dataRefs[0].get(
'camera', immediate=
True)
490 oldWcsList.append(result.wcs)
491 visit_ccd_to_dataRef[result.key] = ref
492 filters.append(result.filter)
493 filters = collections.Counter(filters)
495 associations.computeCommonTangentPoint()
500 bbox = associations.getRaDecBBox()
501 bboxCenter = bbox.getCenter()
502 center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
503 bboxMax = bbox.getMax()
504 corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
505 radius = center.separation(corner).asRadians()
510 raise RuntimeError(
"astrometry_net_data is not setup")
513 defaultFilter = filters.most_common(1)[0][0]
514 self.log.debug(
"Using %s band for reference flux", defaultFilter)
517 tract = dataRefs[0].dataId[
'tract']
519 if self.config.doAstrometry:
523 referenceSelector=self.astrometryReferenceSelector,
525 profile_jointcal=profile_jointcal,
531 if self.config.doPhotometry:
535 referenceSelector=self.photometryReferenceSelector,
537 profile_jointcal=profile_jointcal,
540 reject_bad_fluxes=
True)
545 return pipeBase.Struct(dataRefs=dataRefs,
546 oldWcsList=oldWcsList,
550 defaultFilter=defaultFilter,
551 exitStatus=exitStatus)
553 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
554 name="", refObjLoader=None, referenceSelector=None,
555 filters=[], fit_function=None,
556 tract=None, profile_jointcal=False, match_cut=3.0,
557 reject_bad_fluxes=False):
558 """Load reference catalog, perform the fit, and return the result. 562 associations : `lsst.jointcal.Associations` 563 The star/reference star associations to fit. 564 defaultFilter : `str` 565 filter to load from reference catalog. 566 center : `lsst.afw.geom.SpherePoint` 567 ICRS center of field to load from reference catalog. 568 radius : `lsst.afw.geom.Angle` 569 On-sky radius to load from reference catalog. 571 Name of thing being fit: "Astrometry" or "Photometry". 572 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 573 Reference object loader to load from for fit. 574 filters : `list` of `str`, optional 575 List of filters to load from the reference catalog. 576 fit_function : callable 577 Function to call to perform fit (takes associations object). 579 Name of tract currently being fit. 580 profile_jointcal : `bool`, optional 581 Separately profile the fitting step. 582 match_cut : `float`, optional 583 Radius in arcseconds to find cross-catalog matches to during 584 associations.associateCatalogs. 585 reject_bad_fluxes : `bool`, optional 586 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 590 result : `Photometry` or `Astrometry` 591 Result of `fit_function()` 593 self.log.info(
"====== Now processing %s...", name)
596 associations.associateCatalogs(match_cut)
598 associations.fittedStarListSize())
600 applyColorterms =
False if name ==
"Astrometry" else self.config.applyColorTerms
601 if name ==
"Astrometry":
602 referenceSelector = self.config.astrometryReferenceSelector
603 elif name ==
"Photometry":
604 referenceSelector = self.config.photometryReferenceSelector
606 center, radius, defaultFilter,
607 applyColorterms=applyColorterms)
609 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
610 fluxField, reject_bad_fluxes)
612 associations.refStarListSize())
614 associations.prepareFittedStars(self.config.minMeasurements)
618 associations.nFittedStarsWithAssociatedRefStar())
620 associations.fittedStarListSize())
622 associations.nCcdImagesValidForFit())
624 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 625 dataName =
"{}_{}".format(tract, defaultFilter)
626 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
627 result = fit_function(associations, dataName)
630 if self.config.writeChi2FilesInitialFinal:
631 baseName = f
"{name}_final_chi2-{dataName}" 632 result.fit.saveChi2Contributions(baseName+
"{type}")
636 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
637 applyColorterms=False):
638 """Load the necessary reference catalog sources, convert fluxes to 639 correct units, and apply color term corrections if requested. 643 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 644 The reference catalog loader to use to get the data. 645 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 646 Source selector to apply to loaded reference catalog. 647 center : `lsst.geom.SpherePoint` 648 The center around which to load sources. 649 radius : `lsst.geom.Angle` 650 The radius around ``center`` to load sources in. 652 The name of the camera filter to load fluxes for. 653 applyColorterms : `bool` 654 Apply colorterm corrections to the refcat for ``filterName``? 658 refCat : `lsst.afw.table.SimpleCatalog` 659 The loaded reference catalog. 661 The name of the reference catalog flux field appropriate for ``filterName``. 663 skyCircle = refObjLoader.loadSkyCircle(center,
664 afwGeom.Angle(radius, afwGeom.radians),
667 selected = referenceSelector.run(skyCircle.refCat)
669 if not selected.sourceCat.isContiguous():
670 refCat = selected.sourceCat.copy(deep=
True)
672 refCat = selected.sourceCat
676 refCatName = refObjLoader.ref_dataset_name
677 except AttributeError:
679 raise RuntimeError(
"Cannot perform colorterm corrections with a.net refcats.")
680 self.log.info(
"Applying color terms for filterName=%r reference catalog=%s",
681 filterName, refCatName)
682 colorterm = self.config.colorterms.getColorterm(
683 filterName=filterName, photoCatName=refCatName, doRaise=
True)
685 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
686 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
688 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
690 return refCat, skyCircle.fluxField
692 def _check_star_lists(self, associations, name):
694 if associations.nCcdImagesValidForFit() == 0:
695 raise RuntimeError(
'No images in the ccdImageList!')
696 if associations.fittedStarListSize() == 0:
697 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
698 if associations.refStarListSize() == 0:
699 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
701 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
703 """Compute chi2, log it, validate the model, and return chi2. 707 associations : `lsst.jointcal.Associations` 708 The star/reference star associations to fit. 709 fit : `lsst.jointcal.FitterBase` 710 The fitter to use for minimization. 711 model : `lsst.jointcal.Model` 713 chi2Label : str, optional 714 Label to describe the chi2 (e.g. "Initialized", "Final"). 715 writeChi2Name : `str`, optional 716 Filename prefix to write the chi2 contributions to. 717 Do not supply an extension: an appropriate one will be added. 721 chi2: `lsst.jointcal.Chi2Accumulator` 722 The chi2 object for the current fitter and model. 727 Raised if chi2 is infinite or NaN. 729 Raised if the model is not valid. 731 if writeChi2Name
is not None:
732 fit.saveChi2Contributions(writeChi2Name+
"{type}")
733 self.log.info(
"Wrote chi2 contributions files: %s", writeChi2Name)
735 chi2 = fit.computeChi2()
736 self.log.info(
"%s %s", chi2Label, chi2)
738 if not np.isfinite(chi2.chi2):
739 raise FloatingPointError(
'%s chi2 is invalid: %s', chi2Label, chi2)
740 if not model.validate(associations.getCcdImageList()):
741 raise ValueError(
"Model is not valid: check log messages for warnings.")
744 def _fit_photometry(self, associations, dataName=None):
746 Fit the photometric data. 750 associations : `lsst.jointcal.Associations` 751 The star/reference star associations to fit. 753 Name of the data being processed (e.g. "1234_HSC-Y"), for 754 identifying debugging files. 758 fit_result : `namedtuple` 759 fit : `lsst.jointcal.PhotometryFit` 760 The photometric fitter used to perform the fit. 761 model : `lsst.jointcal.PhotometryModel` 762 The photometric model that was fit. 764 self.log.info(
"=== Starting photometric fitting...")
767 if self.config.photometryModel ==
"constrainedFlux":
770 visitOrder=self.config.photometryVisitOrder,
771 errorPedestal=self.config.photometryErrorPedestal)
773 doLineSearch = self.config.allowLineSearch
774 elif self.config.photometryModel ==
"constrainedMagnitude":
777 visitOrder=self.config.photometryVisitOrder,
778 errorPedestal=self.config.photometryErrorPedestal)
780 doLineSearch = self.config.allowLineSearch
781 elif self.config.photometryModel ==
"simpleFlux":
783 errorPedestal=self.config.photometryErrorPedestal)
785 elif self.config.photometryModel ==
"simpleMagnitude":
787 errorPedestal=self.config.photometryErrorPedestal)
793 if self.config.writeChi2FilesInitialFinal:
794 baseName = f
"photometry_initial_chi2-{dataName}" 799 def getChi2Name(whatToFit):
800 if self.config.writeChi2FilesOuterLoop:
801 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
807 dumpMatrixFile =
"photometry_preinit" if self.config.writeInitMatrix
else "" 808 if self.config.photometryModel.startswith(
"constrained"):
811 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
812 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"ModelVisit"))
815 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
818 fit.minimize(
"Fluxes")
821 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
823 writeChi2Name=getChi2Name(
"ModelFluxes"))
825 model.freezeErrorTransform()
826 self.log.debug(
"Photometry error scales are frozen.")
830 self.config.maxPhotometrySteps,
833 doRankUpdate=self.config.photometryDoRankUpdate,
834 doLineSearch=doLineSearch,
841 def _fit_astrometry(self, associations, dataName=None):
843 Fit the astrometric data. 847 associations : `lsst.jointcal.Associations` 848 The star/reference star associations to fit. 850 Name of the data being processed (e.g. "1234_HSC-Y"), for 851 identifying debugging files. 855 fit_result : `namedtuple` 856 fit : `lsst.jointcal.AstrometryFit` 857 The astrometric fitter used to perform the fit. 858 model : `lsst.jointcal.AstrometryModel` 859 The astrometric model that was fit. 860 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 861 The model for the sky to tangent plane projection that was used in the fit. 864 self.log.info(
"=== Starting astrometric fitting...")
866 associations.deprojectFittedStars()
873 if self.config.astrometryModel ==
"constrained":
875 sky_to_tan_projection,
876 chipOrder=self.config.astrometryChipOrder,
877 visitOrder=self.config.astrometryVisitOrder)
878 elif self.config.astrometryModel ==
"simple":
880 sky_to_tan_projection,
881 self.config.useInputWcs,
883 order=self.config.astrometrySimpleOrder)
888 if self.config.writeChi2FilesInitialFinal:
889 baseName = f
"astrometry_initial_chi2-{dataName}" 894 def getChi2Name(whatToFit):
895 if self.config.writeChi2FilesOuterLoop:
896 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
900 dumpMatrixFile =
"astrometry_preinit" if self.config.writeInitMatrix
else "" 903 if self.config.astrometryModel ==
"constrained":
904 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
905 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"DistortionsVisit"))
908 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
909 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"Distortions"))
911 fit.minimize(
"Positions")
914 fit.minimize(
"Distortions Positions")
916 writeChi2Name=getChi2Name(
"DistortionsPositions"))
920 self.config.maxAstrometrySteps,
922 "Distortions Positions",
923 doRankUpdate=self.config.astrometryDoRankUpdate,
929 return Astrometry(fit, model, sky_to_tan_projection)
931 def _check_stars(self, associations):
932 """Count measured and reference stars per ccd and warn/log them.""" 933 for ccdImage
in associations.getCcdImageList():
934 nMeasuredStars, nRefStars = ccdImage.countStars()
935 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
936 ccdImage.getName(), nMeasuredStars, nRefStars)
937 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
938 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
939 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
940 if nRefStars < self.config.minRefStarsPerCcd:
941 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
942 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
944 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
948 """Run fitter.minimize up to max_steps times, returning the final chi2. 952 associations : `lsst.jointcal.Associations` 953 The star/reference star associations to fit. 954 fitter : `lsst.jointcal.FitterBase` 955 The fitter to use for minimization. 957 Maximum number of steps to run outlier rejection before declaring 959 name : {'photometry' or 'astrometry'} 960 What type of data are we fitting (for logs and debugging files). 962 Passed to ``fitter.minimize()`` to define the parameters to fit. 963 dataName : `str`, optional 964 Descriptive name for this dataset (e.g. tract and filter), 966 doRankUpdate : `bool`, optional 967 Do an Eigen rank update during minimization, or recompute the full 969 doLineSearch : `bool`, optional 970 Do a line search for the optimum step during minimization? 974 chi2: `lsst.jointcal.Chi2Statistic` 975 The final chi2 after the fit converges, or is forced to end. 980 Raised if the fitter fails with a non-finite value. 982 Raised if the fitter fails for some other reason; 983 log messages will provide further details. 985 dumpMatrixFile =
"%s_postinit" % name
if self.config.writeInitMatrix
else "" 986 for i
in range(max_steps):
987 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}" 988 result = fitter.minimize(whatToFit,
989 self.config.outlierRejectSigma,
990 doRankUpdate=doRankUpdate,
991 doLineSearch=doLineSearch,
992 dumpMatrixFile=dumpMatrixFile)
995 writeChi2Name=writeChi2Name)
997 if result == MinimizeResult.Converged:
999 self.log.debug(
"fit has converged - no more outliers - redo minimization " 1000 "one more time in case we have lost accuracy in rank update.")
1002 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1006 if chi2.chi2/chi2.ndof >= 4.0:
1007 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1010 elif result == MinimizeResult.Chi2Increased:
1011 self.log.warn(
"still some outliers but chi2 increases - retry")
1012 elif result == MinimizeResult.NonFinite:
1013 filename =
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName)
1015 fitter.saveChi2Contributions(filename)
1016 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 1017 raise FloatingPointError(msg.format(filename))
1018 elif result == MinimizeResult.Failed:
1019 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1021 raise RuntimeError(
"Unxepected return code from minimize().")
1023 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1027 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1029 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 1033 associations : `lsst.jointcal.Associations` 1034 The star/reference star associations to fit. 1035 model : `lsst.jointcal.AstrometryModel` 1036 The astrometric model that was fit. 1037 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1038 Dict of ccdImage identifiers to dataRefs that were fit. 1041 ccdImageList = associations.getCcdImageList()
1042 for ccdImage
in ccdImageList:
1044 ccd = ccdImage.ccdId
1045 visit = ccdImage.visit
1046 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1047 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
1048 skyWcs = model.makeSkyWcs(ccdImage)
1050 dataRef.put(skyWcs,
'jointcal_wcs')
1051 except pexExceptions.Exception
as e:
1052 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1055 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1057 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 1061 associations : `lsst.jointcal.Associations` 1062 The star/reference star associations to fit. 1063 model : `lsst.jointcal.PhotometryModel` 1064 The photoometric model that was fit. 1065 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1066 Dict of ccdImage identifiers to dataRefs that were fit. 1069 ccdImageList = associations.getCcdImageList()
1070 for ccdImage
in ccdImageList:
1072 ccd = ccdImage.ccdId
1073 visit = ccdImage.visit
1074 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1075 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1076 photoCalib = model.toPhotoCalib(ccdImage)
1078 dataRef.put(photoCalib,
'jointcal_photoCalib')
1079 except pexExceptions.Exception
as e:
1080 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def runDataRef(self, dataRefs, profile_jointcal=False)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", writeChi2Name=None)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, referenceSelector=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def __init__(self, butler=None, profile_jointcal=False, kwargs)