26 import astropy.units
as u
30 import lsst.pex.config
as pexConfig
37 from lsst.pipe.tasks.colorterms
import ColortermLibrary
38 from lsst.verify
import Job, Measurement
43 from .dataIds
import PerTractCcdDataIdContainer
48 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
50 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
51 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
56 meas = Measurement(job.metrics[name], value)
57 job.measurements.insert(meas)
61 """Subclass of TaskRunner for jointcalTask 63 jointcalTask.runDataRef() takes a number of arguments, one of which is a list of dataRefs 64 extracted from the command line (whereas most CmdLineTasks' runDataRef methods take 65 single dataRef, are are called repeatedly). This class transforms the processed 66 arguments generated by the ArgumentParser into the arguments expected by 67 Jointcal.runDataRef(). 69 See pipeBase.TaskRunner for more information. 75 Return a list of tuples per tract, each containing (dataRefs, kwargs). 77 Jointcal operates on lists of dataRefs simultaneously. 79 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
80 kwargs[
'butler'] = parsedCmd.butler
84 for ref
in parsedCmd.id.refList:
85 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
87 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
95 Arguments for Task.runDataRef() 100 if self.doReturnResults is False: 102 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 104 if self.doReturnResults is True: 106 - ``result``: the result of calling jointcal.runDataRef() 107 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 112 dataRefList, kwargs = args
113 butler = kwargs.pop(
'butler')
114 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
117 result = task.runDataRef(dataRefList, **kwargs)
118 exitStatus = result.exitStatus
119 job_path = butler.get(
'verify_job_filename')
120 result.job.write(job_path[0])
121 except Exception
as e:
126 eName = type(e).__name__
127 tract = dataRefList[0].dataId[
'tract']
128 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
131 kwargs[
'butler'] = butler
132 if self.doReturnResults:
133 return pipeBase.Struct(result=result, exitStatus=exitStatus)
135 return pipeBase.Struct(exitStatus=exitStatus)
139 """Configuration for JointcalTask""" 141 doAstrometry = pexConfig.Field(
142 doc=
"Fit astrometry and write the fitted result.",
146 doPhotometry = pexConfig.Field(
147 doc=
"Fit photometry and write the fitted result.",
151 coaddName = pexConfig.Field(
152 doc=
"Type of coadd, typically deep or goodSeeing",
156 positionErrorPedestal = pexConfig.Field(
157 doc=
"Systematic term to apply to the measured position error (pixels)",
161 photometryErrorPedestal = pexConfig.Field(
162 doc=
"Systematic term to apply to the measured error on flux or magnitude as a " 163 "fraction of source flux or magnitude delta (e.g. 0.05 is 5% of flux or +50 millimag).",
168 matchCut = pexConfig.Field(
169 doc=
"Matching radius between fitted and reference stars (arcseconds)",
173 minMeasurements = pexConfig.Field(
174 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
178 minMeasuredStarsPerCcd = pexConfig.Field(
179 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
183 minRefStarsPerCcd = pexConfig.Field(
184 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
188 allowLineSearch = pexConfig.Field(
189 doc=
"Allow a line search during minimization, if it is reasonable for the model" 190 " (models with a significant non-linear component, e.g. constrainedPhotometry).",
194 astrometrySimpleOrder = pexConfig.Field(
195 doc=
"Polynomial order for fitting the simple astrometry model.",
199 astrometryChipOrder = pexConfig.Field(
200 doc=
"Order of the per-chip transform for the constrained astrometry model.",
204 astrometryVisitOrder = pexConfig.Field(
205 doc=
"Order of the per-visit transform for the constrained astrometry model.",
209 useInputWcs = pexConfig.Field(
210 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
214 astrometryModel = pexConfig.ChoiceField(
215 doc=
"Type of model to fit to astrometry",
217 default=
"constrained",
218 allowed={
"simple":
"One polynomial per ccd",
219 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
221 photometryModel = pexConfig.ChoiceField(
222 doc=
"Type of model to fit to photometry",
224 default=
"constrainedMagnitude",
225 allowed={
"simpleFlux":
"One constant zeropoint per ccd and visit, fitting in flux space.",
226 "constrainedFlux":
"Constrained zeropoint per ccd, and one polynomial per visit," 227 " fitting in flux space.",
228 "simpleMagnitude":
"One constant zeropoint per ccd and visit," 229 " fitting in magnitude space.",
230 "constrainedMagnitude":
"Constrained zeropoint per ccd, and one polynomial per visit," 231 " fitting in magnitude space.",
234 applyColorTerms = pexConfig.Field(
235 doc=
"Apply photometric color terms to reference stars?" 236 "Requires that colorterms be set to a ColortermLibrary",
240 colorterms = pexConfig.ConfigField(
241 doc=
"Library of photometric reference catalog name to color term dict.",
242 dtype=ColortermLibrary,
244 photometryVisitOrder = pexConfig.Field(
245 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
249 photometryDoRankUpdate = pexConfig.Field(
250 doc=(
"Do the rank update step during minimization. " 251 "Skipping this can help deal with models that are too non-linear."),
255 astrometryDoRankUpdate = pexConfig.Field(
256 doc=(
"Do the rank update step during minimization (should not change the astrometry fit). " 257 "Skipping this can help deal with models that are too non-linear."),
261 outlierRejectSigma = pexConfig.Field(
262 doc=
"How many sigma to reject outliers at during minimization.",
266 maxPhotometrySteps = pexConfig.Field(
267 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
271 maxAstrometrySteps = pexConfig.Field(
272 doc=
"Maximum number of minimize iterations to take when fitting photometry.",
276 astrometryRefObjLoader = pexConfig.ConfigurableField(
277 target=LoadIndexedReferenceObjectsTask,
278 doc=
"Reference object loader for astrometric fit",
280 photometryRefObjLoader = pexConfig.ConfigurableField(
281 target=LoadIndexedReferenceObjectsTask,
282 doc=
"Reference object loader for photometric fit",
284 sourceSelector = sourceSelectorRegistry.makeField(
285 doc=
"How to select sources for cross-matching",
288 astrometryReferenceSelector = pexConfig.ConfigurableField(
289 target=ReferenceSourceSelectorTask,
290 doc=
"How to down-select the loaded astrometry reference catalog.",
292 photometryReferenceSelector = pexConfig.ConfigurableField(
293 target=ReferenceSourceSelectorTask,
294 doc=
"How to down-select the loaded photometry reference catalog.",
296 astrometryReferenceErr = pexConfig.Field(
297 doc=(
"Uncertainty on reference catalog coordinates [mas] to use in place of the `coord_*Err` fields. " 298 "If None, then raise an exception if the reference catalog is missing coordinate errors. " 299 "If specified, overrides any existing `coord_*Err` values."),
304 writeInitMatrix = pexConfig.Field(
306 doc=(
"Write the pre/post-initialization Hessian and gradient to text files, for debugging. " 307 "The output files will be of the form 'astrometry_preinit-mat.txt', in the current directory. " 308 "Note that these files are the dense versions of the matrix, and so may be very large."),
311 writeChi2FilesInitialFinal = pexConfig.Field(
313 doc=
"Write .csv files containing the contributions to chi2 for the initialization and final fit.",
316 writeChi2FilesOuterLoop = pexConfig.Field(
318 doc=
"Write .csv files containing the contributions to chi2 for the outer fit loop.",
321 writeInitialModel = pexConfig.Field(
323 doc=(
"Write the pre-initialization model to text files, for debugging." 324 " Output is written to `initial[Astro|Photo]metryModel.txt` in the current working directory."),
327 debugOutputPath = pexConfig.Field(
330 doc=(
"Path to write debug output files to. Used by " 331 "`writeInitialModel`, `writeChi2Files*`, `writeInitMatrix`.")
333 sourceFluxType = pexConfig.Field(
335 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
342 msg =
"applyColorTerms=True requires the `colorterms` field be set to a ColortermLibrary." 343 raise pexConfig.FieldValidationError(JointcalConfig.colorterms, self, msg)
345 msg = (
"Only doing astrometry, but Colorterms are not applied for astrometry;" 346 "applyColorTerms=True will be ignored.")
360 fluxField = f
"slot_{self.sourceFluxType}Flux_instFlux" 361 self.
sourceSelector[
'science'].signalToNoise.fluxField = fluxField
362 self.
sourceSelector[
'science'].signalToNoise.errField = fluxField +
"Err" 368 badFlags = [
'base_PixelFlags_flag_edge',
'base_PixelFlags_flag_saturated',
369 'base_PixelFlags_flag_interpolatedCenter',
'base_SdssCentroid_flag',
370 'base_PsfFlux_flag',
'base_PixelFlags_flag_suspectCenter']
375 """Write model to outfile.""" 376 with open(filename,
"w")
as file:
377 file.write(repr(model))
378 log.info(
"Wrote %s to file: %s", model, filename)
382 """Jointly astrometrically and photometrically calibrate a group of images.""" 384 ConfigClass = JointcalConfig
385 RunnerClass = JointcalRunner
386 _DefaultName =
"jointcal" 388 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
390 Instantiate a JointcalTask. 394 butler : `lsst.daf.persistence.Butler` 395 The butler is passed to the refObjLoader constructor in case it is 396 needed. Ignored if the refObjLoader argument provides a loader directly. 397 Used to initialize the astrometry and photometry refObjLoaders. 398 profile_jointcal : `bool` 399 Set to True to profile different stages of this jointcal run. 401 pipeBase.CmdLineTask.__init__(self, **kwargs)
403 self.makeSubtask(
"sourceSelector")
404 if self.config.doAstrometry:
405 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
406 self.makeSubtask(
"astrometryReferenceSelector")
409 if self.config.doPhotometry:
410 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
411 self.makeSubtask(
"photometryReferenceSelector")
416 self.
job = Job.load_metrics_package(subset=
'jointcal')
421 def _getMetadataName(self):
425 def _makeArgumentParser(cls):
426 """Create an argument parser""" 428 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
429 help=
"Profile steps of jointcal separately.")
430 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
431 ContainerClass=PerTractCcdDataIdContainer)
434 def _build_ccdImage(self, dataRef, associations, jointcalControl):
436 Extract the necessary things from this dataRef to add a new ccdImage. 440 dataRef : `lsst.daf.persistence.ButlerDataRef` 441 DataRef to extract info from. 442 associations : `lsst.jointcal.Associations` 443 Object to add the info to, to construct a new CcdImage 444 jointcalControl : `jointcal.JointcalControl` 445 Control object for associations management 451 The TAN WCS of this image, read from the calexp 452 (`lsst.afw.geom.SkyWcs`). 454 A key to identify this dataRef by its visit and ccd ids 457 This calexp's filter (`str`). 459 if "visit" in dataRef.dataId.keys():
460 visit = dataRef.dataId[
"visit"]
462 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
464 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
466 visitInfo = dataRef.get(
'calexp_visitInfo')
467 detector = dataRef.get(
'calexp_detector')
468 ccdId = detector.getId()
469 photoCalib = dataRef.get(
'calexp_photoCalib')
470 tanWcs = dataRef.get(
'calexp_wcs')
471 bbox = dataRef.get(
'calexp_bbox')
472 filt = dataRef.get(
'calexp_filter')
473 filterName = filt.getName()
475 goodSrc = self.sourceSelector.run(src)
477 if len(goodSrc.sourceCat) == 0:
478 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
480 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
481 associations.createCcdImage(goodSrc.sourceCat,
492 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
493 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
494 return Result(tanWcs, Key(visit, ccdId), filterName)
496 def _getDebugPath(self, filename):
497 """Constructs a path to filename using the configured debug path. 499 return os.path.join(self.config.debugOutputPath, filename)
504 Jointly calibrate the astrometry and photometry across a set of images. 508 dataRefs : `list` of `lsst.daf.persistence.ButlerDataRef` 509 List of data references to the exposures to be fit. 510 profile_jointcal : `bool` 511 Profile the individual steps of jointcal. 515 result : `lsst.pipe.base.Struct` 516 Struct of metadata from the fit, containing: 519 The provided data references that were fit (with updated WCSs) 521 The original WCS from each dataRef 523 Dictionary of internally-computed metrics for testing/validation. 525 if len(dataRefs) == 0:
526 raise ValueError(
'Need a non-empty list of data references!')
530 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
534 visit_ccd_to_dataRef = {}
537 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 538 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
541 camera = dataRefs[0].get(
'camera', immediate=
True)
545 oldWcsList.append(result.wcs)
546 visit_ccd_to_dataRef[result.key] = ref
547 filters.append(result.filter)
548 filters = collections.Counter(filters)
550 associations.computeCommonTangentPoint()
552 boundingCircle = associations.computeBoundingCircle()
554 radius =
lsst.geom.Angle(boundingCircle.getOpeningAngle().asRadians(), lsst.geom.radians)
557 defaultFilter = filters.most_common(1)[0][0]
558 self.log.debug(
"Using %s band for reference flux", defaultFilter)
561 tract = dataRefs[0].dataId[
'tract']
563 if self.config.doAstrometry:
567 referenceSelector=self.astrometryReferenceSelector,
569 profile_jointcal=profile_jointcal,
575 if self.config.doPhotometry:
579 referenceSelector=self.photometryReferenceSelector,
581 profile_jointcal=profile_jointcal,
584 reject_bad_fluxes=
True)
589 return pipeBase.Struct(dataRefs=dataRefs,
590 oldWcsList=oldWcsList,
594 defaultFilter=defaultFilter,
595 exitStatus=exitStatus)
597 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
599 tract="", profile_jointcal=False, match_cut=3.0,
600 reject_bad_fluxes=False, *,
601 name="", refObjLoader=None, referenceSelector=None,
603 """Load reference catalog, perform the fit, and return the result. 607 associations : `lsst.jointcal.Associations` 608 The star/reference star associations to fit. 609 defaultFilter : `str` 610 filter to load from reference catalog. 611 center : `lsst.geom.SpherePoint` 612 ICRS center of field to load from reference catalog. 613 radius : `lsst.geom.Angle` 614 On-sky radius to load from reference catalog. 616 Name of thing being fit: "astrometry" or "photometry". 617 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 618 Reference object loader to use to load a reference catalog. 619 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 620 Selector to use to pick objects from the loaded reference catalog. 621 fit_function : callable 622 Function to call to perform fit (takes Associations object). 623 filters : `list` [`str`], optional 624 List of filters to load from the reference catalog. 625 tract : `str`, optional 626 Name of tract currently being fit. 627 profile_jointcal : `bool`, optional 628 Separately profile the fitting step. 629 match_cut : `float`, optional 630 Radius in arcseconds to find cross-catalog matches to during 631 associations.associateCatalogs. 632 reject_bad_fluxes : `bool`, optional 633 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 637 result : `Photometry` or `Astrometry` 638 Result of `fit_function()` 640 self.log.info(
"====== Now processing %s...", name)
643 associations.associateCatalogs(match_cut)
645 associations.fittedStarListSize())
647 applyColorterms =
False if name.lower() ==
"astrometry" else self.config.applyColorTerms
649 center, radius, defaultFilter,
650 applyColorterms=applyColorterms)
652 if self.config.astrometryReferenceErr
is None:
653 refCoordErr = float(
'nan')
655 refCoordErr = self.config.astrometryReferenceErr
657 associations.collectRefStars(refCat,
658 self.config.matchCut*lsst.geom.arcseconds,
660 refCoordinateErr=refCoordErr,
661 rejectBadFluxes=reject_bad_fluxes)
663 associations.refStarListSize())
665 associations.prepareFittedStars(self.config.minMeasurements)
669 associations.nFittedStarsWithAssociatedRefStar())
671 associations.fittedStarListSize())
673 associations.nCcdImagesValidForFit())
675 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 676 dataName =
"{}_{}".format(tract, defaultFilter)
677 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
678 result = fit_function(associations, dataName)
681 if self.config.writeChi2FilesInitialFinal:
682 baseName = self.
_getDebugPath(f
"{name}_final_chi2-{dataName}")
683 result.fit.saveChi2Contributions(baseName+
"{type}")
684 self.log.info(
"Wrote chi2 contributions files: %s", baseName)
688 def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName,
689 applyColorterms=False):
690 """Load the necessary reference catalog sources, convert fluxes to 691 correct units, and apply color term corrections if requested. 695 refObjLoader : `lsst.meas.algorithms.LoadReferenceObjectsTask` 696 The reference catalog loader to use to get the data. 697 referenceSelector : `lsst.meas.algorithms.ReferenceSourceSelectorTask` 698 Source selector to apply to loaded reference catalog. 699 center : `lsst.geom.SpherePoint` 700 The center around which to load sources. 701 radius : `lsst.geom.Angle` 702 The radius around ``center`` to load sources in. 704 The name of the camera filter to load fluxes for. 705 applyColorterms : `bool` 706 Apply colorterm corrections to the refcat for ``filterName``? 710 refCat : `lsst.afw.table.SimpleCatalog` 711 The loaded reference catalog. 713 The name of the reference catalog flux field appropriate for ``filterName``. 715 skyCircle = refObjLoader.loadSkyCircle(center,
719 selected = referenceSelector.run(skyCircle.refCat)
721 if not selected.sourceCat.isContiguous():
722 refCat = selected.sourceCat.copy(deep=
True)
724 refCat = selected.sourceCat
726 if self.config.astrometryReferenceErr
is None and 'coord_raErr' not in refCat.schema:
727 msg = (
"Reference catalog does not contain coordinate errors, " 728 "and config.astrometryReferenceErr not supplied.")
729 raise pexConfig.FieldValidationError(JointcalConfig.astrometryReferenceErr,
733 if self.config.astrometryReferenceErr
is not None and 'coord_raErr' in refCat.schema:
734 self.log.warn(
"Overriding reference catalog coordinate errors with %f/coordinate [mas]",
735 self.config.astrometryReferenceErr)
739 refCatName = refObjLoader.ref_dataset_name
740 except AttributeError:
742 raise RuntimeError(
"Cannot perform colorterm corrections with a.net refcats.")
743 self.log.info(
"Applying color terms for filterName=%r reference catalog=%s",
744 filterName, refCatName)
745 colorterm = self.config.colorterms.getColorterm(
746 filterName=filterName, photoCatName=refCatName, doRaise=
True)
748 refMag, refMagErr = colorterm.getCorrectedMagnitudes(refCat, filterName)
749 refCat[skyCircle.fluxField] = u.Magnitude(refMag, u.ABmag).to_value(u.nJy)
751 refCat[skyCircle.fluxField+
'Err'] = fluxErrFromABMagErr(refMagErr, refMag) * 1e9
753 return refCat, skyCircle.fluxField
755 def _check_star_lists(self, associations, name):
757 if associations.nCcdImagesValidForFit() == 0:
758 raise RuntimeError(
'No images in the ccdImageList!')
759 if associations.fittedStarListSize() == 0:
760 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
761 if associations.refStarListSize() == 0:
762 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
764 def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model",
766 """Compute chi2, log it, validate the model, and return chi2. 770 associations : `lsst.jointcal.Associations` 771 The star/reference star associations to fit. 772 fit : `lsst.jointcal.FitterBase` 773 The fitter to use for minimization. 774 model : `lsst.jointcal.Model` 776 chi2Label : str, optional 777 Label to describe the chi2 (e.g. "Initialized", "Final"). 778 writeChi2Name : `str`, optional 779 Filename prefix to write the chi2 contributions to. 780 Do not supply an extension: an appropriate one will be added. 784 chi2: `lsst.jointcal.Chi2Accumulator` 785 The chi2 object for the current fitter and model. 790 Raised if chi2 is infinite or NaN. 792 Raised if the model is not valid. 794 if writeChi2Name
is not None:
796 fit.saveChi2Contributions(fullpath+
"{type}")
797 self.log.info(
"Wrote chi2 contributions files: %s", fullpath)
799 chi2 = fit.computeChi2()
800 self.log.info(
"%s %s", chi2Label, chi2)
802 if not np.isfinite(chi2.chi2):
803 raise FloatingPointError(f
'{chi2Label} chi2 is invalid: {chi2}')
804 if not model.validate(associations.getCcdImageList(), chi2.ndof):
805 raise ValueError(
"Model is not valid: check log messages for warnings.")
808 def _fit_photometry(self, associations, dataName=None):
810 Fit the photometric data. 814 associations : `lsst.jointcal.Associations` 815 The star/reference star associations to fit. 817 Name of the data being processed (e.g. "1234_HSC-Y"), for 818 identifying debugging files. 822 fit_result : `namedtuple` 823 fit : `lsst.jointcal.PhotometryFit` 824 The photometric fitter used to perform the fit. 825 model : `lsst.jointcal.PhotometryModel` 826 The photometric model that was fit. 828 self.log.info(
"=== Starting photometric fitting...")
831 if self.config.photometryModel ==
"constrainedFlux":
834 visitOrder=self.config.photometryVisitOrder,
835 errorPedestal=self.config.photometryErrorPedestal)
837 doLineSearch = self.config.allowLineSearch
838 elif self.config.photometryModel ==
"constrainedMagnitude":
841 visitOrder=self.config.photometryVisitOrder,
842 errorPedestal=self.config.photometryErrorPedestal)
844 doLineSearch = self.config.allowLineSearch
845 elif self.config.photometryModel ==
"simpleFlux":
847 errorPedestal=self.config.photometryErrorPedestal)
849 elif self.config.photometryModel ==
"simpleMagnitude":
851 errorPedestal=self.config.photometryErrorPedestal)
857 if self.config.writeChi2FilesInitialFinal:
858 baseName = f
"photometry_initial_chi2-{dataName}" 861 if self.config.writeInitialModel:
866 def getChi2Name(whatToFit):
867 if self.config.writeChi2FilesOuterLoop:
868 return f
"photometry_init-%s_chi2-{dataName}" % whatToFit
874 dumpMatrixFile = self.
_getDebugPath(
"photometry_preinit")
if self.config.writeInitMatrix
else "" 875 if self.config.photometryModel.startswith(
"constrained"):
878 fit.minimize(
"ModelVisit", dumpMatrixFile=dumpMatrixFile)
879 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"ModelVisit"))
882 fit.minimize(
"Model", doLineSearch=doLineSearch, dumpMatrixFile=dumpMatrixFile)
885 fit.minimize(
"Fluxes")
888 fit.minimize(
"Model Fluxes", doLineSearch=doLineSearch)
890 writeChi2Name=getChi2Name(
"ModelFluxes"))
892 model.freezeErrorTransform()
893 self.log.debug(
"Photometry error scales are frozen.")
897 self.config.maxPhotometrySteps,
900 doRankUpdate=self.config.photometryDoRankUpdate,
901 doLineSearch=doLineSearch,
908 def _fit_astrometry(self, associations, dataName=None):
910 Fit the astrometric data. 914 associations : `lsst.jointcal.Associations` 915 The star/reference star associations to fit. 917 Name of the data being processed (e.g. "1234_HSC-Y"), for 918 identifying debugging files. 922 fit_result : `namedtuple` 923 fit : `lsst.jointcal.AstrometryFit` 924 The astrometric fitter used to perform the fit. 925 model : `lsst.jointcal.AstrometryModel` 926 The astrometric model that was fit. 927 sky_to_tan_projection : `lsst.jointcal.ProjectionHandler` 928 The model for the sky to tangent plane projection that was used in the fit. 931 self.log.info(
"=== Starting astrometric fitting...")
933 associations.deprojectFittedStars()
940 if self.config.astrometryModel ==
"constrained":
942 sky_to_tan_projection,
943 chipOrder=self.config.astrometryChipOrder,
944 visitOrder=self.config.astrometryVisitOrder)
945 elif self.config.astrometryModel ==
"simple":
947 sky_to_tan_projection,
948 self.config.useInputWcs,
950 order=self.config.astrometrySimpleOrder)
955 if self.config.writeChi2FilesInitialFinal:
956 baseName = f
"astrometry_initial_chi2-{dataName}" 959 if self.config.writeInitialModel:
964 def getChi2Name(whatToFit):
965 if self.config.writeChi2FilesOuterLoop:
966 return f
"astrometry_init-%s_chi2-{dataName}" % whatToFit
970 dumpMatrixFile = self.
_getDebugPath(
"astrometry_preinit")
if self.config.writeInitMatrix
else "" 973 if self.config.astrometryModel ==
"constrained":
974 fit.minimize(
"DistortionsVisit", dumpMatrixFile=dumpMatrixFile)
975 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"DistortionsVisit"))
978 fit.minimize(
"Distortions", dumpMatrixFile=dumpMatrixFile)
979 self.
_logChi2AndValidate(associations, fit, model, writeChi2Name=getChi2Name(
"Distortions"))
981 fit.minimize(
"Positions")
984 fit.minimize(
"Distortions Positions")
986 writeChi2Name=getChi2Name(
"DistortionsPositions"))
990 self.config.maxAstrometrySteps,
992 "Distortions Positions",
993 doRankUpdate=self.config.astrometryDoRankUpdate,
999 return Astrometry(fit, model, sky_to_tan_projection)
1001 def _check_stars(self, associations):
1002 """Count measured and reference stars per ccd and warn/log them.""" 1003 for ccdImage
in associations.getCcdImageList():
1004 nMeasuredStars, nRefStars = ccdImage.countStars()
1005 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
1006 ccdImage.getName(), nMeasuredStars, nRefStars)
1007 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
1008 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
1009 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
1010 if nRefStars < self.config.minRefStarsPerCcd:
1011 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
1012 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
1014 def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit,
1017 doLineSearch=False):
1018 """Run fitter.minimize up to max_steps times, returning the final chi2. 1022 associations : `lsst.jointcal.Associations` 1023 The star/reference star associations to fit. 1024 fitter : `lsst.jointcal.FitterBase` 1025 The fitter to use for minimization. 1027 Maximum number of steps to run outlier rejection before declaring 1028 convergence failure. 1029 name : {'photometry' or 'astrometry'} 1030 What type of data are we fitting (for logs and debugging files). 1032 Passed to ``fitter.minimize()`` to define the parameters to fit. 1033 dataName : `str`, optional 1034 Descriptive name for this dataset (e.g. tract and filter), 1036 doRankUpdate : `bool`, optional 1037 Do an Eigen rank update during minimization, or recompute the full 1038 matrix and gradient? 1039 doLineSearch : `bool`, optional 1040 Do a line search for the optimum step during minimization? 1044 chi2: `lsst.jointcal.Chi2Statistic` 1045 The final chi2 after the fit converges, or is forced to end. 1050 Raised if the fitter fails with a non-finite value. 1052 Raised if the fitter fails for some other reason; 1053 log messages will provide further details. 1055 dumpMatrixFile = self.
_getDebugPath(f
"{name}_postinit")
if self.config.writeInitMatrix
else "" 1056 for i
in range(max_steps):
1057 if self.config.writeChi2FilesOuterLoop:
1058 writeChi2Name = f
"{name}_iterate_{i}_chi2-{dataName}" 1060 writeChi2Name =
None 1061 result = fitter.minimize(whatToFit,
1062 self.config.outlierRejectSigma,
1063 doRankUpdate=doRankUpdate,
1064 doLineSearch=doLineSearch,
1065 dumpMatrixFile=dumpMatrixFile)
1068 writeChi2Name=writeChi2Name)
1070 if result == MinimizeResult.Converged:
1072 self.log.debug(
"fit has converged - no more outliers - redo minimization " 1073 "one more time in case we have lost accuracy in rank update.")
1075 result = fitter.minimize(whatToFit, self.config.outlierRejectSigma)
1079 if chi2.chi2/chi2.ndof >= 4.0:
1080 self.log.error(
"Potentially bad fit: High chi-squared/ndof.")
1083 elif result == MinimizeResult.Chi2Increased:
1084 self.log.warn(
"still some outliers but chi2 increases - retry")
1085 elif result == MinimizeResult.NonFinite:
1086 filename = self.
_getDebugPath(
"{}_failure-nonfinite_chi2-{}.csv".format(name, dataName))
1088 fitter.saveChi2Contributions(filename+
"{type}")
1089 msg =
"Nonfinite value in chi2 minimization, cannot complete fit. Dumped star tables to: {}" 1090 raise FloatingPointError(msg.format(filename))
1091 elif result == MinimizeResult.Failed:
1092 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
1094 raise RuntimeError(
"Unxepected return code from minimize().")
1096 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
1100 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
1102 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 1106 associations : `lsst.jointcal.Associations` 1107 The star/reference star associations to fit. 1108 model : `lsst.jointcal.AstrometryModel` 1109 The astrometric model that was fit. 1110 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1111 Dict of ccdImage identifiers to dataRefs that were fit. 1114 ccdImageList = associations.getCcdImageList()
1115 for ccdImage
in ccdImageList:
1117 ccd = ccdImage.ccdId
1118 visit = ccdImage.visit
1119 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1120 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
1121 skyWcs = model.makeSkyWcs(ccdImage)
1123 dataRef.put(skyWcs,
'jointcal_wcs')
1124 except pexExceptions.Exception
as e:
1125 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
1128 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
1130 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 1134 associations : `lsst.jointcal.Associations` 1135 The star/reference star associations to fit. 1136 model : `lsst.jointcal.PhotometryModel` 1137 The photoometric model that was fit. 1138 visit_ccd_to_dataRef : `dict` of Key: `lsst.daf.persistence.ButlerDataRef` 1139 Dict of ccdImage identifiers to dataRefs that were fit. 1142 ccdImageList = associations.getCcdImageList()
1143 for ccdImage
in ccdImageList:
1145 ccd = ccdImage.ccdId
1146 visit = ccdImage.visit
1147 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
1148 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
1149 photoCalib = model.toPhotoCalib(ccdImage)
1151 dataRef.put(photoCalib,
'jointcal_photoCalib')
1152 except pexExceptions.Exception
as e:
1153 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def runDataRef(self, dataRefs, profile_jointcal=False)
def _load_reference_catalog(self, refObjLoader, referenceSelector, center, radius, filterName, applyColorterms=False)
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, filters=[], tract="", profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False, name="", refObjLoader=None, referenceSelector=None, fit_function=None)
def _fit_photometry(self, associations, dataName=None)
def writeModel(model, filename, log)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
def _logChi2AndValidate(self, associations, fit, model, chi2Label="Model", writeChi2Name=None)
A model where there is one independent transform per CcdImage.
def _iterate_fit(self, associations, fitter, max_steps, name, whatToFit, dataName="", doRankUpdate=True, doLineSearch=False)
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _check_stars(self, associations)
def _getDebugPath(self, filename)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
def add_measurement(job, name, value)
A multi-component model, fitting mappings for sensors and visits simultaneously.
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def __init__(self, butler=None, profile_jointcal=False, kwargs)