3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
18 from lsst.verify
import Job, Measurement
23 from .dataIds
import PerTractCcdDataIdContainer
28 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
30 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
31 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
36 meas = Measurement(job.metrics[name], value)
37 job.measurements.insert(meas)
41 """Subclass of TaskRunner for jointcalTask 43 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 44 extracted from the command line (whereas most CmdLineTasks' run methods take 45 single dataRef, are are called repeatedly). This class transforms the processed 46 arguments generated by the ArgumentParser into the arguments expected by 49 See pipeBase.TaskRunner for more information. 55 Return a list of tuples per tract, each containing (dataRefs, kwargs). 57 Jointcal operates on lists of dataRefs simultaneously. 59 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
60 kwargs[
'butler'] = parsedCmd.butler
64 for ref
in parsedCmd.id.refList:
65 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
67 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
75 Arguments for Task.run() 80 if self.doReturnResults is False: 82 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 84 if self.doReturnResults is True: 86 - ``result``: the result of calling jointcal.run() 87 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 92 dataRefList, kwargs = args
93 butler = kwargs.pop(
'butler')
94 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
97 result = task.run(dataRefList, **kwargs)
98 exitStatus = result.exitStatus
99 job_path = butler.get(
'verify_job_filename')
100 result.job.write(job_path[0])
101 except Exception
as e:
106 eName = type(e).__name__
107 tract = dataRefList[0].dataId[
'tract']
108 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
110 if self.doReturnResults:
111 return pipeBase.Struct(result=result, exitStatus=exitStatus)
113 return pipeBase.Struct(exitStatus=exitStatus)
117 """Config for JointcalTask""" 119 doAstrometry = pexConfig.Field(
120 doc=
"Fit astrometry and write the fitted result.",
124 doPhotometry = pexConfig.Field(
125 doc=
"Fit photometry and write the fitted result.",
129 coaddName = pexConfig.Field(
130 doc=
"Type of coadd, typically deep or goodSeeing",
134 posError = pexConfig.Field(
135 doc=
"Constant term for error on position (in pixel unit)",
140 matchCut = pexConfig.Field(
141 doc=
"Matching radius between fitted and reference stars (arcseconds)",
145 minMeasurements = pexConfig.Field(
146 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
150 astrometrySimpleDegree = pexConfig.Field(
151 doc=
"Polynomial degree for fitting the simple astrometry model.",
155 astrometryChipDegree = pexConfig.Field(
156 doc=
"Degree of the per-chip transform for the constrained astrometry model.",
160 astrometryVisitDegree = pexConfig.Field(
161 doc=
"Degree of the per-visit transform for the constrained astrometry model.",
165 useInputWcs = pexConfig.Field(
166 doc=
"Use the input calexp WCSs to initialize the astrometryModel.",
170 astrometryModel = pexConfig.ChoiceField(
171 doc=
"Type of model to fit to astrometry",
173 default=
"simplePoly",
174 allowed={
"simplePoly":
"One polynomial per ccd",
175 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
177 photometryModel = pexConfig.ChoiceField(
178 doc=
"Type of model to fit to photometry",
181 allowed={
"simple":
"One constant zeropoint per ccd and visit",
182 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
184 photometryVisitDegree = pexConfig.Field(
185 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
189 astrometryRefObjLoader = pexConfig.ConfigurableField(
190 target=LoadIndexedReferenceObjectsTask,
191 doc=
"Reference object loader for astrometric fit",
193 photometryRefObjLoader = pexConfig.ConfigurableField(
194 target=LoadIndexedReferenceObjectsTask,
195 doc=
"Reference object loader for photometric fit",
197 sourceSelector = sourceSelectorRegistry.makeField(
198 doc=
"How to select sources for cross-matching",
201 writeChi2ContributionFiles = pexConfig.Field(
203 doc=
"Write initial/final fit files containing the contributions to chi2.",
209 sourceSelector.setDefaults()
211 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
213 sourceSelector.sourceFluxType =
'Calib' 217 """Jointly astrometrically and photometrically calibrate a group of images.""" 219 ConfigClass = JointcalConfig
220 RunnerClass = JointcalRunner
221 _DefaultName =
"jointcal" 223 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
225 Instantiate a JointcalTask. 229 butler : lsst.daf.persistence.Butler 230 The butler is passed to the refObjLoader constructor in case it is 231 needed. Ignored if the refObjLoader argument provides a loader directly. 232 Used to initialize the astrometry and photometry refObjLoaders. 233 profile_jointcal : bool 234 set to True to profile different stages of this jointcal run. 236 pipeBase.CmdLineTask.__init__(self, **kwargs)
238 self.makeSubtask(
"sourceSelector")
239 if self.config.doAstrometry:
240 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
241 if self.config.doPhotometry:
242 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
245 self.
job = Job.load_metrics_package(subset=
'jointcal')
249 def _getConfigName(self):
252 def _getMetadataName(self):
256 def _makeArgumentParser(cls):
257 """Create an argument parser""" 259 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
260 help=
"Profile steps of jointcal separately.")
261 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
262 ContainerClass=PerTractCcdDataIdContainer)
265 def _build_ccdImage(self, dataRef, associations, jointcalControl):
267 Extract the necessary things from this dataRef to add a new ccdImage. 271 dataRef : lsst.daf.persistence.ButlerDataRef 272 dataRef to extract info from. 273 associations : lsst.jointcal.Associations 274 object to add the info to, to construct a new CcdImage 275 jointcalControl : jointcal.JointcalControl 276 control object for associations management 281 wcs : lsst.afw.geom.SkyWcs 282 the TAN WCS of this image, read from the calexp 284 a key to identify this dataRef by its visit and ccd ids 288 if "visit" in dataRef.dataId.keys():
289 visit = dataRef.dataId[
"visit"]
291 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
293 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
295 visitInfo = dataRef.get(
'calexp_visitInfo')
296 detector = dataRef.get(
'calexp_detector')
297 ccdId = detector.getId()
298 calib = dataRef.get(
'calexp_calib')
299 tanWcs = dataRef.get(
'calexp_wcs')
300 bbox = dataRef.get(
'calexp_bbox')
301 filt = dataRef.get(
'calexp_filter')
302 filterName = filt.getName()
303 fluxMag0 = calib.getFluxMag0()
304 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
306 goodSrc = self.sourceSelector.selectSources(src)
308 if len(goodSrc.sourceCat) == 0:
309 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
311 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
312 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
313 visit, ccdId, jointcalControl)
315 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
316 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
317 return Result(tanWcs, Key(visit, ccdId), filterName)
320 def run(self, dataRefs, profile_jointcal=False):
322 Jointly calibrate the astrometry and photometry across a set of images. 326 dataRefs : list of lsst.daf.persistence.ButlerDataRef 327 List of data references to the exposures to be fit. 328 profile_jointcal : bool 329 Profile the individual steps of jointcal. 335 * dataRefs: the provided data references that were fit (with updated WCSs) 336 * oldWcsList: the original WCS from each dataRef 337 * metrics: dictionary of internally-computed metrics for testing/validation. 339 if len(dataRefs) == 0:
340 raise ValueError(
'Need a non-empty list of data references!')
344 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
348 visit_ccd_to_dataRef = {}
351 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 352 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
355 camera = dataRefs[0].get(
'camera', immediate=
True)
359 oldWcsList.append(result.wcs)
360 visit_ccd_to_dataRef[result.key] = ref
361 filters.append(result.filter)
362 filters = collections.Counter(filters)
364 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
366 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition(afwGeom.degrees))
367 associations.setCommonTangentPoint(commonTangentPoint.getPosition(afwGeom.degrees))
372 bbox = associations.getRaDecBBox()
374 bboxCenter = bbox.getCenter()
376 bboxMax = bbox.getMax()
378 radius = center.separation(corner).asRadians()
383 raise RuntimeError(
"astrometry_net_data is not setup")
386 defaultFilter = filters.most_common(1)[0][0]
387 self.log.debug(
"Using %s band for reference flux", defaultFilter)
390 tract = dataRefs[0].dataId[
'tract']
392 if self.config.doAstrometry:
395 refObjLoader=self.astrometryRefObjLoader,
397 profile_jointcal=profile_jointcal,
403 if self.config.doPhotometry:
406 refObjLoader=self.photometryRefObjLoader,
408 profile_jointcal=profile_jointcal,
411 reject_bad_fluxes=
True)
416 return pipeBase.Struct(dataRefs=dataRefs,
417 oldWcsList=oldWcsList,
419 exitStatus=exitStatus)
421 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
422 name="", refObjLoader=None, filters=[], fit_function=None,
423 tract=None, profile_jointcal=False, match_cut=3.0,
424 reject_bad_fluxes=False):
425 """Load reference catalog, perform the fit, and return the result. 429 associations : lsst.jointcal.Associations 430 The star/reference star associations to fit. 432 filter to load from reference catalog. 433 center : lsst.afw.geom.SpherePoint 434 ICRS center of field to load from reference catalog. 435 radius : lsst.afw.geom.Angle 436 On-sky radius to load from reference catalog. 438 Name of thing being fit: "Astrometry" or "Photometry". 439 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 440 Reference object loader to load from for fit. 441 filters : list of str, optional 442 List of filters to load from the reference catalog. 443 fit_function : function 444 function to call to perform fit (takes associations object). 446 Name of tract currently being fit. 447 profile_jointcal : bool, optional 448 Separately profile the fitting step. 449 match_cut : float, optional 450 Radius in arcseconds to find cross-catalog matches to during 451 associations.associateCatalogs. 452 reject_bad_fluxes : bool, optional 453 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 457 Result of `fit_function()` 459 self.log.info(
"====== Now processing %s...", name)
462 associations.associateCatalogs(match_cut)
464 associations.fittedStarListSize())
466 skyCircle = refObjLoader.loadSkyCircle(center,
471 if not skyCircle.refCat.isContiguous():
472 refCat = skyCircle.refCat.copy(deep=
True)
474 refCat = skyCircle.refCat
481 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
482 refFluxes[filt] = refCat.get(filtKeys[0])
483 refFluxErrs[filt] = refCat.get(filtKeys[1])
485 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
486 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
488 associations.refStarListSize())
490 associations.prepareFittedStars(self.config.minMeasurements)
494 associations.nFittedStarsWithAssociatedRefStar())
496 associations.fittedStarListSize())
498 associations.nCcdImagesValidForFit())
500 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 501 dataName =
"{}_{}".format(tract, defaultFilter)
502 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
503 result = fit_function(associations, dataName)
506 if self.config.writeChi2ContributionFiles:
507 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
508 result.fit.saveChi2Contributions(baseName)
512 def _check_star_lists(self, associations, name):
514 if associations.nCcdImagesValidForFit() == 0:
515 raise RuntimeError(
'No images in the ccdImageList!')
516 if associations.fittedStarListSize() == 0:
517 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
518 if associations.refStarListSize() == 0:
519 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
521 def _fit_photometry(self, associations, dataName=None):
523 Fit the photometric data. 527 associations : lsst.jointcal.Associations 528 The star/reference star associations to fit. 530 Name of the data being processed (e.g. "1234_HSC-Y"), for 531 identifying debugging files. 536 fit : lsst.jointcal.PhotometryFit 537 The photometric fitter used to perform the fit. 538 model : lsst.jointcal.PhotometryModel 539 The photometric model that was fit. 541 self.log.info(
"=== Starting photometric fitting...")
544 if self.config.photometryModel ==
"constrained":
547 visitDegree=self.config.photometryVisitDegree)
548 elif self.config.photometryModel ==
"simple":
552 chi2 = fit.computeChi2()
555 if self.config.writeChi2ContributionFiles:
556 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
557 fit.saveChi2Contributions(baseName)
559 if not np.isfinite(chi2.chi2):
560 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
561 self.log.info(
"Initialized: %s", str(chi2))
562 fit.minimize(
"Model")
563 chi2 = fit.computeChi2()
564 self.log.info(str(chi2))
565 fit.minimize(
"Fluxes")
566 chi2 = fit.computeChi2()
567 self.log.info(str(chi2))
568 fit.minimize(
"Model Fluxes")
569 chi2 = fit.computeChi2()
570 if not np.isfinite(chi2.chi2):
571 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
572 self.log.info(
"Fit prepared with %s", str(chi2))
574 model.freezeErrorTransform()
575 self.log.debug(
"Photometry error scales are frozen.")
577 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
583 def _fit_astrometry(self, associations, dataName=None):
585 Fit the astrometric data. 589 associations : lsst.jointcal.Associations 590 The star/reference star associations to fit. 592 Name of the data being processed (e.g. "1234_HSC-Y"), for 593 identifying debugging files. 598 fit : lsst.jointcal.AstrometryFit 599 The astrometric fitter used to perform the fit. 600 model : lsst.jointcal.AstrometryModel 601 The astrometric model that was fit. 602 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 603 The model for the sky to tangent plane projection that was used in the fit. 606 self.log.info(
"=== Starting astrometric fitting...")
608 associations.deprojectFittedStars()
615 if self.config.astrometryModel ==
"constrainedPoly":
617 sky_to_tan_projection, self.config.useInputWcs, 0,
618 chipDegree=self.config.astrometryChipDegree,
619 visitDegree=self.config.astrometryVisitDegree)
620 elif self.config.astrometryModel ==
"simplePoly":
622 sky_to_tan_projection, self.config.useInputWcs, 0,
623 self.config.astrometrySimpleDegree)
626 chi2 = fit.computeChi2()
629 if self.config.writeChi2ContributionFiles:
630 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
631 fit.saveChi2Contributions(baseName)
633 if not np.isfinite(chi2.chi2):
634 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
635 self.log.info(
"Initialized: %s", str(chi2))
636 fit.minimize(
"Distortions")
637 chi2 = fit.computeChi2()
638 self.log.info(str(chi2))
639 fit.minimize(
"Positions")
640 chi2 = fit.computeChi2()
641 self.log.info(str(chi2))
642 fit.minimize(
"Distortions Positions")
643 chi2 = fit.computeChi2()
644 self.log.info(str(chi2))
645 if not np.isfinite(chi2.chi2):
646 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
647 self.log.info(
"Fit prepared with %s", str(chi2))
649 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
654 return Astrometry(fit, model, sky_to_tan_projection)
656 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
657 """Run fit.minimize up to max_steps times, returning the final chi2.""" 659 for i
in range(max_steps):
660 r = fit.minimize(whatToFit, 5)
661 chi2 = fit.computeChi2()
662 if not np.isfinite(chi2.chi2):
663 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
664 self.log.info(str(chi2))
665 if r == MinimizeResult.Converged:
666 self.log.debug(
"fit has converged - no more outliers - redo minimization " 667 "one more time in case we have lost accuracy in rank update.")
669 r = fit.minimize(whatToFit, 5)
670 chi2 = fit.computeChi2()
671 self.log.info(
"Fit completed with: %s", str(chi2))
673 elif r == MinimizeResult.Chi2Increased:
674 self.log.warn(
"still some ouliers but chi2 increases - retry")
675 elif r == MinimizeResult.Failed:
676 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
678 raise RuntimeError(
"Unxepected return code from minimize().")
680 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
684 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
686 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 690 associations : lsst.jointcal.Associations 691 The star/reference star associations to fit. 692 model : lsst.jointcal.AstrometryModel 693 The astrometric model that was fit. 694 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 695 dict of ccdImage identifiers to dataRefs that were fit 698 ccdImageList = associations.getCcdImageList()
699 for ccdImage
in ccdImageList:
702 visit = ccdImage.visit
703 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
704 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
705 tanSip = model.produceSipWcs(ccdImage)
708 dataRef.put(wcs,
'jointcal_wcs')
709 except pexExceptions.Exception
as e:
710 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
713 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
715 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 719 associations : lsst.jointcal.Associations 720 The star/reference star associations to fit. 721 model : lsst.jointcal.PhotometryModel 722 The photoometric model that was fit. 723 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 724 dict of ccdImage identifiers to dataRefs that were fit 727 ccdImageList = associations.getCcdImageList()
728 for ccdImage
in ccdImageList:
731 visit = ccdImage.visit
732 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
733 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
734 photoCalib = model.toPhotoCalib(ccdImage)
736 dataRef.put(photoCalib,
'jointcal_photoCalib')
737 except pexExceptions.Exception
as e:
738 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
std::shared_ptr< lsst::afw::geom::SkyWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
SpherePoint averageSpherePoint(std::vector< SpherePoint > const &coords)
def __init__(self, butler=None, profile_jointcal=False, kwargs)