13 from lsst.verify
import Job, Measurement
18 from .dataIds
import PerTractCcdDataIdContainer
23 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
25 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
26 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
31 meas = Measurement(job.metrics[name], value)
32 job.measurements.insert(meas)
36 """Subclass of TaskRunner for jointcalTask 38 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 39 extracted from the command line (whereas most CmdLineTasks' run methods take 40 single dataRef, are are called repeatedly). This class transforms the processed 41 arguments generated by the ArgumentParser into the arguments expected by 44 See pipeBase.TaskRunner for more information. 50 Return a list of tuples per tract, each containing (dataRefs, kwargs). 52 Jointcal operates on lists of dataRefs simultaneously. 54 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
55 kwargs[
'butler'] = parsedCmd.butler
59 for ref
in parsedCmd.id.refList:
60 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
62 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
70 Arguments for Task.run() 75 if self.doReturnResults is False: 77 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 79 if self.doReturnResults is True: 81 - ``result``: the result of calling jointcal.run() 82 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 87 dataRefList, kwargs = args
88 butler = kwargs.pop(
'butler')
89 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
92 result = task.run(dataRefList, **kwargs)
93 exitStatus = result.exitStatus
94 job_path = butler.get(
'verify_job_filename')
95 result.job.write(job_path[0])
96 except Exception
as e:
101 eName = type(e).__name__
102 tract = dataRefList[0].dataId[
'tract']
103 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
105 if self.doReturnResults:
106 return pipeBase.Struct(result=result, exitStatus=exitStatus)
108 return pipeBase.Struct(exitStatus=exitStatus)
112 """Config for JointcalTask""" 114 doAstrometry = pexConfig.Field(
115 doc=
"Fit astrometry and write the fitted result.",
119 doPhotometry = pexConfig.Field(
120 doc=
"Fit photometry and write the fitted result.",
124 coaddName = pexConfig.Field(
125 doc=
"Type of coadd, typically deep or goodSeeing",
129 posError = pexConfig.Field(
130 doc=
"Constant term for error on position (in pixel unit)",
135 matchCut = pexConfig.Field(
136 doc=
"Matching radius between fitted and reference stars (arcseconds)",
140 minMeasurements = pexConfig.Field(
141 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
145 minMeasuredStarsPerCcd = pexConfig.Field(
146 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
150 minRefStarsPerCcd = pexConfig.Field(
151 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
155 astrometrySimpleOrder = pexConfig.Field(
156 doc=
"Polynomial order for fitting the simple astrometry model.",
160 astrometryChipOrder = pexConfig.Field(
161 doc=
"Order of the per-chip transform for the constrained astrometry model.",
165 astrometryVisitOrder = pexConfig.Field(
166 doc=
"Order of the per-visit transform for the constrained astrometry model.",
170 useInputWcs = pexConfig.Field(
171 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
175 astrometryModel = pexConfig.ChoiceField(
176 doc=
"Type of model to fit to astrometry",
179 allowed={
"simple":
"One polynomial per ccd",
180 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
182 photometryModel = pexConfig.ChoiceField(
183 doc=
"Type of model to fit to photometry",
186 allowed={
"simple":
"One constant zeropoint per ccd and visit",
187 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
189 photometryVisitOrder = pexConfig.Field(
190 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
194 astrometryRefObjLoader = pexConfig.ConfigurableField(
195 target=LoadIndexedReferenceObjectsTask,
196 doc=
"Reference object loader for astrometric fit",
198 photometryRefObjLoader = pexConfig.ConfigurableField(
199 target=LoadIndexedReferenceObjectsTask,
200 doc=
"Reference object loader for photometric fit",
202 sourceSelector = sourceSelectorRegistry.makeField(
203 doc=
"How to select sources for cross-matching",
206 writeChi2ContributionFiles = pexConfig.Field(
208 doc=
"Write initial/final fit files containing the contributions to chi2.",
211 sourceFluxType = pexConfig.Field(
213 doc=
"Source flux field to use in source selection and to get fluxes from the catalog.",
219 sourceSelector.setDefaults()
221 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
227 """Jointly astrometrically and photometrically calibrate a group of images.""" 229 ConfigClass = JointcalConfig
230 RunnerClass = JointcalRunner
231 _DefaultName =
"jointcal" 233 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
235 Instantiate a JointcalTask. 239 butler : lsst.daf.persistence.Butler 240 The butler is passed to the refObjLoader constructor in case it is 241 needed. Ignored if the refObjLoader argument provides a loader directly. 242 Used to initialize the astrometry and photometry refObjLoaders. 243 profile_jointcal : bool 244 set to True to profile different stages of this jointcal run. 246 pipeBase.CmdLineTask.__init__(self, **kwargs)
248 self.makeSubtask(
"sourceSelector")
249 if self.config.doAstrometry:
250 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
251 if self.config.doPhotometry:
252 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
255 self.
job = Job.load_metrics_package(subset=
'jointcal')
259 def _getConfigName(self):
262 def _getMetadataName(self):
266 def _makeArgumentParser(cls):
267 """Create an argument parser""" 269 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
270 help=
"Profile steps of jointcal separately.")
271 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
272 ContainerClass=PerTractCcdDataIdContainer)
275 def _build_ccdImage(self, dataRef, associations, jointcalControl):
277 Extract the necessary things from this dataRef to add a new ccdImage. 281 dataRef : lsst.daf.persistence.ButlerDataRef 282 dataRef to extract info from. 283 associations : lsst.jointcal.Associations 284 object to add the info to, to construct a new CcdImage 285 jointcalControl : jointcal.JointcalControl 286 control object for associations management 291 wcs : lsst.afw.geom.SkyWcs 292 the TAN WCS of this image, read from the calexp 294 a key to identify this dataRef by its visit and ccd ids 298 if "visit" in dataRef.dataId.keys():
299 visit = dataRef.dataId[
"visit"]
301 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
303 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
305 visitInfo = dataRef.get(
'calexp_visitInfo')
306 detector = dataRef.get(
'calexp_detector')
307 ccdId = detector.getId()
308 calib = dataRef.get(
'calexp_calib')
309 tanWcs = dataRef.get(
'calexp_wcs')
310 bbox = dataRef.get(
'calexp_bbox')
311 filt = dataRef.get(
'calexp_filter')
312 filterName = filt.getName()
313 fluxMag0 = calib.getFluxMag0()
314 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
316 goodSrc = self.sourceSelector.
run(src)
318 if len(goodSrc.sourceCat) == 0:
319 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
321 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
322 associations.createCcdImage(goodSrc.sourceCat,
333 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
334 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
335 return Result(tanWcs, Key(visit, ccdId), filterName)
338 def run(self, dataRefs, profile_jointcal=False):
340 Jointly calibrate the astrometry and photometry across a set of images. 344 dataRefs : list of lsst.daf.persistence.ButlerDataRef 345 List of data references to the exposures to be fit. 346 profile_jointcal : bool 347 Profile the individual steps of jointcal. 353 * dataRefs: the provided data references that were fit (with updated WCSs) 354 * oldWcsList: the original WCS from each dataRef 355 * metrics: dictionary of internally-computed metrics for testing/validation. 357 if len(dataRefs) == 0:
358 raise ValueError(
'Need a non-empty list of data references!')
362 sourceFluxField =
"slot_%sFlux" % (self.config.sourceFluxType,)
366 visit_ccd_to_dataRef = {}
369 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 370 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
373 camera = dataRefs[0].get(
'camera', immediate=
True)
377 oldWcsList.append(result.wcs)
378 visit_ccd_to_dataRef[result.key] = ref
379 filters.append(result.filter)
380 filters = collections.Counter(filters)
382 associations.computeCommonTangentPoint()
387 bbox = associations.getRaDecBBox()
389 bboxCenter = bbox.getCenter()
390 center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
391 bboxMax = bbox.getMax()
392 corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
393 radius = center.separation(corner).asRadians()
398 raise RuntimeError(
"astrometry_net_data is not setup")
401 defaultFilter = filters.most_common(1)[0][0]
402 self.log.debug(
"Using %s band for reference flux", defaultFilter)
405 tract = dataRefs[0].dataId[
'tract']
407 if self.config.doAstrometry:
410 refObjLoader=self.astrometryRefObjLoader,
412 profile_jointcal=profile_jointcal,
418 if self.config.doPhotometry:
421 refObjLoader=self.photometryRefObjLoader,
423 profile_jointcal=profile_jointcal,
426 reject_bad_fluxes=
True)
431 return pipeBase.Struct(dataRefs=dataRefs,
432 oldWcsList=oldWcsList,
434 exitStatus=exitStatus)
436 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
437 name="", refObjLoader=None, filters=[], fit_function=None,
438 tract=None, profile_jointcal=False, match_cut=3.0,
439 reject_bad_fluxes=False):
440 """Load reference catalog, perform the fit, and return the result. 444 associations : lsst.jointcal.Associations 445 The star/reference star associations to fit. 447 filter to load from reference catalog. 448 center : lsst.afw.geom.SpherePoint 449 ICRS center of field to load from reference catalog. 450 radius : lsst.afw.geom.Angle 451 On-sky radius to load from reference catalog. 453 Name of thing being fit: "Astrometry" or "Photometry". 454 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 455 Reference object loader to load from for fit. 456 filters : list of str, optional 457 List of filters to load from the reference catalog. 458 fit_function : function 459 function to call to perform fit (takes associations object). 461 Name of tract currently being fit. 462 profile_jointcal : bool, optional 463 Separately profile the fitting step. 464 match_cut : float, optional 465 Radius in arcseconds to find cross-catalog matches to during 466 associations.associateCatalogs. 467 reject_bad_fluxes : bool, optional 468 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 472 Result of `fit_function()` 474 self.log.info(
"====== Now processing %s...", name)
477 associations.associateCatalogs(match_cut)
479 associations.fittedStarListSize())
481 skyCircle = refObjLoader.loadSkyCircle(center,
482 afwGeom.Angle(radius, afwGeom.radians),
486 if not skyCircle.refCat.isContiguous():
487 refCat = skyCircle.refCat.copy(deep=
True)
489 refCat = skyCircle.refCat
496 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
497 refFluxes[filt] = refCat.get(filtKeys[0])
498 refFluxErrs[filt] = refCat.get(filtKeys[1])
500 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
501 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
503 associations.refStarListSize())
505 associations.prepareFittedStars(self.config.minMeasurements)
509 associations.nFittedStarsWithAssociatedRefStar())
511 associations.fittedStarListSize())
513 associations.nCcdImagesValidForFit())
515 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 516 dataName =
"{}_{}".format(tract, defaultFilter)
517 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
518 result = fit_function(associations, dataName)
521 if self.config.writeChi2ContributionFiles:
522 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
523 result.fit.saveChi2Contributions(baseName)
527 def _check_star_lists(self, associations, name):
529 if associations.nCcdImagesValidForFit() == 0:
530 raise RuntimeError(
'No images in the ccdImageList!')
531 if associations.fittedStarListSize() == 0:
532 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
533 if associations.refStarListSize() == 0:
534 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
536 def _fit_photometry(self, associations, dataName=None):
538 Fit the photometric data. 542 associations : lsst.jointcal.Associations 543 The star/reference star associations to fit. 545 Name of the data being processed (e.g. "1234_HSC-Y"), for 546 identifying debugging files. 551 fit : lsst.jointcal.PhotometryFit 552 The photometric fitter used to perform the fit. 553 model : lsst.jointcal.PhotometryModel 554 The photometric model that was fit. 556 self.log.info(
"=== Starting photometric fitting...")
559 if self.config.photometryModel ==
"constrained":
562 visitOrder=self.config.photometryVisitOrder)
563 elif self.config.photometryModel ==
"simple":
567 chi2 = fit.computeChi2()
570 if self.config.writeChi2ContributionFiles:
571 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
572 fit.saveChi2Contributions(baseName)
574 if not np.isfinite(chi2.chi2):
575 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
576 self.log.info(
"Initialized: %s", str(chi2))
579 if self.config.photometryModel ==
"constrained":
581 fit.minimize(
"ModelVisit")
582 chi2 = fit.computeChi2()
583 self.log.info(str(chi2))
584 fit.minimize(
"Model")
585 chi2 = fit.computeChi2()
586 self.log.info(str(chi2))
587 fit.minimize(
"Fluxes")
588 chi2 = fit.computeChi2()
589 self.log.info(str(chi2))
590 fit.minimize(
"Model Fluxes")
591 chi2 = fit.computeChi2()
592 if not np.isfinite(chi2.chi2):
593 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
594 self.log.info(
"Fit prepared with %s", str(chi2))
596 model.freezeErrorTransform()
597 self.log.debug(
"Photometry error scales are frozen.")
599 chi2 = self.
_iterate_fit(associations, fit, model, 20,
"photometry",
"Model Fluxes")
605 def _fit_astrometry(self, associations, dataName=None):
607 Fit the astrometric data. 611 associations : lsst.jointcal.Associations 612 The star/reference star associations to fit. 614 Name of the data being processed (e.g. "1234_HSC-Y"), for 615 identifying debugging files. 620 fit : lsst.jointcal.AstrometryFit 621 The astrometric fitter used to perform the fit. 622 model : lsst.jointcal.AstrometryModel 623 The astrometric model that was fit. 624 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 625 The model for the sky to tangent plane projection that was used in the fit. 628 self.log.info(
"=== Starting astrometric fitting...")
630 associations.deprojectFittedStars()
637 if self.config.astrometryModel ==
"constrained":
639 sky_to_tan_projection,
640 chipOrder=self.config.astrometryChipOrder,
641 visitOrder=self.config.astrometryVisitOrder)
642 elif self.config.astrometryModel ==
"simple":
644 sky_to_tan_projection,
645 self.config.useInputWcs,
647 order=self.config.astrometrySimpleOrder)
650 chi2 = fit.computeChi2()
653 if self.config.writeChi2ContributionFiles:
654 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
655 fit.saveChi2Contributions(baseName)
657 if not np.isfinite(chi2.chi2):
658 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
659 self.log.info(
"Initialized: %s", str(chi2))
662 if self.config.astrometryModel ==
"constrained":
663 fit.minimize(
"DistortionsVisit")
664 chi2 = fit.computeChi2()
665 self.log.info(str(chi2))
666 fit.minimize(
"Distortions")
667 chi2 = fit.computeChi2()
668 self.log.info(str(chi2))
669 fit.minimize(
"Positions")
670 chi2 = fit.computeChi2()
671 self.log.info(str(chi2))
672 fit.minimize(
"Distortions Positions")
673 chi2 = fit.computeChi2()
674 self.log.info(str(chi2))
675 if not np.isfinite(chi2.chi2):
676 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
677 self.log.info(
"Fit prepared with %s", str(chi2))
679 chi2 = self.
_iterate_fit(associations, fit, model, 20,
"astrometry",
"Distortions Positions")
684 return Astrometry(fit, model, sky_to_tan_projection)
686 def _check_stars(self, associations):
687 """Count measured and reference stars per ccd and warn/log them.""" 688 for ccdImage
in associations.getCcdImageList():
689 nMeasuredStars, nRefStars = ccdImage.countStars()
690 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
691 ccdImage.getName(), nMeasuredStars, nRefStars)
692 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
693 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
694 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
695 if nRefStars < self.config.minRefStarsPerCcd:
696 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
697 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
699 def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit):
700 """Run fit.minimize up to max_steps times, returning the final chi2.""" 702 for i
in range(max_steps):
703 r = fit.minimize(whatToFit, 5)
704 chi2 = fit.computeChi2()
706 if not np.isfinite(chi2.chi2):
707 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
708 self.log.info(str(chi2))
709 if r == MinimizeResult.Converged:
710 self.log.debug(
"fit has converged - no more outliers - redo minimization " 711 "one more time in case we have lost accuracy in rank update.")
713 r = fit.minimize(whatToFit, 5)
714 chi2 = fit.computeChi2()
715 self.log.info(
"Fit completed with: %s", str(chi2))
717 elif r == MinimizeResult.Chi2Increased:
718 self.log.warn(
"still some ouliers but chi2 increases - retry")
719 elif r == MinimizeResult.Failed:
720 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
722 raise RuntimeError(
"Unxepected return code from minimize().")
724 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
728 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
730 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 734 associations : lsst.jointcal.Associations 735 The star/reference star associations to fit. 736 model : lsst.jointcal.AstrometryModel 737 The astrometric model that was fit. 738 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 739 dict of ccdImage identifiers to dataRefs that were fit 742 ccdImageList = associations.getCcdImageList()
743 for ccdImage
in ccdImageList:
746 visit = ccdImage.visit
747 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
748 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
749 skyWcs = model.makeSkyWcs(ccdImage)
751 dataRef.put(skyWcs,
'jointcal_wcs')
752 except pexExceptions.Exception
as e:
753 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
756 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
758 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 762 associations : lsst.jointcal.Associations 763 The star/reference star associations to fit. 764 model : lsst.jointcal.PhotometryModel 765 The photoometric model that was fit. 766 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 767 dict of ccdImage identifiers to dataRefs that were fit 770 ccdImageList = associations.getCcdImageList()
771 for ccdImage
in ccdImageList:
774 visit = ccdImage.visit
775 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
776 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
777 photoCalib = model.toPhotoCalib(ccdImage)
779 dataRef.put(photoCalib,
'jointcal_photoCalib')
780 except pexExceptions.Exception
as e:
781 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit)
def _check_stars(self, associations)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)