13 from lsst.verify
import Job, Measurement
18 from .dataIds
import PerTractCcdDataIdContainer
23 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
25 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
26 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
31 meas = Measurement(job.metrics[name], value)
32 job.measurements.insert(meas)
36 """Subclass of TaskRunner for jointcalTask 38 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 39 extracted from the command line (whereas most CmdLineTasks' run methods take 40 single dataRef, are are called repeatedly). This class transforms the processed 41 arguments generated by the ArgumentParser into the arguments expected by 44 See pipeBase.TaskRunner for more information. 50 Return a list of tuples per tract, each containing (dataRefs, kwargs). 52 Jointcal operates on lists of dataRefs simultaneously. 54 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
55 kwargs[
'butler'] = parsedCmd.butler
59 for ref
in parsedCmd.id.refList:
60 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
62 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
70 Arguments for Task.run() 75 if self.doReturnResults is False: 77 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 79 if self.doReturnResults is True: 81 - ``result``: the result of calling jointcal.run() 82 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 87 dataRefList, kwargs = args
88 butler = kwargs.pop(
'butler')
89 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
92 result = task.run(dataRefList, **kwargs)
93 exitStatus = result.exitStatus
94 job_path = butler.get(
'verify_job_filename')
95 result.job.write(job_path[0])
96 except Exception
as e:
101 eName = type(e).__name__
102 tract = dataRefList[0].dataId[
'tract']
103 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
105 if self.doReturnResults:
106 return pipeBase.Struct(result=result, exitStatus=exitStatus)
108 return pipeBase.Struct(exitStatus=exitStatus)
112 """Config for JointcalTask""" 114 doAstrometry = pexConfig.Field(
115 doc=
"Fit astrometry and write the fitted result.",
119 doPhotometry = pexConfig.Field(
120 doc=
"Fit photometry and write the fitted result.",
124 coaddName = pexConfig.Field(
125 doc=
"Type of coadd, typically deep or goodSeeing",
129 posError = pexConfig.Field(
130 doc=
"Constant term for error on position (in pixel unit)",
135 matchCut = pexConfig.Field(
136 doc=
"Matching radius between fitted and reference stars (arcseconds)",
140 minMeasurements = pexConfig.Field(
141 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
145 minMeasuredStarsPerCcd = pexConfig.Field(
146 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
150 minRefStarsPerCcd = pexConfig.Field(
151 doc=
"Minimum number of measuredStars per ccdImage before printing warnings",
155 astrometrySimpleOrder = pexConfig.Field(
156 doc=
"Polynomial order for fitting the simple astrometry model.",
160 astrometryChipOrder = pexConfig.Field(
161 doc=
"Order of the per-chip transform for the constrained astrometry model.",
165 astrometryVisitOrder = pexConfig.Field(
166 doc=
"Order of the per-visit transform for the constrained astrometry model.",
170 useInputWcs = pexConfig.Field(
171 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
175 astrometryModel = pexConfig.ChoiceField(
176 doc=
"Type of model to fit to astrometry",
179 allowed={
"simple":
"One polynomial per ccd",
180 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
182 photometryModel = pexConfig.ChoiceField(
183 doc=
"Type of model to fit to photometry",
186 allowed={
"simple":
"One constant zeropoint per ccd and visit",
187 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
189 photometryVisitOrder = pexConfig.Field(
190 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
194 astrometryRefObjLoader = pexConfig.ConfigurableField(
195 target=LoadIndexedReferenceObjectsTask,
196 doc=
"Reference object loader for astrometric fit",
198 photometryRefObjLoader = pexConfig.ConfigurableField(
199 target=LoadIndexedReferenceObjectsTask,
200 doc=
"Reference object loader for photometric fit",
202 sourceSelector = sourceSelectorRegistry.makeField(
203 doc=
"How to select sources for cross-matching",
206 writeChi2ContributionFiles = pexConfig.Field(
208 doc=
"Write initial/final fit files containing the contributions to chi2.",
214 sourceSelector.setDefaults()
216 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
218 sourceSelector.sourceFluxType =
'Calib' 222 """Jointly astrometrically and photometrically calibrate a group of images.""" 224 ConfigClass = JointcalConfig
225 RunnerClass = JointcalRunner
226 _DefaultName =
"jointcal" 228 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
230 Instantiate a JointcalTask. 234 butler : lsst.daf.persistence.Butler 235 The butler is passed to the refObjLoader constructor in case it is 236 needed. Ignored if the refObjLoader argument provides a loader directly. 237 Used to initialize the astrometry and photometry refObjLoaders. 238 profile_jointcal : bool 239 set to True to profile different stages of this jointcal run. 241 pipeBase.CmdLineTask.__init__(self, **kwargs)
243 self.makeSubtask(
"sourceSelector")
244 if self.config.doAstrometry:
245 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
246 if self.config.doPhotometry:
247 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
250 self.
job = Job.load_metrics_package(subset=
'jointcal')
254 def _getConfigName(self):
257 def _getMetadataName(self):
261 def _makeArgumentParser(cls):
262 """Create an argument parser""" 264 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
265 help=
"Profile steps of jointcal separately.")
266 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
267 ContainerClass=PerTractCcdDataIdContainer)
270 def _build_ccdImage(self, dataRef, associations, jointcalControl):
272 Extract the necessary things from this dataRef to add a new ccdImage. 276 dataRef : lsst.daf.persistence.ButlerDataRef 277 dataRef to extract info from. 278 associations : lsst.jointcal.Associations 279 object to add the info to, to construct a new CcdImage 280 jointcalControl : jointcal.JointcalControl 281 control object for associations management 286 wcs : lsst.afw.geom.SkyWcs 287 the TAN WCS of this image, read from the calexp 289 a key to identify this dataRef by its visit and ccd ids 293 if "visit" in dataRef.dataId.keys():
294 visit = dataRef.dataId[
"visit"]
296 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
298 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
300 visitInfo = dataRef.get(
'calexp_visitInfo')
301 detector = dataRef.get(
'calexp_detector')
302 ccdId = detector.getId()
303 calib = dataRef.get(
'calexp_calib')
304 tanWcs = dataRef.get(
'calexp_wcs')
305 bbox = dataRef.get(
'calexp_bbox')
306 filt = dataRef.get(
'calexp_filter')
307 filterName = filt.getName()
308 fluxMag0 = calib.getFluxMag0()
309 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
311 goodSrc = self.sourceSelector.selectSources(src)
313 if len(goodSrc.sourceCat) == 0:
314 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
316 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
317 associations.createCcdImage(goodSrc.sourceCat,
328 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
329 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
330 return Result(tanWcs, Key(visit, ccdId), filterName)
333 def run(self, dataRefs, profile_jointcal=False):
335 Jointly calibrate the astrometry and photometry across a set of images. 339 dataRefs : list of lsst.daf.persistence.ButlerDataRef 340 List of data references to the exposures to be fit. 341 profile_jointcal : bool 342 Profile the individual steps of jointcal. 348 * dataRefs: the provided data references that were fit (with updated WCSs) 349 * oldWcsList: the original WCS from each dataRef 350 * metrics: dictionary of internally-computed metrics for testing/validation. 352 if len(dataRefs) == 0:
353 raise ValueError(
'Need a non-empty list of data references!')
357 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
361 visit_ccd_to_dataRef = {}
364 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 365 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
368 camera = dataRefs[0].get(
'camera', immediate=
True)
372 oldWcsList.append(result.wcs)
373 visit_ccd_to_dataRef[result.key] = ref
374 filters.append(result.filter)
375 filters = collections.Counter(filters)
377 associations.computeCommonTangentPoint()
382 bbox = associations.getRaDecBBox()
384 bboxCenter = bbox.getCenter()
385 center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
386 bboxMax = bbox.getMax()
387 corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
388 radius = center.separation(corner).asRadians()
393 raise RuntimeError(
"astrometry_net_data is not setup")
396 defaultFilter = filters.most_common(1)[0][0]
397 self.log.debug(
"Using %s band for reference flux", defaultFilter)
400 tract = dataRefs[0].dataId[
'tract']
402 if self.config.doAstrometry:
405 refObjLoader=self.astrometryRefObjLoader,
407 profile_jointcal=profile_jointcal,
413 if self.config.doPhotometry:
416 refObjLoader=self.photometryRefObjLoader,
418 profile_jointcal=profile_jointcal,
421 reject_bad_fluxes=
True)
426 return pipeBase.Struct(dataRefs=dataRefs,
427 oldWcsList=oldWcsList,
429 exitStatus=exitStatus)
431 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
432 name="", refObjLoader=None, filters=[], fit_function=None,
433 tract=None, profile_jointcal=False, match_cut=3.0,
434 reject_bad_fluxes=False):
435 """Load reference catalog, perform the fit, and return the result. 439 associations : lsst.jointcal.Associations 440 The star/reference star associations to fit. 442 filter to load from reference catalog. 443 center : lsst.afw.geom.SpherePoint 444 ICRS center of field to load from reference catalog. 445 radius : lsst.afw.geom.Angle 446 On-sky radius to load from reference catalog. 448 Name of thing being fit: "Astrometry" or "Photometry". 449 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 450 Reference object loader to load from for fit. 451 filters : list of str, optional 452 List of filters to load from the reference catalog. 453 fit_function : function 454 function to call to perform fit (takes associations object). 456 Name of tract currently being fit. 457 profile_jointcal : bool, optional 458 Separately profile the fitting step. 459 match_cut : float, optional 460 Radius in arcseconds to find cross-catalog matches to during 461 associations.associateCatalogs. 462 reject_bad_fluxes : bool, optional 463 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 467 Result of `fit_function()` 469 self.log.info(
"====== Now processing %s...", name)
472 associations.associateCatalogs(match_cut)
474 associations.fittedStarListSize())
476 skyCircle = refObjLoader.loadSkyCircle(center,
477 afwGeom.Angle(radius, afwGeom.radians),
481 if not skyCircle.refCat.isContiguous():
482 refCat = skyCircle.refCat.copy(deep=
True)
484 refCat = skyCircle.refCat
491 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
492 refFluxes[filt] = refCat.get(filtKeys[0])
493 refFluxErrs[filt] = refCat.get(filtKeys[1])
495 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
496 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
498 associations.refStarListSize())
500 associations.prepareFittedStars(self.config.minMeasurements)
504 associations.nFittedStarsWithAssociatedRefStar())
506 associations.fittedStarListSize())
508 associations.nCcdImagesValidForFit())
510 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 511 dataName =
"{}_{}".format(tract, defaultFilter)
512 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
513 result = fit_function(associations, dataName)
516 if self.config.writeChi2ContributionFiles:
517 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
518 result.fit.saveChi2Contributions(baseName)
522 def _check_star_lists(self, associations, name):
524 if associations.nCcdImagesValidForFit() == 0:
525 raise RuntimeError(
'No images in the ccdImageList!')
526 if associations.fittedStarListSize() == 0:
527 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
528 if associations.refStarListSize() == 0:
529 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
531 def _fit_photometry(self, associations, dataName=None):
533 Fit the photometric data. 537 associations : lsst.jointcal.Associations 538 The star/reference star associations to fit. 540 Name of the data being processed (e.g. "1234_HSC-Y"), for 541 identifying debugging files. 546 fit : lsst.jointcal.PhotometryFit 547 The photometric fitter used to perform the fit. 548 model : lsst.jointcal.PhotometryModel 549 The photometric model that was fit. 551 self.log.info(
"=== Starting photometric fitting...")
554 if self.config.photometryModel ==
"constrained":
557 visitOrder=self.config.photometryVisitOrder)
558 elif self.config.photometryModel ==
"simple":
562 chi2 = fit.computeChi2()
565 if self.config.writeChi2ContributionFiles:
566 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
567 fit.saveChi2Contributions(baseName)
569 if not np.isfinite(chi2.chi2):
570 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
571 self.log.info(
"Initialized: %s", str(chi2))
574 if self.config.photometryModel ==
"constrained":
576 fit.minimize(
"ModelVisit")
577 chi2 = fit.computeChi2()
578 self.log.info(str(chi2))
579 fit.minimize(
"Model")
580 chi2 = fit.computeChi2()
581 self.log.info(str(chi2))
582 fit.minimize(
"Fluxes")
583 chi2 = fit.computeChi2()
584 self.log.info(str(chi2))
585 fit.minimize(
"Model Fluxes")
586 chi2 = fit.computeChi2()
587 if not np.isfinite(chi2.chi2):
588 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
589 self.log.info(
"Fit prepared with %s", str(chi2))
591 model.freezeErrorTransform()
592 self.log.debug(
"Photometry error scales are frozen.")
594 chi2 = self.
_iterate_fit(associations, fit, model, 20,
"photometry",
"Model Fluxes")
600 def _fit_astrometry(self, associations, dataName=None):
602 Fit the astrometric data. 606 associations : lsst.jointcal.Associations 607 The star/reference star associations to fit. 609 Name of the data being processed (e.g. "1234_HSC-Y"), for 610 identifying debugging files. 615 fit : lsst.jointcal.AstrometryFit 616 The astrometric fitter used to perform the fit. 617 model : lsst.jointcal.AstrometryModel 618 The astrometric model that was fit. 619 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 620 The model for the sky to tangent plane projection that was used in the fit. 623 self.log.info(
"=== Starting astrometric fitting...")
625 associations.deprojectFittedStars()
632 if self.config.astrometryModel ==
"constrained":
634 sky_to_tan_projection,
635 chipOrder=self.config.astrometryChipOrder,
636 visitOrder=self.config.astrometryVisitOrder)
637 elif self.config.astrometryModel ==
"simple":
639 sky_to_tan_projection,
640 self.config.useInputWcs,
642 order=self.config.astrometrySimpleOrder)
645 chi2 = fit.computeChi2()
648 if self.config.writeChi2ContributionFiles:
649 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
650 fit.saveChi2Contributions(baseName)
652 if not np.isfinite(chi2.chi2):
653 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
654 self.log.info(
"Initialized: %s", str(chi2))
657 if self.config.astrometryModel ==
"constrained":
658 fit.minimize(
"DistortionsVisit")
659 chi2 = fit.computeChi2()
660 self.log.info(str(chi2))
661 fit.minimize(
"Distortions")
662 chi2 = fit.computeChi2()
663 self.log.info(str(chi2))
664 fit.minimize(
"Positions")
665 chi2 = fit.computeChi2()
666 self.log.info(str(chi2))
667 fit.minimize(
"Distortions Positions")
668 chi2 = fit.computeChi2()
669 self.log.info(str(chi2))
670 if not np.isfinite(chi2.chi2):
671 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
672 self.log.info(
"Fit prepared with %s", str(chi2))
674 chi2 = self.
_iterate_fit(associations, fit, model, 20,
"astrometry",
"Distortions Positions")
679 return Astrometry(fit, model, sky_to_tan_projection)
681 def _check_stars(self, associations):
682 """Count measured and reference stars per ccd and warn/log them.""" 683 for ccdImage
in associations.getCcdImageList():
684 nMeasuredStars, nRefStars = ccdImage.countStars()
685 self.log.debug(
"ccdImage %s has %s measured and %s reference stars",
686 ccdImage.getName(), nMeasuredStars, nRefStars)
687 if nMeasuredStars < self.config.minMeasuredStarsPerCcd:
688 self.log.warn(
"ccdImage %s has only %s measuredStars (desired %s)",
689 ccdImage.getName(), nMeasuredStars, self.config.minMeasuredStarsPerCcd)
690 if nRefStars < self.config.minRefStarsPerCcd:
691 self.log.warn(
"ccdImage %s has only %s RefStars (desired %s)",
692 ccdImage.getName(), nRefStars, self.config.minRefStarsPerCcd)
694 def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit):
695 """Run fit.minimize up to max_steps times, returning the final chi2.""" 697 for i
in range(max_steps):
698 r = fit.minimize(whatToFit, 5)
699 chi2 = fit.computeChi2()
701 if not np.isfinite(chi2.chi2):
702 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
703 self.log.info(str(chi2))
704 if r == MinimizeResult.Converged:
705 self.log.debug(
"fit has converged - no more outliers - redo minimization " 706 "one more time in case we have lost accuracy in rank update.")
708 r = fit.minimize(whatToFit, 5)
709 chi2 = fit.computeChi2()
710 self.log.info(
"Fit completed with: %s", str(chi2))
712 elif r == MinimizeResult.Chi2Increased:
713 self.log.warn(
"still some ouliers but chi2 increases - retry")
714 elif r == MinimizeResult.Failed:
715 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
717 raise RuntimeError(
"Unxepected return code from minimize().")
719 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
723 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
725 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 729 associations : lsst.jointcal.Associations 730 The star/reference star associations to fit. 731 model : lsst.jointcal.AstrometryModel 732 The astrometric model that was fit. 733 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 734 dict of ccdImage identifiers to dataRefs that were fit 737 ccdImageList = associations.getCcdImageList()
738 for ccdImage
in ccdImageList:
741 visit = ccdImage.visit
742 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
743 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
744 skyWcs = model.makeSkyWcs(ccdImage)
746 dataRef.put(skyWcs,
'jointcal_wcs')
747 except pexExceptions.Exception
as e:
748 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
751 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
753 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 757 associations : lsst.jointcal.Associations 758 The star/reference star associations to fit. 759 model : lsst.jointcal.PhotometryModel 760 The photoometric model that was fit. 761 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 762 dict of ccdImage identifiers to dataRefs that were fit 765 ccdImageList = associations.getCcdImageList()
766 for ccdImage
in ccdImageList:
769 visit = ccdImage.visit
770 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
771 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
772 photoCalib = model.toPhotoCalib(ccdImage)
774 dataRef.put(photoCalib,
'jointcal_photoCalib')
775 except pexExceptions.Exception
as e:
776 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
def _iterate_fit(self, associations, fit, model, max_steps, name, whatToFit)
def _check_stars(self, associations)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)