3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
19 from lsst.verify
import Job, Measurement
24 from .dataIds
import PerTractCcdDataIdContainer
29 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
31 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
32 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
37 meas = Measurement(job.metrics[name], value)
38 job.measurements.insert(meas)
42 """Subclass of TaskRunner for jointcalTask 44 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 45 extracted from the command line (whereas most CmdLineTasks' run methods take 46 single dataRef, are are called repeatedly). This class transforms the processed 47 arguments generated by the ArgumentParser into the arguments expected by 50 See pipeBase.TaskRunner for more information. 56 Return a list of tuples per tract, each containing (dataRefs, kwargs). 58 Jointcal operates on lists of dataRefs simultaneously. 60 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
61 kwargs[
'butler'] = parsedCmd.butler
65 for ref
in parsedCmd.id.refList:
66 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
68 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
76 Arguments for Task.run() 81 if self.doReturnResults is False: 83 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 85 if self.doReturnResults is True: 87 - ``result``: the result of calling jointcal.run() 88 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 93 dataRefList, kwargs = args
94 butler = kwargs.pop(
'butler')
95 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
98 result = task.run(dataRefList, **kwargs)
99 exitStatus = result.exitStatus
100 job_path = butler.get(
'verify_job_filename')
101 result.job.write(job_path[0])
102 except Exception
as e:
107 eName = type(e).__name__
108 tract = dataRefList[0].dataId[
'tract']
109 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
111 if self.doReturnResults:
112 return pipeBase.Struct(result=result, exitStatus=exitStatus)
114 return pipeBase.Struct(exitStatus=exitStatus)
118 """Config for JointcalTask""" 120 doAstrometry = pexConfig.Field(
121 doc=
"Fit astrometry and write the fitted result.",
125 doPhotometry = pexConfig.Field(
126 doc=
"Fit photometry and write the fitted result.",
130 coaddName = pexConfig.Field(
131 doc=
"Type of coadd, typically deep or goodSeeing",
135 posError = pexConfig.Field(
136 doc=
"Constant term for error on position (in pixel unit)",
141 matchCut = pexConfig.Field(
142 doc=
"Matching radius between fitted and reference stars (arcseconds)",
146 minMeasurements = pexConfig.Field(
147 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
151 astrometrySimpleDegree = pexConfig.Field(
152 doc=
"Polynomial degree for fitting the simple astrometry model.",
156 astrometryChipDegree = pexConfig.Field(
157 doc=
"Degree of the per-chip transform for the constrained astrometry model.",
161 astrometryVisitDegree = pexConfig.Field(
162 doc=
"Degree of the per-visit transform for the constrained astrometry model.",
166 useInputWcs = pexConfig.Field(
167 doc=
"Use the input calexp WCSs to initialize the astrometryModel.",
171 astrometryModel = pexConfig.ChoiceField(
172 doc=
"Type of model to fit to astrometry",
174 default=
"simplePoly",
175 allowed={
"simplePoly":
"One polynomial per ccd",
176 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
178 photometryModel = pexConfig.ChoiceField(
179 doc=
"Type of model to fit to photometry",
182 allowed={
"simple":
"One constant zeropoint per ccd and visit",
183 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
185 photometryVisitDegree = pexConfig.Field(
186 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
190 astrometryRefObjLoader = pexConfig.ConfigurableField(
191 target=LoadIndexedReferenceObjectsTask,
192 doc=
"Reference object loader for astrometric fit",
194 photometryRefObjLoader = pexConfig.ConfigurableField(
195 target=LoadIndexedReferenceObjectsTask,
196 doc=
"Reference object loader for photometric fit",
198 sourceSelector = sourceSelectorRegistry.makeField(
199 doc=
"How to select sources for cross-matching",
202 writeChi2ContributionFiles = pexConfig.Field(
204 doc=
"Write initial/final fit files containing the contributions to chi2.",
210 sourceSelector.setDefaults()
212 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
214 sourceSelector.sourceFluxType =
'Calib' 218 """Jointly astrometrically and photometrically calibrate a group of images.""" 220 ConfigClass = JointcalConfig
221 RunnerClass = JointcalRunner
222 _DefaultName =
"jointcal" 224 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
226 Instantiate a JointcalTask. 230 butler : lsst.daf.persistence.Butler 231 The butler is passed to the refObjLoader constructor in case it is 232 needed. Ignored if the refObjLoader argument provides a loader directly. 233 Used to initialize the astrometry and photometry refObjLoaders. 234 profile_jointcal : bool 235 set to True to profile different stages of this jointcal run. 237 pipeBase.CmdLineTask.__init__(self, **kwargs)
239 self.makeSubtask(
"sourceSelector")
240 if self.config.doAstrometry:
241 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
242 if self.config.doPhotometry:
243 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
246 self.
job = Job.load_metrics_package(subset=
'jointcal')
250 def _getConfigName(self):
253 def _getMetadataName(self):
257 def _makeArgumentParser(cls):
258 """Create an argument parser""" 260 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
261 help=
"Profile steps of jointcal separately.")
262 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
263 ContainerClass=PerTractCcdDataIdContainer)
266 def _build_ccdImage(self, dataRef, associations, jointcalControl):
268 Extract the necessary things from this dataRef to add a new ccdImage. 272 dataRef : lsst.daf.persistence.ButlerDataRef 273 dataRef to extract info from. 274 associations : lsst.jointcal.Associations 275 object to add the info to, to construct a new CcdImage 276 jointcalControl : jointcal.JointcalControl 277 control object for associations management 282 wcs : lsst.afw.geom.SkyWcs 283 the TAN WCS of this image, read from the calexp 285 a key to identify this dataRef by its visit and ccd ids 289 if "visit" in dataRef.dataId.keys():
290 visit = dataRef.dataId[
"visit"]
292 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
294 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
296 visitInfo = dataRef.get(
'calexp_visitInfo')
297 detector = dataRef.get(
'calexp_detector')
298 ccdId = detector.getId()
299 calib = dataRef.get(
'calexp_calib')
300 tanWcs = dataRef.get(
'calexp_wcs')
301 bbox = dataRef.get(
'calexp_bbox')
302 filt = dataRef.get(
'calexp_filter')
303 filterName = filt.getName()
304 fluxMag0 = calib.getFluxMag0()
305 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
307 goodSrc = self.sourceSelector.selectSources(src)
309 if len(goodSrc.sourceCat) == 0:
310 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
312 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
313 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
314 visit, ccdId, jointcalControl)
316 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
317 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
318 return Result(tanWcs, Key(visit, ccdId), filterName)
321 def run(self, dataRefs, profile_jointcal=False):
323 Jointly calibrate the astrometry and photometry across a set of images. 327 dataRefs : list of lsst.daf.persistence.ButlerDataRef 328 List of data references to the exposures to be fit. 329 profile_jointcal : bool 330 Profile the individual steps of jointcal. 336 * dataRefs: the provided data references that were fit (with updated WCSs) 337 * oldWcsList: the original WCS from each dataRef 338 * metrics: dictionary of internally-computed metrics for testing/validation. 340 if len(dataRefs) == 0:
341 raise ValueError(
'Need a non-empty list of data references!')
345 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
349 visit_ccd_to_dataRef = {}
352 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 353 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
356 camera = dataRefs[0].get(
'camera', immediate=
True)
360 oldWcsList.append(result.wcs)
361 visit_ccd_to_dataRef[result.key] = ref
362 filters.append(result.filter)
363 filters = collections.Counter(filters)
365 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
367 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
368 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
373 bbox = associations.getRaDecBBox()
374 center = afwCoord.IcrsCoord(bbox.getCenter(), afwGeom.degrees)
375 corner = afwCoord.IcrsCoord(bbox.getMax(), afwGeom.degrees)
376 radius = center.angularSeparation(corner).asRadians()
381 raise RuntimeError(
"astrometry_net_data is not setup")
384 defaultFilter = filters.most_common(1)[0][0]
385 self.log.debug(
"Using %s band for reference flux", defaultFilter)
388 tract = dataRefs[0].dataId[
'tract']
390 if self.config.doAstrometry:
393 refObjLoader=self.astrometryRefObjLoader,
395 profile_jointcal=profile_jointcal,
401 if self.config.doPhotometry:
404 refObjLoader=self.photometryRefObjLoader,
406 profile_jointcal=profile_jointcal,
409 reject_bad_fluxes=
True)
414 return pipeBase.Struct(dataRefs=dataRefs,
415 oldWcsList=oldWcsList,
417 exitStatus=exitStatus)
419 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
420 name="", refObjLoader=None, filters=[], fit_function=None,
421 tract=None, profile_jointcal=False, match_cut=3.0,
422 reject_bad_fluxes=False):
423 """Load reference catalog, perform the fit, and return the result. 427 associations : lsst.jointcal.Associations 428 The star/reference star associations to fit. 430 filter to load from reference catalog. 431 center : lsst.afw.coord.IcrsCoord 432 Center of field to load from reference catalog. 433 radius : lsst.afw.geom.Angle 434 On-sky radius to load from reference catalog. 436 Name of thing being fit: "Astrometry" or "Photometry". 437 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 438 Reference object loader to load from for fit. 439 filters : list of str, optional 440 List of filters to load from the reference catalog. 441 fit_function : function 442 function to call to perform fit (takes associations object). 444 Name of tract currently being fit. 445 profile_jointcal : bool, optional 446 Separately profile the fitting step. 447 match_cut : float, optional 448 Radius in arcseconds to find cross-catalog matches to during 449 associations.associateCatalogs. 450 reject_bad_fluxes : bool, optional 451 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 455 Result of `fit_function()` 457 self.log.info(
"====== Now processing %s...", name)
460 associations.associateCatalogs(match_cut)
462 associations.fittedStarListSize())
464 skyCircle = refObjLoader.loadSkyCircle(center,
469 if not skyCircle.refCat.isContiguous():
470 refCat = skyCircle.refCat.copy(deep=
True)
472 refCat = skyCircle.refCat
479 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
480 refFluxes[filt] = refCat.get(filtKeys[0])
481 refFluxErrs[filt] = refCat.get(filtKeys[1])
483 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
484 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
486 associations.refStarListSize())
488 associations.prepareFittedStars(self.config.minMeasurements)
492 associations.nFittedStarsWithAssociatedRefStar())
494 associations.fittedStarListSize())
496 associations.nCcdImagesValidForFit())
498 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 499 dataName =
"{}_{}".format(tract, defaultFilter)
500 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
501 result = fit_function(associations, dataName)
504 if self.config.writeChi2ContributionFiles:
505 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
506 result.fit.saveChi2Contributions(baseName)
510 def _check_star_lists(self, associations, name):
512 if associations.nCcdImagesValidForFit() == 0:
513 raise RuntimeError(
'No images in the ccdImageList!')
514 if associations.fittedStarListSize() == 0:
515 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
516 if associations.refStarListSize() == 0:
517 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
519 def _fit_photometry(self, associations, dataName=None):
521 Fit the photometric data. 525 associations : lsst.jointcal.Associations 526 The star/reference star associations to fit. 528 Name of the data being processed (e.g. "1234_HSC-Y"), for 529 identifying debugging files. 534 fit : lsst.jointcal.PhotometryFit 535 The photometric fitter used to perform the fit. 536 model : lsst.jointcal.PhotometryModel 537 The photometric model that was fit. 539 self.log.info(
"=== Starting photometric fitting...")
542 if self.config.photometryModel ==
"constrained":
545 visitDegree=self.config.photometryVisitDegree)
546 elif self.config.photometryModel ==
"simple":
550 chi2 = fit.computeChi2()
553 if self.config.writeChi2ContributionFiles:
554 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
555 fit.saveChi2Contributions(baseName)
557 if not np.isfinite(chi2.chi2):
558 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
559 self.log.info(
"Initialized: %s", str(chi2))
560 fit.minimize(
"Model")
561 chi2 = fit.computeChi2()
562 self.log.info(str(chi2))
563 fit.minimize(
"Fluxes")
564 chi2 = fit.computeChi2()
565 self.log.info(str(chi2))
566 fit.minimize(
"Model Fluxes")
567 chi2 = fit.computeChi2()
568 if not np.isfinite(chi2.chi2):
569 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
570 self.log.info(
"Fit prepared with %s", str(chi2))
572 model.freezeErrorTransform()
573 self.log.debug(
"Photometry error scales are frozen.")
575 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
581 def _fit_astrometry(self, associations, dataName=None):
583 Fit the astrometric data. 587 associations : lsst.jointcal.Associations 588 The star/reference star associations to fit. 590 Name of the data being processed (e.g. "1234_HSC-Y"), for 591 identifying debugging files. 596 fit : lsst.jointcal.AstrometryFit 597 The astrometric fitter used to perform the fit. 598 model : lsst.jointcal.AstrometryModel 599 The astrometric model that was fit. 600 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 601 The model for the sky to tangent plane projection that was used in the fit. 604 self.log.info(
"=== Starting astrometric fitting...")
606 associations.deprojectFittedStars()
613 if self.config.astrometryModel ==
"constrainedPoly":
615 sky_to_tan_projection, self.config.useInputWcs, 0,
616 chipDegree=self.config.astrometryChipDegree,
617 visitDegree=self.config.astrometryVisitDegree)
618 elif self.config.astrometryModel ==
"simplePoly":
620 sky_to_tan_projection, self.config.useInputWcs, 0,
621 self.config.astrometrySimpleDegree)
624 chi2 = fit.computeChi2()
627 if self.config.writeChi2ContributionFiles:
628 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
629 fit.saveChi2Contributions(baseName)
631 if not np.isfinite(chi2.chi2):
632 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
633 self.log.info(
"Initialized: %s", str(chi2))
634 fit.minimize(
"Distortions")
635 chi2 = fit.computeChi2()
636 self.log.info(str(chi2))
637 fit.minimize(
"Positions")
638 chi2 = fit.computeChi2()
639 self.log.info(str(chi2))
640 fit.minimize(
"Distortions Positions")
641 chi2 = fit.computeChi2()
642 self.log.info(str(chi2))
643 if not np.isfinite(chi2.chi2):
644 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
645 self.log.info(
"Fit prepared with %s", str(chi2))
647 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
652 return Astrometry(fit, model, sky_to_tan_projection)
654 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
655 """Run fit.minimize up to max_steps times, returning the final chi2.""" 657 for i
in range(max_steps):
658 r = fit.minimize(whatToFit, 5)
659 chi2 = fit.computeChi2()
660 if not np.isfinite(chi2.chi2):
661 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
662 self.log.info(str(chi2))
663 if r == MinimizeResult.Converged:
664 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 665 "one more time in case we have lost accuracy in rank update")
667 r = fit.minimize(whatToFit, 5)
668 chi2 = fit.computeChi2()
669 self.log.info(
"Fit completed with: %s", str(chi2))
671 elif r == MinimizeResult.Chi2Increased:
672 self.log.warn(
"still some ouliers but chi2 increases - retry")
673 elif r == MinimizeResult.Failed:
674 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
676 raise RuntimeError(
"Unxepected return code from minimize().")
678 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
682 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
684 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 688 associations : lsst.jointcal.Associations 689 The star/reference star associations to fit. 690 model : lsst.jointcal.AstrometryModel 691 The astrometric model that was fit. 692 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 693 dict of ccdImage identifiers to dataRefs that were fit 696 ccdImageList = associations.getCcdImageList()
697 for ccdImage
in ccdImageList:
700 visit = ccdImage.visit
701 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
702 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
703 tanSip = model.produceSipWcs(ccdImage)
706 dataRef.put(wcs,
'jointcal_wcs')
707 except pexExceptions.Exception
as e:
708 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
711 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
713 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 717 associations : lsst.jointcal.Associations 718 The star/reference star associations to fit. 719 model : lsst.jointcal.PhotometryModel 720 The photoometric model that was fit. 721 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 722 dict of ccdImage identifiers to dataRefs that were fit 725 ccdImageList = associations.getCcdImageList()
726 for ccdImage
in ccdImageList:
729 visit = ccdImage.visit
730 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
731 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
732 photoCalib = model.toPhotoCalib(ccdImage)
734 dataRef.put(photoCalib,
'jointcal_photoCalib')
735 except pexExceptions.Exception
as e:
736 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
std::shared_ptr< lsst::afw::geom::SkyWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)