3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
23 from .dataIds
import PerTractCcdDataIdContainer
28 __all__ = [
"JointcalConfig",
"JointcalTask"]
30 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
31 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
35 """Subclass of TaskRunner for jointcalTask 37 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 38 extracted from the command line (whereas most CmdLineTasks' run methods take 39 single dataRef, are are called repeatedly). This class transforms the processed 40 arguments generated by the ArgumentParser into the arguments expected by 43 See pipeBase.TaskRunner for more information. 49 Return a list of tuples per tract, each containing (dataRefs, kwargs). 51 Jointcal operates on lists of dataRefs simultaneously. 53 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
54 kwargs[
'butler'] = parsedCmd.butler
58 for ref
in parsedCmd.id.refList:
59 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
61 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
66 @param args Arguments for Task.run() 69 - A pipe.base.Struct containing these fields if self.doReturnResults is False: 70 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 71 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 72 - ``result``: the result of calling jointcal.run() 73 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 78 dataRefList, kwargs = args
79 butler = kwargs.pop(
'butler')
80 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
83 result = task.run(dataRefList, **kwargs)
84 exitStatus = result.exitStatus
85 except Exception
as e:
90 eName = type(e).__name__
91 tract = dataRefList[0].dataId[
'tract']
92 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
94 if self.doReturnResults:
95 return pipeBase.Struct(result=result, exitStatus=exitStatus)
97 return pipeBase.Struct(exitStatus=exitStatus)
101 """Config for jointcalTask""" 103 doAstrometry = pexConfig.Field(
104 doc=
"Fit astrometry and write the fitted result.",
108 doPhotometry = pexConfig.Field(
109 doc=
"Fit photometry and write the fitted result.",
113 coaddName = pexConfig.Field(
114 doc=
"Type of coadd, typically deep or goodSeeing",
118 posError = pexConfig.Field(
119 doc=
"Constant term for error on position (in pixel unit)",
124 matchCut = pexConfig.Field(
125 doc=
"Matching radius between fitted and reference stars (arcseconds)",
129 minMeasurements = pexConfig.Field(
130 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
134 astrometrySimpleDegree = pexConfig.Field(
135 doc=
"Polynomial degree for fitting the simple astrometry model.",
139 astrometryChipDegree = pexConfig.Field(
140 doc=
"Degree of the per-chip transform for the constrained astrometry model.",
144 astrometryVisitDegree = pexConfig.Field(
145 doc=
"Degree of the per-visit transform for the constrained astrometry model.",
149 astrometryModel = pexConfig.ChoiceField(
150 doc=
"Type of model to fit to astrometry",
152 default=
"simplePoly",
153 allowed={
"simplePoly":
"One polynomial per ccd",
154 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
156 photometryModel = pexConfig.ChoiceField(
157 doc=
"Type of model to fit to photometry",
160 allowed={
"simple":
"One constant zeropoint per ccd and visit",
161 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
163 photometryVisitDegree = pexConfig.Field(
164 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
168 astrometryRefObjLoader = pexConfig.ConfigurableField(
169 target=LoadIndexedReferenceObjectsTask,
170 doc=
"Reference object loader for astrometric fit",
172 photometryRefObjLoader = pexConfig.ConfigurableField(
173 target=LoadIndexedReferenceObjectsTask,
174 doc=
"Reference object loader for photometric fit",
176 sourceSelector = sourceSelectorRegistry.makeField(
177 doc=
"How to select sources for cross-matching",
180 writeChi2ContributionFiles = pexConfig.Field(
182 doc=
"Write initial/final fit files containing the contributions to chi2.",
188 sourceSelector.setDefaults()
190 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
192 sourceSelector.sourceFluxType =
'Calib' 196 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 198 ConfigClass = JointcalConfig
199 RunnerClass = JointcalRunner
200 _DefaultName =
"jointcal" 202 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
204 Instantiate a JointcalTask. 208 butler : lsst.daf.persistence.Butler 209 The butler is passed to the refObjLoader constructor in case it is 210 needed. Ignored if the refObjLoader argument provides a loader directly. 211 Used to initialize the astrometry and photometry refObjLoaders. 212 profile_jointcal : bool 213 set to True to profile different stages of this jointcal run. 215 pipeBase.CmdLineTask.__init__(self, **kwargs)
217 self.makeSubtask(
"sourceSelector")
218 if self.config.doAstrometry:
219 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
220 if self.config.doPhotometry:
221 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
228 def _getConfigName(self):
231 def _getMetadataName(self):
235 def _makeArgumentParser(cls):
236 """Create an argument parser""" 238 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
239 help=
"Profile steps of jointcal separately.")
240 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
241 ContainerClass=PerTractCcdDataIdContainer)
244 def _build_ccdImage(self, dataRef, associations, jointcalControl):
246 Extract the necessary things from this dataRef to add a new ccdImage. 250 dataRef : lsst.daf.persistence.ButlerDataRef 251 dataRef to extract info from. 252 associations : lsst.jointcal.Associations 253 object to add the info to, to construct a new CcdImage 254 jointcalControl : jointcal.JointcalControl 255 control object for associations management 260 wcs : lsst.afw.image.TanWcs 261 the TAN WCS of this image, read from the calexp 263 a key to identify this dataRef by its visit and ccd ids 267 if "visit" in dataRef.dataId.keys():
268 visit = dataRef.dataId[
"visit"]
270 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
272 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
274 visitInfo = dataRef.get(
'calexp_visitInfo')
275 detector = dataRef.get(
'calexp_detector')
276 ccdId = detector.getId()
277 calib = dataRef.get(
'calexp_calib')
278 tanWcs = dataRef.get(
'calexp_wcs')
279 bbox = dataRef.get(
'calexp_bbox')
280 filt = dataRef.get(
'calexp_filter')
281 filterName = filt.getName()
282 fluxMag0 = calib.getFluxMag0()
283 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
285 goodSrc = self.sourceSelector.selectSources(src)
287 if len(goodSrc.sourceCat) == 0:
288 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
290 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
291 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
292 visit, ccdId, jointcalControl)
294 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
295 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
296 return Result(tanWcs, Key(visit, ccdId), filterName)
299 def run(self, dataRefs, profile_jointcal=False):
301 Jointly calibrate the astrometry and photometry across a set of images. 305 dataRefs : list of lsst.daf.persistence.ButlerDataRef 306 List of data references to the exposures to be fit. 307 profile_jointcal : bool 308 Profile the individual steps of jointcal. 314 * dataRefs: the provided data references that were fit (with updated WCSs) 315 * oldWcsList: the original WCS from each dataRef 316 * metrics: dictionary of internally-computed metrics for testing/validation. 318 if len(dataRefs) == 0:
319 raise ValueError(
'Need a non-empty list of data references!')
323 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
327 visit_ccd_to_dataRef = {}
330 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 331 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
334 camera = dataRefs[0].get(
'camera', immediate=
True)
338 oldWcsList.append(result.wcs)
339 visit_ccd_to_dataRef[result.key] = ref
340 filters.append(result.filter)
341 filters = collections.Counter(filters)
343 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
345 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
346 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
351 bbox = associations.getRaDecBBox()
352 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
353 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
354 radius = center.angularSeparation(corner).asRadians()
359 raise RuntimeError(
"astrometry_net_data is not setup")
362 defaultFilter = filters.most_common(1)[0][0]
363 self.log.debug(
"Using %s band for reference flux", defaultFilter)
366 tract = dataRefs[0].dataId[
'tract']
368 if self.config.doAstrometry:
371 refObjLoader=self.astrometryRefObjLoader,
373 profile_jointcal=profile_jointcal,
379 if self.config.doPhotometry:
382 refObjLoader=self.photometryRefObjLoader,
384 profile_jointcal=profile_jointcal,
387 reject_bad_fluxes=
True)
392 return pipeBase.Struct(dataRefs=dataRefs,
393 oldWcsList=oldWcsList,
395 exitStatus=exitStatus)
397 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
398 name="", refObjLoader=None, filters=[], fit_function=None,
399 tract=None, profile_jointcal=False, match_cut=3.0,
400 reject_bad_fluxes=False):
401 """Load reference catalog, perform the fit, and return the result. 405 associations : lsst.jointcal.Associations 406 The star/reference star associations to fit. 408 filter to load from reference catalog. 409 center : lsst.afw.coord.Coord 410 Center of field to load from reference catalog. 411 radius : lsst.afw.geom.Angle 412 On-sky radius to load from reference catalog. 414 Name of thing being fit: "Astrometry" or "Photometry". 415 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 416 Reference object loader to load from for fit. 417 filters : list of str, optional 418 List of filters to load from the reference catalog. 419 fit_function : function 420 function to call to perform fit (takes associations object). 422 Name of tract currently being fit. 423 profile_jointcal : bool, optional 424 Separately profile the fitting step. 425 match_cut : float, optional 426 Radius in arcseconds to find cross-catalog matches to during 427 associations.associateCatalogs. 428 reject_bad_fluxes : bool, optional 429 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 433 Result of `fit_function()` 435 self.log.info(
"====== Now processing %s...", name)
438 associations.associateCatalogs(match_cut)
439 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
441 skyCircle = refObjLoader.loadSkyCircle(center,
446 if not skyCircle.refCat.isContiguous():
447 refCat = skyCircle.refCat.copy(deep=
True)
449 refCat = skyCircle.refCat
456 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
457 refFluxes[filt] = refCat.get(filtKeys[0])
458 refFluxErrs[filt] = refCat.get(filtKeys[1])
460 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
461 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
462 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
464 associations.selectFittedStars(self.config.minMeasurements)
466 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
467 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
468 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
470 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 471 dataName =
"{}_{}".format(tract, defaultFilter)
472 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
473 result = fit_function(associations, dataName)
476 if self.config.writeChi2ContributionFiles:
477 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
478 result.fit.saveChi2Contributions(baseName)
482 def _check_star_lists(self, associations, name):
484 if associations.nCcdImagesValidForFit() == 0:
485 raise RuntimeError(
'No images in the ccdImageList!')
486 if associations.fittedStarListSize() == 0:
487 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
488 if associations.refStarListSize() == 0:
489 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
491 def _fit_photometry(self, associations, dataName=None):
493 Fit the photometric data. 497 associations : lsst.jointcal.Associations 498 The star/reference star associations to fit. 500 Name of the data being processed (e.g. "1234_HSC-Y"), for 501 identifying debugging files. 506 fit : lsst.jointcal.PhotometryFit 507 The photometric fitter used to perform the fit. 508 model : lsst.jointcal.PhotometryModel 509 The photometric model that was fit. 511 self.log.info(
"=== Starting photometric fitting...")
514 if self.config.photometryModel ==
"constrained":
517 visitDegree=self.config.photometryVisitDegree)
518 elif self.config.photometryModel ==
"simple":
522 chi2 = fit.computeChi2()
525 if self.config.writeChi2ContributionFiles:
526 baseName =
"Photometry_initial_chi2-{}.csv".format(dataName)
527 fit.saveChi2Contributions(baseName)
529 if not np.isfinite(chi2.chi2):
530 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
531 self.log.info(
"Initialized: %s", str(chi2))
532 fit.minimize(
"Model")
533 chi2 = fit.computeChi2()
534 self.log.info(str(chi2))
535 fit.minimize(
"Fluxes")
536 chi2 = fit.computeChi2()
537 self.log.info(str(chi2))
538 fit.minimize(
"Model Fluxes")
539 chi2 = fit.computeChi2()
540 if not np.isfinite(chi2.chi2):
541 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
542 self.log.info(
"Fit prepared with %s", str(chi2))
544 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
546 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
547 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
550 def _fit_astrometry(self, associations, dataName=None):
552 Fit the astrometric data. 556 associations : lsst.jointcal.Associations 557 The star/reference star associations to fit. 559 Name of the data being processed (e.g. "1234_HSC-Y"), for 560 identifying debugging files. 565 fit : lsst.jointcal.AstrometryFit 566 The astrometric fitter used to perform the fit. 567 model : lsst.jointcal.AstrometryModel 568 The astrometric model that was fit. 569 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 570 The model for the sky to tangent plane projection that was used in the fit. 573 self.log.info(
"=== Starting astrometric fitting...")
575 associations.deprojectFittedStars()
582 if self.config.astrometryModel ==
"constrainedPoly":
584 sky_to_tan_projection,
True, 0,
585 chipDegree=self.config.astrometryChipDegree,
586 visitDegree=self.config.astrometryVisitDegree)
587 elif self.config.astrometryModel ==
"simplePoly":
589 sky_to_tan_projection,
590 True, 0, self.config.astrometrySimpleDegree)
593 chi2 = fit.computeChi2()
596 if self.config.writeChi2ContributionFiles:
597 baseName =
"Astrometry_initial_chi2-{}.csv".format(dataName)
598 fit.saveChi2Contributions(baseName)
600 if not np.isfinite(chi2.chi2):
601 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
602 self.log.info(
"Initialized: %s", str(chi2))
603 fit.minimize(
"Distortions")
604 chi2 = fit.computeChi2()
605 self.log.info(str(chi2))
606 fit.minimize(
"Positions")
607 chi2 = fit.computeChi2()
608 self.log.info(str(chi2))
609 fit.minimize(
"Distortions Positions")
610 chi2 = fit.computeChi2()
611 self.log.info(str(chi2))
612 if not np.isfinite(chi2.chi2):
613 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
614 self.log.info(
"Fit prepared with %s", str(chi2))
616 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
618 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
619 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
621 return Astrometry(fit, model, sky_to_tan_projection)
623 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
624 """Run fit.minimize up to max_steps times, returning the final chi2.""" 626 for i
in range(max_steps):
627 r = fit.minimize(whatToFit, 5)
628 chi2 = fit.computeChi2()
629 if not np.isfinite(chi2.chi2):
630 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
631 self.log.info(str(chi2))
632 if r == MinimizeResult.Converged:
633 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 634 "one more time in case we have lost accuracy in rank update")
636 r = fit.minimize(whatToFit, 5)
637 chi2 = fit.computeChi2()
638 self.log.info(
"Fit completed with: %s", str(chi2))
640 elif r == MinimizeResult.Chi2Increased:
641 self.log.warn(
"still some ouliers but chi2 increases - retry")
642 elif r == MinimizeResult.Failed:
643 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
645 raise RuntimeError(
"Unxepected return code from minimize().")
647 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
651 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
653 Write the fitted astrometric results to a new 'wcs' dataRef. 657 associations : lsst.jointcal.Associations 658 The star/reference star associations to fit. 659 model : lsst.jointcal.AstrometryModel 660 The astrometric model that was fit. 661 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 662 dict of ccdImage identifiers to dataRefs that were fit 665 ccdImageList = associations.getCcdImageList()
666 for ccdImage
in ccdImageList:
669 visit = ccdImage.visit
670 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
671 exp = afwImage.ExposureI(0, 0)
672 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
673 tanSip = model.produceSipWcs(ccdImage)
677 dataRef.put(exp,
'wcs')
678 except pexExceptions.Exception
as e:
679 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
682 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
684 Write the fitted photometric results to a new 'photoCalib' dataRef. 688 associations : lsst.jointcal.Associations 689 The star/reference star associations to fit. 690 model : lsst.jointcal.PhotometryModel 691 The photoometric model that was fit. 692 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 693 dict of ccdImage identifiers to dataRefs that were fit 696 ccdImageList = associations.getCcdImageList()
697 for ccdImage
in ccdImageList:
700 visit = ccdImage.visit
701 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
702 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
703 photoCalib = model.toPhotoCalib(ccdImage)
705 dataRef.put(photoCalib,
'photoCalib')
706 except pexExceptions.Exception
as e:
707 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)