3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
19 from lsst.verify
import Job, Measurement
24 from .dataIds
import PerTractCcdDataIdContainer
29 __all__ = [
"JointcalConfig",
"JointcalTask"]
31 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
32 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
37 meas = Measurement(job.metrics[name], value)
38 job.measurements.insert(meas)
42 """Subclass of TaskRunner for jointcalTask 44 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 45 extracted from the command line (whereas most CmdLineTasks' run methods take 46 single dataRef, are are called repeatedly). This class transforms the processed 47 arguments generated by the ArgumentParser into the arguments expected by 50 See pipeBase.TaskRunner for more information. 56 Return a list of tuples per tract, each containing (dataRefs, kwargs). 58 Jointcal operates on lists of dataRefs simultaneously. 60 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
61 kwargs[
'butler'] = parsedCmd.butler
65 for ref
in parsedCmd.id.refList:
66 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
68 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
73 @param args Arguments for Task.run() 76 - A pipe.base.Struct containing these fields if self.doReturnResults is False: 77 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 78 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 79 - ``result``: the result of calling jointcal.run() 80 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 85 dataRefList, kwargs = args
86 butler = kwargs.pop(
'butler')
87 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90 result = task.run(dataRefList, **kwargs)
91 exitStatus = result.exitStatus
92 job_path = butler.get(
'verify_job_filename')
93 result.job.write(job_path[0])
94 except Exception
as e:
99 eName = type(e).__name__
100 tract = dataRefList[0].dataId[
'tract']
101 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
103 if self.doReturnResults:
104 return pipeBase.Struct(result=result, exitStatus=exitStatus)
106 return pipeBase.Struct(exitStatus=exitStatus)
110 """Config for jointcalTask""" 112 doAstrometry = pexConfig.Field(
113 doc=
"Fit astrometry and write the fitted result.",
117 doPhotometry = pexConfig.Field(
118 doc=
"Fit photometry and write the fitted result.",
122 coaddName = pexConfig.Field(
123 doc=
"Type of coadd, typically deep or goodSeeing",
127 posError = pexConfig.Field(
128 doc=
"Constant term for error on position (in pixel unit)",
133 matchCut = pexConfig.Field(
134 doc=
"Matching radius between fitted and reference stars (arcseconds)",
138 minMeasurements = pexConfig.Field(
139 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
143 astrometrySimpleDegree = pexConfig.Field(
144 doc=
"Polynomial degree for fitting the simple astrometry model.",
148 astrometryChipDegree = pexConfig.Field(
149 doc=
"Degree of the per-chip transform for the constrained astrometry model.",
153 astrometryVisitDegree = pexConfig.Field(
154 doc=
"Degree of the per-visit transform for the constrained astrometry model.",
158 useInputWcs = pexConfig.Field(
159 doc=
"Use the input calexp WCSs to initialize the astrometryModel.",
163 astrometryModel = pexConfig.ChoiceField(
164 doc=
"Type of model to fit to astrometry",
166 default=
"simplePoly",
167 allowed={
"simplePoly":
"One polynomial per ccd",
168 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
170 photometryModel = pexConfig.ChoiceField(
171 doc=
"Type of model to fit to photometry",
174 allowed={
"simple":
"One constant zeropoint per ccd and visit",
175 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
177 photometryVisitDegree = pexConfig.Field(
178 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
182 astrometryRefObjLoader = pexConfig.ConfigurableField(
183 target=LoadIndexedReferenceObjectsTask,
184 doc=
"Reference object loader for astrometric fit",
186 photometryRefObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 doc=
"Reference object loader for photometric fit",
190 sourceSelector = sourceSelectorRegistry.makeField(
191 doc=
"How to select sources for cross-matching",
194 writeChi2ContributionFiles = pexConfig.Field(
196 doc=
"Write initial/final fit files containing the contributions to chi2.",
202 sourceSelector.setDefaults()
204 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
206 sourceSelector.sourceFluxType =
'Calib' 210 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 212 ConfigClass = JointcalConfig
213 RunnerClass = JointcalRunner
214 _DefaultName =
"jointcal" 216 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
218 Instantiate a JointcalTask. 222 butler : lsst.daf.persistence.Butler 223 The butler is passed to the refObjLoader constructor in case it is 224 needed. Ignored if the refObjLoader argument provides a loader directly. 225 Used to initialize the astrometry and photometry refObjLoaders. 226 profile_jointcal : bool 227 set to True to profile different stages of this jointcal run. 229 pipeBase.CmdLineTask.__init__(self, **kwargs)
231 self.makeSubtask(
"sourceSelector")
232 if self.config.doAstrometry:
233 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
234 if self.config.doPhotometry:
235 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
238 self.
job = Job.load_metrics_package(subset=
'jointcal')
242 def _getConfigName(self):
245 def _getMetadataName(self):
249 def _makeArgumentParser(cls):
250 """Create an argument parser""" 252 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
253 help=
"Profile steps of jointcal separately.")
254 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
255 ContainerClass=PerTractCcdDataIdContainer)
258 def _build_ccdImage(self, dataRef, associations, jointcalControl):
260 Extract the necessary things from this dataRef to add a new ccdImage. 264 dataRef : lsst.daf.persistence.ButlerDataRef 265 dataRef to extract info from. 266 associations : lsst.jointcal.Associations 267 object to add the info to, to construct a new CcdImage 268 jointcalControl : jointcal.JointcalControl 269 control object for associations management 274 wcs : lsst.afw.geom.SkyWcs 275 the TAN WCS of this image, read from the calexp 277 a key to identify this dataRef by its visit and ccd ids 281 if "visit" in dataRef.dataId.keys():
282 visit = dataRef.dataId[
"visit"]
284 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
286 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
288 visitInfo = dataRef.get(
'calexp_visitInfo')
289 detector = dataRef.get(
'calexp_detector')
290 ccdId = detector.getId()
291 calib = dataRef.get(
'calexp_calib')
292 tanWcs = dataRef.get(
'calexp_wcs')
293 bbox = dataRef.get(
'calexp_bbox')
294 filt = dataRef.get(
'calexp_filter')
295 filterName = filt.getName()
296 fluxMag0 = calib.getFluxMag0()
297 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
299 goodSrc = self.sourceSelector.selectSources(src)
301 if len(goodSrc.sourceCat) == 0:
302 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
304 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
305 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
306 visit, ccdId, jointcalControl)
308 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
309 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
310 return Result(tanWcs, Key(visit, ccdId), filterName)
313 def run(self, dataRefs, profile_jointcal=False):
315 Jointly calibrate the astrometry and photometry across a set of images. 319 dataRefs : list of lsst.daf.persistence.ButlerDataRef 320 List of data references to the exposures to be fit. 321 profile_jointcal : bool 322 Profile the individual steps of jointcal. 328 * dataRefs: the provided data references that were fit (with updated WCSs) 329 * oldWcsList: the original WCS from each dataRef 330 * metrics: dictionary of internally-computed metrics for testing/validation. 332 if len(dataRefs) == 0:
333 raise ValueError(
'Need a non-empty list of data references!')
337 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
341 visit_ccd_to_dataRef = {}
344 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 345 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
348 camera = dataRefs[0].get(
'camera', immediate=
True)
352 oldWcsList.append(result.wcs)
353 visit_ccd_to_dataRef[result.key] = ref
354 filters.append(result.filter)
355 filters = collections.Counter(filters)
357 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
359 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
360 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
365 bbox = associations.getRaDecBBox()
366 center = afwCoord.IcrsCoord(bbox.getCenter(), afwGeom.degrees)
367 corner = afwCoord.IcrsCoord(bbox.getMax(), afwGeom.degrees)
368 radius = center.angularSeparation(corner).asRadians()
373 raise RuntimeError(
"astrometry_net_data is not setup")
376 defaultFilter = filters.most_common(1)[0][0]
377 self.log.debug(
"Using %s band for reference flux", defaultFilter)
380 tract = dataRefs[0].dataId[
'tract']
382 if self.config.doAstrometry:
385 refObjLoader=self.astrometryRefObjLoader,
387 profile_jointcal=profile_jointcal,
393 if self.config.doPhotometry:
396 refObjLoader=self.photometryRefObjLoader,
398 profile_jointcal=profile_jointcal,
401 reject_bad_fluxes=
True)
406 return pipeBase.Struct(dataRefs=dataRefs,
407 oldWcsList=oldWcsList,
409 exitStatus=exitStatus)
411 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
412 name="", refObjLoader=None, filters=[], fit_function=None,
413 tract=None, profile_jointcal=False, match_cut=3.0,
414 reject_bad_fluxes=False):
415 """Load reference catalog, perform the fit, and return the result. 419 associations : lsst.jointcal.Associations 420 The star/reference star associations to fit. 422 filter to load from reference catalog. 423 center : lsst.afw.coord.IcrsCoord 424 Center of field to load from reference catalog. 425 radius : lsst.afw.geom.Angle 426 On-sky radius to load from reference catalog. 428 Name of thing being fit: "Astrometry" or "Photometry". 429 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 430 Reference object loader to load from for fit. 431 filters : list of str, optional 432 List of filters to load from the reference catalog. 433 fit_function : function 434 function to call to perform fit (takes associations object). 436 Name of tract currently being fit. 437 profile_jointcal : bool, optional 438 Separately profile the fitting step. 439 match_cut : float, optional 440 Radius in arcseconds to find cross-catalog matches to during 441 associations.associateCatalogs. 442 reject_bad_fluxes : bool, optional 443 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 447 Result of `fit_function()` 449 self.log.info(
"====== Now processing %s...", name)
452 associations.associateCatalogs(match_cut)
454 associations.fittedStarListSize())
456 skyCircle = refObjLoader.loadSkyCircle(center,
461 if not skyCircle.refCat.isContiguous():
462 refCat = skyCircle.refCat.copy(deep=
True)
464 refCat = skyCircle.refCat
471 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
472 refFluxes[filt] = refCat.get(filtKeys[0])
473 refFluxErrs[filt] = refCat.get(filtKeys[1])
475 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
476 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
478 associations.refStarListSize())
480 associations.prepareFittedStars(self.config.minMeasurements)
484 associations.nFittedStarsWithAssociatedRefStar())
486 associations.fittedStarListSize())
488 associations.nCcdImagesValidForFit())
490 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 491 dataName =
"{}_{}".format(tract, defaultFilter)
492 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
493 result = fit_function(associations, dataName)
496 if self.config.writeChi2ContributionFiles:
497 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
498 result.fit.saveChi2Contributions(baseName)
502 def _check_star_lists(self, associations, name):
504 if associations.nCcdImagesValidForFit() == 0:
505 raise RuntimeError(
'No images in the ccdImageList!')
506 if associations.fittedStarListSize() == 0:
507 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
508 if associations.refStarListSize() == 0:
509 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
511 def _fit_photometry(self, associations, dataName=None):
513 Fit the photometric data. 517 associations : lsst.jointcal.Associations 518 The star/reference star associations to fit. 520 Name of the data being processed (e.g. "1234_HSC-Y"), for 521 identifying debugging files. 526 fit : lsst.jointcal.PhotometryFit 527 The photometric fitter used to perform the fit. 528 model : lsst.jointcal.PhotometryModel 529 The photometric model that was fit. 531 self.log.info(
"=== Starting photometric fitting...")
534 if self.config.photometryModel ==
"constrained":
537 visitDegree=self.config.photometryVisitDegree)
538 elif self.config.photometryModel ==
"simple":
542 chi2 = fit.computeChi2()
545 if self.config.writeChi2ContributionFiles:
546 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
547 fit.saveChi2Contributions(baseName)
549 if not np.isfinite(chi2.chi2):
550 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
551 self.log.info(
"Initialized: %s", str(chi2))
552 fit.minimize(
"Model")
553 chi2 = fit.computeChi2()
554 self.log.info(str(chi2))
555 fit.minimize(
"Fluxes")
556 chi2 = fit.computeChi2()
557 self.log.info(str(chi2))
558 fit.minimize(
"Model Fluxes")
559 chi2 = fit.computeChi2()
560 if not np.isfinite(chi2.chi2):
561 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
562 self.log.info(
"Fit prepared with %s", str(chi2))
564 model.freezeErrorTransform()
565 self.log.debug(
"Photometry error scales are frozen.")
567 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
573 def _fit_astrometry(self, associations, dataName=None):
575 Fit the astrometric data. 579 associations : lsst.jointcal.Associations 580 The star/reference star associations to fit. 582 Name of the data being processed (e.g. "1234_HSC-Y"), for 583 identifying debugging files. 588 fit : lsst.jointcal.AstrometryFit 589 The astrometric fitter used to perform the fit. 590 model : lsst.jointcal.AstrometryModel 591 The astrometric model that was fit. 592 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 593 The model for the sky to tangent plane projection that was used in the fit. 596 self.log.info(
"=== Starting astrometric fitting...")
598 associations.deprojectFittedStars()
605 if self.config.astrometryModel ==
"constrainedPoly":
607 sky_to_tan_projection, self.config.useInputWcs, 0,
608 chipDegree=self.config.astrometryChipDegree,
609 visitDegree=self.config.astrometryVisitDegree)
610 elif self.config.astrometryModel ==
"simplePoly":
612 sky_to_tan_projection, self.config.useInputWcs, 0,
613 self.config.astrometrySimpleDegree)
616 chi2 = fit.computeChi2()
619 if self.config.writeChi2ContributionFiles:
620 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
621 fit.saveChi2Contributions(baseName)
623 if not np.isfinite(chi2.chi2):
624 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
625 self.log.info(
"Initialized: %s", str(chi2))
626 fit.minimize(
"Distortions")
627 chi2 = fit.computeChi2()
628 self.log.info(str(chi2))
629 fit.minimize(
"Positions")
630 chi2 = fit.computeChi2()
631 self.log.info(str(chi2))
632 fit.minimize(
"Distortions Positions")
633 chi2 = fit.computeChi2()
634 self.log.info(str(chi2))
635 if not np.isfinite(chi2.chi2):
636 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
637 self.log.info(
"Fit prepared with %s", str(chi2))
639 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
644 return Astrometry(fit, model, sky_to_tan_projection)
646 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
647 """Run fit.minimize up to max_steps times, returning the final chi2.""" 649 for i
in range(max_steps):
650 r = fit.minimize(whatToFit, 5)
651 chi2 = fit.computeChi2()
652 if not np.isfinite(chi2.chi2):
653 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
654 self.log.info(str(chi2))
655 if r == MinimizeResult.Converged:
656 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 657 "one more time in case we have lost accuracy in rank update")
659 r = fit.minimize(whatToFit, 5)
660 chi2 = fit.computeChi2()
661 self.log.info(
"Fit completed with: %s", str(chi2))
663 elif r == MinimizeResult.Chi2Increased:
664 self.log.warn(
"still some ouliers but chi2 increases - retry")
665 elif r == MinimizeResult.Failed:
666 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
668 raise RuntimeError(
"Unxepected return code from minimize().")
670 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
674 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
676 Write the fitted astrometric results to a new 'wcs' dataRef. 680 associations : lsst.jointcal.Associations 681 The star/reference star associations to fit. 682 model : lsst.jointcal.AstrometryModel 683 The astrometric model that was fit. 684 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 685 dict of ccdImage identifiers to dataRefs that were fit 688 ccdImageList = associations.getCcdImageList()
689 for ccdImage
in ccdImageList:
692 visit = ccdImage.visit
693 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
694 exp = afwImage.ExposureI(0, 0)
695 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
696 tanSip = model.produceSipWcs(ccdImage)
700 dataRef.put(exp,
'wcs')
701 except pexExceptions.Exception
as e:
702 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
705 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
707 Write the fitted photometric results to a new 'photoCalib' dataRef. 711 associations : lsst.jointcal.Associations 712 The star/reference star associations to fit. 713 model : lsst.jointcal.PhotometryModel 714 The photoometric model that was fit. 715 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 716 dict of ccdImage identifiers to dataRefs that were fit 719 ccdImageList = associations.getCcdImageList()
720 for ccdImage
in ccdImageList:
723 visit = ccdImage.visit
724 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
725 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
726 photoCalib = model.toPhotoCalib(ccdImage)
728 dataRef.put(photoCalib,
'photoCalib')
729 except pexExceptions.Exception
as e:
730 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
std::shared_ptr< lsst::afw::geom::SkyWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)