3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
18 from lsst.verify
import Job, Measurement
23 from .dataIds
import PerTractCcdDataIdContainer
28 __all__ = [
"JointcalConfig",
"JointcalRunner",
"JointcalTask"]
30 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
31 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
36 meas = Measurement(job.metrics[name], value)
37 job.measurements.insert(meas)
41 """Subclass of TaskRunner for jointcalTask 43 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 44 extracted from the command line (whereas most CmdLineTasks' run methods take 45 single dataRef, are are called repeatedly). This class transforms the processed 46 arguments generated by the ArgumentParser into the arguments expected by 49 See pipeBase.TaskRunner for more information. 55 Return a list of tuples per tract, each containing (dataRefs, kwargs). 57 Jointcal operates on lists of dataRefs simultaneously. 59 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
60 kwargs[
'butler'] = parsedCmd.butler
64 for ref
in parsedCmd.id.refList:
65 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
67 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
75 Arguments for Task.run() 80 if self.doReturnResults is False: 82 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 84 if self.doReturnResults is True: 86 - ``result``: the result of calling jointcal.run() 87 - ``exitStatus``: 0 if the task completed successfully, 1 otherwise. 92 dataRefList, kwargs = args
93 butler = kwargs.pop(
'butler')
94 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
97 result = task.run(dataRefList, **kwargs)
98 exitStatus = result.exitStatus
99 job_path = butler.get(
'verify_job_filename')
100 result.job.write(job_path[0])
101 except Exception
as e:
106 eName = type(e).__name__
107 tract = dataRefList[0].dataId[
'tract']
108 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
110 if self.doReturnResults:
111 return pipeBase.Struct(result=result, exitStatus=exitStatus)
113 return pipeBase.Struct(exitStatus=exitStatus)
117 """Config for JointcalTask""" 119 doAstrometry = pexConfig.Field(
120 doc=
"Fit astrometry and write the fitted result.",
124 doPhotometry = pexConfig.Field(
125 doc=
"Fit photometry and write the fitted result.",
129 coaddName = pexConfig.Field(
130 doc=
"Type of coadd, typically deep or goodSeeing",
134 posError = pexConfig.Field(
135 doc=
"Constant term for error on position (in pixel unit)",
140 matchCut = pexConfig.Field(
141 doc=
"Matching radius between fitted and reference stars (arcseconds)",
145 minMeasurements = pexConfig.Field(
146 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
150 astrometrySimpleOrder = pexConfig.Field(
151 doc=
"Polynomial order for fitting the simple astrometry model.",
155 astrometryChipOrder = pexConfig.Field(
156 doc=
"Order of the per-chip transform for the constrained astrometry model.",
160 astrometryVisitOrder = pexConfig.Field(
161 doc=
"Order of the per-visit transform for the constrained astrometry model.",
165 useInputWcs = pexConfig.Field(
166 doc=
"Use the input calexp WCSs to initialize a SimpleAstrometryModel.",
170 astrometryModel = pexConfig.ChoiceField(
171 doc=
"Type of model to fit to astrometry",
174 allowed={
"simple":
"One polynomial per ccd",
175 "constrained":
"One polynomial per ccd, and one polynomial per visit"}
177 photometryModel = pexConfig.ChoiceField(
178 doc=
"Type of model to fit to photometry",
181 allowed={
"simple":
"One constant zeropoint per ccd and visit",
182 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
184 photometryVisitOrder = pexConfig.Field(
185 doc=
"Order of the per-visit polynomial transform for the constrained photometry model.",
189 astrometryRefObjLoader = pexConfig.ConfigurableField(
190 target=LoadIndexedReferenceObjectsTask,
191 doc=
"Reference object loader for astrometric fit",
193 photometryRefObjLoader = pexConfig.ConfigurableField(
194 target=LoadIndexedReferenceObjectsTask,
195 doc=
"Reference object loader for photometric fit",
197 sourceSelector = sourceSelectorRegistry.makeField(
198 doc=
"How to select sources for cross-matching",
201 writeChi2ContributionFiles = pexConfig.Field(
203 doc=
"Write initial/final fit files containing the contributions to chi2.",
209 sourceSelector.setDefaults()
211 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
213 sourceSelector.sourceFluxType =
'Calib' 217 """Jointly astrometrically and photometrically calibrate a group of images.""" 219 ConfigClass = JointcalConfig
220 RunnerClass = JointcalRunner
221 _DefaultName =
"jointcal" 223 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
225 Instantiate a JointcalTask. 229 butler : lsst.daf.persistence.Butler 230 The butler is passed to the refObjLoader constructor in case it is 231 needed. Ignored if the refObjLoader argument provides a loader directly. 232 Used to initialize the astrometry and photometry refObjLoaders. 233 profile_jointcal : bool 234 set to True to profile different stages of this jointcal run. 236 pipeBase.CmdLineTask.__init__(self, **kwargs)
238 self.makeSubtask(
"sourceSelector")
239 if self.config.doAstrometry:
240 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
241 if self.config.doPhotometry:
242 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
245 self.
job = Job.load_metrics_package(subset=
'jointcal')
249 def _getConfigName(self):
252 def _getMetadataName(self):
256 def _makeArgumentParser(cls):
257 """Create an argument parser""" 259 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
260 help=
"Profile steps of jointcal separately.")
261 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
262 ContainerClass=PerTractCcdDataIdContainer)
265 def _build_ccdImage(self, dataRef, associations, jointcalControl):
267 Extract the necessary things from this dataRef to add a new ccdImage. 271 dataRef : lsst.daf.persistence.ButlerDataRef 272 dataRef to extract info from. 273 associations : lsst.jointcal.Associations 274 object to add the info to, to construct a new CcdImage 275 jointcalControl : jointcal.JointcalControl 276 control object for associations management 281 wcs : lsst.afw.geom.SkyWcs 282 the TAN WCS of this image, read from the calexp 284 a key to identify this dataRef by its visit and ccd ids 288 if "visit" in dataRef.dataId.keys():
289 visit = dataRef.dataId[
"visit"]
291 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
293 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
295 visitInfo = dataRef.get(
'calexp_visitInfo')
296 detector = dataRef.get(
'calexp_detector')
297 ccdId = detector.getId()
298 calib = dataRef.get(
'calexp_calib')
299 tanWcs = dataRef.get(
'calexp_wcs')
300 bbox = dataRef.get(
'calexp_bbox')
301 filt = dataRef.get(
'calexp_filter')
302 filterName = filt.getName()
303 fluxMag0 = calib.getFluxMag0()
304 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
306 goodSrc = self.sourceSelector.selectSources(src)
308 if len(goodSrc.sourceCat) == 0:
309 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
311 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
312 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
313 visit, ccdId, jointcalControl)
315 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
316 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
317 return Result(tanWcs, Key(visit, ccdId), filterName)
320 def run(self, dataRefs, profile_jointcal=False):
322 Jointly calibrate the astrometry and photometry across a set of images. 326 dataRefs : list of lsst.daf.persistence.ButlerDataRef 327 List of data references to the exposures to be fit. 328 profile_jointcal : bool 329 Profile the individual steps of jointcal. 335 * dataRefs: the provided data references that were fit (with updated WCSs) 336 * oldWcsList: the original WCS from each dataRef 337 * metrics: dictionary of internally-computed metrics for testing/validation. 339 if len(dataRefs) == 0:
340 raise ValueError(
'Need a non-empty list of data references!')
344 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
348 visit_ccd_to_dataRef = {}
351 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 352 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
355 camera = dataRefs[0].get(
'camera', immediate=
True)
359 oldWcsList.append(result.wcs)
360 visit_ccd_to_dataRef[result.key] = ref
361 filters.append(result.filter)
362 filters = collections.Counter(filters)
364 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
365 commonTangentPoint = afwGeom.averageSpherePoint(centers)
366 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition(afwGeom.degrees))
367 associations.setCommonTangentPoint(commonTangentPoint.getPosition(afwGeom.degrees))
372 bbox = associations.getRaDecBBox()
374 bboxCenter = bbox.getCenter()
375 center = afwGeom.SpherePoint(bboxCenter[0], bboxCenter[1], afwGeom.degrees)
376 bboxMax = bbox.getMax()
377 corner = afwGeom.SpherePoint(bboxMax[0], bboxMax[1], afwGeom.degrees)
378 radius = center.separation(corner).asRadians()
383 raise RuntimeError(
"astrometry_net_data is not setup")
386 defaultFilter = filters.most_common(1)[0][0]
387 self.log.debug(
"Using %s band for reference flux", defaultFilter)
390 tract = dataRefs[0].dataId[
'tract']
392 if self.config.doAstrometry:
395 refObjLoader=self.astrometryRefObjLoader,
397 profile_jointcal=profile_jointcal,
403 if self.config.doPhotometry:
406 refObjLoader=self.photometryRefObjLoader,
408 profile_jointcal=profile_jointcal,
411 reject_bad_fluxes=
True)
416 return pipeBase.Struct(dataRefs=dataRefs,
417 oldWcsList=oldWcsList,
419 exitStatus=exitStatus)
421 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
422 name="", refObjLoader=None, filters=[], fit_function=None,
423 tract=None, profile_jointcal=False, match_cut=3.0,
424 reject_bad_fluxes=False):
425 """Load reference catalog, perform the fit, and return the result. 429 associations : lsst.jointcal.Associations 430 The star/reference star associations to fit. 432 filter to load from reference catalog. 433 center : lsst.afw.geom.SpherePoint 434 ICRS center of field to load from reference catalog. 435 radius : lsst.afw.geom.Angle 436 On-sky radius to load from reference catalog. 438 Name of thing being fit: "Astrometry" or "Photometry". 439 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 440 Reference object loader to load from for fit. 441 filters : list of str, optional 442 List of filters to load from the reference catalog. 443 fit_function : function 444 function to call to perform fit (takes associations object). 446 Name of tract currently being fit. 447 profile_jointcal : bool, optional 448 Separately profile the fitting step. 449 match_cut : float, optional 450 Radius in arcseconds to find cross-catalog matches to during 451 associations.associateCatalogs. 452 reject_bad_fluxes : bool, optional 453 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 457 Result of `fit_function()` 459 self.log.info(
"====== Now processing %s...", name)
462 associations.associateCatalogs(match_cut)
464 associations.fittedStarListSize())
466 skyCircle = refObjLoader.loadSkyCircle(center,
467 afwGeom.Angle(radius, afwGeom.radians),
471 if not skyCircle.refCat.isContiguous():
472 refCat = skyCircle.refCat.copy(deep=
True)
474 refCat = skyCircle.refCat
481 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
482 refFluxes[filt] = refCat.get(filtKeys[0])
483 refFluxErrs[filt] = refCat.get(filtKeys[1])
485 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
486 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
488 associations.refStarListSize())
490 associations.prepareFittedStars(self.config.minMeasurements)
494 associations.nFittedStarsWithAssociatedRefStar())
496 associations.fittedStarListSize())
498 associations.nCcdImagesValidForFit())
500 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 501 dataName =
"{}_{}".format(tract, defaultFilter)
502 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
503 result = fit_function(associations, dataName)
506 if self.config.writeChi2ContributionFiles:
507 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
508 result.fit.saveChi2Contributions(baseName)
512 def _check_star_lists(self, associations, name):
514 if associations.nCcdImagesValidForFit() == 0:
515 raise RuntimeError(
'No images in the ccdImageList!')
516 if associations.fittedStarListSize() == 0:
517 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
518 if associations.refStarListSize() == 0:
519 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
521 def _fit_photometry(self, associations, dataName=None):
523 Fit the photometric data. 527 associations : lsst.jointcal.Associations 528 The star/reference star associations to fit. 530 Name of the data being processed (e.g. "1234_HSC-Y"), for 531 identifying debugging files. 536 fit : lsst.jointcal.PhotometryFit 537 The photometric fitter used to perform the fit. 538 model : lsst.jointcal.PhotometryModel 539 The photometric model that was fit. 541 self.log.info(
"=== Starting photometric fitting...")
544 if self.config.photometryModel ==
"constrained":
547 visitOrder=self.config.photometryVisitOrder)
548 elif self.config.photometryModel ==
"simple":
552 chi2 = fit.computeChi2()
555 if self.config.writeChi2ContributionFiles:
556 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
557 fit.saveChi2Contributions(baseName)
559 if not np.isfinite(chi2.chi2):
560 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
561 self.log.info(
"Initialized: %s", str(chi2))
564 if self.config.photometryModel ==
"constrained":
566 fit.minimize(
"ModelVisit")
567 chi2 = fit.computeChi2()
568 self.log.info(str(chi2))
569 fit.minimize(
"Model")
570 chi2 = fit.computeChi2()
571 self.log.info(str(chi2))
572 fit.minimize(
"Fluxes")
573 chi2 = fit.computeChi2()
574 self.log.info(str(chi2))
575 fit.minimize(
"Model Fluxes")
576 chi2 = fit.computeChi2()
577 if not np.isfinite(chi2.chi2):
578 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
579 self.log.info(
"Fit prepared with %s", str(chi2))
581 model.freezeErrorTransform()
582 self.log.debug(
"Photometry error scales are frozen.")
584 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
590 def _fit_astrometry(self, associations, dataName=None):
592 Fit the astrometric data. 596 associations : lsst.jointcal.Associations 597 The star/reference star associations to fit. 599 Name of the data being processed (e.g. "1234_HSC-Y"), for 600 identifying debugging files. 605 fit : lsst.jointcal.AstrometryFit 606 The astrometric fitter used to perform the fit. 607 model : lsst.jointcal.AstrometryModel 608 The astrometric model that was fit. 609 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 610 The model for the sky to tangent plane projection that was used in the fit. 613 self.log.info(
"=== Starting astrometric fitting...")
615 associations.deprojectFittedStars()
622 if self.config.astrometryModel ==
"constrained":
624 sky_to_tan_projection,
625 chipOrder=self.config.astrometryChipOrder,
626 visitOrder=self.config.astrometryVisitOrder)
627 elif self.config.astrometryModel ==
"simple":
629 sky_to_tan_projection,
630 self.config.useInputWcs,
632 order=self.config.astrometrySimpleOrder)
635 chi2 = fit.computeChi2()
638 if self.config.writeChi2ContributionFiles:
639 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
640 fit.saveChi2Contributions(baseName)
642 if not np.isfinite(chi2.chi2):
643 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
644 self.log.info(
"Initialized: %s", str(chi2))
647 if self.config.astrometryModel ==
"constrained":
648 fit.minimize(
"DistortionsVisit")
649 chi2 = fit.computeChi2()
650 self.log.info(str(chi2))
651 fit.minimize(
"Distortions")
652 chi2 = fit.computeChi2()
653 self.log.info(str(chi2))
654 fit.minimize(
"Positions")
655 chi2 = fit.computeChi2()
656 self.log.info(str(chi2))
657 fit.minimize(
"Distortions Positions")
658 chi2 = fit.computeChi2()
659 self.log.info(str(chi2))
660 if not np.isfinite(chi2.chi2):
661 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
662 self.log.info(
"Fit prepared with %s", str(chi2))
664 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
669 return Astrometry(fit, model, sky_to_tan_projection)
671 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
672 """Run fit.minimize up to max_steps times, returning the final chi2.""" 674 for i
in range(max_steps):
675 r = fit.minimize(whatToFit, 5)
676 chi2 = fit.computeChi2()
677 if not np.isfinite(chi2.chi2):
678 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
679 self.log.info(str(chi2))
680 if r == MinimizeResult.Converged:
681 self.log.debug(
"fit has converged - no more outliers - redo minimization " 682 "one more time in case we have lost accuracy in rank update.")
684 r = fit.minimize(whatToFit, 5)
685 chi2 = fit.computeChi2()
686 self.log.info(
"Fit completed with: %s", str(chi2))
688 elif r == MinimizeResult.Chi2Increased:
689 self.log.warn(
"still some ouliers but chi2 increases - retry")
690 elif r == MinimizeResult.Failed:
691 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
693 raise RuntimeError(
"Unxepected return code from minimize().")
695 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
699 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
701 Write the fitted astrometric results to a new 'jointcal_wcs' dataRef. 705 associations : lsst.jointcal.Associations 706 The star/reference star associations to fit. 707 model : lsst.jointcal.AstrometryModel 708 The astrometric model that was fit. 709 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 710 dict of ccdImage identifiers to dataRefs that were fit 713 ccdImageList = associations.getCcdImageList()
714 for ccdImage
in ccdImageList:
717 visit = ccdImage.visit
718 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
719 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
720 skyWcs = model.makeSkyWcs(ccdImage)
722 dataRef.put(skyWcs,
'jointcal_wcs')
723 except pexExceptions.Exception
as e:
724 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
727 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
729 Write the fitted photometric results to a new 'jointcal_photoCalib' dataRef. 733 associations : lsst.jointcal.Associations 734 The star/reference star associations to fit. 735 model : lsst.jointcal.PhotometryModel 736 The photoometric model that was fit. 737 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 738 dict of ccdImage identifiers to dataRefs that were fit 741 ccdImageList = associations.getCcdImageList()
742 for ccdImage
in ccdImageList:
745 visit = ccdImage.visit
746 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
747 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
748 photoCalib = model.toPhotoCalib(ccdImage)
750 dataRef.put(photoCalib,
'jointcal_photoCalib')
751 except pexExceptions.Exception
as e:
752 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)