3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - None if self.doReturnResults is False 69 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 70 - dataRef: the provided data references, with update post-fit WCS's. 73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask""" 84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 photometryVisitDegree = pexConfig.Field(
135 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
139 astrometryRefObjLoader = pexConfig.ConfigurableField(
140 target=LoadIndexedReferenceObjectsTask,
141 doc=
"Reference object loader for astrometric fit",
143 photometryRefObjLoader = pexConfig.ConfigurableField(
144 target=LoadIndexedReferenceObjectsTask,
145 doc=
"Reference object loader for photometric fit",
147 sourceSelector = sourceSelectorRegistry.makeField(
148 doc=
"How to select sources for cross-matching",
154 sourceSelector.setDefaults()
156 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
158 sourceSelector.sourceFluxType =
'Calib' 162 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 164 ConfigClass = JointcalConfig
165 RunnerClass = JointcalRunner
166 _DefaultName =
"jointcal" 168 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
170 Instantiate a JointcalTask. 174 butler : lsst.daf.persistence.Butler 175 The butler is passed to the refObjLoader constructor in case it is 176 needed. Ignored if the refObjLoader argument provides a loader directly. 177 Used to initialize the astrometry and photometry refObjLoaders. 178 profile_jointcal : bool 179 set to True to profile different stages of this jointcal run. 181 pipeBase.CmdLineTask.__init__(self, **kwargs)
183 self.makeSubtask(
"sourceSelector")
184 if self.config.doAstrometry:
185 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
186 if self.config.doPhotometry:
187 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
194 def _getConfigName(self):
197 def _getMetadataName(self):
201 def _makeArgumentParser(cls):
202 """Create an argument parser""" 204 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
205 help=
"Profile steps of jointcal separately.")
206 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
207 ContainerClass=PerTractCcdDataIdContainer)
210 def _build_ccdImage(self, dataRef, associations, jointcalControl):
212 Extract the necessary things from this dataRef to add a new ccdImage. 216 dataRef : lsst.daf.persistence.ButlerDataRef 217 dataRef to extract info from. 218 associations : lsst.jointcal.Associations 219 object to add the info to, to construct a new CcdImage 220 jointcalControl : jointcal.JointcalControl 221 control object for associations management 226 wcs : lsst.afw.image.TanWcs 227 the TAN WCS of this image, read from the calexp 229 a key to identify this dataRef by its visit and ccd ids 233 if "visit" in dataRef.dataId.keys():
234 visit = dataRef.dataId[
"visit"]
236 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
238 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
240 visitInfo = dataRef.get(
'calexp_visitInfo')
241 detector = dataRef.get(
'calexp_detector')
242 ccdname = detector.getId()
243 calib = dataRef.get(
'calexp_calib')
244 tanWcs = dataRef.get(
'calexp_wcs')
245 bbox = dataRef.get(
'calexp_bbox')
246 filt = dataRef.get(
'calexp_filter')
247 filterName = filt.getName()
248 fluxMag0 = calib.getFluxMag0()
249 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
251 goodSrc = self.sourceSelector.selectSources(src)
253 if len(goodSrc.sourceCat) == 0:
254 self.log.warn(
"no stars selected in ", visit, ccdname)
256 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
257 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
258 visit, ccdname, jointcalControl)
260 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
261 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
262 return Result(tanWcs, Key(visit, ccdname), filterName)
265 def run(self, dataRefs, profile_jointcal=False):
267 Jointly calibrate the astrometry and photometry across a set of images. 271 dataRefs : list of lsst.daf.persistence.ButlerDataRef 272 List of data references to the exposures to be fit. 273 profile_jointcal : bool 274 Profile the individual steps of jointcal. 280 * dataRefs: the provided data references that were fit (with updated WCSs) 281 * oldWcsList: the original WCS from each dataRef 282 * metrics: dictionary of internally-computed metrics for testing/validation. 284 if len(dataRefs) == 0:
285 raise ValueError(
'Need a list of data references!')
287 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
291 visit_ccd_to_dataRef = {}
294 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 295 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
298 camera = dataRefs[0].get(
'camera', immediate=
True)
302 oldWcsList.append(result.wcs)
303 visit_ccd_to_dataRef[result.key] = ref
304 filters.append(result.filter)
305 filters = collections.Counter(filters)
307 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
309 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
310 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
315 bbox = associations.getRaDecBBox()
316 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
317 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
318 radius = center.angularSeparation(corner).asRadians()
323 raise RuntimeError(
"astrometry_net_data is not setup")
326 defaultFilter = filters.most_common(1)[0][0]
327 self.log.debug(
"Using %s band for reference flux", defaultFilter)
330 tract = dataRefs[0].dataId[
'tract']
332 if self.config.doAstrometry:
335 refObjLoader=self.astrometryRefObjLoader,
337 profile_jointcal=profile_jointcal,
342 if self.config.doPhotometry:
345 refObjLoader=self.photometryRefObjLoader,
347 profile_jointcal=profile_jointcal,
353 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 354 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
355 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
357 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
359 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
360 name="", refObjLoader=None, filters=[], fit_function=None,
361 tract=None, profile_jointcal=False, match_cut=3.0):
362 """Load reference catalog, perform the fit, and return the result. 366 associations : lsst.jointcal.Associations 367 The star/reference star associations to fit. 369 filter to load from reference catalog. 370 center : lsst.afw.coord.Coord 371 Center of field to load from reference catalog. 372 radius : lsst.afw.geom.Angle 373 On-sky radius to load from reference catalog. 375 Name of thing being fit: "Astrometry" or "Photometry". 376 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 377 Reference object loader to load from for fit. 378 filters : list of str, optional 379 List of filters to load from the reference catalog. 380 fit_function : function 381 function to call to perform fit (takes associations object). 383 Name of tract currently being fit. 384 profile_jointcal : bool, optional 385 Separately profile the fitting step. 386 match_cut : float, optional 387 Radius in arcseconds to find cross-catalog matches to during 388 associations.associateCatalogs. 392 Result of `fit_function()` 394 self.log.info(
"====== Now processing %s...", name)
397 associations.associateCatalogs(match_cut)
398 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
400 skyCircle = refObjLoader.loadSkyCircle(center,
405 if not skyCircle.refCat.isContiguous():
406 refCat = skyCircle.refCat.copy(deep=
True)
408 refCat = skyCircle.refCat
415 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
416 refFluxes[filt] = refCat.get(filtKeys[0])
417 refFluxErrs[filt] = refCat.get(filtKeys[1])
419 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
420 skyCircle.fluxField, refFluxes, refFluxErrs)
421 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
423 associations.selectFittedStars(self.config.minMeasurements)
425 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
426 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
427 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
429 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 430 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
431 result = fit_function(associations)
434 tupleName =
"{}_res_{}.list".format(name, tract)
435 result.fit.saveResultTuples(tupleName)
439 def _check_star_lists(self, associations, name):
441 if associations.nCcdImagesValidForFit() == 0:
442 raise RuntimeError(
'No images in the ccdImageList!')
443 if associations.fittedStarListSize() == 0:
444 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
445 if associations.refStarListSize() == 0:
446 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
448 def _fit_photometry(self, associations):
450 Fit the photometric data. 454 associations : lsst.jointcal.Associations 455 The star/reference star associations to fit. 460 fit : lsst.jointcal.PhotometryFit 461 The photometric fitter used to perform the fit. 462 model : lsst.jointcal.PhotometryModel 463 The photometric model that was fit. 465 self.log.info(
"=== Starting photometric fitting...")
468 if self.config.photometryModel ==
"constrained":
471 visitDegree=self.config.photometryVisitDegree)
472 elif self.config.photometryModel ==
"simple":
476 chi2 = fit.computeChi2()
477 self.log.info(
"Initialized: %s", str(chi2))
478 fit.minimize(
"Model")
479 chi2 = fit.computeChi2()
480 self.log.info(str(chi2))
481 fit.minimize(
"Fluxes")
482 chi2 = fit.computeChi2()
483 self.log.info(str(chi2))
484 fit.minimize(
"Model Fluxes")
485 chi2 = fit.computeChi2()
486 self.log.info(
"Fit prepared with %s", str(chi2))
488 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
490 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
491 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
494 def _fit_astrometry(self, associations):
496 Fit the astrometric data. 500 associations : lsst.jointcal.Associations 501 The star/reference star associations to fit. 506 fit : lsst.jointcal.AstrometryFit 507 The astrometric fitter used to perform the fit. 508 model : lsst.jointcal.AstrometryModel 509 The astrometric model that was fit. 510 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 511 The model for the sky to tangent plane projection that was used in the fit. 514 self.log.info(
"=== Starting astrometric fitting...")
516 associations.deprojectFittedStars()
523 if self.config.astrometryModel ==
"constrainedPoly":
525 sky_to_tan_projection,
True, 0)
526 elif self.config.astrometryModel ==
"simplePoly":
528 sky_to_tan_projection,
529 True, 0, self.config.polyOrder)
532 chi2 = fit.computeChi2()
533 self.log.info(
"Initialized: %s", str(chi2))
534 fit.minimize(
"Distortions")
535 chi2 = fit.computeChi2()
536 self.log.info(str(chi2))
537 fit.minimize(
"Positions")
538 chi2 = fit.computeChi2()
539 self.log.info(str(chi2))
540 fit.minimize(
"Distortions Positions")
541 chi2 = fit.computeChi2()
542 self.log.info(str(chi2))
544 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
546 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
547 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
549 return Astrometry(fit, model, sky_to_tan_projection)
551 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
552 """Run fit.minimize up to max_steps times, returning the final chi2.""" 554 for i
in range(max_steps):
555 r = fit.minimize(whatToFit, 5)
556 chi2 = fit.computeChi2()
557 self.log.info(str(chi2))
558 if r == MinimizeResult.Converged:
559 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 560 "one more time in case we have lost accuracy in rank update")
562 r = fit.minimize(whatToFit, 5)
563 chi2 = fit.computeChi2()
564 self.log.info(
"Fit completed with: %s", str(chi2))
566 elif r == MinimizeResult.Failed:
567 self.log.warn(
"minimization failed")
569 elif r == MinimizeResult.Chi2Increased:
570 self.log.warn(
"still some ouliers but chi2 increases - retry")
572 self.log.error(
"unxepected return code from minimize")
575 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
579 def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef):
581 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 585 associations : lsst.jointcal.Associations 586 The star/reference star associations to fit. 587 astrometry_model : lsst.jointcal.AstrometryModel 588 The astrometric model that was fit. 589 photometry_model : lsst.jointcal.PhotometryModel 590 The photometric model that was fit. 591 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 592 dict of ccdImage identifiers to dataRefs that were fit 595 ccdImageList = associations.getCcdImageList()
596 for ccdImage
in ccdImageList:
599 visit = ccdImage.visit
600 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
601 exp = afwImage.ExposureI(0, 0)
602 if self.config.doAstrometry:
603 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
604 tanSip = astrometry_model.produceSipWcs(ccdImage)
608 dataRef.put(exp,
'wcs')
609 except pexExceptions.Exception
as e:
610 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
612 if self.config.doPhotometry:
613 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
614 photoCalib = photometry_model.toPhotoCalib(ccdImage)
616 dataRef.put(photoCalib,
'photoCalib')
617 except pexExceptions.Exception
as e:
618 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def getTargetList(parsedCmd, kwargs)
def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)