3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
10 import lsst.pex.config
as pexConfig
11 import lsst.pipe.base
as pipeBase
12 import lsst.afw.image
as afwImage
13 import lsst.afw.geom
as afwGeom
14 import lsst.afw.coord
as afwCoord
15 import lsst.pex.exceptions
as pexExceptions
17 import lsst.meas.algorithms
19 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
20 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - None if self.doReturnResults is False 69 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 70 - dataRef: the provided data references, with update post-fit WCS's. 73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask""" 84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 astrometryRefObjLoader = pexConfig.ConfigurableField(
135 target=LoadAstrometryNetObjectsTask,
136 doc=
"Reference object loader for astrometric fit",
138 photometryRefObjLoader = pexConfig.ConfigurableField(
139 target=LoadAstrometryNetObjectsTask,
140 doc=
"Reference object loader for photometric fit",
142 sourceSelector = sourceSelectorRegistry.makeField(
143 doc=
"How to select sources for cross-matching",
149 sourceSelector.setDefaults()
151 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
153 sourceSelector.sourceFluxType =
'Calib' 157 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 159 ConfigClass = JointcalConfig
160 RunnerClass = JointcalRunner
161 _DefaultName =
"jointcal" 163 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
165 Instantiate a JointcalTask. 169 butler : lsst.daf.persistence.Butler 170 The butler is passed to the refObjLoader constructor in case it is 171 needed. Ignored if the refObjLoader argument provides a loader directly. 172 Used to initialize the astrometry and photometry refObjLoaders. 173 profile_jointcal : bool 174 set to True to profile different stages of this jointcal run. 176 pipeBase.CmdLineTask.__init__(self, **kwargs)
178 self.makeSubtask(
"sourceSelector")
179 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
180 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
187 def _getConfigName(self):
190 def _getMetadataName(self):
194 def _makeArgumentParser(cls):
195 """Create an argument parser""" 197 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
198 help=
"Profile steps of jointcal separately.")
199 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
200 ContainerClass=PerTractCcdDataIdContainer)
203 def _build_ccdImage(self, dataRef, associations, jointcalControl):
205 Extract the necessary things from this dataRef to add a new ccdImage. 209 dataRef : lsst.daf.persistence.ButlerDataRef 210 dataRef to extract info from. 211 associations : lsst.jointcal.Associations 212 object to add the info to, to construct a new CcdImage 213 jointcalControl : jointcal.JointcalControl 214 control object for associations management 219 wcs : lsst.afw.image.TanWcs 220 the TAN WCS of this image, read from the calexp 222 a key to identify this dataRef by its visit and ccd ids 226 visit = dataRef.dataId[
"visit"]
227 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
228 calexp = dataRef.get(
"calexp", immediate=
True)
229 visitInfo = calexp.getInfo().getVisitInfo()
230 ccdname = calexp.getDetector().getId()
232 calib = calexp.getCalib()
233 tanWcs = calexp.getWcs()
234 bbox = calexp.getBBox()
235 filt = calexp.getInfo().getFilter().getName()
236 fluxMag0 = calib.getFluxMag0()
237 photoCalib = afwImage.PhotoCalib(fluxMag0[0], fluxMag0[1], bbox)
239 goodSrc = self.sourceSelector.selectSources(src)
241 if len(goodSrc.sourceCat) == 0:
242 self.log.warn(
"no stars selected in ", visit, ccdname)
244 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
245 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, photoCalib,
246 visit, ccdname, jointcalControl)
248 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
249 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
250 return Result(tanWcs, Key(visit, ccdname), filt)
253 def run(self, dataRefs, profile_jointcal=False):
255 Jointly calibrate the astrometry and photometry across a set of images. 259 dataRefs : list of lsst.daf.persistence.ButlerDataRef 260 List of data references to the exposures to be fit. 261 profile_jointcal : bool 262 Profile the individual steps of jointcal. 268 * dataRefs: the provided data references that were fit (with updated WCSs) 269 * oldWcsList: the original WCS from each dataRef 270 * metrics: dictionary of internally-computed metrics for testing/validation. 272 if len(dataRefs) == 0:
273 raise ValueError(
'Need a list of data references!')
275 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
279 visit_ccd_to_dataRef = {}
282 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 283 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
286 oldWcsList.append(result.wcs)
287 visit_ccd_to_dataRef[result.key] = ref
288 filters.append(result.filter)
289 filters = collections.Counter(filters)
291 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
292 commonTangentPoint = lsst.afw.coord.averageCoord(centers)
293 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
294 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
299 bbox = associations.getRaDecBBox()
300 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
301 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
302 radius = center.angularSeparation(corner).asRadians()
305 anDir = lsst.utils.getPackageDir(
'astrometry_net_data')
307 raise RuntimeError(
"astrometry_net_data is not setup")
310 defaultFilter = filters.most_common(1)[0][0]
311 self.log.debug(
"Using %s band for reference flux", defaultFilter)
314 tract = dataRefs[0].dataId[
'tract']
316 if self.config.doAstrometry:
319 refObjLoader=self.astrometryRefObjLoader,
321 profile_jointcal=profile_jointcal,
326 if self.config.doPhotometry:
329 refObjLoader=self.photometryRefObjLoader,
331 profile_jointcal=profile_jointcal,
337 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 338 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
339 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
341 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
343 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
344 name="", refObjLoader=None, filters=[], fit_function=None,
345 tract=None, profile_jointcal=False, match_cut=3.0):
346 """Load reference catalog, perform the fit, and return the result. 350 associations : lsst.jointcal.Associations 351 The star/reference star associations to fit. 353 filter to load from reference catalog. 354 center : lsst.afw.coord.Coord 355 Center of field to load from reference catalog. 356 radius : lsst.afw.geom.Angle 357 On-sky radius to load from reference catalog. 359 Name of thing being fit: "Astrometry" or "Photometry". 360 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 361 Reference object loader to load from for fit. 362 filters : list of str, optional 363 List of filters to load from the reference catalog. 364 fit_function : function 365 function to call to perform fit (takes associations object). 367 Name of tract currently being fit. 368 profile_jointcal : bool, optional 369 Separately profile the fitting step. 370 match_cut : float, optional 371 Radius in arcseconds to find cross-catalog matches to during 372 associations.associateCatalogs. 376 Result of `fit_function()` 378 self.log.info(
"====== Now processing %s...", name)
381 associations.associateCatalogs(match_cut)
382 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
384 skyCircle = refObjLoader.loadSkyCircle(center,
385 afwGeom.Angle(radius, afwGeom.radians),
389 if not skyCircle.refCat.isContiguous():
390 refCat = skyCircle.refCat.copy(deep=
True)
392 refCat = skyCircle.refCat
399 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
400 refFluxes[filt] = refCat.get(filtKeys[0])
401 refFluxErrs[filt] = refCat.get(filtKeys[1])
403 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
404 skyCircle.fluxField, refFluxes, refFluxErrs)
405 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
407 associations.selectFittedStars(self.config.minMeasurements)
409 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
410 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
411 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
413 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 414 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
415 result = fit_function(associations)
418 tupleName =
"{}_res_{}.list".format(name, tract)
419 result.fit.saveResultTuples(tupleName)
423 def _check_star_lists(self, associations, name):
425 if associations.nCcdImagesValidForFit() == 0:
426 raise RuntimeError(
'No images in the ccdImageList!')
427 if associations.fittedStarListSize() == 0:
428 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
429 if associations.refStarListSize() == 0:
430 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
432 def _fit_photometry(self, associations):
434 Fit the photometric data. 438 associations : lsst.jointcal.Associations 439 The star/reference star associations to fit. 444 fit : lsst.jointcal.PhotometryFit 445 The photometric fitter used to perform the fit. 446 model : lsst.jointcal.PhotometryModel 447 The photometric model that was fit. 449 self.log.info(
"=== Starting photometric fitting...")
452 if self.config.photometryModel ==
"constrained":
454 elif self.config.photometryModel ==
"simple":
458 chi2 = fit.computeChi2()
459 self.log.info(
"Initialized: %s", str(chi2))
460 fit.minimize(
"Model")
461 chi2 = fit.computeChi2()
462 self.log.info(str(chi2))
463 fit.minimize(
"Fluxes")
464 chi2 = fit.computeChi2()
465 self.log.info(str(chi2))
466 fit.minimize(
"Model Fluxes")
467 chi2 = fit.computeChi2()
468 self.log.info(
"Fit prepared with %s", str(chi2))
470 chi2 = self.
_iterate_fit(fit, 20,
"photometry",
"Model Fluxes")
472 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
473 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
476 def _fit_astrometry(self, associations):
478 Fit the astrometric data. 482 associations : lsst.jointcal.Associations 483 The star/reference star associations to fit. 488 fit : lsst.jointcal.AstrometryFit 489 The astrometric fitter used to perform the fit. 490 model : lsst.jointcal.AstrometryModel 491 The astrometric model that was fit. 492 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 493 The model for the sky to tangent plane projection that was used in the fit. 496 self.log.info(
"=== Starting astrometric fitting...")
498 associations.deprojectFittedStars()
505 if self.config.astrometryModel ==
"constrainedPoly":
507 sky_to_tan_projection,
True, 0)
508 elif self.config.astrometryModel ==
"simplePoly":
510 sky_to_tan_projection,
511 True, 0, self.config.polyOrder)
514 chi2 = fit.computeChi2()
515 self.log.info(
"Initialized: %s", str(chi2))
516 fit.minimize(
"Distortions")
517 chi2 = fit.computeChi2()
518 self.log.info(str(chi2))
519 fit.minimize(
"Positions")
520 chi2 = fit.computeChi2()
521 self.log.info(str(chi2))
522 fit.minimize(
"Distortions Positions")
523 chi2 = fit.computeChi2()
524 self.log.info(str(chi2))
526 chi2 = self.
_iterate_fit(fit, 20,
"astrometry",
"Distortions Positions")
528 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
529 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
531 return Astrometry(fit, model, sky_to_tan_projection)
533 def _iterate_fit(self, fit, max_steps, name, whatToFit):
534 """Run fit.minimize up to max_steps times, returning the final chi2.""" 536 for i
in range(max_steps):
537 r = fit.minimize(whatToFit, 5)
538 chi2 = fit.computeChi2()
539 self.log.info(str(chi2))
540 if r == MinimizeResult.Converged:
541 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 542 "one more time in case we have lost accuracy in rank update")
544 r = fit.minimize(whatToFit, 5)
545 chi2 = fit.computeChi2()
546 self.log.info(
"Fit completed with: %s", str(chi2))
548 elif r == MinimizeResult.Failed:
549 self.log.warn(
"minimization failed")
551 elif r == MinimizeResult.Chi2Increased:
552 self.log.warn(
"still some ouliers but chi2 increases - retry")
554 self.log.error(
"unxepected return code from minimize")
557 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
561 def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef):
563 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 567 associations : lsst.jointcal.Associations 568 The star/reference star associations to fit. 569 astrom_model : lsst.jointcal.AstrometryModel 570 The astrometric model that was fit. 571 photom_model : lsst.jointcal.PhotometryModel 572 The photometric model that was fit. 573 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 574 dict of ccdImage identifiers to dataRefs that were fit 577 ccdImageList = associations.getCcdImageList()
578 for ccdImage
in ccdImageList:
581 visit = ccdImage.visit
582 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
583 exp = afwImage.ExposureI(0, 0)
584 if self.config.doAstrometry:
585 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
586 tanSip = astrom_model.produceSipWcs(ccdImage)
589 if self.config.doPhotometry:
590 self.log.info(
"Updating Calib for visit: %d, ccd: %d", visit, ccd)
592 fluxMag0 = ccdImage.getPhotoCalib().getInstFluxMag0()
593 fluxMag0Err = ccdImage.getPhotoCalib().getInstFluxMag0Err()
594 exp.getCalib().setFluxMag0(fluxMag0/photom_model.photomFactor(ccdImage), fluxMag0Err)
596 dataRef.put(exp,
'wcs')
597 except pexExceptions.Exception
as e:
598 self.log.fatal(
'Failed to write updated Wcs and Calib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def _iterate_fit(self, fit, max_steps, name, whatToFit)
def getTargetList(parsedCmd, kwargs)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)