3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
10 import lsst.pex.config
as pexConfig
11 import lsst.pipe.base
as pipeBase
12 import lsst.afw.image
as afwImage
13 import lsst.afw.geom
as afwGeom
14 import lsst.afw.coord
as afwCoord
15 import lsst.pex.exceptions
as pexExceptions
17 import lsst.meas.algorithms
19 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
20 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - None if self.doReturnResults is False 69 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 70 - dataRef: the provided data references, with update post-fit WCS's. 73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask""" 84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 astrometryRefObjLoader = pexConfig.ConfigurableField(
135 target=LoadAstrometryNetObjectsTask,
136 doc=
"Reference object loader for astrometric fit",
138 photometryRefObjLoader = pexConfig.ConfigurableField(
139 target=LoadAstrometryNetObjectsTask,
140 doc=
"Reference object loader for photometric fit",
142 sourceSelector = sourceSelectorRegistry.makeField(
143 doc=
"How to select sources for cross-matching",
149 sourceSelector.setDefaults()
151 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
153 sourceSelector.sourceFluxType =
'Calib' 157 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 159 ConfigClass = JointcalConfig
160 RunnerClass = JointcalRunner
161 _DefaultName =
"jointcal" 163 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
165 Instantiate a JointcalTask. 169 butler : lsst.daf.persistence.Butler 170 The butler is passed to the refObjLoader constructor in case it is 171 needed. Ignored if the refObjLoader argument provides a loader directly. 172 Used to initialize the astrometry and photometry refObjLoaders. 173 profile_jointcal : bool 174 set to True to profile different stages of this jointcal run. 176 pipeBase.CmdLineTask.__init__(self, **kwargs)
178 self.makeSubtask(
"sourceSelector")
179 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
180 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
187 def _getConfigName(self):
190 def _getMetadataName(self):
194 def _makeArgumentParser(cls):
195 """Create an argument parser""" 197 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
198 help=
"Profile steps of jointcal separately.")
199 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
200 ContainerClass=PerTractCcdDataIdContainer)
203 def _build_ccdImage(self, dataRef, associations, jointcalControl):
205 Extract the necessary things from this dataRef to add a new ccdImage. 209 dataRef : lsst.daf.persistence.ButlerDataRef 210 dataRef to extract info from. 211 associations : lsst.jointcal.Associations 212 object to add the info to, to construct a new CcdImage 213 jointcalControl : jointcal.JointcalControl 214 control object for associations management 219 wcs : lsst.afw.image.TanWcs 220 the TAN WCS of this image, read from the calexp 222 a key to identify this dataRef by its visit and ccd ids 226 if "visit" in dataRef.dataId.keys():
227 visit = dataRef.dataId[
"visit"]
229 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
230 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
231 calexp = dataRef.get(
"calexp", immediate=
True)
232 visitInfo = calexp.getInfo().getVisitInfo()
233 ccdname = calexp.getDetector().getId()
235 calib = calexp.getCalib()
236 tanWcs = calexp.getWcs()
237 bbox = calexp.getBBox()
238 filt = calexp.getInfo().getFilter().getName()
239 fluxMag0 = calib.getFluxMag0()
240 photoCalib = afwImage.PhotoCalib(fluxMag0[0], fluxMag0[1], bbox)
242 goodSrc = self.sourceSelector.selectSources(src)
244 if len(goodSrc.sourceCat) == 0:
245 self.log.warn(
"no stars selected in ", visit, ccdname)
247 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
248 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, photoCalib,
249 visit, ccdname, jointcalControl)
251 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
252 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
253 return Result(tanWcs, Key(visit, ccdname), filt)
256 def run(self, dataRefs, profile_jointcal=False):
258 Jointly calibrate the astrometry and photometry across a set of images. 262 dataRefs : list of lsst.daf.persistence.ButlerDataRef 263 List of data references to the exposures to be fit. 264 profile_jointcal : bool 265 Profile the individual steps of jointcal. 271 * dataRefs: the provided data references that were fit (with updated WCSs) 272 * oldWcsList: the original WCS from each dataRef 273 * metrics: dictionary of internally-computed metrics for testing/validation. 275 if len(dataRefs) == 0:
276 raise ValueError(
'Need a list of data references!')
278 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
282 visit_ccd_to_dataRef = {}
285 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 286 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
289 oldWcsList.append(result.wcs)
290 visit_ccd_to_dataRef[result.key] = ref
291 filters.append(result.filter)
292 filters = collections.Counter(filters)
294 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
295 commonTangentPoint = lsst.afw.coord.averageCoord(centers)
296 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
297 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
302 bbox = associations.getRaDecBBox()
303 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
304 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
305 radius = center.angularSeparation(corner).asRadians()
308 anDir = lsst.utils.getPackageDir(
'astrometry_net_data')
310 raise RuntimeError(
"astrometry_net_data is not setup")
313 defaultFilter = filters.most_common(1)[0][0]
314 self.log.debug(
"Using %s band for reference flux", defaultFilter)
317 tract = dataRefs[0].dataId[
'tract']
319 if self.config.doAstrometry:
322 refObjLoader=self.astrometryRefObjLoader,
324 profile_jointcal=profile_jointcal,
329 if self.config.doPhotometry:
332 refObjLoader=self.photometryRefObjLoader,
334 profile_jointcal=profile_jointcal,
340 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 341 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
342 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
344 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
346 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
347 name="", refObjLoader=None, filters=[], fit_function=None,
348 tract=None, profile_jointcal=False, match_cut=3.0):
349 """Load reference catalog, perform the fit, and return the result. 353 associations : lsst.jointcal.Associations 354 The star/reference star associations to fit. 356 filter to load from reference catalog. 357 center : lsst.afw.coord.Coord 358 Center of field to load from reference catalog. 359 radius : lsst.afw.geom.Angle 360 On-sky radius to load from reference catalog. 362 Name of thing being fit: "Astrometry" or "Photometry". 363 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 364 Reference object loader to load from for fit. 365 filters : list of str, optional 366 List of filters to load from the reference catalog. 367 fit_function : function 368 function to call to perform fit (takes associations object). 370 Name of tract currently being fit. 371 profile_jointcal : bool, optional 372 Separately profile the fitting step. 373 match_cut : float, optional 374 Radius in arcseconds to find cross-catalog matches to during 375 associations.associateCatalogs. 379 Result of `fit_function()` 381 self.log.info(
"====== Now processing %s...", name)
384 associations.associateCatalogs(match_cut)
385 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
387 skyCircle = refObjLoader.loadSkyCircle(center,
388 afwGeom.Angle(radius, afwGeom.radians),
392 if not skyCircle.refCat.isContiguous():
393 refCat = skyCircle.refCat.copy(deep=
True)
395 refCat = skyCircle.refCat
402 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
403 refFluxes[filt] = refCat.get(filtKeys[0])
404 refFluxErrs[filt] = refCat.get(filtKeys[1])
406 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
407 skyCircle.fluxField, refFluxes, refFluxErrs)
408 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
410 associations.selectFittedStars(self.config.minMeasurements)
412 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
413 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
414 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
416 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 417 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
418 result = fit_function(associations)
421 tupleName =
"{}_res_{}.list".format(name, tract)
422 result.fit.saveResultTuples(tupleName)
426 def _check_star_lists(self, associations, name):
428 if associations.nCcdImagesValidForFit() == 0:
429 raise RuntimeError(
'No images in the ccdImageList!')
430 if associations.fittedStarListSize() == 0:
431 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
432 if associations.refStarListSize() == 0:
433 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
435 def _fit_photometry(self, associations):
437 Fit the photometric data. 441 associations : lsst.jointcal.Associations 442 The star/reference star associations to fit. 447 fit : lsst.jointcal.PhotometryFit 448 The photometric fitter used to perform the fit. 449 model : lsst.jointcal.PhotometryModel 450 The photometric model that was fit. 452 self.log.info(
"=== Starting photometric fitting...")
455 if self.config.photometryModel ==
"constrained":
457 elif self.config.photometryModel ==
"simple":
461 chi2 = fit.computeChi2()
462 self.log.info(
"Initialized: %s", str(chi2))
463 fit.minimize(
"Model")
464 chi2 = fit.computeChi2()
465 self.log.info(str(chi2))
466 fit.minimize(
"Fluxes")
467 chi2 = fit.computeChi2()
468 self.log.info(str(chi2))
469 fit.minimize(
"Model Fluxes")
470 chi2 = fit.computeChi2()
471 self.log.info(
"Fit prepared with %s", str(chi2))
473 chi2 = self.
_iterate_fit(fit, 20,
"photometry",
"Model Fluxes")
475 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
476 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
479 def _fit_astrometry(self, associations):
481 Fit the astrometric data. 485 associations : lsst.jointcal.Associations 486 The star/reference star associations to fit. 491 fit : lsst.jointcal.AstrometryFit 492 The astrometric fitter used to perform the fit. 493 model : lsst.jointcal.AstrometryModel 494 The astrometric model that was fit. 495 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 496 The model for the sky to tangent plane projection that was used in the fit. 499 self.log.info(
"=== Starting astrometric fitting...")
501 associations.deprojectFittedStars()
508 if self.config.astrometryModel ==
"constrainedPoly":
510 sky_to_tan_projection,
True, 0)
511 elif self.config.astrometryModel ==
"simplePoly":
513 sky_to_tan_projection,
514 True, 0, self.config.polyOrder)
517 chi2 = fit.computeChi2()
518 self.log.info(
"Initialized: %s", str(chi2))
519 fit.minimize(
"Distortions")
520 chi2 = fit.computeChi2()
521 self.log.info(str(chi2))
522 fit.minimize(
"Positions")
523 chi2 = fit.computeChi2()
524 self.log.info(str(chi2))
525 fit.minimize(
"Distortions Positions")
526 chi2 = fit.computeChi2()
527 self.log.info(str(chi2))
529 chi2 = self.
_iterate_fit(fit, 20,
"astrometry",
"Distortions Positions")
531 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
532 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
534 return Astrometry(fit, model, sky_to_tan_projection)
536 def _iterate_fit(self, fit, max_steps, name, whatToFit):
537 """Run fit.minimize up to max_steps times, returning the final chi2.""" 539 for i
in range(max_steps):
540 r = fit.minimize(whatToFit, 5)
541 chi2 = fit.computeChi2()
542 self.log.info(str(chi2))
543 if r == MinimizeResult.Converged:
544 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 545 "one more time in case we have lost accuracy in rank update")
547 r = fit.minimize(whatToFit, 5)
548 chi2 = fit.computeChi2()
549 self.log.info(
"Fit completed with: %s", str(chi2))
551 elif r == MinimizeResult.Failed:
552 self.log.warn(
"minimization failed")
554 elif r == MinimizeResult.Chi2Increased:
555 self.log.warn(
"still some ouliers but chi2 increases - retry")
557 self.log.error(
"unxepected return code from minimize")
560 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
564 def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef):
566 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 570 associations : lsst.jointcal.Associations 571 The star/reference star associations to fit. 572 astrom_model : lsst.jointcal.AstrometryModel 573 The astrometric model that was fit. 574 photom_model : lsst.jointcal.PhotometryModel 575 The photometric model that was fit. 576 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 577 dict of ccdImage identifiers to dataRefs that were fit 580 ccdImageList = associations.getCcdImageList()
581 for ccdImage
in ccdImageList:
584 visit = ccdImage.visit
585 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
586 exp = afwImage.ExposureI(0, 0)
587 if self.config.doAstrometry:
588 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
589 tanSip = astrom_model.produceSipWcs(ccdImage)
592 if self.config.doPhotometry:
593 self.log.info(
"Updating Calib for visit: %d, ccd: %d", visit, ccd)
595 fluxMag0 = ccdImage.getPhotoCalib().getInstFluxMag0()
596 fluxMag0Err = ccdImage.getPhotoCalib().getInstFluxMag0Err()
597 exp.getCalib().setFluxMag0(fluxMag0/photom_model.photomFactor(ccdImage), fluxMag0Err)
599 dataRef.put(exp,
'wcs')
600 except pexExceptions.Exception
as e:
601 self.log.fatal(
'Failed to write updated Wcs and Calib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def _iterate_fit(self, fit, max_steps, name, whatToFit)
def getTargetList(parsedCmd, kwargs)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)