3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
10 import lsst.pex.config
as pexConfig
11 import lsst.pipe.base
as pipeBase
12 import lsst.afw.image
as afwImage
13 import lsst.afw.geom
as afwGeom
14 import lsst.afw.coord
as afwCoord
15 import lsst.pex.exceptions
as pexExceptions
17 import lsst.meas.algorithms
19 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
20 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask
36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
37 extracted from the command line (whereas most CmdLineTasks' run methods take
38 single dataRef, are are called repeatedly). This class transforms the processed
39 arguments generated by the ArgumentParser into the arguments expected by
42 See pipeBase.TaskRunner for more information.
48 Return a list of tuples per tract, each containing (dataRefs, kwargs).
50 Jointcal operates on lists of dataRefs simultaneously.
52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run()
68 - None if self.doReturnResults is False
69 - A pipe.base.Struct containing these fields if self.doReturnResults is True:
70 - dataRef: the provided data references, with update post-fit WCS's.
73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask"""
84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 astrometryRefObjLoader = pexConfig.ConfigurableField(
135 target=LoadAstrometryNetObjectsTask,
136 doc=
"Reference object loader for astrometric fit",
138 photometryRefObjLoader = pexConfig.ConfigurableField(
139 target=LoadAstrometryNetObjectsTask,
140 doc=
"Reference object loader for photometric fit",
142 sourceSelector = sourceSelectorRegistry.makeField(
143 doc=
"How to select sources for cross-matching",
149 sourceSelector.setDefaults()
151 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
153 sourceSelector.sourceFluxType =
'Calib'
157 """Jointly astrometrically (photometrically later) calibrate a group of images."""
159 ConfigClass = JointcalConfig
160 RunnerClass = JointcalRunner
161 _DefaultName =
"jointcal"
163 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
165 Instantiate a JointcalTask.
169 butler : lsst.daf.persistence.Butler
170 The butler is passed to the refObjLoader constructor in case it is
171 needed. Ignored if the refObjLoader argument provides a loader directly.
172 Used to initialize the astrometry and photometry refObjLoaders.
173 profile_jointcal : bool
174 set to True to profile different stages of this jointcal run.
176 pipeBase.CmdLineTask.__init__(self, **kwargs)
178 self.makeSubtask(
"sourceSelector")
179 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
180 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
187 def _getConfigName(self):
190 def _getMetadataName(self):
194 def _makeArgumentParser(cls):
195 """Create an argument parser"""
196 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
197 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
198 help=
"Profile steps of jointcal separately.")
199 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
200 ContainerClass=PerTractCcdDataIdContainer)
203 def _build_ccdImage(self, dataRef, associations, jointcalControl):
205 Extract the necessary things from this dataRef to add a new ccdImage.
209 dataRef : lsst.daf.persistence.ButlerDataRef
210 dataRef to extract info from.
211 associations : lsst.jointcal.Associations
212 object to add the info to, to construct a new CcdImage
213 jointcalControl : jointcal.JointcalControl
214 control object for associations management
219 wcs : lsst.afw.image.TanWcs
220 the TAN WCS of this image, read from the calexp
222 a key to identify this dataRef by its visit and ccd ids
226 visit = dataRef.dataId[
"visit"]
227 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
228 calexp = dataRef.get(
"calexp", immediate=
True)
229 visitInfo = calexp.getInfo().getVisitInfo()
230 ccdname = calexp.getDetector().getId()
232 calib = calexp.getCalib()
233 tanWcs = calexp.getWcs()
234 bbox = calexp.getBBox()
235 filt = calexp.getInfo().getFilter().getName()
236 fluxMag0 = calib.getFluxMag0()
237 photoCalib = afwImage.PhotoCalib(fluxMag0[0], fluxMag0[1], bbox)
239 goodSrc = self.sourceSelector.selectSources(src)
241 if len(goodSrc.sourceCat) == 0:
242 self.log.warn(
"no stars selected in ", visit, ccdname)
244 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
245 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, photoCalib,
246 visit, ccdname, jointcalControl)
248 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
249 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
250 return Result(tanWcs, Key(visit, ccdname), filt)
253 def run(self, dataRefs, profile_jointcal=False):
255 Jointly calibrate the astrometry and photometry across a set of images.
259 dataRefs : list of lsst.daf.persistence.ButlerDataRef
260 List of data references to the exposures to be fit.
261 profile_jointcal : bool
262 Profile the individual steps of jointcal.
268 * dataRefs: the provided data references that were fit (with updated WCSs)
269 * oldWcsList: the original WCS from each dataRef
270 * metrics: dictionary of internally-computed metrics for testing/validation.
272 if len(dataRefs) == 0:
273 raise ValueError(
'Need a list of data references!')
275 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
279 visit_ccd_to_dataRef = {}
282 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else ''
283 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
286 oldWcsList.append(result.wcs)
287 visit_ccd_to_dataRef[result.key] = ref
288 filters.append(result.filter)
289 filters = collections.Counter(filters)
291 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
292 commonTangentPoint = lsst.afw.coord.averageCoord(centers)
293 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
294 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
299 bbox = associations.getRaDecBBox()
300 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
301 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
302 radius = center.angularSeparation(corner).asRadians()
305 anDir = lsst.utils.getPackageDir(
'astrometry_net_data')
307 raise RuntimeError(
"astrometry_net_data is not setup")
310 defaultFilter = filters.most_common(1)[0][0]
311 self.log.debug(
"Using %s band for reference flux", defaultFilter)
314 tract = dataRefs[0].dataId[
'tract']
316 if self.config.doAstrometry:
319 refObjLoader=self.astrometryRefObjLoader,
321 profile_jointcal=profile_jointcal,
326 if self.config.doPhotometry:
329 refObjLoader=self.photometryRefObjLoader,
331 profile_jointcal=profile_jointcal,
337 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else ''
338 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
339 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
341 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
343 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
344 name=
"", refObjLoader=
None, filters=[], fit_function=
None,
345 tract=
None, profile_jointcal=
False, match_cut=3.0):
346 """Load reference catalog, perform the fit, and return the result.
350 associations : lsst.jointcal.Associations
351 The star/reference star associations to fit.
353 filter to load from reference catalog.
354 center : lsst.afw.coord.Coord
355 Center of field to load from reference catalog.
356 radius : lsst.afw.geom.Angle
357 On-sky radius to load from reference catalog.
359 Name of thing being fit: "Astrometry" or "Photometry".
360 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
361 Reference object loader to load from for fit.
362 filters : list of str, optional
363 List of filters to load from the reference catalog.
364 fit_function : function
365 function to call to perform fit (takes associations object).
367 Name of tract currently being fit.
368 profile_jointcal : bool, optional
369 Separately profile the fitting step.
370 match_cut : float, optional
371 Radius in arcseconds to find cross-catalog matches to during
372 associations.associateCatalogs.
376 Result of `fit_function()`
378 self.log.info(
"====== Now processing %s...", name)
381 associations.associateCatalogs(match_cut)
382 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
384 skyCircle = refObjLoader.loadSkyCircle(center,
385 afwGeom.Angle(radius, afwGeom.radians),
389 if not skyCircle.refCat.isContiguous():
390 refCat = skyCircle.refCat.copy(deep=
True)
392 refCat = skyCircle.refCat
399 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
400 refFluxes[filt] = refCat.get(filtKeys[0])
401 refFluxErrs[filt] = refCat.get(filtKeys[1])
403 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
404 skyCircle.fluxField, refFluxes, refFluxErrs)
405 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
407 associations.selectFittedStars(self.config.minMeasurements)
409 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
410 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
411 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
413 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else ''
414 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
415 result = fit_function(associations)
418 tupleName =
"{}_res_{}.list".format(name, tract)
419 result.fit.saveResultTuples(tupleName)
423 def _check_star_lists(self, associations, name):
425 if associations.nCcdImagesValidForFit() == 0:
426 raise RuntimeError(
'No images in the ccdImageList!')
427 if associations.fittedStarListSize() == 0:
428 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
429 if associations.refStarListSize() == 0:
430 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
432 def _fit_photometry(self, associations):
434 Fit the photometric data.
438 associations : lsst.jointcal.Associations
439 The star/reference star associations to fit.
444 fit : lsst.jointcal.PhotometryFit
445 The photometric fitter used to perform the fit.
446 model : lsst.jointcal.PhotometryModel
447 The photometric model that was fit.
449 self.log.info(
"=== Starting photometric fitting...")
452 if self.config.photometryModel ==
"constrained":
454 elif self.config.photometryModel ==
"simple":
458 chi2 = fit.computeChi2()
459 self.log.info(
"Initialized: %s", str(chi2))
460 fit.minimize(
"Model")
461 chi2 = fit.computeChi2()
462 self.log.info(str(chi2))
463 fit.minimize(
"Fluxes")
464 chi2 = fit.computeChi2()
465 self.log.info(str(chi2))
466 fit.minimize(
"Model Fluxes")
467 chi2 = fit.computeChi2()
468 self.log.info(
"Fit prepared with %s", str(chi2))
470 chi2 = self.
_iterate_fit(fit, 20,
"photometry",
"Model Fluxes")
472 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
473 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
476 def _fit_astrometry(self, associations):
478 Fit the astrometric data.
482 associations : lsst.jointcal.Associations
483 The star/reference star associations to fit.
488 fit : lsst.jointcal.AstrometryFit
489 The astrometric fitter used to perform the fit.
490 model : lsst.jointcal.AstrometryModel
491 The astrometric model that was fit.
492 sky_to_tan_projection : lsst.jointcal.ProjectionHandler
493 The model for the sky to tangent plane projection that was used in the fit.
496 self.log.info(
"=== Starting astrometric fitting...")
498 associations.deprojectFittedStars()
505 if self.config.astrometryModel ==
"constrainedPoly":
507 sky_to_tan_projection,
True, 0)
508 elif self.config.astrometryModel ==
"simplePoly":
510 sky_to_tan_projection,
511 True, 0, self.config.polyOrder)
514 chi2 = fit.computeChi2()
515 self.log.info(
"Initialized: %s", str(chi2))
516 fit.minimize(
"Distortions")
517 chi2 = fit.computeChi2()
518 self.log.info(str(chi2))
519 fit.minimize(
"Positions")
520 chi2 = fit.computeChi2()
521 self.log.info(str(chi2))
522 fit.minimize(
"Distortions Positions")
523 chi2 = fit.computeChi2()
524 self.log.info(str(chi2))
526 chi2 = self.
_iterate_fit(fit, 20,
"astrometry",
"Distortions Positions")
528 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
529 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
531 return Astrometry(fit, model, sky_to_tan_projection)
533 def _iterate_fit(self, fit, max_steps, name, whatToFit):
534 """Run fit.minimize up to max_steps times, returning the final chi2."""
536 for i
in range(max_steps):
537 r = fit.minimize(whatToFit, 5)
538 chi2 = fit.computeChi2()
539 self.log.info(str(chi2))
540 if r == MinimizeResult.Converged:
541 self.log.debug(
"fit has converged - no more outliers - redo minimixation"
542 "one more time in case we have lost accuracy in rank update")
544 r = fit.minimize(whatToFit, 5)
545 chi2 = fit.computeChi2()
546 self.log.info(
"Fit completed with: %s", str(chi2))
548 elif r == MinimizeResult.Failed:
549 self.log.warn(
"minimization failed")
551 elif r == MinimizeResult.Chi2Increased:
552 self.log.warn(
"still some ouliers but chi2 increases - retry")
554 self.log.error(
"unxepected return code from minimize")
557 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
561 def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef):
563 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef.
567 associations : lsst.jointcal.Associations
568 The star/reference star associations to fit.
569 astrom_model : lsst.jointcal.AstrometryModel
570 The astrometric model that was fit.
571 photom_model : lsst.jointcal.PhotometryModel
572 The photometric model that was fit.
573 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
574 dict of ccdImage identifiers to dataRefs that were fit
577 ccdImageList = associations.getCcdImageList()
578 for ccdImage
in ccdImageList:
581 visit = ccdImage.visit
582 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
583 exp = afwImage.ExposureI(0, 0)
584 if self.config.doAstrometry:
585 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
586 tanSip = astrom_model.produceSipWcs(ccdImage)
589 if self.config.doPhotometry:
590 self.log.info(
"Updating Calib for visit: %d, ccd: %d", visit, ccd)
592 fluxMag0 = ccdImage.getPhotoCalib().getInstFluxMag0()
593 fluxMag0Err = ccdImage.getPhotoCalib().getInstFluxMag0Err()
594 exp.getCalib().setFluxMag0(fluxMag0/photom_model.photomFactor(ccdImage), fluxMag0Err)
596 dataRef.put(exp,
'wcs')
597 except pexExceptions.Exception
as e:
598 self.log.fatal(
'Failed to write updated Wcs and Calib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _do_load_refcat_and_fit
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
Class that handles the photometric least squares problem.
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometric response model which has a single photometric factor per CcdImage.