3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
10 import lsst.pex.config
as pexConfig
11 import lsst.pipe.base
as pipeBase
12 import lsst.afw.image
as afwImage
13 import lsst.afw.geom
as afwGeom
14 import lsst.afw.coord
as afwCoord
15 import lsst.pex.exceptions
as pexExceptions
18 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
19 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
21 from .dataIds
import PerTractCcdDataIdContainer
25 __all__ = [
"JointcalConfig",
"JointcalTask"]
27 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
28 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
32 """Subclass of TaskRunner for jointcalTask
34 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs
35 extracted from the command line (whereas most CmdLineTasks' run methods take
36 single dataRef, are are called repeatedly). This class transforms the processed
37 arguments generated by the ArgumentParser into the arguments expected by
40 See pipeBase.TaskRunner for more information.
46 Return a list of tuples per tract, each containing (dataRefs, kwargs).
48 Jointcal operates on lists of dataRefs simultaneously.
50 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
51 kwargs[
'butler'] = parsedCmd.butler
55 for ref
in parsedCmd.id.refList:
56 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
58 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
63 @param args Arguments for Task.run()
66 - None if self.doReturnResults is False
67 - A pipe.base.Struct containing these fields if self.doReturnResults is True:
68 - dataRef: the provided data references, with update post-fit WCS's.
71 dataRefList, kwargs = args
72 butler = kwargs.pop(
'butler')
73 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
74 result = task.run(dataRefList, **kwargs)
75 if self.doReturnResults:
76 return pipeBase.Struct(result=result)
80 """Config for jointcalTask"""
82 doAstrometry = pexConfig.Field(
83 doc=
"Fit astrometry and write the fitted result.",
87 doPhotometry = pexConfig.Field(
88 doc=
"Fit photometry and write the fitted result.",
92 coaddName = pexConfig.Field(
93 doc=
"Type of coadd, typically deep or goodSeeing",
97 posError = pexConfig.Field(
98 doc=
"Constant term for error on position (in pixel unit)",
103 matchCut = pexConfig.Field(
104 doc=
"Matching radius between fitted and reference stars (arcseconds)",
108 minMeasurements = pexConfig.Field(
109 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
113 polyOrder = pexConfig.Field(
114 doc=
"Polynomial order for fitting distorsion",
118 astrometryModel = pexConfig.ChoiceField(
119 doc=
"Type of model to fit to astrometry",
121 default=
"simplePoly",
122 allowed={
"simplePoly":
"One polynomial per ccd",
123 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
125 astrometryRefObjLoader = pexConfig.ConfigurableField(
126 target=LoadAstrometryNetObjectsTask,
127 doc=
"Reference object loader for astrometric fit",
129 photometryRefObjLoader = pexConfig.ConfigurableField(
130 target=LoadAstrometryNetObjectsTask,
131 doc=
"Reference object loader for photometric fit",
133 sourceSelector = sourceSelectorRegistry.makeField(
134 doc=
"How to select sources for cross-matching",
140 sourceSelector.setDefaults()
142 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
144 sourceSelector.sourceFluxType =
'Calib'
148 """Jointly astrometrically (photometrically later) calibrate a group of images."""
150 ConfigClass = JointcalConfig
151 RunnerClass = JointcalRunner
152 _DefaultName =
"jointcal"
154 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
156 Instantiate a JointcalTask.
160 butler : lsst.daf.persistence.Butler
161 The butler is passed to the refObjLoader constructor in case it is
162 needed. Ignored if the refObjLoader argument provides a loader directly.
163 Used to initialize the astrometry and photometry refObjLoaders.
164 profile_jointcal : bool
165 set to True to profile different stages of this jointcal run.
167 pipeBase.CmdLineTask.__init__(self, **kwargs)
169 self.makeSubtask(
"sourceSelector")
170 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
171 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
178 def _getConfigName(self):
181 def _getMetadataName(self):
185 def _makeArgumentParser(cls):
186 """Create an argument parser"""
187 parser = pipeBase.ArgumentParser(name=cls._DefaultName)
188 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
189 help=
"Profile steps of jointcal separately.")
190 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
191 ContainerClass=PerTractCcdDataIdContainer)
194 def _build_ccdImage(self, dataRef, associations, jointcalControl):
196 Extract the necessary things from this dataRef to add a new ccdImage.
200 dataRef : lsst.daf.persistence.ButlerDataRef
201 dataRef to extract info from.
202 associations : lsst.jointcal.Associations
203 object to add the info to, to construct a new CcdImage
204 jointcalControl : jointcal.JointcalControl
205 control object for associations management
210 wcs : lsst.afw.image.TanWcs
211 the TAN WCS of this image, read from the calexp
213 a key to identify this dataRef by its visit and ccd ids
217 visit = dataRef.dataId[
"visit"]
218 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
219 calexp = dataRef.get(
"calexp", immediate=
True)
220 visitInfo = calexp.getInfo().getVisitInfo()
221 ccdname = calexp.getDetector().getId()
223 calib = calexp.getCalib()
224 tanWcs = calexp.getWcs()
225 bbox = calexp.getBBox()
226 filt = calexp.getInfo().getFilter().getName()
228 goodSrc = self.sourceSelector.selectSources(src)
230 if len(goodSrc.sourceCat) == 0:
231 self.log.warn(
"no stars selected in ", visit, ccdname)
233 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
234 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, calib,
235 visit, ccdname, jointcalControl)
237 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
238 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
239 return Result(tanWcs, Key(visit, ccdname), filt)
242 def run(self, dataRefs, profile_jointcal=False):
244 Jointly calibrate the astrometry and photometry across a set of images.
248 dataRefs : list of lsst.daf.persistence.ButlerDataRef
249 List of data references to the exposures to be fit.
250 profile_jointcal : bool
251 Profile the individual steps of jointcal.
257 * dataRefs: the provided data references that were fit (with updated WCSs)
258 * oldWcsList: the original WCS from each dataRef
259 * metrics: dictionary of internally-computed metrics for testing/validation.
261 if len(dataRefs) == 0:
262 raise ValueError(
'Need a list of data references!')
264 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
268 visit_ccd_to_dataRef = {}
271 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else ''
272 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
275 oldWcsList.append(result.wcs)
276 visit_ccd_to_dataRef[result.key] = ref
277 filters.append(result.filter)
278 filters = collections.Counter(filters)
280 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
281 commonTangentPoint = lsst.afw.coord.averageCoord(centers)
282 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
283 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
288 bbox = associations.getRaDecBBox()
289 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
290 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
291 radius = center.angularSeparation(corner).asRadians()
294 anDir = lsst.utils.getPackageDir(
'astrometry_net_data')
296 raise RuntimeError(
"astrometry_net_data is not setup")
299 defaultFilter = filters.most_common(1)[0][0]
300 self.log.debug(
"Using %s band for reference flux", defaultFilter)
303 tract = dataRefs[0].dataId[
'tract']
305 if self.config.doAstrometry:
308 refObjLoader=self.astrometryRefObjLoader,
310 profile_jointcal=profile_jointcal,
315 if self.config.doPhotometry:
318 refObjLoader=self.photometryRefObjLoader,
320 profile_jointcal=profile_jointcal,
325 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else ''
326 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
327 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
329 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
331 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
332 name=
"", refObjLoader=
None, fit_function=
None, tract=
None,
333 profile_jointcal=
False, match_cut=3.0):
334 """Load reference catalog, perform the fit, and return the result.
338 associations : lsst.jointcal.Associations
339 The star/reference star associations to fit.
341 filter to load from reference catalog.
342 center : lsst.afw.coord.Coord
343 Center of field to load from reference catalog.
344 radius : lsst.afw.geom.Angle
345 On-sky radius to load from reference catalog.
347 Name of thing being fit: "Astrometry" or "Photometry".
348 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask
349 Reference object loader to load from for fit.
350 fit_function : function
351 function to call to perform fit (takes associations object).
353 Name of tract currently being fit.
354 profile_jointcal : bool, optional
355 Separately profile the fitting step.
356 match_cut : float, optional
357 Radius in arcseconds to find cross-catalog matches to during
358 associations.associateCatalogs.
362 Result of `fit_function()`
364 self.log.info(
"====== Now processing %s...", name)
367 associations.associateCatalogs(match_cut)
368 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
370 skyCircle = refObjLoader.loadSkyCircle(center,
371 afwGeom.Angle(radius, afwGeom.radians),
373 associations.collectRefStars(skyCircle.refCat,
374 self.config.matchCut*afwGeom.arcseconds,
376 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
378 associations.selectFittedStars(self.config.minMeasurements)
380 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
381 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
382 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
384 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else ''
385 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
386 result = fit_function(associations)
390 tupleName =
"{}_res_{}.list".format(name, tract)
391 result.fit.makeResTuple(tupleName)
395 def _check_star_lists(self, associations, name):
397 if associations.nCcdImagesValidForFit() == 0:
398 raise RuntimeError(
'No images in the ccdImageList!')
399 if associations.fittedStarListSize() == 0:
400 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
401 if associations.refStarListSize() == 0:
402 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
404 def _fit_photometry(self, associations):
406 Fit the photometric data.
410 associations : lsst.jointcal.Associations
411 The star/reference star associations to fit.
416 fit : lsst.jointcal.PhotometryFit
417 The photometric fitter used to perform the fit.
418 model : lsst.jointcal.PhotometryModel
419 The photometric model that was fit.
422 self.log.info(
"=== Starting photometric fitting...")
426 fit.minimize(
"Model")
427 chi2 = fit.computeChi2()
428 self.log.info(str(chi2))
429 fit.minimize(
"Fluxes")
430 chi2 = fit.computeChi2()
431 self.log.info(str(chi2))
432 fit.minimize(
"Model Fluxes")
433 chi2 = fit.computeChi2()
434 self.log.info(
"Fit completed with %s", str(chi2))
436 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
437 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
440 def _fit_astrometry(self, associations):
442 Fit the astrometric data.
446 associations : lsst.jointcal.Associations
447 The star/reference star associations to fit.
452 fit : lsst.jointcal.AstrometryFit
453 The astrometric fitter used to perform the fit.
454 model : lsst.jointcal.AstrometryModel
455 The astrometric model that was fit.
456 sky_to_tan_projection : lsst.jointcal.ProjectionHandler
457 The model for the sky to tangent plane projection that was used in the fit.
460 self.log.info(
"=== Starting astrometric fitting...")
462 associations.deprojectFittedStars()
469 if self.config.astrometryModel ==
"constrainedPoly":
471 sky_to_tan_projection,
True, 0)
472 elif self.config.astrometryModel ==
"simplePoly":
474 sky_to_tan_projection,
475 True, 0, self.config.polyOrder)
478 fit.minimize(
"Distortions")
479 chi2 = fit.computeChi2()
480 self.log.info(str(chi2))
481 fit.minimize(
"Positions")
482 chi2 = fit.computeChi2()
483 self.log.info(str(chi2))
484 fit.minimize(
"Distortions Positions")
485 chi2 = fit.computeChi2()
486 self.log.info(str(chi2))
489 for i
in range(max_steps):
490 r = fit.minimize(
"Distortions Positions", 5)
491 chi2 = fit.computeChi2()
492 self.log.info(str(chi2))
494 self.log.debug(
"""fit has converged - no more outliers - redo minimixation\
495 one more time in case we have lost accuracy in rank update""")
497 r = fit.minimize(
"Distortions Positions", 5)
498 chi2 = fit.computeChi2()
499 self.log.info(
"Fit completed with: %s", str(chi2))
502 self.log.warn(
"minimization failed")
504 self.log.warn(
"still some ouliers but chi2 increases - retry")
507 self.log.error(
"unxepected return code from minimize")
509 self.log.error(
"astrometry failed to converge after %d steps", max_steps)
511 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
512 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
514 return Astrometry(fit, model, sky_to_tan_projection)
516 def _write_results(self, associations, astrom_model, photom_model, visit_ccd_to_dataRef):
518 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef.
522 associations : lsst.jointcal.Associations
523 The star/reference star associations to fit.
524 astrom_model : lsst.jointcal.AstrometryModel
525 The astrometric model that was fit.
526 photom_model : lsst.jointcal.PhotometryModel
527 The photometric model that was fit.
528 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef
529 dict of ccdImage identifiers to dataRefs that were fit
532 ccdImageList = associations.getCcdImageList()
533 for ccdImage
in ccdImageList:
536 visit = ccdImage.visit
537 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
538 exp = afwImage.ExposureI(0, 0)
539 if self.config.doAstrometry:
540 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
541 tanSip = astrom_model.produceSipWcs(ccdImage)
544 if self.config.doPhotometry:
545 self.log.info(
"Updating Calib for visit: %d, ccd: %d", visit, ccd)
547 fluxMag0, fluxMag0Sigma = ccdImage.getCalib().getFluxMag0()
548 exp.getCalib().setFluxMag0(fluxMag0*photom_model.photomFactor(ccdImage), fluxMag0Sigma)
550 dataRef.put(exp,
'wcs')
551 except pexExceptions.Exception
as e:
552 self.log.fatal(
'Failed to write updated Wcs and Calib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _do_load_refcat_and_fit
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
Class that handles the photometric least squares problem.
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometric response model which has a single photometric factor per CcdImage.