3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
19 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - None if self.doReturnResults is False 69 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 70 - dataRef: the provided data references, with update post-fit WCS's. 73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask""" 84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 photometryVisitDegree = pexConfig.Field(
135 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
139 astrometryRefObjLoader = pexConfig.ConfigurableField(
140 target=LoadAstrometryNetObjectsTask,
141 doc=
"Reference object loader for astrometric fit",
143 photometryRefObjLoader = pexConfig.ConfigurableField(
144 target=LoadAstrometryNetObjectsTask,
145 doc=
"Reference object loader for photometric fit",
147 sourceSelector = sourceSelectorRegistry.makeField(
148 doc=
"How to select sources for cross-matching",
154 sourceSelector.setDefaults()
156 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
158 sourceSelector.sourceFluxType =
'Calib' 162 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 164 ConfigClass = JointcalConfig
165 RunnerClass = JointcalRunner
166 _DefaultName =
"jointcal" 168 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
170 Instantiate a JointcalTask. 174 butler : lsst.daf.persistence.Butler 175 The butler is passed to the refObjLoader constructor in case it is 176 needed. Ignored if the refObjLoader argument provides a loader directly. 177 Used to initialize the astrometry and photometry refObjLoaders. 178 profile_jointcal : bool 179 set to True to profile different stages of this jointcal run. 181 pipeBase.CmdLineTask.__init__(self, **kwargs)
183 self.makeSubtask(
"sourceSelector")
184 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
185 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
192 def _getConfigName(self):
195 def _getMetadataName(self):
199 def _makeArgumentParser(cls):
200 """Create an argument parser""" 202 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
203 help=
"Profile steps of jointcal separately.")
204 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
205 ContainerClass=PerTractCcdDataIdContainer)
208 def _build_ccdImage(self, dataRef, associations, jointcalControl):
210 Extract the necessary things from this dataRef to add a new ccdImage. 214 dataRef : lsst.daf.persistence.ButlerDataRef 215 dataRef to extract info from. 216 associations : lsst.jointcal.Associations 217 object to add the info to, to construct a new CcdImage 218 jointcalControl : jointcal.JointcalControl 219 control object for associations management 224 wcs : lsst.afw.image.TanWcs 225 the TAN WCS of this image, read from the calexp 227 a key to identify this dataRef by its visit and ccd ids 231 if "visit" in dataRef.dataId.keys():
232 visit = dataRef.dataId[
"visit"]
234 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
236 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
238 visitInfo = dataRef.get(
'calexp_visitInfo')
239 detector = dataRef.get(
'calexp_detector')
240 ccdname = detector.getId()
241 calib = dataRef.get(
'calexp_calib')
242 tanWcs = dataRef.get(
'calexp_wcs')
243 bbox = dataRef.get(
'calexp_bbox')
244 filt = dataRef.get(
'calexp_filter')
245 filterName = filt.getName()
246 fluxMag0 = calib.getFluxMag0()
247 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
249 goodSrc = self.sourceSelector.selectSources(src)
251 if len(goodSrc.sourceCat) == 0:
252 self.log.warn(
"no stars selected in ", visit, ccdname)
254 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
255 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
256 visit, ccdname, jointcalControl)
258 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
259 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
260 return Result(tanWcs, Key(visit, ccdname), filterName)
263 def run(self, dataRefs, profile_jointcal=False):
265 Jointly calibrate the astrometry and photometry across a set of images. 269 dataRefs : list of lsst.daf.persistence.ButlerDataRef 270 List of data references to the exposures to be fit. 271 profile_jointcal : bool 272 Profile the individual steps of jointcal. 278 * dataRefs: the provided data references that were fit (with updated WCSs) 279 * oldWcsList: the original WCS from each dataRef 280 * metrics: dictionary of internally-computed metrics for testing/validation. 282 if len(dataRefs) == 0:
283 raise ValueError(
'Need a list of data references!')
285 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
289 visit_ccd_to_dataRef = {}
292 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 293 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
296 camera = dataRefs[0].get(
'camera', immediate=
True)
300 oldWcsList.append(result.wcs)
301 visit_ccd_to_dataRef[result.key] = ref
302 filters.append(result.filter)
303 filters = collections.Counter(filters)
305 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
307 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
308 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
313 bbox = associations.getRaDecBBox()
314 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
315 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
316 radius = center.angularSeparation(corner).asRadians()
321 raise RuntimeError(
"astrometry_net_data is not setup")
324 defaultFilter = filters.most_common(1)[0][0]
325 self.log.debug(
"Using %s band for reference flux", defaultFilter)
328 tract = dataRefs[0].dataId[
'tract']
330 if self.config.doAstrometry:
333 refObjLoader=self.astrometryRefObjLoader,
335 profile_jointcal=profile_jointcal,
340 if self.config.doPhotometry:
343 refObjLoader=self.photometryRefObjLoader,
345 profile_jointcal=profile_jointcal,
351 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 352 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
353 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
355 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
357 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
358 name="", refObjLoader=None, filters=[], fit_function=None,
359 tract=None, profile_jointcal=False, match_cut=3.0):
360 """Load reference catalog, perform the fit, and return the result. 364 associations : lsst.jointcal.Associations 365 The star/reference star associations to fit. 367 filter to load from reference catalog. 368 center : lsst.afw.coord.Coord 369 Center of field to load from reference catalog. 370 radius : lsst.afw.geom.Angle 371 On-sky radius to load from reference catalog. 373 Name of thing being fit: "Astrometry" or "Photometry". 374 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 375 Reference object loader to load from for fit. 376 filters : list of str, optional 377 List of filters to load from the reference catalog. 378 fit_function : function 379 function to call to perform fit (takes associations object). 381 Name of tract currently being fit. 382 profile_jointcal : bool, optional 383 Separately profile the fitting step. 384 match_cut : float, optional 385 Radius in arcseconds to find cross-catalog matches to during 386 associations.associateCatalogs. 390 Result of `fit_function()` 392 self.log.info(
"====== Now processing %s...", name)
395 associations.associateCatalogs(match_cut)
396 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
398 skyCircle = refObjLoader.loadSkyCircle(center,
403 if not skyCircle.refCat.isContiguous():
404 refCat = skyCircle.refCat.copy(deep=
True)
406 refCat = skyCircle.refCat
413 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
414 refFluxes[filt] = refCat.get(filtKeys[0])
415 refFluxErrs[filt] = refCat.get(filtKeys[1])
417 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
418 skyCircle.fluxField, refFluxes, refFluxErrs)
419 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
421 associations.selectFittedStars(self.config.minMeasurements)
423 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
424 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
425 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
427 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 428 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
429 result = fit_function(associations)
432 tupleName =
"{}_res_{}.list".format(name, tract)
433 result.fit.saveResultTuples(tupleName)
437 def _check_star_lists(self, associations, name):
439 if associations.nCcdImagesValidForFit() == 0:
440 raise RuntimeError(
'No images in the ccdImageList!')
441 if associations.fittedStarListSize() == 0:
442 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
443 if associations.refStarListSize() == 0:
444 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
446 def _fit_photometry(self, associations):
448 Fit the photometric data. 452 associations : lsst.jointcal.Associations 453 The star/reference star associations to fit. 458 fit : lsst.jointcal.PhotometryFit 459 The photometric fitter used to perform the fit. 460 model : lsst.jointcal.PhotometryModel 461 The photometric model that was fit. 463 self.log.info(
"=== Starting photometric fitting...")
466 if self.config.photometryModel ==
"constrained":
469 visitDegree=self.config.photometryVisitDegree)
470 elif self.config.photometryModel ==
"simple":
474 chi2 = fit.computeChi2()
475 self.log.info(
"Initialized: %s", str(chi2))
476 fit.minimize(
"Model")
477 chi2 = fit.computeChi2()
478 self.log.info(str(chi2))
479 fit.minimize(
"Fluxes")
480 chi2 = fit.computeChi2()
481 self.log.info(str(chi2))
482 fit.minimize(
"Model Fluxes")
483 chi2 = fit.computeChi2()
484 self.log.info(
"Fit prepared with %s", str(chi2))
486 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
488 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
489 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
492 def _fit_astrometry(self, associations):
494 Fit the astrometric data. 498 associations : lsst.jointcal.Associations 499 The star/reference star associations to fit. 504 fit : lsst.jointcal.AstrometryFit 505 The astrometric fitter used to perform the fit. 506 model : lsst.jointcal.AstrometryModel 507 The astrometric model that was fit. 508 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 509 The model for the sky to tangent plane projection that was used in the fit. 512 self.log.info(
"=== Starting astrometric fitting...")
514 associations.deprojectFittedStars()
521 if self.config.astrometryModel ==
"constrainedPoly":
523 sky_to_tan_projection,
True, 0)
524 elif self.config.astrometryModel ==
"simplePoly":
526 sky_to_tan_projection,
527 True, 0, self.config.polyOrder)
530 chi2 = fit.computeChi2()
531 self.log.info(
"Initialized: %s", str(chi2))
532 fit.minimize(
"Distortions")
533 chi2 = fit.computeChi2()
534 self.log.info(str(chi2))
535 fit.minimize(
"Positions")
536 chi2 = fit.computeChi2()
537 self.log.info(str(chi2))
538 fit.minimize(
"Distortions Positions")
539 chi2 = fit.computeChi2()
540 self.log.info(str(chi2))
542 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
544 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
545 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
547 return Astrometry(fit, model, sky_to_tan_projection)
549 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
550 """Run fit.minimize up to max_steps times, returning the final chi2.""" 552 for i
in range(max_steps):
553 r = fit.minimize(whatToFit, 5)
554 chi2 = fit.computeChi2()
555 self.log.info(str(chi2))
556 if r == MinimizeResult.Converged:
557 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 558 "one more time in case we have lost accuracy in rank update")
560 r = fit.minimize(whatToFit, 5)
561 chi2 = fit.computeChi2()
562 self.log.info(
"Fit completed with: %s", str(chi2))
564 elif r == MinimizeResult.Failed:
565 self.log.warn(
"minimization failed")
567 elif r == MinimizeResult.Chi2Increased:
568 self.log.warn(
"still some ouliers but chi2 increases - retry")
570 self.log.error(
"unxepected return code from minimize")
573 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
577 def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef):
579 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 583 associations : lsst.jointcal.Associations 584 The star/reference star associations to fit. 585 astrometry_model : lsst.jointcal.AstrometryModel 586 The astrometric model that was fit. 587 photometry_model : lsst.jointcal.PhotometryModel 588 The photometric model that was fit. 589 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 590 dict of ccdImage identifiers to dataRefs that were fit 593 ccdImageList = associations.getCcdImageList()
594 for ccdImage
in ccdImageList:
597 visit = ccdImage.visit
598 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
599 exp = afwImage.ExposureI(0, 0)
600 if self.config.doAstrometry:
601 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
602 tanSip = astrometry_model.produceSipWcs(ccdImage)
606 dataRef.put(exp,
'wcs')
607 except pexExceptions.Exception
as e:
608 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
610 if self.config.doPhotometry:
611 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
612 photoCalib = photometry_model.toPhotoCalib(ccdImage)
614 dataRef.put(photoCalib,
'photoCalib')
615 except pexExceptions.Exception
as e:
616 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def getTargetList(parsedCmd, kwargs)
def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)