3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
19 from lsst.meas.extensions.astrometryNet
import LoadAstrometryNetObjectsTask
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - None if self.doReturnResults is False 69 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 70 - dataRef: the provided data references, with update post-fit WCS's. 73 dataRefList, kwargs = args
74 butler = kwargs.pop(
'butler')
75 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
76 result = task.run(dataRefList, **kwargs)
77 if self.doReturnResults:
78 return pipeBase.Struct(result=result)
82 """Config for jointcalTask""" 84 doAstrometry = pexConfig.Field(
85 doc=
"Fit astrometry and write the fitted result.",
89 doPhotometry = pexConfig.Field(
90 doc=
"Fit photometry and write the fitted result.",
94 coaddName = pexConfig.Field(
95 doc=
"Type of coadd, typically deep or goodSeeing",
99 posError = pexConfig.Field(
100 doc=
"Constant term for error on position (in pixel unit)",
105 matchCut = pexConfig.Field(
106 doc=
"Matching radius between fitted and reference stars (arcseconds)",
110 minMeasurements = pexConfig.Field(
111 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
115 polyOrder = pexConfig.Field(
116 doc=
"Polynomial order for fitting distorsion",
120 astrometryModel = pexConfig.ChoiceField(
121 doc=
"Type of model to fit to astrometry",
123 default=
"simplePoly",
124 allowed={
"simplePoly":
"One polynomial per ccd",
125 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
127 photometryModel = pexConfig.ChoiceField(
128 doc=
"Type of model to fit to photometry",
131 allowed={
"simple":
"One constant zeropoint per ccd and visit",
132 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
134 photometryVisitDegree = pexConfig.Field(
135 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
139 astrometryRefObjLoader = pexConfig.ConfigurableField(
140 target=LoadAstrometryNetObjectsTask,
141 doc=
"Reference object loader for astrometric fit",
143 photometryRefObjLoader = pexConfig.ConfigurableField(
144 target=LoadAstrometryNetObjectsTask,
145 doc=
"Reference object loader for photometric fit",
147 sourceSelector = sourceSelectorRegistry.makeField(
148 doc=
"How to select sources for cross-matching",
154 sourceSelector.setDefaults()
156 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
158 sourceSelector.sourceFluxType =
'Calib' 162 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 164 ConfigClass = JointcalConfig
165 RunnerClass = JointcalRunner
166 _DefaultName =
"jointcal" 168 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
170 Instantiate a JointcalTask. 174 butler : lsst.daf.persistence.Butler 175 The butler is passed to the refObjLoader constructor in case it is 176 needed. Ignored if the refObjLoader argument provides a loader directly. 177 Used to initialize the astrometry and photometry refObjLoaders. 178 profile_jointcal : bool 179 set to True to profile different stages of this jointcal run. 181 pipeBase.CmdLineTask.__init__(self, **kwargs)
183 self.makeSubtask(
"sourceSelector")
184 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
185 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
192 def _getConfigName(self):
195 def _getMetadataName(self):
199 def _makeArgumentParser(cls):
200 """Create an argument parser""" 202 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
203 help=
"Profile steps of jointcal separately.")
204 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
205 ContainerClass=PerTractCcdDataIdContainer)
208 def _build_ccdImage(self, dataRef, associations, jointcalControl):
210 Extract the necessary things from this dataRef to add a new ccdImage. 214 dataRef : lsst.daf.persistence.ButlerDataRef 215 dataRef to extract info from. 216 associations : lsst.jointcal.Associations 217 object to add the info to, to construct a new CcdImage 218 jointcalControl : jointcal.JointcalControl 219 control object for associations management 224 wcs : lsst.afw.image.TanWcs 225 the TAN WCS of this image, read from the calexp 227 a key to identify this dataRef by its visit and ccd ids 231 if "visit" in dataRef.dataId.keys():
232 visit = dataRef.dataId[
"visit"]
234 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
235 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
236 calexp = dataRef.get(
"calexp", immediate=
True)
237 visitInfo = calexp.getInfo().getVisitInfo()
238 detector = calexp.getDetector()
239 ccdname = detector.getId()
241 calib = calexp.getCalib()
242 tanWcs = calexp.getWcs()
243 bbox = calexp.getBBox()
244 filt = calexp.getInfo().getFilter().getName()
245 fluxMag0 = calib.getFluxMag0()
246 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
248 goodSrc = self.sourceSelector.selectSources(src)
250 if len(goodSrc.sourceCat) == 0:
251 self.log.warn(
"no stars selected in ", visit, ccdname)
253 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
254 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filt, photoCalib, detector,
255 visit, ccdname, jointcalControl)
257 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
258 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
259 return Result(tanWcs, Key(visit, ccdname), filt)
262 def run(self, dataRefs, profile_jointcal=False):
264 Jointly calibrate the astrometry and photometry across a set of images. 268 dataRefs : list of lsst.daf.persistence.ButlerDataRef 269 List of data references to the exposures to be fit. 270 profile_jointcal : bool 271 Profile the individual steps of jointcal. 277 * dataRefs: the provided data references that were fit (with updated WCSs) 278 * oldWcsList: the original WCS from each dataRef 279 * metrics: dictionary of internally-computed metrics for testing/validation. 281 if len(dataRefs) == 0:
282 raise ValueError(
'Need a list of data references!')
284 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
288 visit_ccd_to_dataRef = {}
291 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 292 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
295 camera = dataRefs[0].get(
'camera', immediate=
True)
299 oldWcsList.append(result.wcs)
300 visit_ccd_to_dataRef[result.key] = ref
301 filters.append(result.filter)
302 filters = collections.Counter(filters)
304 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
306 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
307 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
312 bbox = associations.getRaDecBBox()
313 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
314 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
315 radius = center.angularSeparation(corner).asRadians()
320 raise RuntimeError(
"astrometry_net_data is not setup")
323 defaultFilter = filters.most_common(1)[0][0]
324 self.log.debug(
"Using %s band for reference flux", defaultFilter)
327 tract = dataRefs[0].dataId[
'tract']
329 if self.config.doAstrometry:
332 refObjLoader=self.astrometryRefObjLoader,
334 profile_jointcal=profile_jointcal,
339 if self.config.doPhotometry:
342 refObjLoader=self.photometryRefObjLoader,
344 profile_jointcal=profile_jointcal,
350 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 351 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
352 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
354 return pipeBase.Struct(dataRefs=dataRefs, oldWcsList=oldWcsList, metrics=self.
metrics)
356 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
357 name="", refObjLoader=None, filters=[], fit_function=None,
358 tract=None, profile_jointcal=False, match_cut=3.0):
359 """Load reference catalog, perform the fit, and return the result. 363 associations : lsst.jointcal.Associations 364 The star/reference star associations to fit. 366 filter to load from reference catalog. 367 center : lsst.afw.coord.Coord 368 Center of field to load from reference catalog. 369 radius : lsst.afw.geom.Angle 370 On-sky radius to load from reference catalog. 372 Name of thing being fit: "Astrometry" or "Photometry". 373 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 374 Reference object loader to load from for fit. 375 filters : list of str, optional 376 List of filters to load from the reference catalog. 377 fit_function : function 378 function to call to perform fit (takes associations object). 380 Name of tract currently being fit. 381 profile_jointcal : bool, optional 382 Separately profile the fitting step. 383 match_cut : float, optional 384 Radius in arcseconds to find cross-catalog matches to during 385 associations.associateCatalogs. 389 Result of `fit_function()` 391 self.log.info(
"====== Now processing %s...", name)
394 associations.associateCatalogs(match_cut)
395 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
397 skyCircle = refObjLoader.loadSkyCircle(center,
402 if not skyCircle.refCat.isContiguous():
403 refCat = skyCircle.refCat.copy(deep=
True)
405 refCat = skyCircle.refCat
412 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
413 refFluxes[filt] = refCat.get(filtKeys[0])
414 refFluxErrs[filt] = refCat.get(filtKeys[1])
416 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
417 skyCircle.fluxField, refFluxes, refFluxErrs)
418 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
420 associations.selectFittedStars(self.config.minMeasurements)
422 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
423 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
424 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
426 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 427 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
428 result = fit_function(associations)
431 tupleName =
"{}_res_{}.list".format(name, tract)
432 result.fit.saveResultTuples(tupleName)
436 def _check_star_lists(self, associations, name):
438 if associations.nCcdImagesValidForFit() == 0:
439 raise RuntimeError(
'No images in the ccdImageList!')
440 if associations.fittedStarListSize() == 0:
441 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
442 if associations.refStarListSize() == 0:
443 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
445 def _fit_photometry(self, associations):
447 Fit the photometric data. 451 associations : lsst.jointcal.Associations 452 The star/reference star associations to fit. 457 fit : lsst.jointcal.PhotometryFit 458 The photometric fitter used to perform the fit. 459 model : lsst.jointcal.PhotometryModel 460 The photometric model that was fit. 462 self.log.info(
"=== Starting photometric fitting...")
465 if self.config.photometryModel ==
"constrained":
468 visitDegree=self.config.photometryVisitDegree)
469 elif self.config.photometryModel ==
"simple":
473 chi2 = fit.computeChi2()
474 self.log.info(
"Initialized: %s", str(chi2))
475 fit.minimize(
"Model")
476 chi2 = fit.computeChi2()
477 self.log.info(str(chi2))
478 fit.minimize(
"Fluxes")
479 chi2 = fit.computeChi2()
480 self.log.info(str(chi2))
481 fit.minimize(
"Model Fluxes")
482 chi2 = fit.computeChi2()
483 self.log.info(
"Fit prepared with %s", str(chi2))
485 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
487 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
488 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
491 def _fit_astrometry(self, associations):
493 Fit the astrometric data. 497 associations : lsst.jointcal.Associations 498 The star/reference star associations to fit. 503 fit : lsst.jointcal.AstrometryFit 504 The astrometric fitter used to perform the fit. 505 model : lsst.jointcal.AstrometryModel 506 The astrometric model that was fit. 507 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 508 The model for the sky to tangent plane projection that was used in the fit. 511 self.log.info(
"=== Starting astrometric fitting...")
513 associations.deprojectFittedStars()
520 if self.config.astrometryModel ==
"constrainedPoly":
522 sky_to_tan_projection,
True, 0)
523 elif self.config.astrometryModel ==
"simplePoly":
525 sky_to_tan_projection,
526 True, 0, self.config.polyOrder)
529 chi2 = fit.computeChi2()
530 self.log.info(
"Initialized: %s", str(chi2))
531 fit.minimize(
"Distortions")
532 chi2 = fit.computeChi2()
533 self.log.info(str(chi2))
534 fit.minimize(
"Positions")
535 chi2 = fit.computeChi2()
536 self.log.info(str(chi2))
537 fit.minimize(
"Distortions Positions")
538 chi2 = fit.computeChi2()
539 self.log.info(str(chi2))
541 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
543 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
544 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
546 return Astrometry(fit, model, sky_to_tan_projection)
548 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
549 """Run fit.minimize up to max_steps times, returning the final chi2.""" 551 for i
in range(max_steps):
552 r = fit.minimize(whatToFit, 5)
553 chi2 = fit.computeChi2()
554 self.log.info(str(chi2))
555 if r == MinimizeResult.Converged:
556 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 557 "one more time in case we have lost accuracy in rank update")
559 r = fit.minimize(whatToFit, 5)
560 chi2 = fit.computeChi2()
561 self.log.info(
"Fit completed with: %s", str(chi2))
563 elif r == MinimizeResult.Failed:
564 self.log.warn(
"minimization failed")
566 elif r == MinimizeResult.Chi2Increased:
567 self.log.warn(
"still some ouliers but chi2 increases - retry")
569 self.log.error(
"unxepected return code from minimize")
572 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
576 def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef):
578 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 582 associations : lsst.jointcal.Associations 583 The star/reference star associations to fit. 584 astrometry_model : lsst.jointcal.AstrometryModel 585 The astrometric model that was fit. 586 photometry_model : lsst.jointcal.PhotometryModel 587 The photometric model that was fit. 588 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 589 dict of ccdImage identifiers to dataRefs that were fit 592 ccdImageList = associations.getCcdImageList()
593 for ccdImage
in ccdImageList:
596 visit = ccdImage.visit
597 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
598 exp = afwImage.ExposureI(0, 0)
599 if self.config.doAstrometry:
600 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
601 tanSip = astrometry_model.produceSipWcs(ccdImage)
605 dataRef.put(exp,
'wcs')
606 except pexExceptions.Exception
as e:
607 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
609 if self.config.doPhotometry:
610 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
611 photoCalib = photometry_model.toPhotoCalib(ccdImage)
613 dataRef.put(photoCalib,
'photoCalib')
614 except pexExceptions.Exception
as e:
615 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def getTargetList(parsedCmd, kwargs)
def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)