3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
22 from .dataIds
import PerTractCcdDataIdContainer
27 __all__ = [
"JointcalConfig",
"JointcalTask"]
29 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
30 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
34 """Subclass of TaskRunner for jointcalTask 36 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 37 extracted from the command line (whereas most CmdLineTasks' run methods take 38 single dataRef, are are called repeatedly). This class transforms the processed 39 arguments generated by the ArgumentParser into the arguments expected by 42 See pipeBase.TaskRunner for more information. 48 Return a list of tuples per tract, each containing (dataRefs, kwargs). 50 Jointcal operates on lists of dataRefs simultaneously. 52 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
53 kwargs[
'butler'] = parsedCmd.butler
57 for ref
in parsedCmd.id.refList:
58 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
60 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
65 @param args Arguments for Task.run() 68 - A pipe.base.Struct containing these fields if self.doReturnResults is False: 69 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 70 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 71 - ``result``: the result of calling jointcal.run() 72 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 77 dataRefList, kwargs = args
78 butler = kwargs.pop(
'butler')
79 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
82 result = task.run(dataRefList, **kwargs)
83 exitStatus = result.exitStatus
84 except Exception
as e:
89 eName = type(e).__name__
90 task.log.fatal(
"Failed on dataIds=%s: %s: %s", dataRefList, eName, e)
92 if self.doReturnResults:
93 return pipeBase.Struct(result=result, exitStatus=exitStatus)
95 return pipeBase.Struct(exitStatus=exitStatus)
99 """Config for jointcalTask""" 101 doAstrometry = pexConfig.Field(
102 doc=
"Fit astrometry and write the fitted result.",
106 doPhotometry = pexConfig.Field(
107 doc=
"Fit photometry and write the fitted result.",
111 coaddName = pexConfig.Field(
112 doc=
"Type of coadd, typically deep or goodSeeing",
116 posError = pexConfig.Field(
117 doc=
"Constant term for error on position (in pixel unit)",
122 matchCut = pexConfig.Field(
123 doc=
"Matching radius between fitted and reference stars (arcseconds)",
127 minMeasurements = pexConfig.Field(
128 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
132 polyOrder = pexConfig.Field(
133 doc=
"Polynomial order for fitting distorsion",
137 astrometryModel = pexConfig.ChoiceField(
138 doc=
"Type of model to fit to astrometry",
140 default=
"simplePoly",
141 allowed={
"simplePoly":
"One polynomial per ccd",
142 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
144 photometryModel = pexConfig.ChoiceField(
145 doc=
"Type of model to fit to photometry",
148 allowed={
"simple":
"One constant zeropoint per ccd and visit",
149 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
151 photometryVisitDegree = pexConfig.Field(
152 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
156 astrometryRefObjLoader = pexConfig.ConfigurableField(
157 target=LoadIndexedReferenceObjectsTask,
158 doc=
"Reference object loader for astrometric fit",
160 photometryRefObjLoader = pexConfig.ConfigurableField(
161 target=LoadIndexedReferenceObjectsTask,
162 doc=
"Reference object loader for photometric fit",
164 sourceSelector = sourceSelectorRegistry.makeField(
165 doc=
"How to select sources for cross-matching",
171 sourceSelector.setDefaults()
173 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
175 sourceSelector.sourceFluxType =
'Calib' 179 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 181 ConfigClass = JointcalConfig
182 RunnerClass = JointcalRunner
183 _DefaultName =
"jointcal" 185 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
187 Instantiate a JointcalTask. 191 butler : lsst.daf.persistence.Butler 192 The butler is passed to the refObjLoader constructor in case it is 193 needed. Ignored if the refObjLoader argument provides a loader directly. 194 Used to initialize the astrometry and photometry refObjLoaders. 195 profile_jointcal : bool 196 set to True to profile different stages of this jointcal run. 198 pipeBase.CmdLineTask.__init__(self, **kwargs)
200 self.makeSubtask(
"sourceSelector")
201 if self.config.doAstrometry:
202 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
203 if self.config.doPhotometry:
204 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
211 def _getConfigName(self):
214 def _getMetadataName(self):
218 def _makeArgumentParser(cls):
219 """Create an argument parser""" 221 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
222 help=
"Profile steps of jointcal separately.")
223 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
224 ContainerClass=PerTractCcdDataIdContainer)
227 def _build_ccdImage(self, dataRef, associations, jointcalControl):
229 Extract the necessary things from this dataRef to add a new ccdImage. 233 dataRef : lsst.daf.persistence.ButlerDataRef 234 dataRef to extract info from. 235 associations : lsst.jointcal.Associations 236 object to add the info to, to construct a new CcdImage 237 jointcalControl : jointcal.JointcalControl 238 control object for associations management 243 wcs : lsst.afw.image.TanWcs 244 the TAN WCS of this image, read from the calexp 246 a key to identify this dataRef by its visit and ccd ids 250 if "visit" in dataRef.dataId.keys():
251 visit = dataRef.dataId[
"visit"]
253 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
255 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
257 visitInfo = dataRef.get(
'calexp_visitInfo')
258 detector = dataRef.get(
'calexp_detector')
259 ccdname = detector.getId()
260 calib = dataRef.get(
'calexp_calib')
261 tanWcs = dataRef.get(
'calexp_wcs')
262 bbox = dataRef.get(
'calexp_bbox')
263 filt = dataRef.get(
'calexp_filter')
264 filterName = filt.getName()
265 fluxMag0 = calib.getFluxMag0()
266 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
268 goodSrc = self.sourceSelector.selectSources(src)
270 if len(goodSrc.sourceCat) == 0:
271 self.log.warn(
"No stars selected in visit %s ccd %s", visit, ccdname)
273 self.log.info(
"%d stars selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdname)
274 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
275 visit, ccdname, jointcalControl)
277 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
278 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
279 return Result(tanWcs, Key(visit, ccdname), filterName)
282 def run(self, dataRefs, profile_jointcal=False):
284 Jointly calibrate the astrometry and photometry across a set of images. 288 dataRefs : list of lsst.daf.persistence.ButlerDataRef 289 List of data references to the exposures to be fit. 290 profile_jointcal : bool 291 Profile the individual steps of jointcal. 297 * dataRefs: the provided data references that were fit (with updated WCSs) 298 * oldWcsList: the original WCS from each dataRef 299 * metrics: dictionary of internally-computed metrics for testing/validation. 301 if len(dataRefs) == 0:
302 raise ValueError(
'Need a non-empty list of data references!')
306 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
310 visit_ccd_to_dataRef = {}
313 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 314 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
317 camera = dataRefs[0].get(
'camera', immediate=
True)
321 oldWcsList.append(result.wcs)
322 visit_ccd_to_dataRef[result.key] = ref
323 filters.append(result.filter)
324 filters = collections.Counter(filters)
326 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
328 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
329 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
334 bbox = associations.getRaDecBBox()
335 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
336 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
337 radius = center.angularSeparation(corner).asRadians()
342 raise RuntimeError(
"astrometry_net_data is not setup")
345 defaultFilter = filters.most_common(1)[0][0]
346 self.log.debug(
"Using %s band for reference flux", defaultFilter)
349 tract = dataRefs[0].dataId[
'tract']
351 if self.config.doAstrometry:
354 refObjLoader=self.astrometryRefObjLoader,
356 profile_jointcal=profile_jointcal,
361 if self.config.doPhotometry:
364 refObjLoader=self.photometryRefObjLoader,
366 profile_jointcal=profile_jointcal,
372 load_cat_prof_file =
'jointcal_write_results.prof' if profile_jointcal
else '' 373 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
374 self.
_write_results(associations, astrometry.model, photometry.model, visit_ccd_to_dataRef)
376 return pipeBase.Struct(dataRefs=dataRefs,
377 oldWcsList=oldWcsList,
379 exitStatus=exitStatus)
381 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
382 name="", refObjLoader=None, filters=[], fit_function=None,
383 tract=None, profile_jointcal=False, match_cut=3.0):
384 """Load reference catalog, perform the fit, and return the result. 388 associations : lsst.jointcal.Associations 389 The star/reference star associations to fit. 391 filter to load from reference catalog. 392 center : lsst.afw.coord.Coord 393 Center of field to load from reference catalog. 394 radius : lsst.afw.geom.Angle 395 On-sky radius to load from reference catalog. 397 Name of thing being fit: "Astrometry" or "Photometry". 398 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 399 Reference object loader to load from for fit. 400 filters : list of str, optional 401 List of filters to load from the reference catalog. 402 fit_function : function 403 function to call to perform fit (takes associations object). 405 Name of tract currently being fit. 406 profile_jointcal : bool, optional 407 Separately profile the fitting step. 408 match_cut : float, optional 409 Radius in arcseconds to find cross-catalog matches to during 410 associations.associateCatalogs. 414 Result of `fit_function()` 416 self.log.info(
"====== Now processing %s...", name)
419 associations.associateCatalogs(match_cut)
420 self.
metrics[
'associated%sFittedStars' % name] = associations.fittedStarListSize()
422 skyCircle = refObjLoader.loadSkyCircle(center,
427 if not skyCircle.refCat.isContiguous():
428 refCat = skyCircle.refCat.copy(deep=
True)
430 refCat = skyCircle.refCat
437 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
438 refFluxes[filt] = refCat.get(filtKeys[0])
439 refFluxErrs[filt] = refCat.get(filtKeys[1])
441 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
442 skyCircle.fluxField, refFluxes, refFluxErrs)
443 self.
metrics[
'collected%sRefStars' % name] = associations.refStarListSize()
445 associations.selectFittedStars(self.config.minMeasurements)
447 self.
metrics[
'selected%sRefStars' % name] = associations.refStarListSize()
448 self.
metrics[
'selected%sFittedStars' % name] = associations.fittedStarListSize()
449 self.
metrics[
'selected%sCcdImageList' % name] = associations.nCcdImagesValidForFit()
451 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 452 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
453 result = fit_function(associations)
456 tupleName =
"{}_res_{}.list".format(name, tract)
457 result.fit.saveResultTuples(tupleName)
461 def _check_star_lists(self, associations, name):
463 if associations.nCcdImagesValidForFit() == 0:
464 raise RuntimeError(
'No images in the ccdImageList!')
465 if associations.fittedStarListSize() == 0:
466 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
467 if associations.refStarListSize() == 0:
468 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
470 def _fit_photometry(self, associations):
472 Fit the photometric data. 476 associations : lsst.jointcal.Associations 477 The star/reference star associations to fit. 482 fit : lsst.jointcal.PhotometryFit 483 The photometric fitter used to perform the fit. 484 model : lsst.jointcal.PhotometryModel 485 The photometric model that was fit. 487 self.log.info(
"=== Starting photometric fitting...")
490 if self.config.photometryModel ==
"constrained":
493 visitDegree=self.config.photometryVisitDegree)
494 elif self.config.photometryModel ==
"simple":
498 chi2 = fit.computeChi2()
499 self.log.info(
"Initialized: %s", str(chi2))
500 fit.minimize(
"Model")
501 chi2 = fit.computeChi2()
502 self.log.info(str(chi2))
503 fit.minimize(
"Fluxes")
504 chi2 = fit.computeChi2()
505 self.log.info(str(chi2))
506 fit.minimize(
"Model Fluxes")
507 chi2 = fit.computeChi2()
508 self.log.info(
"Fit prepared with %s", str(chi2))
510 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
512 self.
metrics[
'photometryFinalChi2'] = chi2.chi2
513 self.
metrics[
'photometryFinalNdof'] = chi2.ndof
516 def _fit_astrometry(self, associations):
518 Fit the astrometric data. 522 associations : lsst.jointcal.Associations 523 The star/reference star associations to fit. 528 fit : lsst.jointcal.AstrometryFit 529 The astrometric fitter used to perform the fit. 530 model : lsst.jointcal.AstrometryModel 531 The astrometric model that was fit. 532 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 533 The model for the sky to tangent plane projection that was used in the fit. 536 self.log.info(
"=== Starting astrometric fitting...")
538 associations.deprojectFittedStars()
545 if self.config.astrometryModel ==
"constrainedPoly":
547 sky_to_tan_projection,
True, 0)
548 elif self.config.astrometryModel ==
"simplePoly":
550 sky_to_tan_projection,
551 True, 0, self.config.polyOrder)
554 chi2 = fit.computeChi2()
555 self.log.info(
"Initialized: %s", str(chi2))
556 fit.minimize(
"Distortions")
557 chi2 = fit.computeChi2()
558 self.log.info(str(chi2))
559 fit.minimize(
"Positions")
560 chi2 = fit.computeChi2()
561 self.log.info(str(chi2))
562 fit.minimize(
"Distortions Positions")
563 chi2 = fit.computeChi2()
564 self.log.info(str(chi2))
566 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
568 self.
metrics[
'astrometryFinalChi2'] = chi2.chi2
569 self.
metrics[
'astrometryFinalNdof'] = chi2.ndof
571 return Astrometry(fit, model, sky_to_tan_projection)
573 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
574 """Run fit.minimize up to max_steps times, returning the final chi2.""" 576 for i
in range(max_steps):
577 r = fit.minimize(whatToFit, 5)
578 chi2 = fit.computeChi2()
579 self.log.info(str(chi2))
580 if r == MinimizeResult.Converged:
581 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 582 "one more time in case we have lost accuracy in rank update")
584 r = fit.minimize(whatToFit, 5)
585 chi2 = fit.computeChi2()
586 self.log.info(
"Fit completed with: %s", str(chi2))
588 elif r == MinimizeResult.Failed:
589 self.log.warn(
"minimization failed")
591 elif r == MinimizeResult.Chi2Increased:
592 self.log.warn(
"still some ouliers but chi2 increases - retry")
594 self.log.error(
"unxepected return code from minimize")
597 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
601 def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef):
603 Write the fitted results (photometric and astrometric) to a new 'wcs' dataRef. 607 associations : lsst.jointcal.Associations 608 The star/reference star associations to fit. 609 astrometry_model : lsst.jointcal.AstrometryModel 610 The astrometric model that was fit. 611 photometry_model : lsst.jointcal.PhotometryModel 612 The photometric model that was fit. 613 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 614 dict of ccdImage identifiers to dataRefs that were fit 617 ccdImageList = associations.getCcdImageList()
618 for ccdImage
in ccdImageList:
621 visit = ccdImage.visit
622 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
623 exp = afwImage.ExposureI(0, 0)
624 if self.config.doAstrometry:
625 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
626 tanSip = astrometry_model.produceSipWcs(ccdImage)
630 dataRef.put(exp,
'wcs')
631 except pexExceptions.Exception
as e:
632 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
634 if self.config.doPhotometry:
635 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
636 photoCalib = photometry_model.toPhotoCalib(ccdImage)
638 dataRef.put(photoCalib,
'photoCalib')
639 except pexExceptions.Exception
as e:
640 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations)
def getTargetList(parsedCmd, kwargs)
def _write_results(self, associations, astrometry_model, photometry_model, visit_ccd_to_dataRef)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0)
Class that handles the photometric least squares problem.
def _fit_astrometry(self, associations)
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)