3 from __future__
import division, absolute_import, print_function
4 from builtins
import str
5 from builtins
import range
19 from lsst.verify
import Job, Measurement
24 from .dataIds
import PerTractCcdDataIdContainer
29 __all__ = [
"JointcalConfig",
"JointcalTask"]
31 Photometry = collections.namedtuple(
'Photometry', (
'fit',
'model'))
32 Astrometry = collections.namedtuple(
'Astrometry', (
'fit',
'model',
'sky_to_tan_projection'))
37 meas = Measurement(job.metrics[name], value)
38 job.measurements.insert(meas)
42 """Subclass of TaskRunner for jointcalTask 44 jointcalTask.run() takes a number of arguments, one of which is a list of dataRefs 45 extracted from the command line (whereas most CmdLineTasks' run methods take 46 single dataRef, are are called repeatedly). This class transforms the processed 47 arguments generated by the ArgumentParser into the arguments expected by 50 See pipeBase.TaskRunner for more information. 56 Return a list of tuples per tract, each containing (dataRefs, kwargs). 58 Jointcal operates on lists of dataRefs simultaneously. 60 kwargs[
'profile_jointcal'] = parsedCmd.profile_jointcal
61 kwargs[
'butler'] = parsedCmd.butler
65 for ref
in parsedCmd.id.refList:
66 refListDict.setdefault(ref.dataId[
"tract"], []).append(ref)
68 result = [(refListDict[tract], kwargs)
for tract
in sorted(refListDict.keys())]
73 @param args Arguments for Task.run() 76 - A pipe.base.Struct containing these fields if self.doReturnResults is False: 77 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 78 - A pipe.base.Struct containing these fields if self.doReturnResults is True: 79 - ``result``: the result of calling jointcal.run() 80 - ``exitStatus`: 0 if the task completed successfully, 1 otherwise. 85 dataRefList, kwargs = args
86 butler = kwargs.pop(
'butler')
87 task = self.TaskClass(config=self.config, log=self.log, butler=butler)
90 result = task.run(dataRefList, **kwargs)
91 exitStatus = result.exitStatus
92 job_path = butler.get(
'verify_job_filename')
93 result.job.write(job_path[0])
94 except Exception
as e:
99 eName = type(e).__name__
100 tract = dataRefList[0].dataId[
'tract']
101 task.log.fatal(
"Failed processing tract %s, %s: %s", tract, eName, e)
103 if self.doReturnResults:
104 return pipeBase.Struct(result=result, exitStatus=exitStatus)
106 return pipeBase.Struct(exitStatus=exitStatus)
110 """Config for jointcalTask""" 112 doAstrometry = pexConfig.Field(
113 doc=
"Fit astrometry and write the fitted result.",
117 doPhotometry = pexConfig.Field(
118 doc=
"Fit photometry and write the fitted result.",
122 coaddName = pexConfig.Field(
123 doc=
"Type of coadd, typically deep or goodSeeing",
127 posError = pexConfig.Field(
128 doc=
"Constant term for error on position (in pixel unit)",
133 matchCut = pexConfig.Field(
134 doc=
"Matching radius between fitted and reference stars (arcseconds)",
138 minMeasurements = pexConfig.Field(
139 doc=
"Minimum number of associated measured stars for a fitted star to be included in the fit",
143 astrometrySimpleDegree = pexConfig.Field(
144 doc=
"Polynomial degree for fitting the simple astrometry model.",
148 astrometryChipDegree = pexConfig.Field(
149 doc=
"Degree of the per-chip transform for the constrained astrometry model.",
153 astrometryVisitDegree = pexConfig.Field(
154 doc=
"Degree of the per-visit transform for the constrained astrometry model.",
158 useInputWcs = pexConfig.Field(
159 doc=
"Use the input calexp WCSs to initialize the astrometryModel.",
163 astrometryModel = pexConfig.ChoiceField(
164 doc=
"Type of model to fit to astrometry",
166 default=
"simplePoly",
167 allowed={
"simplePoly":
"One polynomial per ccd",
168 "constrainedPoly":
"One polynomial per ccd, and one polynomial per visit"}
170 photometryModel = pexConfig.ChoiceField(
171 doc=
"Type of model to fit to photometry",
174 allowed={
"simple":
"One constant zeropoint per ccd and visit",
175 "constrained":
"Constrained zeropoint per ccd, and one polynomial per visit"}
177 photometryVisitDegree = pexConfig.Field(
178 doc=
"Degree of the per-visit polynomial transform for the constrained photometry model.",
182 astrometryRefObjLoader = pexConfig.ConfigurableField(
183 target=LoadIndexedReferenceObjectsTask,
184 doc=
"Reference object loader for astrometric fit",
186 photometryRefObjLoader = pexConfig.ConfigurableField(
187 target=LoadIndexedReferenceObjectsTask,
188 doc=
"Reference object loader for photometric fit",
190 sourceSelector = sourceSelectorRegistry.makeField(
191 doc=
"How to select sources for cross-matching",
194 writeChi2ContributionFiles = pexConfig.Field(
196 doc=
"Write initial/final fit files containing the contributions to chi2.",
202 sourceSelector.setDefaults()
204 sourceSelector.badFlags.extend([
"slot_Shape_flag"])
206 sourceSelector.sourceFluxType =
'Calib' 210 """Jointly astrometrically (photometrically later) calibrate a group of images.""" 212 ConfigClass = JointcalConfig
213 RunnerClass = JointcalRunner
214 _DefaultName =
"jointcal" 216 def __init__(self, butler=None, profile_jointcal=False, **kwargs):
218 Instantiate a JointcalTask. 222 butler : lsst.daf.persistence.Butler 223 The butler is passed to the refObjLoader constructor in case it is 224 needed. Ignored if the refObjLoader argument provides a loader directly. 225 Used to initialize the astrometry and photometry refObjLoaders. 226 profile_jointcal : bool 227 set to True to profile different stages of this jointcal run. 229 pipeBase.CmdLineTask.__init__(self, **kwargs)
231 self.makeSubtask(
"sourceSelector")
232 if self.config.doAstrometry:
233 self.makeSubtask(
'astrometryRefObjLoader', butler=butler)
234 if self.config.doPhotometry:
235 self.makeSubtask(
'photometryRefObjLoader', butler=butler)
238 self.
job = Job.load_metrics_package(subset=
'jointcal')
242 def _getConfigName(self):
245 def _getMetadataName(self):
249 def _makeArgumentParser(cls):
250 """Create an argument parser""" 252 parser.add_argument(
"--profile_jointcal", default=
False, action=
"store_true",
253 help=
"Profile steps of jointcal separately.")
254 parser.add_id_argument(
"--id",
"calexp", help=
"data ID, e.g. --id visit=6789 ccd=0..9",
255 ContainerClass=PerTractCcdDataIdContainer)
258 def _build_ccdImage(self, dataRef, associations, jointcalControl):
260 Extract the necessary things from this dataRef to add a new ccdImage. 264 dataRef : lsst.daf.persistence.ButlerDataRef 265 dataRef to extract info from. 266 associations : lsst.jointcal.Associations 267 object to add the info to, to construct a new CcdImage 268 jointcalControl : jointcal.JointcalControl 269 control object for associations management 274 wcs : lsst.afw.image.TanWcs 275 the TAN WCS of this image, read from the calexp 277 a key to identify this dataRef by its visit and ccd ids 281 if "visit" in dataRef.dataId.keys():
282 visit = dataRef.dataId[
"visit"]
284 visit = dataRef.getButler().queryMetadata(
"calexp", (
"visit"), dataRef.dataId)[0]
286 src = dataRef.get(
"src", flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS, immediate=
True)
288 visitInfo = dataRef.get(
'calexp_visitInfo')
289 detector = dataRef.get(
'calexp_detector')
290 ccdId = detector.getId()
291 calib = dataRef.get(
'calexp_calib')
292 tanWcs = dataRef.get(
'calexp_wcs')
293 bbox = dataRef.get(
'calexp_bbox')
294 filt = dataRef.get(
'calexp_filter')
295 filterName = filt.getName()
296 fluxMag0 = calib.getFluxMag0()
297 photoCalib = afwImage.PhotoCalib(1.0/fluxMag0[0], fluxMag0[1]/fluxMag0[0]**2, bbox)
299 goodSrc = self.sourceSelector.selectSources(src)
301 if len(goodSrc.sourceCat) == 0:
302 self.log.warn(
"No sources selected in visit %s ccd %s", visit, ccdId)
304 self.log.info(
"%d sources selected in visit %d ccd %d", len(goodSrc.sourceCat), visit, ccdId)
305 associations.addImage(goodSrc.sourceCat, tanWcs, visitInfo, bbox, filterName, photoCalib, detector,
306 visit, ccdId, jointcalControl)
308 Result = collections.namedtuple(
'Result_from_build_CcdImage', (
'wcs',
'key',
'filter'))
309 Key = collections.namedtuple(
'Key', (
'visit',
'ccd'))
310 return Result(tanWcs, Key(visit, ccdId), filterName)
313 def run(self, dataRefs, profile_jointcal=False):
315 Jointly calibrate the astrometry and photometry across a set of images. 319 dataRefs : list of lsst.daf.persistence.ButlerDataRef 320 List of data references to the exposures to be fit. 321 profile_jointcal : bool 322 Profile the individual steps of jointcal. 328 * dataRefs: the provided data references that were fit (with updated WCSs) 329 * oldWcsList: the original WCS from each dataRef 330 * metrics: dictionary of internally-computed metrics for testing/validation. 332 if len(dataRefs) == 0:
333 raise ValueError(
'Need a non-empty list of data references!')
337 sourceFluxField =
"slot_%sFlux" % (self.sourceSelector.config.sourceFluxType,)
341 visit_ccd_to_dataRef = {}
344 load_cat_prof_file =
'jointcal_build_ccdImage.prof' if profile_jointcal
else '' 345 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
348 camera = dataRefs[0].get(
'camera', immediate=
True)
352 oldWcsList.append(result.wcs)
353 visit_ccd_to_dataRef[result.key] = ref
354 filters.append(result.filter)
355 filters = collections.Counter(filters)
357 centers = [ccdImage.getBoresightRaDec()
for ccdImage
in associations.getCcdImageList()]
359 self.log.debug(
"Using common tangent point: %s", commonTangentPoint.getPosition())
360 associations.setCommonTangentPoint(commonTangentPoint.getPosition())
365 bbox = associations.getRaDecBBox()
366 center = afwCoord.Coord(bbox.getCenter(), afwGeom.degrees)
367 corner = afwCoord.Coord(bbox.getMax(), afwGeom.degrees)
368 radius = center.angularSeparation(corner).asRadians()
373 raise RuntimeError(
"astrometry_net_data is not setup")
376 defaultFilter = filters.most_common(1)[0][0]
377 self.log.debug(
"Using %s band for reference flux", defaultFilter)
380 tract = dataRefs[0].dataId[
'tract']
382 if self.config.doAstrometry:
385 refObjLoader=self.astrometryRefObjLoader,
387 profile_jointcal=profile_jointcal,
393 if self.config.doPhotometry:
396 refObjLoader=self.photometryRefObjLoader,
398 profile_jointcal=profile_jointcal,
401 reject_bad_fluxes=
True)
406 return pipeBase.Struct(dataRefs=dataRefs,
407 oldWcsList=oldWcsList,
409 exitStatus=exitStatus)
411 def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius,
412 name="", refObjLoader=None, filters=[], fit_function=None,
413 tract=None, profile_jointcal=False, match_cut=3.0,
414 reject_bad_fluxes=False):
415 """Load reference catalog, perform the fit, and return the result. 419 associations : lsst.jointcal.Associations 420 The star/reference star associations to fit. 422 filter to load from reference catalog. 423 center : lsst.afw.coord.Coord 424 Center of field to load from reference catalog. 425 radius : lsst.afw.geom.Angle 426 On-sky radius to load from reference catalog. 428 Name of thing being fit: "Astrometry" or "Photometry". 429 refObjLoader : lsst.meas.algorithms.LoadReferenceObjectsTask 430 Reference object loader to load from for fit. 431 filters : list of str, optional 432 List of filters to load from the reference catalog. 433 fit_function : function 434 function to call to perform fit (takes associations object). 436 Name of tract currently being fit. 437 profile_jointcal : bool, optional 438 Separately profile the fitting step. 439 match_cut : float, optional 440 Radius in arcseconds to find cross-catalog matches to during 441 associations.associateCatalogs. 442 reject_bad_fluxes : bool, optional 443 Reject refCat sources with NaN/inf flux or NaN/0 fluxErr. 447 Result of `fit_function()` 449 self.log.info(
"====== Now processing %s...", name)
452 associations.associateCatalogs(match_cut)
454 associations.fittedStarListSize())
456 skyCircle = refObjLoader.loadSkyCircle(center,
461 if not skyCircle.refCat.isContiguous():
462 refCat = skyCircle.refCat.copy(deep=
True)
464 refCat = skyCircle.refCat
471 filtKeys = lsst.meas.algorithms.getRefFluxKeys(refCat.schema, filt)
472 refFluxes[filt] = refCat.get(filtKeys[0])
473 refFluxErrs[filt] = refCat.get(filtKeys[1])
475 associations.collectRefStars(refCat, self.config.matchCut*afwGeom.arcseconds,
476 skyCircle.fluxField, refFluxes, refFluxErrs, reject_bad_fluxes)
478 associations.refStarListSize())
480 associations.selectFittedStars(self.config.minMeasurements)
483 associations.refStarListSize())
485 associations.fittedStarListSize())
487 associations.nCcdImagesValidForFit())
489 load_cat_prof_file =
'jointcal_fit_%s.prof'%name
if profile_jointcal
else '' 490 dataName =
"{}_{}".format(tract, defaultFilter)
491 with pipeBase.cmdLineTask.profile(load_cat_prof_file):
492 result = fit_function(associations, dataName)
495 if self.config.writeChi2ContributionFiles:
496 baseName =
"{}_final_chi2-{}.csv".format(name, dataName)
497 result.fit.saveChi2Contributions(baseName)
501 def _check_star_lists(self, associations, name):
503 if associations.nCcdImagesValidForFit() == 0:
504 raise RuntimeError(
'No images in the ccdImageList!')
505 if associations.fittedStarListSize() == 0:
506 raise RuntimeError(
'No stars in the {} fittedStarList!'.format(name))
507 if associations.refStarListSize() == 0:
508 raise RuntimeError(
'No stars in the {} reference star list!'.format(name))
510 def _fit_photometry(self, associations, dataName=None):
512 Fit the photometric data. 516 associations : lsst.jointcal.Associations 517 The star/reference star associations to fit. 519 Name of the data being processed (e.g. "1234_HSC-Y"), for 520 identifying debugging files. 525 fit : lsst.jointcal.PhotometryFit 526 The photometric fitter used to perform the fit. 527 model : lsst.jointcal.PhotometryModel 528 The photometric model that was fit. 530 self.log.info(
"=== Starting photometric fitting...")
533 if self.config.photometryModel ==
"constrained":
536 visitDegree=self.config.photometryVisitDegree)
537 elif self.config.photometryModel ==
"simple":
541 chi2 = fit.computeChi2()
544 if self.config.writeChi2ContributionFiles:
545 baseName =
"photometry_initial_chi2-{}.csv".format(dataName)
546 fit.saveChi2Contributions(baseName)
548 if not np.isfinite(chi2.chi2):
549 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
550 self.log.info(
"Initialized: %s", str(chi2))
551 fit.minimize(
"Model")
552 chi2 = fit.computeChi2()
553 self.log.info(str(chi2))
554 fit.minimize(
"Fluxes")
555 chi2 = fit.computeChi2()
556 self.log.info(str(chi2))
557 fit.minimize(
"Model Fluxes")
558 chi2 = fit.computeChi2()
559 if not np.isfinite(chi2.chi2):
560 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
561 self.log.info(
"Fit prepared with %s", str(chi2))
563 model.freezeErrorTransform()
564 self.log.debug(
"Photometry error scales are frozen.")
566 chi2 = self.
_iterate_fit(fit, model, 20,
"photometry",
"Model Fluxes")
572 def _fit_astrometry(self, associations, dataName=None):
574 Fit the astrometric data. 578 associations : lsst.jointcal.Associations 579 The star/reference star associations to fit. 581 Name of the data being processed (e.g. "1234_HSC-Y"), for 582 identifying debugging files. 587 fit : lsst.jointcal.AstrometryFit 588 The astrometric fitter used to perform the fit. 589 model : lsst.jointcal.AstrometryModel 590 The astrometric model that was fit. 591 sky_to_tan_projection : lsst.jointcal.ProjectionHandler 592 The model for the sky to tangent plane projection that was used in the fit. 595 self.log.info(
"=== Starting astrometric fitting...")
597 associations.deprojectFittedStars()
604 if self.config.astrometryModel ==
"constrainedPoly":
606 sky_to_tan_projection, self.config.useInputWcs, 0,
607 chipDegree=self.config.astrometryChipDegree,
608 visitDegree=self.config.astrometryVisitDegree)
609 elif self.config.astrometryModel ==
"simplePoly":
611 sky_to_tan_projection, self.config.useInputWcs, 0,
612 self.config.astrometrySimpleDegree)
615 chi2 = fit.computeChi2()
618 if self.config.writeChi2ContributionFiles:
619 baseName =
"astrometry_initial_chi2-{}.csv".format(dataName)
620 fit.saveChi2Contributions(baseName)
622 if not np.isfinite(chi2.chi2):
623 raise FloatingPointError(
'Initial chi2 is invalid: %s'%chi2)
624 self.log.info(
"Initialized: %s", str(chi2))
625 fit.minimize(
"Distortions")
626 chi2 = fit.computeChi2()
627 self.log.info(str(chi2))
628 fit.minimize(
"Positions")
629 chi2 = fit.computeChi2()
630 self.log.info(str(chi2))
631 fit.minimize(
"Distortions Positions")
632 chi2 = fit.computeChi2()
633 self.log.info(str(chi2))
634 if not np.isfinite(chi2.chi2):
635 raise FloatingPointError(
'Pre-iteration chi2 is invalid: %s'%chi2)
636 self.log.info(
"Fit prepared with %s", str(chi2))
638 chi2 = self.
_iterate_fit(fit, model, 20,
"astrometry",
"Distortions Positions")
643 return Astrometry(fit, model, sky_to_tan_projection)
645 def _iterate_fit(self, fit, model, max_steps, name, whatToFit):
646 """Run fit.minimize up to max_steps times, returning the final chi2.""" 648 for i
in range(max_steps):
649 r = fit.minimize(whatToFit, 5)
650 chi2 = fit.computeChi2()
651 if not np.isfinite(chi2.chi2):
652 raise FloatingPointError(
'Fit iteration chi2 is invalid: %s'%chi2)
653 self.log.info(str(chi2))
654 if r == MinimizeResult.Converged:
655 self.log.debug(
"fit has converged - no more outliers - redo minimixation" 656 "one more time in case we have lost accuracy in rank update")
658 r = fit.minimize(whatToFit, 5)
659 chi2 = fit.computeChi2()
660 self.log.info(
"Fit completed with: %s", str(chi2))
662 elif r == MinimizeResult.Chi2Increased:
663 self.log.warn(
"still some ouliers but chi2 increases - retry")
664 elif r == MinimizeResult.Failed:
665 raise RuntimeError(
"Chi2 minimization failure, cannot complete fit.")
667 raise RuntimeError(
"Unxepected return code from minimize().")
669 self.log.error(
"%s failed to converge after %d steps"%(name, max_steps))
673 def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef):
675 Write the fitted astrometric results to a new 'wcs' dataRef. 679 associations : lsst.jointcal.Associations 680 The star/reference star associations to fit. 681 model : lsst.jointcal.AstrometryModel 682 The astrometric model that was fit. 683 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 684 dict of ccdImage identifiers to dataRefs that were fit 687 ccdImageList = associations.getCcdImageList()
688 for ccdImage
in ccdImageList:
691 visit = ccdImage.visit
692 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
693 exp = afwImage.ExposureI(0, 0)
694 self.log.info(
"Updating WCS for visit: %d, ccd: %d", visit, ccd)
695 tanSip = model.produceSipWcs(ccdImage)
699 dataRef.put(exp,
'wcs')
700 except pexExceptions.Exception
as e:
701 self.log.fatal(
'Failed to write updated Wcs: %s', str(e))
704 def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef):
706 Write the fitted photometric results to a new 'photoCalib' dataRef. 710 associations : lsst.jointcal.Associations 711 The star/reference star associations to fit. 712 model : lsst.jointcal.PhotometryModel 713 The photoometric model that was fit. 714 visit_ccd_to_dataRef : dict of Key: lsst.daf.persistence.ButlerDataRef 715 dict of ccdImage identifiers to dataRefs that were fit 718 ccdImageList = associations.getCcdImageList()
719 for ccdImage
in ccdImageList:
722 visit = ccdImage.visit
723 dataRef = visit_ccd_to_dataRef[(visit, ccd)]
724 self.log.info(
"Updating PhotoCalib for visit: %d, ccd: %d", visit, ccd)
725 photoCalib = model.toPhotoCalib(ccdImage)
727 dataRef.put(photoCalib,
'photoCalib')
728 except pexExceptions.Exception
as e:
729 self.log.fatal(
'Failed to write updated PhotoCalib: %s', str(e))
this is the model used to fit independent CCDs, meaning that there is no instrument model...
def _build_ccdImage(self, dataRef, associations, jointcalControl)
def _fit_photometry(self, associations, dataName=None)
def getTargetList(parsedCmd, kwargs)
def _fit_astrometry(self, associations, dataName=None)
def _iterate_fit(self, fit, model, max_steps, name, whatToFit)
def _check_star_lists(self, associations, name)
The class that implements the relations between MeasuredStar and FittedStar.
A projection handler in which all CCDs from the same visit have the same tangent point.
std::string getPackageDir(std::string const &packageName)
This is the model used to fit mappings as the combination of a transformation depending on the chip n...
def _write_photometry_results(self, associations, model, visit_ccd_to_dataRef)
Class that handles the photometric least squares problem.
boost::shared_ptr< lsst::afw::image::TanWcs > gtransfoToTanWcs(const lsst::jointcal::TanSipPix2RaDec wcsTransfo, const lsst::jointcal::Frame &ccdFrame, const bool noLowOrderSipTerms=false)
Transform the other way around.
Class that handles the astrometric least squares problem.
Photometry model with constraints, .
def add_measurement(job, name, value)
def _do_load_refcat_and_fit(self, associations, defaultFilter, center, radius, name="", refObjLoader=None, filters=[], fit_function=None, tract=None, profile_jointcal=False, match_cut=3.0, reject_bad_fluxes=False)
def _write_astrometry_results(self, associations, model, visit_ccd_to_dataRef)
def run(self, dataRefs, profile_jointcal=False)
std::shared_ptr< Coord > averageCoord(std::vector< std::shared_ptr< Coord const >> const coords, CoordSystem system=UNKNOWN)
Photometric response model which has a single photometric factor per CcdImage.
def __init__(self, butler=None, profile_jointcal=False, kwargs)