1 from __future__
import absolute_import, division, print_function
4 from scipy.spatial
import cKDTree
5 from scipy.stats
import sigmaclip
8 import lsst.pipe.base
as pipeBase
11 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
13 from .matchOptimisticB
import MatchTolerance
15 from .pessimistic_pattern_matcher_b_3D
import PessimisticPatternMatcherB
17 __all__ = [
"MatchPessimisticBTask",
"MatchPessimisticBConfig",
18 "MatchTolerancePessimistic"]
22 """Stores match tolerances for use in AstrometryTask and later 23 iterations of the matcher. 27 maxMatchDist : lsst.afw.geom.Angle 28 autoMaxMatchDist : lsst.afw.geom.Angle 29 maxShift : lsst.afw.geom.Angle 30 lastMatchedPattern : int 31 failedPatternList : list of ints 34 def __init__(self, maxMatchDist=None, autoMaxMatchDist=None,
35 maxShift=None, lastMatchedPattern=None,
36 failedPatternList=None):
37 """Construct a MatchPessimisticTolerance 39 MatchPessimisticBTask relies on several state variables to be 40 preserved over different iterations in the 41 AstrometryTask.matchAndFitWcs loop of AstrometryTask. 45 maxMatchDist : afw.geom.Angle 46 Current 2 sigma scatter from the previous matched wcs (if it 47 exists. It is None if this is the first iteration.) 48 autoMatxMatchDist : afw.geom.Angle 49 Result of the automated match tolerance generation. 50 maxShift : afw.geom.Angle 51 None for the first iteration or is the magnitude of the previous 52 iteration's wcs shift. 53 lastMatchedPattern : int 54 Reference to the position in the magnitude sorted source array 55 where a successful pattern match was found. 56 failedPatternList : list of ints 57 List of ints specifying indicies in the magnitude sourced source 58 array to skip. These are skipped are previous iterations that are 59 likely false positives due to the code having to soften after a 66 if failedPatternList
is None:
71 """Configuration for MatchPessimisticBTask 73 numBrightStars = pexConfig.RangeField(
74 doc=
"Number of bright stars to use. Sets the max number of patterns " 75 "that can be tested.",
80 minMatchedPairs = pexConfig.RangeField(
81 doc=
"Minimum number of matched pairs; see also minFracMatchedPairs.",
86 minFracMatchedPairs = pexConfig.RangeField(
87 doc=
"Minimum number of matched pairs as a fraction of the smaller of " 88 "the number of reference stars or the number of good sources; " 89 "the actual minimum is the smaller of this value or " 96 matcherIterations = pexConfig.RangeField(
97 doc=
"Number of softening iterations in matcher.",
102 maxOffsetPix = pexConfig.RangeField(
103 doc=
"Maximum allowed shift of WCS, due to matching (pixel). " 104 "When changing this value, the " 105 "LoadReferenceObjectsConfig.pixelMargin should also be updated.",
110 maxRotationDeg = pexConfig.RangeField(
111 doc=
"Rotation angle allowed between sources and position reference " 112 "objects (degrees).",
117 numPointsForShape = pexConfig.Field(
118 doc=
"Number of points to define a shape for matching.",
122 numPointsForShapeAttempt = pexConfig.Field(
123 doc=
"Number of points to try for creating a shape. This value should " 124 "be greater than or equal to numPointsForShape. Besides " 125 "loosening the signal to noise cut in the matcherSourceSelector, " 126 "increasing this number will solve CCDs where no match was found.",
130 minMatchDistPixels = pexConfig.RangeField(
131 doc=
"Distance in units of pixels to always consider a source-" 132 "reference pair a match. This prevents the astrometric fitter " 133 "from over-fitting and removing stars that should be matched and " 134 "allows for inclusion of new matches as the wcs improves.",
140 numPatternConsensus = pexConfig.Field(
141 doc=
"Number of implied shift/rotations from patterns that must agree " 142 "before it a given shift/rotation is accepted. This is only used " 143 "after the first softening iteration fails and if both the " 144 "number of reference and source objects is greater than " 149 sourceSelector = sourceSelectorRegistry.makeField(
150 doc=
"How to select sources for cross-matching. The default " 151 "matcherSourceSelector removes objects with low S/N, bad " 152 "saturated objects, edge objects, and interpolated objects.",
153 default=
"matcherPessimistic" 158 sourceSelector.setDefaults()
161 pexConfig.Config.validate(self)
163 raise ValueError(
"numPointsForShapeAttempt must be greater than " 164 "or equal to numPointsForShape.")
177 """!Match sources to reference objects 179 @anchor MatchPessimisticBTask_ 181 @section meas_astrom_MatchPessimisticB_Contents Contents 183 - @ref meas_astrom_MatchPessimisticB_Purpose 184 - @ref meas_astrom_MatchPessimisticB_Initialize 185 - @ref meas_astrom_MatchPessimisticB_IO 186 - @ref meas_astrom_MatchPessimisticB_Config 187 - @ref meas_astrom_MatchPessimisticB_Example 188 - @ref meas_astrom_MatchPessimisticB_Debug 190 @section meas_astrom_MatchPessimisticB_Purpose Description 192 Match sources to reference objects. This is often done as a preliminary 193 step to fitting an astrometric or photometric solution. For details about 194 the matching algorithm see pessimistic_pattern_matcher_b_3D.py 196 @section meas_astrom_MatchPessimisticB_Initialize Task initialization 198 @copydoc \_\_init\_\_ 200 @section meas_astrom_MatchPessimisticB_IO Invoking the Task 202 @copydoc matchObjectsToSources 204 @section meas_astrom_MatchPessimisticB_Config Configuration 207 See @ref MatchPessimisticBConfig 209 To modify the tests for good sources for matching, create a new 210 sourceSelector class in meas_algorithms and use it in the config. 212 @section meas_astrom_MatchPessimisticB_Example A complete example of 213 using MatchPessimisticBTask 215 MatchPessimisticBTask is a subtask of AstrometryTask, which is called by 216 PhotoCalTask. See \ref meas_photocal_photocal_Example. 218 @section meas_astrom_MatchPessimisticB_Debug Debug variables 220 The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink 221 interface supports a flag @c -d to import @b debug.py from your 222 @c PYTHONPATH; see @ref baseDebug for more about @b debug.py files. 224 The available variables in MatchPessimisticBTask are: 226 <DT> @c verbose (bool) 227 <DD> If True then the matcher prints debug messages to stdout 230 To investigate the @ref meas_astrom_MatchPessimisticB_Debug, put something 235 # N.b. lsstDebug.Info(name) would call us recursively 236 debug = lsstDebug.getInfo(name) 237 if name == "lsst.pipe.tasks.astrometry": 242 lsstDebug.Info = DebugInfo 244 into your debug.py file and run this task with the @c --debug flag. 247 ConfigClass = MatchPessimisticBConfig
248 _DefaultName =
"matchObjectsToSources" 251 pipeBase.Task.__init__(self, **kwargs)
252 self.makeSubtask(
"sourceSelector")
256 match_tolerance=None):
257 """!Match sources to position reference stars 259 @param[in] refCat catalog of reference objects that overlap the 260 exposure; reads fields for: 262 - the specified flux field 263 @param[in] sourceCat catalog of sources found on an exposure; 264 Please check the required fields of your specified source selector 265 that the correct flags are present. 266 @param[in] wcs estimated WCS 267 @param[in] refFluxField field of refCat to use for flux 268 @param[in] match_tolerance is a MatchTolerance class object or None. 269 This this class is used to communicate state between AstrometryTask 270 and MatcherTask. AstrometryTask will also set the MatchTolerance 271 class variable maxMatchDist based on the scatter AstrometryTask has 272 found after fitting for the wcs. 273 @return an lsst.pipe.base.Struct with fields: 274 - matches a list of matches, each instance of 275 lsst.afw.table.ReferenceMatch 276 - usableSourcCat a catalog of sources potentially usable for 278 - match_tolerance a MatchTolerance object containing the resulting 279 state variables from the match. 286 if match_tolerance
is None:
290 numSources = len(sourceCat)
291 selectedSources = self.sourceSelector.selectSources(sourceCat)
292 goodSourceCat = selectedSources.sourceCat
293 numUsableSources = len(goodSourceCat)
294 self.log.info(
"Purged %d sources, leaving %d good sources" %
295 (numSources - numUsableSources, numUsableSources))
297 if len(goodSourceCat) == 0:
298 raise pipeBase.TaskError(
"No sources are good")
303 minMatchedPairs = min(self.config.minMatchedPairs,
304 int(self.config.minFracMatchedPairs *
305 min([len(refCat), len(goodSourceCat)])))
309 sourceCat=goodSourceCat,
311 refFluxField=refFluxField,
312 numUsableSources=numUsableSources,
313 minMatchedPairs=minMatchedPairs,
314 match_tolerance=match_tolerance,
315 sourceFluxField=self.sourceSelector.fluxField,
316 verbose=debug.verbose,
318 matches = doMatchReturn.matches
319 match_tolerance = doMatchReturn.match_tolerance
321 if len(matches) == 0:
322 raise RuntimeError(
"Unable to match sources")
324 self.log.info(
"Matched %d sources" % len(matches))
325 if len(matches) < minMatchedPairs:
326 self.log.warn(
"Number of matches is smaller than request")
328 return pipeBase.Struct(
330 usableSourceCat=goodSourceCat,
331 match_tolerance=match_tolerance,
335 def _doMatch(self, refCat, sourceCat, wcs, refFluxField, numUsableSources,
336 minMatchedPairs, match_tolerance, sourceFluxField, verbose):
337 """!Implementation of matching sources to position reference stars 339 Unlike matchObjectsToSources, this method does not check if the sources 342 @param[in] refCat catalog of position reference stars that overlap an 344 @param[in] sourceCat catalog of sources found on the exposure 345 @param[in] wcs estimated WCS of exposure 346 @param[in] refFluxField field of refCat to use for flux 347 @param[in] numUsableSources number of usable sources (sources with 348 known centroid that are not near the edge, but may be saturated) 349 @param[in] minMatchedPairs minimum number of matches 350 @param[in] match_tolerance a MatchTolerance object containing 351 variables specifying matcher tolerances and state from possible 353 @param[in] sourceInfo SourceInfo for the sourceCat 354 @param[in] verbose true to print diagnostic information to std::cout 356 @return a list of matches, an instance of 357 lsst.afw.table.ReferenceMatch, a MatchTolerance object 366 ref_array = np.empty((len(refCat), 4), dtype=np.float64)
367 for ref_idx, refObj
in enumerate(refCat):
368 theta = np.pi / 2 - refObj.getDec().asRadians()
369 phi = refObj.getRa().asRadians()
370 flux = refObj[refFluxField]
371 ref_array[ref_idx, :] = \
374 src_array = np.empty((len(sourceCat), 4), dtype=np.float64)
375 for src_idx, srcObj
in enumerate(sourceCat):
376 coord = wcs.pixelToSky(srcObj.getCentroid())
377 theta = np.pi / 2 - coord.getLatitude().asRadians()
378 phi = coord.getLongitude().asRadians()
379 flux = srcObj.getPsfFlux()
380 src_array[src_idx, :] = \
385 if match_tolerance.maxShift
is None:
386 maxShiftArcseconds = (self.config.maxOffsetPix *
387 wcs.getPixelScale().asArcseconds())
391 maxShiftArcseconds = np.max(
392 (match_tolerance.maxShift.asArcseconds(),
393 self.config.minMatchDistPixels *
394 wcs.getPixelScale().asArcseconds()))
400 if match_tolerance.maxMatchDist
is None:
401 self.log.debug(
"Computing source statistics...")
404 self.log.debug(
"Computing reference statistics...")
407 maxMatchDistArcSec = np.min((maxMatchDistArcSecSrc,
408 maxMatchDistArcSecRef))
409 match_tolerance.autoMaxDist = afwgeom.Angle(maxMatchDistArcSec,
412 maxMatchDistArcSec = np.max(
413 (self.config.minMatchDistPixels *
414 wcs.getPixelScale().asArcseconds(),
415 np.min((match_tolerance.maxMatchDist.asArcseconds(),
416 match_tolerance.autoMaxDist.asArcseconds()))))
421 numConsensus = self.config.numPatternConsensus
422 minObjectsForConsensus = \
423 self.config.numBrightStars + self.config.numPointsForShapeAttempt
424 if ref_array.shape[0] < minObjectsForConsensus
or \
425 src_array.shape[0] < minObjectsForConsensus:
428 self.log.debug(
"Current tol maxDist: %.4f arcsec" %
430 self.log.debug(
"Current shift: %.4f arcsec" %
438 for soften_pattern
in range(self.config.matcherIterations):
439 for soften_dist
in range(self.config.matcherIterations):
440 if soften_pattern == 0
and soften_dist == 0
and \
441 match_tolerance.lastMatchedPattern
is not None:
450 run_n_consent = numConsensus
453 matcher_struct = pyPPMb.match(
454 source_array=src_array,
455 n_check=self.config.numPointsForShapeAttempt + soften_pattern,
456 n_match=self.config.numPointsForShape,
457 n_agree=run_n_consent,
458 max_n_patterns=self.config.numBrightStars,
459 max_shift=maxShiftArcseconds,
460 max_rotation=self.config.maxRotationDeg,
461 max_dist=maxMatchDistArcSec * 2. ** soften_dist,
462 min_matches=minMatchedPairs,
463 pattern_skip_array=np.array(
464 match_tolerance.failedPatternList)
467 if soften_pattern == 0
and soften_dist == 0
and \
468 len(matcher_struct.match_ids) == 0
and \
469 match_tolerance.lastMatchedPattern
is not None:
476 match_tolerance.failedPatternList.append(
477 match_tolerance.lastMatchedPattern)
478 match_tolerance.lastMatchedPattern =
None 479 maxShiftArcseconds = \
480 self.config.maxOffsetPix * wcs.getPixelScale().asArcseconds()
481 elif len(matcher_struct.match_ids) > 0:
484 match_tolerance.maxShift = \
485 matcher_struct.shift * afwgeom.arcseconds
486 match_tolerance.lastMatchedPattern = \
487 matcher_struct.pattern_idx
495 return pipeBase.Struct(
497 match_tolerance=match_tolerance,
509 distances_arcsec = np.degrees(matcher_struct.distances_rad) * 3600
510 clip_max_dist = np.max(
511 (sigmaclip(distances_arcsec, low=100, high=2)[-1],
512 self.config.minMatchDistPixels * wcs.getPixelScale().asArcseconds())
517 if not np.isfinite(clip_max_dist):
518 clip_max_dist = maxMatchDistArcSec
520 if clip_max_dist < maxMatchDistArcSec
and \
521 len(distances_arcsec[distances_arcsec < clip_max_dist]) < \
523 dist_cut_arcsec = maxMatchDistArcSec
525 dist_cut_arcsec = np.min((clip_max_dist, maxMatchDistArcSec))
530 for match_id_pair, dist_arcsec
in zip(matcher_struct.match_ids,
532 if dist_arcsec < dist_cut_arcsec:
534 match.first = refCat[match_id_pair[1]]
535 match.second = sourceCat[match_id_pair[0]]
539 match.distance = match.first.getCoord().angularSeparation(
540 match.second.getCoord()).asArcseconds()
541 matches.append(match)
543 return pipeBase.Struct(
545 match_tolerance=match_tolerance,
548 def _latlong_flux_to_xyz_mag(self, theta, phi, flux):
549 r"""Convert angles theta and phi and a flux into unit sphere 550 x, y, z, and a relative magnitude. 552 Takes in a afw catalog object and converts the catalog object RA, DECs 553 to points on the unit sphere. Also converts the flux into a simple, 554 non-zero-pointed magnitude for relative sorting. 559 Angle from the north pole (z axis) of the sphere 561 Rotation around the sphere 566 Spherical unit vector x, y, z on the unit-sphere. 568 output_array = np.empty(4, dtype=np.float64)
569 output_array[0] = np.sin(theta)*np.cos(phi)
570 output_array[1] = np.sin(theta)*np.sin(phi)
571 output_array[2] = np.cos(theta)
573 output_array[3] = -2.5 * np.log10(flux)
577 output_array[3] = 99.
581 def _get_pair_pattern_statistics(self, cat_array):
582 """ Compute the tolerances for the matcher automatically by comparing 583 pinwheel patterns as we would in the matcher. 585 We test how similar the patterns we can create from a given set of 586 objects by computing the spoke lengths for each pattern and sorting 587 them from smallest to largest. The match tolerance is the average 588 distance per spoke between the closest two patterns in the sorted 593 cat_array : float array 594 array of 3 vectors representing the x, y, z position of catalog 595 objects on the unit sphere. 600 Suggested max match tolerance distance calculated from comparisons 601 between pinwheel patterns used in optimistic/pessimistic pattern 605 self.log.debug(
"Starting automated tolerance calculation...")
609 pattern_array = np.empty(
610 (cat_array.shape[0] - self.config.numPointsForShape,
611 self.config.numPointsForShape - 1))
612 flux_args_array = np.argsort(cat_array[:, -1])
615 tmp_sort_array = cat_array[flux_args_array]
618 for start_idx
in range(cat_array.shape[0] -
619 self.config.numPointsForShape):
620 pattern_points = tmp_sort_array[start_idx:start_idx +
621 self.config.numPointsForShape, :-1]
622 pattern_delta = pattern_points[1:, :] - pattern_points[0, :]
623 pattern_array[start_idx, :] = np.sqrt(
624 pattern_delta[:, 0] ** 2 +
625 pattern_delta[:, 1] ** 2 +
626 pattern_delta[:, 2] ** 2)
631 pattern_array[start_idx, :] = pattern_array[
632 start_idx, np.argsort(pattern_array[start_idx, :])]
638 pattern_array[:, :(self.config.numPointsForShape - 1)])
639 dist_nearest_array, ids = dist_tree.query(
640 pattern_array[:, :(self.config.numPointsForShape - 1)], k=2)
641 dist_nearest_array = dist_nearest_array[:, 1]
642 dist_nearest_array.sort()
646 dist_tol = (np.degrees(dist_nearest_array[dist_idx]) * 3600. /
647 (self.config.numPointsForShape - 1.))
649 self.log.debug(
"Automated tolerance")
650 self.log.debug(
"\tdistance/match tol: %.4f [arcsec]" % dist_tol)
Match sources to reference objects.
def __init__(self, kwargs)
def _doMatch(self, refCat, sourceCat, wcs, refFluxField, numUsableSources, minMatchedPairs, match_tolerance, sourceFluxField, verbose)
Implementation of matching sources to position reference stars.
def __init__(self, maxMatchDist=None, autoMaxMatchDist=None, maxShift=None, lastMatchedPattern=None, failedPatternList=None)
def matchObjectsToSources(self, refCat, sourceCat, wcs, refFluxField, match_tolerance=None)
Match sources to position reference stars.
def _get_pair_pattern_statistics(self, cat_array)
def _latlong_flux_to_xyz_mag(self, theta, phi, flux)