3 from scipy.spatial
import cKDTree
4 from scipy.stats
import sigmaclip
7 import lsst.pipe.base
as pipeBase
10 from lsst.meas.algorithms.sourceSelector
import sourceSelectorRegistry
12 from .matchOptimisticB
import MatchTolerance
14 from .pessimistic_pattern_matcher_b_3D
import PessimisticPatternMatcherB
16 __all__ = [
"MatchPessimisticBTask",
"MatchPessimisticBConfig",
17 "MatchTolerancePessimistic"]
21 """Stores match tolerances for use in AstrometryTask and later 22 iterations of the matcher. 24 MatchPessimisticBTask relies on several state variables to be 25 preserved over different iterations in the 26 AstrometryTask.matchAndFitWcs loop of AstrometryTask. 30 maxMatchDist : `lsst.geom.Angle` 31 Maximum distance to consider a match from the previous match/fit 33 autoMaxMatchDist : `lsst.geom.Angle` 34 Automated estimation of the maxMatchDist from the sky statistics of the 35 source and reference catalogs. 36 maxShift : `lsst.geom.Angle` 37 Maximum shift found in the previous match/fit cycle. 38 lastMatchedPattern : `int` 39 Index of the last source pattern that was matched into the reference 41 failedPatternList : `list` of `int` 42 Previous matches were found to be false positives. 43 PPMbObj : `lsst.meas.astrom.PessimisticPatternMatcherB` 44 Initialized Pessimistic pattern matcher object. Storing this prevents 45 the need for recalculation of the searchable distances in the PPMB. 48 def __init__(self, maxMatchDist=None, autoMaxMatchDist=None,
49 maxShift=None, lastMatchedPattern=None,
50 failedPatternList=None, PPMbObj=None):
56 if failedPatternList
is None:
63 """Configuration for MatchPessimisticBTask 65 numBrightStars = pexConfig.RangeField(
66 doc=
"Number of bright stars to use. Sets the max number of patterns " 67 "that can be tested.",
72 minMatchedPairs = pexConfig.RangeField(
73 doc=
"Minimum number of matched pairs; see also minFracMatchedPairs.",
78 minFracMatchedPairs = pexConfig.RangeField(
79 doc=
"Minimum number of matched pairs as a fraction of the smaller of " 80 "the number of reference stars or the number of good sources; " 81 "the actual minimum is the smaller of this value or " 88 matcherIterations = pexConfig.RangeField(
89 doc=
"Number of softening iterations in matcher.",
94 maxOffsetPix = pexConfig.RangeField(
95 doc=
"Maximum allowed shift of WCS, due to matching (pixel). " 96 "When changing this value, the " 97 "LoadReferenceObjectsConfig.pixelMargin should also be updated.",
102 maxRotationDeg = pexConfig.RangeField(
103 doc=
"Rotation angle allowed between sources and position reference " 104 "objects (degrees).",
109 numPointsForShape = pexConfig.Field(
110 doc=
"Number of points to define a shape for matching.",
114 numPointsForShapeAttempt = pexConfig.Field(
115 doc=
"Number of points to try for creating a shape. This value should " 116 "be greater than or equal to numPointsForShape. Besides " 117 "loosening the signal to noise cut in the matcherSourceSelector, " 118 "increasing this number will solve CCDs where no match was found.",
122 minMatchDistPixels = pexConfig.RangeField(
123 doc=
"Distance in units of pixels to always consider a source-" 124 "reference pair a match. This prevents the astrometric fitter " 125 "from over-fitting and removing stars that should be matched and " 126 "allows for inclusion of new matches as the wcs improves.",
132 numPatternConsensus = pexConfig.Field(
133 doc=
"Number of implied shift/rotations from patterns that must agree " 134 "before it a given shift/rotation is accepted. This is only used " 135 "after the first softening iteration fails and if both the " 136 "number of reference and source objects is greater than " 141 numRefRequireConsensus = pexConfig.Field(
142 doc=
"If the available reference objects exceeds this number, " 143 "consensus/pessimistic mode will enforced regardless of the " 144 "number of available sources. Below this optimistic mode (" 145 "exit at first match rather than requiring numPatternConsensus to " 146 "be matched) can be used. If more sources are required to match, " 147 "decrease the signal to noise cut in the sourceSelector.",
151 maxRefObjects = pexConfig.RangeField(
152 doc=
"Maximum number of reference objects to use for the matcher. The " 153 "absolute maximum allowed for is 2 ** 16 for memory reasons.",
159 sourceSelector = sourceSelectorRegistry.makeField(
160 doc=
"How to select sources for cross-matching. The default " 161 "matcherSourceSelector removes objects with low S/N, bad " 162 "saturated objects, edge objects, and interpolated objects.",
163 default=
"matcherPessimistic" 168 sourceSelector.setDefaults()
171 pexConfig.Config.validate(self)
173 raise ValueError(
"numPointsForShapeAttempt must be greater than " 174 "or equal to numPointsForShape.")
176 raise ValueError(
"numBrightStars must be greater than " 177 "numPointsForShape.")
190 """Match sources to reference objects. 193 ConfigClass = MatchPessimisticBConfig
194 _DefaultName =
"matchObjectsToSources" 197 pipeBase.Task.__init__(self, **kwargs)
198 self.makeSubtask(
"sourceSelector")
202 match_tolerance=None):
203 """Match sources to position reference stars 205 refCat : `lsst.afw.table.SimpleCatalog` 206 catalog of reference objects that overlap the exposure; reads 210 - the specified flux field 212 sourceCat : `lsst.afw.table.SourceCatalog` 213 catalog of sources found on an exposure; Please check the required 214 fields of your specified source selector that the correct flags are present. 215 wcs : `lsst.afw.geom.SkyWcs` 218 field of refCat to use for flux 219 match_tolerance : `lsst.meas.astrom.MatchTolerancePessimistic` 220 is a MatchTolerance class object or `None`. This this class is used 221 to communicate state between AstrometryTask and MatcherTask. 222 AstrometryTask will also set the MatchTolerance class variable 223 maxMatchDist based on the scatter AstrometryTask has found after 228 result : `lsst.pipe.base.Struct` 229 Result struct with components: 231 - ``matches`` : source to reference matches found (`list` of 232 `lsst.afw.table.ReferenceMatch`) 233 - ``usableSourcCat`` : a catalog of sources potentially usable for 234 matching and WCS fitting (`lsst.afw.table.SourceCatalog`). 235 - ``match_tolerance`` : a MatchTolerance object containing the 236 resulting state variables from the match 237 (`lsst.meas.astrom.MatchTolerancePessimistic`). 244 if match_tolerance
is None:
248 numSources = len(sourceCat)
249 selectedSources = self.sourceSelector.run(sourceCat)
250 goodSourceCat = selectedSources.sourceCat
251 numUsableSources = len(goodSourceCat)
252 self.log.info(
"Purged %d sources, leaving %d good sources" %
253 (numSources - numUsableSources, numUsableSources))
255 if len(goodSourceCat) == 0:
256 raise pipeBase.TaskError(
"No sources are good")
261 minMatchedPairs = min(self.config.minMatchedPairs,
262 int(self.config.minFracMatchedPairs *
263 min([len(refCat), len(goodSourceCat)])))
265 if len(refCat) > self.config.maxRefObjects:
267 "WARNING: Reference catalog larger that maximum allowed. " 268 "Trimming to %i" % self.config.maxRefObjects)
271 trimedRefCat = refCat
275 sourceCat=goodSourceCat,
277 refFluxField=refFluxField,
278 numUsableSources=numUsableSources,
279 minMatchedPairs=minMatchedPairs,
280 match_tolerance=match_tolerance,
281 sourceFluxField=self.sourceSelector.fluxField,
282 verbose=debug.verbose,
284 matches = doMatchReturn.matches
285 match_tolerance = doMatchReturn.match_tolerance
287 if len(matches) == 0:
288 raise RuntimeError(
"Unable to match sources")
290 self.log.info(
"Matched %d sources" % len(matches))
291 if len(matches) < minMatchedPairs:
292 self.log.warn(
"Number of matches is smaller than request")
294 return pipeBase.Struct(
296 usableSourceCat=goodSourceCat,
297 match_tolerance=match_tolerance,
300 def _filterRefCat(self, refCat, refFluxField):
301 """Sub-select a number of reference objects starting from the brightest 302 and maxing out at the number specified by maxRefObjects in the config. 304 No trimming is done if len(refCat) > config.maxRefObjects. 308 refCat : `lsst.afw.table.SimpleCatalog` 309 Catalog of reference objects to trim. 311 field of refCat to use for flux 314 outCat : `lsst.afw.table.SimpleCatalog` 315 Catalog trimmed to the number set in the task config from the 319 if len(refCat) <= self.config.maxRefObjects:
321 fluxArray = refCat.get(refFluxField)
322 sortedFluxArray = fluxArray[fluxArray.argsort()]
323 minFlux = sortedFluxArray[-(self.config.maxRefObjects + 1)]
325 selected = (refCat.get(refFluxField) > minFlux)
328 outCat.reserve(self.config.maxRefObjects)
329 outCat.extend(refCat[selected])
334 def _doMatch(self, refCat, sourceCat, wcs, refFluxField, numUsableSources,
335 minMatchedPairs, match_tolerance, sourceFluxField, verbose):
336 """Implementation of matching sources to position reference objects 338 Unlike matchObjectsToSources, this method does not check if the sources 343 refCat : `lsst.afw.table.SimpleCatalog` 344 catalog of position reference objects that overlap an exposure 345 sourceCat : `lsst.afw.table.SourceCatalog` 346 catalog of sources found on the exposure 347 wcs : `lsst.afw.geom.SkyWcs` 348 estimated WCS of exposure 350 field of refCat to use for flux 351 numUsableSources : `int` 352 number of usable sources (sources with known centroid that are not 353 near the edge, but may be saturated) 354 minMatchedPairs : `int` 355 minimum number of matches 356 match_tolerance : `lsst.meas.astrom.MatchTolerancePessimistic` 357 a MatchTolerance object containing variables specifying matcher 358 tolerances and state from possible previous runs. 359 sourceFluxField : `str` 360 Name of the flux field in the source catalog. 362 Set true to print diagnostic information to std::cout 367 Results struct with components: 369 - ``matches`` : a list the matches found 370 (`list` of `lsst.afw.table.ReferenceMatch`). 371 - ``match_tolerance`` : MatchTolerance containing updated values from 372 this fit iteration (`lsst.meas.astrom.MatchTolerancePessimistic`) 381 src_array = np.empty((len(sourceCat), 4), dtype=np.float64)
382 for src_idx, srcObj
in enumerate(sourceCat):
383 coord = wcs.pixelToSky(srcObj.getCentroid())
384 theta = np.pi / 2 - coord.getLatitude().asRadians()
385 phi = coord.getLongitude().asRadians()
386 flux = srcObj.getPsfInstFlux()
387 src_array[src_idx, :] = \
390 if match_tolerance.PPMbObj
is None or \
391 match_tolerance.autoMaxMatchDist
is None:
395 ref_array = np.empty((len(refCat), 4), dtype=np.float64)
396 for ref_idx, refObj
in enumerate(refCat):
397 theta = np.pi / 2 - refObj.getDec().asRadians()
398 phi = refObj.getRa().asRadians()
399 flux = refObj[refFluxField]
400 ref_array[ref_idx, :] = \
404 ref_array[:, :3], self.log)
405 self.log.debug(
"Computing source statistics...")
408 self.log.debug(
"Computing reference statistics...")
411 maxMatchDistArcSec = np.max((
412 self.config.minMatchDistPixels *
413 wcs.getPixelScale().asArcseconds(),
414 np.min((maxMatchDistArcSecSrc,
415 maxMatchDistArcSecRef))))
416 match_tolerance.autoMaxMatchDist = afwgeom.Angle(
417 maxMatchDistArcSec, afwgeom.arcseconds)
421 if match_tolerance.maxShift
is None:
422 maxShiftArcseconds = (self.config.maxOffsetPix *
423 wcs.getPixelScale().asArcseconds())
427 maxShiftArcseconds = np.max(
428 (match_tolerance.maxShift.asArcseconds(),
429 self.config.minMatchDistPixels *
430 wcs.getPixelScale().asArcseconds()))
436 if match_tolerance.maxMatchDist
is None:
437 match_tolerance.maxMatchDist = match_tolerance.autoMaxMatchDist
439 maxMatchDistArcSec = np.max(
440 (self.config.minMatchDistPixels *
441 wcs.getPixelScale().asArcseconds(),
442 np.min((match_tolerance.maxMatchDist.asArcseconds(),
443 match_tolerance.autoMaxMatchDist.asArcseconds()))))
449 numConsensus = self.config.numPatternConsensus
450 if len(refCat) < self.config.numRefRequireConsensus:
451 minObjectsForConsensus = \
452 self.config.numBrightStars + \
453 self.config.numPointsForShapeAttempt
454 if len(refCat) < minObjectsForConsensus
or \
455 len(sourceCat) < minObjectsForConsensus:
458 self.log.debug(
"Current tol maxDist: %.4f arcsec" %
460 self.log.debug(
"Current shift: %.4f arcsec" %
465 for soften_dist
in range(self.config.matcherIterations):
466 if soften_dist == 0
and \
467 match_tolerance.lastMatchedPattern
is not None:
476 run_n_consent = numConsensus
479 matcher_struct = match_tolerance.PPMbObj.match(
480 source_array=src_array,
481 n_check=self.config.numPointsForShapeAttempt,
482 n_match=self.config.numPointsForShape,
483 n_agree=run_n_consent,
484 max_n_patterns=self.config.numBrightStars,
485 max_shift=maxShiftArcseconds,
486 max_rotation=self.config.maxRotationDeg,
487 max_dist=maxMatchDistArcSec * 2. ** soften_dist,
488 min_matches=minMatchedPairs,
489 pattern_skip_array=np.array(
490 match_tolerance.failedPatternList)
493 if soften_dist == 0
and \
494 len(matcher_struct.match_ids) == 0
and \
495 match_tolerance.lastMatchedPattern
is not None:
502 match_tolerance.failedPatternList.append(
503 match_tolerance.lastMatchedPattern)
504 match_tolerance.lastMatchedPattern =
None 505 maxShiftArcseconds = \
506 self.config.maxOffsetPix * wcs.getPixelScale().asArcseconds()
507 elif len(matcher_struct.match_ids) > 0:
510 match_tolerance.maxShift = \
511 matcher_struct.shift * afwgeom.arcseconds
512 match_tolerance.lastMatchedPattern = \
513 matcher_struct.pattern_idx
519 return pipeBase.Struct(
521 match_tolerance=match_tolerance,
533 distances_arcsec = np.degrees(matcher_struct.distances_rad) * 3600
534 clip_max_dist = np.max(
535 (sigmaclip(distances_arcsec, low=100, high=2)[-1],
536 self.config.minMatchDistPixels * wcs.getPixelScale().asArcseconds())
541 if not np.isfinite(clip_max_dist):
542 clip_max_dist = maxMatchDistArcSec
544 if clip_max_dist < maxMatchDistArcSec
and \
545 len(distances_arcsec[distances_arcsec < clip_max_dist]) < \
547 dist_cut_arcsec = maxMatchDistArcSec
549 dist_cut_arcsec = np.min((clip_max_dist, maxMatchDistArcSec))
554 for match_id_pair, dist_arcsec
in zip(matcher_struct.match_ids,
556 if dist_arcsec < dist_cut_arcsec:
558 match.first = refCat[int(match_id_pair[1])]
559 match.second = sourceCat[int(match_id_pair[0])]
563 match.distance = match.first.getCoord().separation(
564 match.second.getCoord()).asArcseconds()
565 matches.append(match)
567 return pipeBase.Struct(
569 match_tolerance=match_tolerance,
572 def _latlong_flux_to_xyz_mag(self, theta, phi, flux):
573 """Convert angles theta and phi and a flux into unit sphere 574 x, y, z, and a relative magnitude. 576 Takes in a afw catalog object and converts the catalog object RA, DECs 577 to points on the unit sphere. Also converts the flux into a simple, 578 non-zero-pointed magnitude for relative sorting. 583 Angle from the north pole (z axis) of the sphere 585 Rotation around the sphere 589 output_array : `numpy.ndarray`, (N, 4) 590 Spherical unit vector x, y, z with flux. 592 output_array = np.empty(4, dtype=np.float64)
593 output_array[0] = np.sin(theta)*np.cos(phi)
594 output_array[1] = np.sin(theta)*np.sin(phi)
595 output_array[2] = np.cos(theta)
597 output_array[3] = -2.5 * np.log10(flux)
601 output_array[3] = 99.
605 def _get_pair_pattern_statistics(self, cat_array):
606 """ Compute the tolerances for the matcher automatically by comparing 607 pinwheel patterns as we would in the matcher. 609 We test how similar the patterns we can create from a given set of 610 objects by computing the spoke lengths for each pattern and sorting 611 them from smallest to largest. The match tolerance is the average 612 distance per spoke between the closest two patterns in the sorted 617 cat_array : `numpy.ndarray`, (N, 3) 618 array of 3 vectors representing the x, y, z position of catalog 619 objects on the unit sphere. 624 Suggested max match tolerance distance calculated from comparisons 625 between pinwheel patterns used in optimistic/pessimistic pattern 629 self.log.debug(
"Starting automated tolerance calculation...")
633 pattern_array = np.empty(
634 (cat_array.shape[0] - self.config.numPointsForShape,
635 self.config.numPointsForShape - 1))
636 flux_args_array = np.argsort(cat_array[:, -1])
639 tmp_sort_array = cat_array[flux_args_array]
642 for start_idx
in range(cat_array.shape[0] -
643 self.config.numPointsForShape):
644 pattern_points = tmp_sort_array[start_idx:start_idx +
645 self.config.numPointsForShape, :-1]
646 pattern_delta = pattern_points[1:, :] - pattern_points[0, :]
647 pattern_array[start_idx, :] = np.sqrt(
648 pattern_delta[:, 0] ** 2 +
649 pattern_delta[:, 1] ** 2 +
650 pattern_delta[:, 2] ** 2)
655 pattern_array[start_idx, :] = pattern_array[
656 start_idx, np.argsort(pattern_array[start_idx, :])]
662 pattern_array[:, :(self.config.numPointsForShape - 1)])
663 dist_nearest_array, ids = dist_tree.query(
664 pattern_array[:, :(self.config.numPointsForShape - 1)], k=2)
665 dist_nearest_array = dist_nearest_array[:, 1]
666 dist_nearest_array.sort()
670 dist_tol = (np.degrees(dist_nearest_array[dist_idx]) * 3600. /
671 (self.config.numPointsForShape - 1.))
673 self.log.debug(
"Automated tolerance")
674 self.log.debug(
"\tdistance/match tol: %.4f [arcsec]" % dist_tol)
def __init__(self, kwargs)
def __init__(self, maxMatchDist=None, autoMaxMatchDist=None, maxShift=None, lastMatchedPattern=None, failedPatternList=None, PPMbObj=None)
def _doMatch(self, refCat, sourceCat, wcs, refFluxField, numUsableSources, minMatchedPairs, match_tolerance, sourceFluxField, verbose)
def _filterRefCat(self, refCat, refFluxField)
def matchObjectsToSources(self, refCat, sourceCat, wcs, refFluxField, match_tolerance=None)
def _get_pair_pattern_statistics(self, cat_array)
def _latlong_flux_to_xyz_mag(self, theta, phi, flux)