Coverage for python/lsst/faro/measurement/MatchedCatalogMeasurementTasks.py : 41%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1import astropy.units as u
2import numpy as np
3from lsst.pipe.base import Struct, Task
4from lsst.pex.config import ChoiceField, Config, Field, ListField
5from lsst.verify import Measurement, Datum
6from lsst.faro.utils.filtermatches import filterMatches
7from lsst.faro.utils.separations import (
8 calcRmsDistances,
9 calcRmsDistancesVsRef,
10 astromResiduals,
11)
12from lsst.faro.utils.phot_repeat import photRepeat
15__all__ = (
16 "PA1Config",
17 "PA1Task",
18 "PF1Config",
19 "PF1Task",
20 "AMxConfig",
21 "AMxTask",
22 "ADxTask",
23 "AFxTask",
24 "AB1Config",
25 "AB1Task",
26 "ModelPhotRepTask",
27)
30filter_dict = {
31 "u": 1,
32 "g": 2,
33 "r": 3,
34 "i": 4,
35 "z": 5,
36 "y": 6,
37 "HSC-U": 1,
38 "HSC-G": 2,
39 "HSC-R": 3,
40 "HSC-I": 4,
41 "HSC-Z": 5,
42 "HSC-Y": 6,
43}
46class PA1Config(Config):
47 """Config fields for the PA1 photometric repeatability metric.
48 """
50 brightSnrMin = Field(
51 doc="Minimum median SNR for a source to be considered bright.",
52 dtype=float,
53 default=200,
54 )
55 brightSnrMax = Field(
56 doc="Maximum median SNR for a source to be considered bright.",
57 dtype=float,
58 default=np.Inf,
59 )
60 nMinPhotRepeat = Field(
61 doc="Minimum number of objects required for photometric repeatability.",
62 dtype=int,
63 default=50,
64 )
65 writeExtras = Field(
66 doc="Write out the magnitude residuals and rms values for debugging.",
67 dtype=bool,
68 default=False,
69 )
72class PA1Task(Task):
73 """A Task that computes the PA1 photometric repeatability metric from an
74 input set of multiple visits of the same field.
76 Notes
77 -----
78 The intended usage is to retarget the run method of
79 `lsst.faro.measurement.TractMatchedMeasurementTask` to PA1Task.
80 This metric is calculated on a set of matched visits, and aggregated at the tract level.
81 """
83 ConfigClass = PA1Config
84 _DefaultName = "PA1Task"
86 def __init__(self, config: PA1Config, *args, **kwargs):
87 super().__init__(*args, config=config, **kwargs)
89 def run(self, matchedCatalog, metricName):
90 """Calculate the photometric repeatability.
92 Parameters
93 ----------
94 matchedCatalog : `lsst.afw.table.base.Catalog`
95 `~lsst.afw.table.base.Catalog` object as created by
96 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
97 metricName : `str`
98 The name of the metric.
100 Returns
101 -------
102 measurement : `lsst.verify.Measurement`
103 Measurement of the repeatability and its associated metadata.
104 """
105 self.log.info("Measuring %s", metricName)
107 pa1 = photRepeat(
108 matchedCatalog,
109 nMinPhotRepeat=self.config.nMinPhotRepeat,
110 snrMax=self.config.brightSnrMax,
111 snrMin=self.config.brightSnrMin,
112 )
114 if "magMean" in pa1.keys():
115 if self.config.writeExtras:
116 extras = {
117 "rms": Datum(
118 pa1["rms"],
119 label="RMS",
120 description="Photometric repeatability rms for each star.",
121 ),
122 "count": Datum(
123 pa1["count"] * u.count,
124 label="count",
125 description="Number of detections used to calculate repeatability.",
126 ),
127 "mean_mag": Datum(
128 pa1["magMean"],
129 label="mean_mag",
130 description="Mean magnitude of each star.",
131 ),
132 }
133 return Struct(
134 measurement=Measurement("PA1", pa1["repeatability"], extras=extras)
135 )
136 else:
137 return Struct(measurement=Measurement("PA1", pa1["repeatability"]))
138 else:
139 return Struct(measurement=Measurement("PA1", np.nan * u.mmag))
142class PF1Config(Config):
143 brightSnrMin = Field(
144 doc="Minimum median SNR for a source to be considered bright.",
145 dtype=float,
146 default=200,
147 )
148 brightSnrMax = Field(
149 doc="Maximum median SNR for a source to be considered bright.",
150 dtype=float,
151 default=np.Inf,
152 )
153 nMinPhotRepeat = Field(
154 doc="Minimum number of objects required for photometric repeatability.",
155 dtype=int,
156 default=50,
157 )
158 # The defaults for threshPA2 correspond to the SRD "design" thresholds.
159 threshPA2 = Field(
160 doc="Threshold in mmag for PF1 calculation.", dtype=float, default=15.0
161 )
164class PF1Task(Task):
165 """A Task that computes PF1, the percentage of photometric repeatability measurements
166 that deviate by more than PA2 mmag from the mean.
168 Notes
169 -----
170 The intended usage is to retarget the run method of
171 `lsst.faro.measurement.TractMatchedMeasurementTask` to PF1Task. This Task uses the
172 same set of photometric residuals that are calculated for the PA1 metric.
173 This metric is calculated on a set of matched visits, and aggregated at the tract level.
174 """
176 ConfigClass = PF1Config
177 _DefaultName = "PF1Task"
179 def __init__(self, config: PF1Config, *args, **kwargs):
180 super().__init__(*args, config=config, **kwargs)
182 def run(self, matchedCatalog, metricName):
183 """Calculate the percentage of outliers in the photometric repeatability values.
185 Parameters
186 ----------
187 matchedCatalog : `lsst.afw.table.base.Catalog`
188 `~lsst.afw.table.base.Catalog` object as created by
189 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
190 metricName : `str`
191 The name of the metric.
193 Returns
194 -------
195 measurement : `lsst.verify.Measurement`
196 Measurement of the percentage of repeatability outliers, and associated metadata.
197 """
198 self.log.info("Measuring %s", metricName)
199 pa2_thresh = self.config.threshPA2 * u.mmag
201 pf1 = photRepeat(
202 matchedCatalog,
203 nMinPhotRepeat=self.config.nMinPhotRepeat,
204 snrMax=self.config.brightSnrMax,
205 snrMin=self.config.brightSnrMin,
206 )
208 if "magResid" in pf1.keys():
209 # Previously, validate_drp used the first random sample from PA1 measurement
210 # Now, use all of them.
211 # Keep only stars with > 2 observations:
212 okrms = pf1["count"] > 2
213 magResid0 = pf1["magResid"]
214 magResid = np.concatenate(magResid0[okrms])
216 percentileAtPA2 = (
217 100 * np.mean(np.abs(magResid.value) > pa2_thresh.value) * u.percent
218 )
220 return Struct(measurement=Measurement("PF1", percentileAtPA2))
221 else:
222 return Struct(measurement=Measurement("PF1", np.nan * u.percent))
225def isSorted(a):
226 return all(a[i] <= a[i + 1] for i in range(len(a) - 1))
229def bins(window, n):
230 delta = window / n
231 return [i * delta for i in range(n + 1)]
234class AMxConfig(Config):
235 annulus_r = Field(
236 doc="Radial distance of the annulus in arcmin (5, 20, or 200 for AM1, AM2, AM3)",
237 dtype=float,
238 default=5.0,
239 )
240 width = Field(doc="Width of annulus in arcmin", dtype=float, default=2.0)
241 bright_mag_cut = Field(
242 doc="Bright limit of catalog entries to include", dtype=float, default=17.0
243 )
244 faint_mag_cut = Field(
245 doc="Faint limit of catalog entries to include", dtype=float, default=21.5
246 )
247 # The defaults for threshADx and threshAFx correspond to the SRD "design" thresholds.
248 threshAD = Field(
249 doc="Threshold in mas for AFx calculation.", dtype=float, default=20.0
250 )
251 threshAF = Field(
252 doc="Percentile of differences that can vary by more than threshAD.",
253 dtype=float,
254 default=10.0,
255 )
256 bins = ListField(
257 doc="Bins for histogram.",
258 dtype=float,
259 minLength=2,
260 maxLength=1500,
261 listCheck=isSorted,
262 default=bins(30, 200),
263 )
266class AMxTask(Task):
267 ConfigClass = AMxConfig
268 _DefaultName = "AMxTask"
270 def run(self, matchedCatalog, metric_name):
271 self.log.info("Measuring %s", metric_name)
273 filteredCat = filterMatches(matchedCatalog)
275 magRange = (
276 np.array([self.config.bright_mag_cut, self.config.faint_mag_cut]) * u.mag
277 )
278 D = self.config.annulus_r * u.arcmin
279 width = self.config.width * u.arcmin
280 annulus = D + (width / 2) * np.array([-1, +1])
282 rmsDistances = calcRmsDistances(filteredCat, annulus, magRange=magRange)
284 values, bins = np.histogram(
285 rmsDistances.to(u.marcsec), bins=self.config.bins * u.marcsec
286 )
287 extras = {
288 "bins": Datum(bins, label="binvalues", description="bins"),
289 "values": Datum(
290 values * u.count, label="counts", description="icounts in bins"
291 ),
292 }
294 if len(rmsDistances) == 0:
295 return Struct(
296 measurement=Measurement(metric_name, np.nan * u.marcsec, extras=extras)
297 )
299 return Struct(
300 measurement=Measurement(
301 metric_name, np.median(rmsDistances.to(u.marcsec)), extras=extras
302 )
303 )
306class ADxTask(Task):
307 ConfigClass = AMxConfig
308 _DefaultName = "ADxTask"
310 def run(self, matchedCatalog, metric_name):
311 self.log.info("Measuring %s", metric_name)
313 sepDistances = astromResiduals(
314 matchedCatalog,
315 self.config.bright_mag_cut,
316 self.config.faint_mag_cut,
317 self.config.annulus_r,
318 self.config.width,
319 )
321 afThresh = self.config.threshAF * u.percent
322 afPercentile = 100.0 * u.percent - afThresh
324 if len(sepDistances) <= 1:
325 return Struct(measurement=Measurement(metric_name, np.nan * u.marcsec))
326 else:
327 # absolute value of the difference between each astrometric rms
328 # and the median astrometric RMS
329 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec)
330 absDiffsMarcsec = (sepDistances - np.median(sepDistances)).to(u.marcsec)
331 return Struct(
332 measurement=Measurement(
333 metric_name,
334 np.percentile(absDiffsMarcsec.value, afPercentile.value)
335 * u.marcsec,
336 )
337 )
340class AFxTask(Task):
341 ConfigClass = AMxConfig
342 _DefaultName = "AFxTask"
344 def run(self, matchedCatalog, metric_name):
345 self.log.info("Measuring %s", metric_name)
347 sepDistances = astromResiduals(
348 matchedCatalog,
349 self.config.bright_mag_cut,
350 self.config.faint_mag_cut,
351 self.config.annulus_r,
352 self.config.width,
353 )
355 adxThresh = self.config.threshAD * u.marcsec
357 if len(sepDistances) <= 1:
358 return Struct(measurement=Measurement(metric_name, np.nan * u.percent))
359 else:
360 # absolute value of the difference between each astrometric rms
361 # and the median astrometric RMS
362 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec)
363 absDiffsMarcsec = (sepDistances - np.median(sepDistances)).to(u.marcsec)
364 percentileAtADx = (
365 100
366 * np.mean(np.abs(absDiffsMarcsec.value) > adxThresh.value)
367 * u.percent
368 )
369 return Struct(measurement=Measurement(metric_name, percentileAtADx))
372class AB1Config(Config):
373 bright_mag_cut = Field(
374 doc="Bright limit of catalog entries to include", dtype=float, default=17.0
375 )
376 faint_mag_cut = Field(
377 doc="Faint limit of catalog entries to include", dtype=float, default=21.5
378 )
379 ref_filter = Field(
380 doc="String representing the filter to use as reference", dtype=str, default="r"
381 )
384class AB1Task(Task):
385 ConfigClass = AB1Config
386 _DefaultName = "AB1Task"
388 def run(self, matchedCatalogMulti, metric_name, in_id, out_id):
389 self.log.info("Measuring %s", metric_name)
391 if self.config.ref_filter not in filter_dict:
392 raise Exception("Reference filter supplied for AB1 not in dictionary.")
394 filteredCat = filterMatches(matchedCatalogMulti)
395 rmsDistancesAll = []
397 if len(filteredCat) > 0:
399 filtnum = filter_dict[self.config.ref_filter]
401 refVisits = set()
402 for id in filteredCat.ids:
403 grptmp = filteredCat[id]
404 filtmch = grptmp["filt"] == filtnum
405 if len(filtmch) > 0:
406 refVisits.update(set(grptmp[filtmch]["visit"]))
408 refVisits = list(refVisits)
410 magRange = (
411 np.array([self.config.bright_mag_cut, self.config.faint_mag_cut])
412 * u.mag
413 )
414 for rv in refVisits:
415 rmsDistances = calcRmsDistancesVsRef(
416 filteredCat, rv, magRange=magRange, band=filter_dict[out_id["band"]]
417 )
418 finiteEntries = np.where(np.isfinite(rmsDistances))[0]
419 if len(finiteEntries) > 0:
420 rmsDistancesAll.append(rmsDistances[finiteEntries])
422 if len(rmsDistancesAll) == 0:
423 return Struct(measurement=Measurement(metric_name, np.nan * u.marcsec))
424 else:
425 rmsDistancesAll = np.concatenate(rmsDistancesAll)
426 return Struct(
427 measurement=Measurement(metric_name, np.mean(rmsDistancesAll))
428 )
430 else:
431 return Struct(measurement=Measurement(metric_name, np.nan * u.marcsec))
434class ModelPhotRepConfig(Config):
435 """Config fields for the *ModelPhotRep photometric repeatability metrics.
436 """
438 index = ChoiceField(
439 doc="Index of the metric definition",
440 dtype=int,
441 allowed={x: f"Nth (N={x}) lowest S/N bin" for x in range(1, 5)},
442 optional=False,
443 )
444 magName = Field(
445 doc="Name of the magnitude column", dtype=str, default="slot_ModelFlux_mag"
446 )
447 nMinPhotRepeat = Field(
448 doc="Minimum number of objects required for photometric repeatability.",
449 dtype=int,
450 default=50,
451 )
452 prefix = Field(doc="Prefix of the metric name", dtype=str, default="model")
453 selectExtended = Field(doc="Whether to select extended sources", dtype=bool)
454 selectSnrMin = Field(
455 doc="Minimum median SNR for a source to be selected.", dtype=float
456 )
457 selectSnrMax = Field(
458 doc="Maximum median SNR for a source to be selected.", dtype=float
459 )
460 writeExtras = Field(
461 doc="Write out the magnitude residuals and rms values for debugging.",
462 dtype=bool,
463 default=False,
464 )
467class ModelPhotRepTask(Task):
468 """A Task that computes a *ModelPhotRep* photometric repeatability metric
469 from an input set of multiple visits of the same field.
471 Notes
472 -----
473 The intended usage is to retarget the run method of
474 `lsst.faro.measurement.TractMatchedMeasurementTask` to ModelPhotRepTask.
475 This metric is calculated on a set of matched visits, and aggregated at the tract level.
476 """
478 ConfigClass = ModelPhotRepConfig
479 _DefaultName = "ModelPhotRepTask"
481 def __init__(self, config: ModelPhotRepConfig, *args, **kwargs):
482 super().__init__(*args, config=config, **kwargs)
484 def run(self, matchedCatalog, metricName):
485 """Calculate the photometric repeatability.
487 Parameters
488 ----------
489 matchedCatalog : `lsst.afw.table.base.Catalog`
490 `~lsst.afw.table.base.Catalog` object as created by
491 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
492 metricName : `str`
493 The name of the metric.
495 Returns
496 -------
497 measurement : `lsst.verify.Measurement`
498 Measurement of the repeatability and its associated metadata.
499 """
500 self.log.info("Measuring %s", metricName)
502 meas = photRepeat(
503 matchedCatalog,
504 nMinPhotRepeat=self.config.nMinPhotRepeat,
505 snrMax=self.config.selectSnrMax,
506 snrMin=self.config.selectSnrMin,
507 magName=self.config.magName,
508 extended=self.config.selectExtended,
509 )
511 name_type = "Gal" if self.config.selectExtended else "Star"
512 name_meas = f"{self.config.prefix}PhotRep{name_type}{self.config.index}"
514 if "magMean" in meas.keys():
515 if self.config.writeExtras:
516 extras = {
517 "rms": Datum(
518 meas["rms"],
519 label="RMS",
520 description="Photometric repeatability rms for each star.",
521 ),
522 "count": Datum(
523 meas["count"] * u.count,
524 label="count",
525 description="Number of detections used to calculate repeatability.",
526 ),
527 "mean_mag": Datum(
528 meas["magMean"],
529 label="mean_mag",
530 description="Mean magnitude of each star.",
531 ),
532 }
533 return Struct(
534 measurement=Measurement(
535 name_meas, meas["repeatability"], extras=extras
536 )
537 )
538 else:
539 return Struct(measurement=Measurement(name_meas, meas["repeatability"]))
540 else:
541 return Struct(measurement=Measurement(name_meas, np.nan * u.mmag))