Coverage for python/lsst/faro/measurement/MatchedCatalogMeasurementTasks.py: 41%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of faro.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import astropy.units as u
23import numpy as np
24from lsst.pipe.base import Struct, Task
25from lsst.pex.config import ChoiceField, Config, Field, ListField
26from lsst.verify import Measurement, Datum
27from lsst.faro.utils.filtermatches import filterMatches
28from lsst.faro.utils.separations import (
29 calcRmsDistances,
30 calcRmsDistancesVsRef,
31 astromResiduals,
32)
33from lsst.faro.utils.phot_repeat import photRepeat
36__all__ = (
37 "PA1Config",
38 "PA1Task",
39 "PF1Config",
40 "PF1Task",
41 "AMxConfig",
42 "AMxTask",
43 "ADxTask",
44 "AFxTask",
45 "AB1Config",
46 "AB1Task",
47 "ModelPhotRepTask",
48)
51filter_dict = {
52 "u": 1,
53 "g": 2,
54 "r": 3,
55 "i": 4,
56 "z": 5,
57 "y": 6,
58 "HSC-U": 1,
59 "HSC-G": 2,
60 "HSC-R": 3,
61 "HSC-I": 4,
62 "HSC-Z": 5,
63 "HSC-Y": 6,
64}
67class PA1Config(Config):
68 """Config fields for the PA1 photometric repeatability metric.
69 """
71 brightSnrMin = Field(
72 doc="Minimum median SNR for a source to be considered bright.",
73 dtype=float,
74 default=200,
75 )
76 brightSnrMax = Field(
77 doc="Maximum median SNR for a source to be considered bright.",
78 dtype=float,
79 default=np.Inf,
80 )
81 nMinPhotRepeat = Field(
82 doc="Minimum number of objects required for photometric repeatability.",
83 dtype=int,
84 default=50,
85 )
86 writeExtras = Field(
87 doc="Write out the magnitude residuals and rms values for debugging.",
88 dtype=bool,
89 default=False,
90 )
93class PA1Task(Task):
94 """A Task that computes the PA1 photometric repeatability metric from an
95 input set of multiple visits of the same field.
97 Notes
98 -----
99 The intended usage is to retarget the run method of
100 `lsst.faro.measurement.TractMatchedMeasurementTask` to PA1Task.
101 This metric is calculated on a set of matched visits, and aggregated at the tract level.
102 """
104 ConfigClass = PA1Config
105 _DefaultName = "PA1Task"
107 def __init__(self, config: PA1Config, *args, **kwargs):
108 super().__init__(*args, config=config, **kwargs)
110 def run(self, metricName, matchedCatalog):
111 """Calculate the photometric repeatability.
113 Parameters
114 ----------
115 matchedCatalog : `lsst.afw.table.base.Catalog`
116 `~lsst.afw.table.base.Catalog` object as created by
117 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
118 metricName : `str`
119 The name of the metric.
121 Returns
122 -------
123 measurement : `lsst.verify.Measurement`
124 Measurement of the repeatability and its associated metadata.
125 """
126 self.log.info("Measuring %s", metricName)
128 pa1 = photRepeat(
129 matchedCatalog,
130 nMinPhotRepeat=self.config.nMinPhotRepeat,
131 snrMax=self.config.brightSnrMax,
132 snrMin=self.config.brightSnrMin,
133 )
135 if "magMean" in pa1.keys():
136 if self.config.writeExtras:
137 extras = {
138 "rms": Datum(
139 pa1["rms"],
140 label="RMS",
141 description="Photometric repeatability rms for each star.",
142 ),
143 "count": Datum(
144 pa1["count"] * u.count,
145 label="count",
146 description="Number of detections used to calculate repeatability.",
147 ),
148 "mean_mag": Datum(
149 pa1["magMean"],
150 label="mean_mag",
151 description="Mean magnitude of each star.",
152 ),
153 }
154 return Struct(
155 measurement=Measurement("PA1", pa1["repeatability"], extras=extras)
156 )
157 else:
158 return Struct(measurement=Measurement("PA1", pa1["repeatability"]))
159 else:
160 return Struct(measurement=Measurement("PA1", np.nan * u.mmag))
163class PF1Config(Config):
164 brightSnrMin = Field(
165 doc="Minimum median SNR for a source to be considered bright.",
166 dtype=float,
167 default=200,
168 )
169 brightSnrMax = Field(
170 doc="Maximum median SNR for a source to be considered bright.",
171 dtype=float,
172 default=np.Inf,
173 )
174 nMinPhotRepeat = Field(
175 doc="Minimum number of objects required for photometric repeatability.",
176 dtype=int,
177 default=50,
178 )
179 # The defaults for threshPA2 correspond to the SRD "design" thresholds.
180 threshPA2 = Field(
181 doc="Threshold in mmag for PF1 calculation.", dtype=float, default=15.0
182 )
185class PF1Task(Task):
186 """A Task that computes PF1, the percentage of photometric repeatability measurements
187 that deviate by more than PA2 mmag from the mean.
189 Notes
190 -----
191 The intended usage is to retarget the run method of
192 `lsst.faro.measurement.TractMatchedMeasurementTask` to PF1Task. This Task uses the
193 same set of photometric residuals that are calculated for the PA1 metric.
194 This metric is calculated on a set of matched visits, and aggregated at the tract level.
195 """
197 ConfigClass = PF1Config
198 _DefaultName = "PF1Task"
200 def __init__(self, config: PF1Config, *args, **kwargs):
201 super().__init__(*args, config=config, **kwargs)
203 def run(self, metricName, matchedCatalog):
204 """Calculate the percentage of outliers in the photometric repeatability values.
206 Parameters
207 ----------
208 matchedCatalog : `lsst.afw.table.base.Catalog`
209 `~lsst.afw.table.base.Catalog` object as created by
210 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
211 metricName : `str`
212 The name of the metric.
214 Returns
215 -------
216 measurement : `lsst.verify.Measurement`
217 Measurement of the percentage of repeatability outliers, and associated metadata.
218 """
219 self.log.info("Measuring %s", metricName)
220 pa2_thresh = self.config.threshPA2 * u.mmag
222 pf1 = photRepeat(
223 matchedCatalog,
224 nMinPhotRepeat=self.config.nMinPhotRepeat,
225 snrMax=self.config.brightSnrMax,
226 snrMin=self.config.brightSnrMin,
227 )
229 if "magResid" in pf1.keys():
230 # Previously, validate_drp used the first random sample from PA1 measurement
231 # Now, use all of them.
232 # Keep only stars with > 2 observations:
233 okrms = pf1["count"] > 2
234 magResid0 = pf1["magResid"]
235 magResid = np.concatenate(magResid0[okrms])
237 percentileAtPA2 = (
238 100 * np.mean(np.abs(magResid.value) > pa2_thresh.value) * u.percent
239 )
241 return Struct(measurement=Measurement("PF1", percentileAtPA2))
242 else:
243 return Struct(measurement=Measurement("PF1", np.nan * u.percent))
246def isSorted(a):
247 return all(a[i] <= a[i + 1] for i in range(len(a) - 1))
250def bins(window, n):
251 delta = window / n
252 return [i * delta for i in range(n + 1)]
255class AMxConfig(Config):
256 annulus_r = Field(
257 doc="Radial distance of the annulus in arcmin (5, 20, or 200 for AM1, AM2, AM3)",
258 dtype=float,
259 default=5.0,
260 )
261 width = Field(doc="Width of annulus in arcmin", dtype=float, default=2.0)
262 bright_mag_cut = Field(
263 doc="Bright limit of catalog entries to include", dtype=float, default=17.0
264 )
265 faint_mag_cut = Field(
266 doc="Faint limit of catalog entries to include", dtype=float, default=21.5
267 )
268 # The defaults for threshADx and threshAFx correspond to the SRD "design" thresholds.
269 threshAD = Field(
270 doc="Threshold in mas for AFx calculation.", dtype=float, default=20.0
271 )
272 threshAF = Field(
273 doc="Percentile of differences that can vary by more than threshAD.",
274 dtype=float,
275 default=10.0,
276 )
277 bins = ListField(
278 doc="Bins for histogram.",
279 dtype=float,
280 minLength=2,
281 maxLength=1500,
282 listCheck=isSorted,
283 default=bins(30, 200),
284 )
287class AMxTask(Task):
288 ConfigClass = AMxConfig
289 _DefaultName = "AMxTask"
291 def run(self, metricName, matchedCatalog):
292 self.log.info("Measuring %s", metricName)
294 filteredCat = filterMatches(matchedCatalog)
296 magRange = (
297 np.array([self.config.bright_mag_cut, self.config.faint_mag_cut]) * u.mag
298 )
299 D = self.config.annulus_r * u.arcmin
300 width = self.config.width * u.arcmin
301 annulus = D + (width / 2) * np.array([-1, +1])
303 rmsDistances = calcRmsDistances(filteredCat, annulus, magRange=magRange)
305 values, bins = np.histogram(
306 rmsDistances.to(u.marcsec), bins=self.config.bins * u.marcsec
307 )
308 extras = {
309 "bins": Datum(bins, label="binvalues", description="bins"),
310 "values": Datum(
311 values * u.count, label="counts", description="icounts in bins"
312 ),
313 }
315 if len(rmsDistances) == 0:
316 return Struct(
317 measurement=Measurement(metricName, np.nan * u.marcsec, extras=extras)
318 )
320 return Struct(
321 measurement=Measurement(
322 metricName, np.median(rmsDistances.to(u.marcsec)), extras=extras
323 )
324 )
327class ADxTask(Task):
328 ConfigClass = AMxConfig
329 _DefaultName = "ADxTask"
331 def run(self, metricName, matchedCatalog):
332 self.log.info("Measuring %s", metricName)
334 sepDistances = astromResiduals(
335 matchedCatalog,
336 self.config.bright_mag_cut,
337 self.config.faint_mag_cut,
338 self.config.annulus_r,
339 self.config.width,
340 )
342 afThresh = self.config.threshAF * u.percent
343 afPercentile = 100.0 * u.percent - afThresh
345 if len(sepDistances) <= 1:
346 return Struct(measurement=Measurement(metricName, np.nan * u.marcsec))
347 else:
348 # absolute value of the difference between each astrometric rms
349 # and the median astrometric RMS
350 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec)
351 absDiffsMarcsec = (sepDistances - np.median(sepDistances)).to(u.marcsec)
352 return Struct(
353 measurement=Measurement(
354 metricName,
355 np.percentile(absDiffsMarcsec.value, afPercentile.value)
356 * u.marcsec,
357 )
358 )
361class AFxTask(Task):
362 ConfigClass = AMxConfig
363 _DefaultName = "AFxTask"
365 def run(self, metricName, matchedCatalog):
366 self.log.info("Measuring %s", metricName)
368 sepDistances = astromResiduals(
369 matchedCatalog,
370 self.config.bright_mag_cut,
371 self.config.faint_mag_cut,
372 self.config.annulus_r,
373 self.config.width,
374 )
376 adxThresh = self.config.threshAD * u.marcsec
378 if len(sepDistances) <= 1:
379 return Struct(measurement=Measurement(metricName, np.nan * u.percent))
380 else:
381 # absolute value of the difference between each astrometric rms
382 # and the median astrometric RMS
383 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec)
384 absDiffsMarcsec = (sepDistances - np.median(sepDistances)).to(u.marcsec)
385 percentileAtADx = (
386 100
387 * np.mean(np.abs(absDiffsMarcsec.value) > adxThresh.value)
388 * u.percent
389 )
390 return Struct(measurement=Measurement(metricName, percentileAtADx))
393class AB1Config(Config):
394 bright_mag_cut = Field(
395 doc="Bright limit of catalog entries to include", dtype=float, default=17.0
396 )
397 faint_mag_cut = Field(
398 doc="Faint limit of catalog entries to include", dtype=float, default=21.5
399 )
400 ref_filter = Field(
401 doc="String representing the filter to use as reference", dtype=str, default="r"
402 )
405class AB1Task(Task):
406 ConfigClass = AB1Config
407 _DefaultName = "AB1Task"
409 def run(self, metricName, matchedCatalogMulti, in_id, out_id):
410 self.log.info("Measuring %s", metricName)
412 if self.config.ref_filter not in filter_dict:
413 raise Exception("Reference filter supplied for AB1 not in dictionary.")
415 filteredCat = filterMatches(matchedCatalogMulti)
416 rmsDistancesAll = []
418 if len(filteredCat) > 0:
420 filtnum = filter_dict[self.config.ref_filter]
422 refVisits = set()
423 for id in filteredCat.ids:
424 grptmp = filteredCat[id]
425 filtmch = grptmp["filt"] == filtnum
426 if len(filtmch) > 0:
427 refVisits.update(set(grptmp[filtmch]["visit"]))
429 refVisits = list(refVisits)
431 magRange = (
432 np.array([self.config.bright_mag_cut, self.config.faint_mag_cut])
433 * u.mag
434 )
435 for rv in refVisits:
436 rmsDistances = calcRmsDistancesVsRef(
437 filteredCat, rv, magRange=magRange, band=filter_dict[out_id["band"]]
438 )
439 finiteEntries = np.where(np.isfinite(rmsDistances))[0]
440 if len(finiteEntries) > 0:
441 rmsDistancesAll.append(rmsDistances[finiteEntries])
443 if len(rmsDistancesAll) == 0:
444 return Struct(measurement=Measurement(metricName, np.nan * u.marcsec))
445 else:
446 rmsDistancesAll = np.concatenate(rmsDistancesAll)
447 return Struct(
448 measurement=Measurement(metricName, np.mean(rmsDistancesAll))
449 )
451 else:
452 return Struct(measurement=Measurement(metricName, np.nan * u.marcsec))
455class ModelPhotRepConfig(Config):
456 """Config fields for the *ModelPhotRep photometric repeatability metrics.
457 """
459 index = ChoiceField(
460 doc="Index of the metric definition",
461 dtype=int,
462 allowed={x: f"Nth (N={x}) lowest S/N bin" for x in range(1, 5)},
463 optional=False,
464 )
465 magName = Field(
466 doc="Name of the magnitude column", dtype=str, default="slot_ModelFlux_mag"
467 )
468 nMinPhotRepeat = Field(
469 doc="Minimum number of objects required for photometric repeatability.",
470 dtype=int,
471 default=50,
472 )
473 prefix = Field(doc="Prefix of the metric name", dtype=str, default="model")
474 selectExtended = Field(doc="Whether to select extended sources", dtype=bool)
475 selectSnrMin = Field(
476 doc="Minimum median SNR for a source to be selected.", dtype=float
477 )
478 selectSnrMax = Field(
479 doc="Maximum median SNR for a source to be selected.", dtype=float
480 )
481 writeExtras = Field(
482 doc="Write out the magnitude residuals and rms values for debugging.",
483 dtype=bool,
484 default=False,
485 )
488class ModelPhotRepTask(Task):
489 """A Task that computes a *ModelPhotRep* photometric repeatability metric
490 from an input set of multiple visits of the same field.
492 Notes
493 -----
494 The intended usage is to retarget the run method of
495 `lsst.faro.measurement.TractMatchedMeasurementTask` to ModelPhotRepTask.
496 This metric is calculated on a set of matched visits, and aggregated at the tract level.
497 """
499 ConfigClass = ModelPhotRepConfig
500 _DefaultName = "ModelPhotRepTask"
502 def __init__(self, config: ModelPhotRepConfig, *args, **kwargs):
503 super().__init__(*args, config=config, **kwargs)
505 def run(self, metricName, matchedCatalog):
506 """Calculate the photometric repeatability.
508 Parameters
509 ----------
510 matchedCatalog : `lsst.afw.table.base.Catalog`
511 `~lsst.afw.table.base.Catalog` object as created by
512 `~lsst.afw.table.multiMatch` matching of sources from multiple visits.
513 metricName : `str`
514 The name of the metric.
516 Returns
517 -------
518 measurement : `lsst.verify.Measurement`
519 Measurement of the repeatability and its associated metadata.
520 """
521 self.log.info("Measuring %s", metricName)
523 meas = photRepeat(
524 matchedCatalog,
525 nMinPhotRepeat=self.config.nMinPhotRepeat,
526 snrMax=self.config.selectSnrMax,
527 snrMin=self.config.selectSnrMin,
528 magName=self.config.magName,
529 extended=self.config.selectExtended,
530 )
532 name_type = "Gal" if self.config.selectExtended else "Star"
533 name_meas = f"{self.config.prefix}PhotRep{name_type}{self.config.index}"
535 if "magMean" in meas.keys():
536 if self.config.writeExtras:
537 extras = {
538 "rms": Datum(
539 meas["rms"],
540 label="RMS",
541 description="Photometric repeatability rms for each star.",
542 ),
543 "count": Datum(
544 meas["count"] * u.count,
545 label="count",
546 description="Number of detections used to calculate repeatability.",
547 ),
548 "mean_mag": Datum(
549 meas["magMean"],
550 label="mean_mag",
551 description="Mean magnitude of each star.",
552 ),
553 }
554 return Struct(
555 measurement=Measurement(
556 name_meas, meas["repeatability"], extras=extras
557 )
558 )
559 else:
560 return Struct(measurement=Measurement(name_meas, meas["repeatability"]))
561 else:
562 return Struct(measurement=Measurement(name_meas, np.nan * u.mmag))