Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import astropy.units as u 

2import numpy as np 

3from lsst.pipe.base import Struct, Task 

4from lsst.pex.config import Config, Field, ListField 

5from lsst.verify import Measurement, ThresholdSpecification, Datum 

6from lsst.faro.utils.filtermatches import filterMatches 

7from lsst.faro.utils.separations import (calcRmsDistances, calcRmsDistancesVsRef, 

8 astromResiduals) 

9from lsst.faro.utils.phot_repeat import photRepeat 

10from lsst.faro.utils.tex import (correlation_function_ellipticity_from_matches, 

11 select_bin_from_corr) 

12 

13__all__ = ("PA1TaskConfig", "PA1Task", "PA2TaskConfig", "PA2Task", "PF1Task", "TExTaskConfig", 

14 "TExTask", "AMxTaskConfig", "AMxTask", "ADxTask", "AFxTask", "AB1TaskConfig", "AB1Task") 

15 

16 

17filter_dict = {'u': 1, 'g': 2, 'r': 3, 'i': 4, 'z': 5, 'y': 6, 

18 'HSC-U': 1, 'HSC-G': 2, 'HSC-R': 3, 'HSC-I': 4, 'HSC-Z': 5, 'HSC-Y': 6} 

19 

20 

21class PA1TaskConfig(Config): 

22 brightSnrMin = Field(doc="Minimum median SNR for a source to be considered bright.", 

23 dtype=float, default=50) 

24 brightSnrMax = Field(doc="Maximum median SNR for a source to be considered bright.", 

25 dtype=float, default=np.Inf) 

26 numRandomShuffles = Field(doc="Number of trials used for random sampling of observation pairs.", 

27 dtype=int, default=50) 

28 randomSeed = Field(doc="Random seed for sampling.", 

29 dtype=int, default=12345) 

30 

31 

32class PA1Task(Task): 

33 

34 ConfigClass = PA1TaskConfig 

35 _DefaultName = "PA1Task" 

36 

37 def __init__(self, config: PA1TaskConfig, *args, **kwargs): 

38 super().__init__(*args, config=config, **kwargs) 

39 self.brightSnrMin = self.config.brightSnrMin 

40 self.brightSnrMax = self.config.brightSnrMax 

41 self.numRandomShuffles = self.config.numRandomShuffles 

42 self.randomSeed = self.config.randomSeed 

43 

44 def run(self, matchedCatalog, metric_name): 

45 self.log.info("Measuring PA1") 

46 

47 pa1 = photRepeat(matchedCatalog, snrMax=self.brightSnrMax, snrMin=self.brightSnrMin, 

48 numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) 

49 

50 if 'magDiff' in pa1.keys(): 

51 return Struct(measurement=Measurement("PA1", pa1['repeatability'])) 

52 else: 

53 return Struct(measurement=Measurement("PA1", np.nan*u.mmag)) 

54 

55 

56class PA2TaskConfig(Config): 

57 brightSnrMin = Field(doc="Minimum median SNR for a source to be considered bright.", 

58 dtype=float, default=50) 

59 brightSnrMax = Field(doc="Maximum median SNR for a source to be considered bright.", 

60 dtype=float, default=np.Inf) 

61 # The defaults for threshPA2 and threshPF1 correspond to the SRD "design" thresholds. 

62 threshPA2 = Field(doc="Threshold in mmag for PF1 calculation.", dtype=float, default=15.0) 

63 threshPF1 = Field(doc="Percentile of differences that can vary by more than threshPA2.", 

64 dtype=float, default=10.0) 

65 numRandomShuffles = Field(doc="Number of trials used for random sampling of observation pairs.", 

66 dtype=int, default=50) 

67 randomSeed = Field(doc="Random seed for sampling.", 

68 dtype=int, default=12345) 

69 

70 

71class PA2Task(Task): 

72 

73 ConfigClass = PA2TaskConfig 

74 _DefaultName = "PA2Task" 

75 

76 def __init__(self, config: PA2TaskConfig, *args, **kwargs): 

77 super().__init__(*args, config=config, **kwargs) 

78 self.brightSnrMin = self.config.brightSnrMin 

79 self.brightSnrMax = self.config.brightSnrMax 

80 self.threshPA2 = self.config.threshPA2 

81 self.threshPF1 = self.config.threshPF1 

82 self.numRandomShuffles = self.config.numRandomShuffles 

83 self.randomSeed = self.config.randomSeed 

84 

85 def run(self, matchedCatalog, metric_name): 

86 self.log.info("Measuring PA2") 

87 pf1_thresh = self.threshPF1 * u.percent 

88 

89 pa2 = photRepeat(matchedCatalog, 

90 numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) 

91 

92 if 'magDiff' in pa2.keys(): 

93 # Previously, validate_drp used the first random sample from PA1 measurement 

94 # Now, use all of them. 

95 magDiffs = pa2['magDiff'] 

96 

97 pf1Percentile = 100.*u.percent - pf1_thresh 

98 return Struct(measurement=Measurement("PA2", np.percentile(np.abs(magDiffs.value), 

99 pf1Percentile.value) * magDiffs.unit)) 

100 else: 

101 return Struct(measurement=Measurement("PA2", np.nan*u.mmag)) 

102 

103 

104class PF1Task(Task): 

105 

106 ConfigClass = PA2TaskConfig 

107 _DefaultName = "PF1Task" 

108 

109 def __init__(self, config: PA2TaskConfig, *args, **kwargs): 

110 super().__init__(*args, config=config, **kwargs) 

111 self.brightSnrMin = self.config.brightSnrMin 

112 self.brightSnrMax = self.config.brightSnrMax 

113 self.threshPA2 = self.config.threshPA2 

114 self.threshPF1 = self.config.threshPF1 

115 self.numRandomShuffles = self.config.numRandomShuffles 

116 self.randomSeed = self.config.randomSeed 

117 

118 def run(self, matchedCatalog, metric_name): 

119 self.log.info("Measuring PF1") 

120 pa2_thresh = self.threshPA2 * u.mmag 

121 

122 pf1 = photRepeat(matchedCatalog, 

123 numRandomShuffles=self.numRandomShuffles, randomSeed=self.randomSeed) 

124 

125 if 'magDiff' in pf1.keys(): 

126 # Previously, validate_drp used the first random sample from PA1 measurement 

127 # Now, use all of them. 

128 magDiffs = pf1['magDiff'] 

129 

130 percentileAtPA2 = 100 * np.mean(np.abs(magDiffs.value) > pa2_thresh.value) * u.percent 

131 

132 return Struct(measurement=Measurement("PF1", percentileAtPA2)) 

133 else: 

134 return Struct(measurement=Measurement("PF1", np.nan*u.percent)) 

135 

136 

137class TExTaskConfig(Config): 

138 annulus_r = Field(doc="Radial size of the annulus in arcmin", 

139 dtype=float, default=1.) 

140 comparison_operator = Field(doc="String representation of the operator to use in comparisons", 

141 dtype=str, default="<=") 

142 

143 

144class TExTask(Task): 

145 ConfigClass = TExTaskConfig 

146 _DefaultName = "TExTask" 

147 

148 def run(self, matchedCatalog, metric_name): 

149 self.log.info(f"Measuring {metric_name}") 

150 

151 D = self.config.annulus_r * u.arcmin 

152 filteredCat = filterMatches(matchedCatalog) 

153 nMinTEx = 50 

154 if filteredCat.count <= nMinTEx: 

155 return Struct(measurement=Measurement(metric_name, np.nan*u.Unit(''))) 

156 

157 radius, xip, xip_err = correlation_function_ellipticity_from_matches(filteredCat) 

158 operator = ThresholdSpecification.convert_operator_str(self.config.comparison_operator) 

159 corr, corr_err = select_bin_from_corr(radius, xip, xip_err, radius=D, operator=operator) 

160 return Struct(measurement=Measurement(metric_name, np.abs(corr)*u.Unit(''))) 

161 

162 

163def isSorted(a): 

164 return all(a[i] <= a[i+1] for i in range(len(a)-1)) 

165 

166 

167def bins(window, n): 

168 delta = window/n 

169 return [i*delta for i in range(n+1)] 

170 

171 

172class AMxTaskConfig(Config): 

173 annulus_r = Field(doc="Radial distance of the annulus in arcmin (5, 20, or 200 for AM1, AM2, AM3)", 

174 dtype=float, default=5.) 

175 width = Field(doc="Width of annulus in arcmin", 

176 dtype=float, default=2.) 

177 bright_mag_cut = Field(doc="Bright limit of catalog entries to include", 

178 dtype=float, default=17.0) 

179 faint_mag_cut = Field(doc="Faint limit of catalog entries to include", 

180 dtype=float, default=21.5) 

181 # The defaults for threshADx and threshAFx correspond to the SRD "design" thresholds. 

182 threshAD = Field(doc="Threshold in mas for AFx calculation.", dtype=float, default=20.0) 

183 threshAF = Field(doc="Percentile of differences that can vary by more than threshAD.", 

184 dtype=float, default=10.0) 

185 bins = ListField(doc="Bins for histogram.", 

186 dtype=float, minLength=2, maxLength=1500, 

187 listCheck=isSorted, default=bins(30, 200)) 

188 

189 

190class AMxTask(Task): 

191 ConfigClass = AMxTaskConfig 

192 _DefaultName = "AMxTask" 

193 

194 def run(self, matchedCatalog, metric_name): 

195 self.log.info(f"Measuring {metric_name}") 

196 

197 filteredCat = filterMatches(matchedCatalog) 

198 

199 magRange = np.array([self.config.bright_mag_cut, self.config.faint_mag_cut]) * u.mag 

200 D = self.config.annulus_r * u.arcmin 

201 width = self.config.width * u.arcmin 

202 annulus = D + (width/2)*np.array([-1, +1]) 

203 

204 rmsDistances = calcRmsDistances( 

205 filteredCat, 

206 annulus, 

207 magRange=magRange) 

208 

209 values, bins = np.histogram(rmsDistances.to(u.marcsec), bins=self.config.bins*u.marcsec) 

210 extras = {'bins': Datum(bins, label='binvalues', description='bins'), 

211 'values': Datum(values*u.count, label='counts', description='icounts in bins')} 

212 

213 if len(rmsDistances) == 0: 

214 return Struct(measurement=Measurement(metric_name, np.nan*u.marcsec, extras=extras)) 

215 

216 return Struct(measurement=Measurement(metric_name, np.median(rmsDistances.to(u.marcsec)), 

217 extras=extras)) 

218 

219 

220class ADxTask(Task): 

221 ConfigClass = AMxTaskConfig 

222 _DefaultName = "ADxTask" 

223 

224 def run(self, matchedCatalog, metric_name): 

225 self.log.info(f"Measuring {metric_name}") 

226 

227 sepDistances = astromResiduals(matchedCatalog, self.config.bright_mag_cut, 

228 self.config.faint_mag_cut, self.config.annulus_r, 

229 self.config.width) 

230 

231 afThresh = self.config.threshAF * u.percent 

232 afPercentile = 100.0*u.percent - afThresh 

233 

234 if len(sepDistances) <= 1: 

235 return Struct(measurement=Measurement(metric_name, np.nan*u.marcsec)) 

236 else: 

237 # absolute value of the difference between each astrometric rms 

238 # and the median astrometric RMS 

239 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec) 

240 absDiffsMarcsec = (sepDistances-np.median(sepDistances)).to(u.marcsec) 

241 return Struct(measurement=Measurement(metric_name, np.percentile(absDiffsMarcsec.value, 

242 afPercentile.value)*u.marcsec)) 

243 

244 

245class AFxTask(Task): 

246 ConfigClass = AMxTaskConfig 

247 _DefaultName = "AFxTask" 

248 

249 def run(self, matchedCatalog, metric_name): 

250 self.log.info(f"Measuring {metric_name}") 

251 

252 sepDistances = astromResiduals(matchedCatalog, self.config.bright_mag_cut, 

253 self.config.faint_mag_cut, self.config.annulus_r, 

254 self.config.width) 

255 

256 adxThresh = self.config.threshAD * u.marcsec 

257 

258 if len(sepDistances) <= 1: 

259 return Struct(measurement=Measurement(metric_name, np.nan*u.percent)) 

260 else: 

261 # absolute value of the difference between each astrometric rms 

262 # and the median astrometric RMS 

263 # absRmsDiffs = np.abs(rmsDistances - np.median(rmsDistances)).to(u.marcsec) 

264 absDiffsMarcsec = (sepDistances-np.median(sepDistances)).to(u.marcsec) 

265 percentileAtADx = 100 * np.mean(np.abs(absDiffsMarcsec.value) > adxThresh.value) * u.percent 

266 return Struct(measurement=Measurement(metric_name, percentileAtADx)) 

267 

268 

269class AB1TaskConfig(Config): 

270 bright_mag_cut = Field(doc="Bright limit of catalog entries to include", 

271 dtype=float, default=17.0) 

272 faint_mag_cut = Field(doc="Faint limit of catalog entries to include", 

273 dtype=float, default=21.5) 

274 ref_filter = Field(doc="String representing the filter to use as reference", 

275 dtype=str, default="r") 

276 

277 

278class AB1Task(Task): 

279 ConfigClass = AB1TaskConfig 

280 _DefaultName = "AB1Task" 

281 

282 def run(self, matchedCatalogMulti, metric_name, in_id, out_id): 

283 self.log.info(f"Measuring {metric_name}") 

284 

285 if self.config.ref_filter not in filter_dict: 

286 raise Exception('Reference filter supplied for AB1 not in dictionary.') 

287 

288 filteredCat = filterMatches(matchedCatalogMulti) 

289 rmsDistancesAll = [] 

290 

291 if len(filteredCat) > 0: 

292 

293 filtnum = filter_dict[self.config.ref_filter] 

294 

295 refVisits = set() 

296 for id in filteredCat.ids: 

297 grptmp = filteredCat[id] 

298 filtmch = (grptmp['filt'] == filtnum) 

299 if len(filtmch) > 0: 

300 refVisits.update(set(grptmp[filtmch]['visit'])) 

301 

302 refVisits = list(refVisits) 

303 

304 magRange = np.array([self.config.bright_mag_cut, self.config.faint_mag_cut]) * u.mag 

305 for rv in refVisits: 

306 rmsDistances = calcRmsDistancesVsRef( 

307 filteredCat, 

308 rv, 

309 magRange=magRange, 

310 band=filter_dict[out_id['band']]) 

311 finiteEntries = np.where(np.isfinite(rmsDistances))[0] 

312 if len(finiteEntries) > 0: 

313 rmsDistancesAll.append(rmsDistances[finiteEntries]) 

314 

315 if len(rmsDistancesAll) == 0: 

316 return Struct(measurement=Measurement(metric_name, np.nan*u.marcsec)) 

317 else: 

318 rmsDistancesAll = np.concatenate(rmsDistancesAll) 

319 return Struct(measurement=Measurement(metric_name, np.mean(rmsDistancesAll)*u.marcsec)) 

320 

321 else: 

322 return Struct(measurement=Measurement(metric_name, np.nan*u.marcsec))