23__all__ = [
"KernelCandidateQa"]
32from .
import diffimLib
33from .utils
import calcCentroid, calcWidth
37 """Quality Assessment class for Kernel Candidates"""
40 """Class to undertake QA of KernelCandidates after modeling of
41 the Psf-matching kernel. Both directly--fitted diffim (LOCAL)
42 and spatially--interpolated kernel diffim (SPATIAL) metrics
43 are calculated, based on the distribution of residuals
in the
44 KernelCandidates stamp.
48 nKernelSpatial : `int`
49 Number of terms
in the spatial model; needed to initialize per-basis QA arrays
52 self.fields.append(afwTable.Field["PointD"](
53 "RegisterRefPosition",
54 "Position of reference object for registration (radians)."))
56 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualBearing",
57 "Angle of residual wrt declination parallel in radians"))
59 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualDistance",
60 "Offset of residual in radians"))
63 for kType
in (
"LOCAL",
"SPATIAL"):
65 commentAndUnit = metricMap[k][
'comment']
66 self.
fields.append(afwTable.Field[metricMap[k][
'type']](k%(kType), *commentAndUnit))
68 self.
fields.append(afwTable.Field[
"I"](
"KCKernelStatus_LOCAL",
69 "Status of the KernelCandidate"))
71 self.
fields.append(afwTable.Field[
"ArrayD"](
"KernelCoeffValues_LOCAL",
72 "Original basis coefficients",
75 self.
fields.append(afwTable.Field[
"F"](
"BackgroundValue_LOCAL",
76 "Evaluation of background model at this point"))
78 self.
fields.append(afwTable.Field[
"F"](
"KCDiffimMseKernel_SPATIAL",
79 "Mean squared error of spatial kernel estimate"))
82 nameList = [
'KCDiffimMean_%s',
'KCDiffimMedian_%s',
'KCDiffimIQR_%s',
'KCDiffimStDev_%s',
83 'KCDiffimKSD_%s',
'KCDiffimKSProb_%s',
'KCDiffimADA2_%s',
'KCDiffimADCrit_%s',
84 'KCDiffimADSig_%s',
'KCDiffimChiSq_%s',
'KCDiffimMseResids_%s',
'KCKernelCentX_%s',
85 'KCKernelCentY_%s',
'KCKernelStdX_%s',
'KCKernelStdY_%s',
'KernelCandidateId_%s']
86 typeList = [
'F',
'F',
'F',
'F',
'F',
'F',
'F',
'ArrayD',
'ArrayD',
'F',
'F',
'F',
89 (
"Mean of KernelCandidate diffim",
"sigma"),
90 (
"Median of KernelCandidate diffim",
"sigma"),
91 (
"Inner quartile range of KernelCandidate diffim",
"sigma"),
92 (
"Standard deviation of KernelCandidate diffim",
"sigma"),
93 (
"D from K-S test of diffim pixels relative to Normal", ),
94 (
"Prob from K-S test of diffim pixels relative to Normal",
"likelihood"),
95 (
"Anderson-Darling test statistic of diffim pixels relative to Normal", ),
96 (
"Critical values for the significance levels in KCDiffimADSig. If A2 is greater "
97 "than this number, hypothesis that the distributions are similar can be rejected.", 5),
98 (
"Anderson-Darling significance levels for the Normal distribution", 5),
99 (
"Reduced chi^2 of the residual.",
"likelihood"),
100 (
"Mean squared error in diffim : Variance + Bias**2",),
101 (
"Centroid in X for this Kernel",
"pixel"),
102 (
"Centroid in Y for this Kernel",
"pixel"),
103 (
"Standard deviation in X for this Kernel",
"pixel"),
104 (
"Standard deviation in Y for this Kernel",
"pixel"),
105 (
"Id for this KernelCandidate",),
108 for name, mtype, comment
in zip(nameList, typeList, commentList):
109 metricMap[name] = {
'type': mtype,
'comment': comment}
114 """Add the to-be-generated QA keys to the Source schema"""
115 schema = inSourceCatalog.getSchema()
117 psfDef = inSourceCatalog.schema.getAliasMap().get(
"slot_PsfFlux")
118 centroidDef = inSourceCatalog.getCentroidDefinition()
119 shapeDef = inSourceCatalog.getShapeDefinition()
120 for n
in schema.getNames():
121 inKeys.append(schema[n].asKey())
124 schema.addField(field)
125 outSourceCatalog = afwTable.SourceCatalog(schema)
126 for source
in inSourceCatalog:
127 rec = outSourceCatalog.addNew()
129 if k.getTypeString() ==
'Coord':
130 rec.setCoord(source.getCoord())
132 setter = getattr(rec,
"set"+k.getTypeString())
133 getter = getattr(source,
"get"+k.getTypeString())
135 outSourceCatalog.definePsfFlux(psfDef)
136 outSourceCatalog.defineCentroid(centroidDef)
137 outSourceCatalog.defineShape(shapeDef)
138 return outSourceCatalog
142 """Calculate the core QA statistics on a difference image"""
147 maskArr &= mask.getPlaneBitMask([
"BAD",
"SAT",
"NO_DATA",
"EDGE"])
150 diArr = ma.array(di.image.array, mask=maskArr)
151 varArr = ma.array(di.variance.array, mask=maskArr)
154 diArr /= np.sqrt(varArr)
159 median = ma.extras.median(diArr)
162 data = ma.getdata(diArr[~diArr.mask])
163 iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
166 chisq = np.sum(np.power(data, 2.))
172 variance = np.power(data, 2.).mean()
173 mseResids = bias**2 + variance
178 rchisq = chisq/(len(data) - 1 - dof)
181 D, prob = scipy.stats.kstest(data,
'norm')
183 A2, crit, sig = scipy.stats.anderson(data,
'norm')
185 if np.isinf(A2)
or np.isnan(A2):
187 except ZeroDivisionError:
195 return {
"mean": mean,
"stdev": stdev,
"median": median,
"iqr": iqr,
196 "D": D,
"prob": prob,
"A2": A2,
"crit": crit,
"sig": sig,
197 "rchisq": rchisq,
"mseResids": mseResids}
200 def apply(cls, candidateList, spatialKernel, spatialBackground, dof=0):
201 """Evaluate the QA metrics for all KernelCandidates in the
202 candidateList; set the values of the metrics in their
203 associated Sources
"""
204 for kernelCandidate
in candidateList:
205 source = kernelCandidate.getSource()
206 schema = source.schema
209 if kernelCandidate.getStatus() != afwMath.SpatialCellCandidate.UNKNOWN:
210 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
211 di = kernelCandidate.getDifferenceImage(kType)
212 kernelValues = kernelCandidate.getKernel(kType).getKernelParameters()
213 kernelValues = np.asarray(kernelValues)
215 lkim = kernelCandidate.getKernelImage(kType)
216 centx, centy = calcCentroid(lkim.array)
217 stdx, stdy = calcWidth(lkim.array, centx, centy)
223 metrics = {
"KCDiffimMean_LOCAL": localResults[
"mean"],
224 "KCDiffimMedian_LOCAL": localResults[
"median"],
225 "KCDiffimIQR_LOCAL": localResults[
"iqr"],
226 "KCDiffimStDev_LOCAL": localResults[
"stdev"],
227 "KCDiffimKSD_LOCAL": localResults[
"D"],
228 "KCDiffimKSProb_LOCAL": localResults[
"prob"],
229 "KCDiffimADA2_LOCAL": localResults[
"A2"],
230 "KCDiffimADCrit_LOCAL": localResults[
"crit"],
231 "KCDiffimADSig_LOCAL": localResults[
"sig"],
232 "KCDiffimChiSq_LOCAL": localResults[
"rchisq"],
233 "KCDiffimMseResids_LOCAL": localResults[
"mseResids"],
234 "KCKernelCentX_LOCAL": centx,
235 "KCKernelCentY_LOCAL": centy,
236 "KCKernelStdX_LOCAL": stdx,
237 "KCKernelStdY_LOCAL": stdy,
238 "KernelCandidateId_LOCAL": kernelCandidate.getId(),
239 "KernelCoeffValues_LOCAL": kernelValues}
241 key = schema[k].asKey()
242 setter = getattr(source,
"set"+key.getTypeString())
243 setter(key, metrics[k])
246 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
247 lkim = kernelCandidate.getKernelImage(kType)
253 skim = afwImage.ImageD(spatialKernel.getDimensions())
254 spatialKernel.computeImage(skim,
False, kernelCandidate.getXCenter(),
255 kernelCandidate.getYCenter())
256 centx, centy = calcCentroid(skim.array)
257 stdx, stdy = calcWidth(skim.array, centx, centy)
260 sbg = spatialBackground(kernelCandidate.getXCenter(), kernelCandidate.getYCenter())
261 di = kernelCandidate.getDifferenceImage(sk, sbg)
267 bias = np.mean(skim.array)
268 variance = np.mean(np.power(skim.array, 2.))
269 mseKernel = bias**2 + variance
273 metrics = {
"KCDiffimMean_SPATIAL": spatialResults[
"mean"],
274 "KCDiffimMedian_SPATIAL": spatialResults[
"median"],
275 "KCDiffimIQR_SPATIAL": spatialResults[
"iqr"],
276 "KCDiffimStDev_SPATIAL": spatialResults[
"stdev"],
277 "KCDiffimKSD_SPATIAL": spatialResults[
"D"],
278 "KCDiffimKSProb_SPATIAL": spatialResults[
"prob"],
279 "KCDiffimADA2_SPATIAL": spatialResults[
"A2"],
280 "KCDiffimADCrit_SPATIAL": spatialResults[
"crit"],
281 "KCDiffimADSig_SPATIAL": spatialResults[
"sig"],
282 "KCDiffimChiSq_SPATIAL": spatialResults[
"rchisq"],
283 "KCDiffimMseResids_SPATIAL": spatialResults[
"mseResids"],
284 "KCDiffimMseKernel_SPATIAL": mseKernel,
285 "KCKernelCentX_SPATIAL": centx,
286 "KCKernelCentY_SPATIAL": centy,
287 "KCKernelStdX_SPATIAL": stdx,
288 "KCKernelStdY_SPATIAL": stdy,
289 "KernelCandidateId_SPATIAL": kernelCandidate.getId()}
291 key = schema[k].asKey()
292 setter = getattr(source,
"set"+key.getTypeString())
293 setter(key, metrics[k])
296 def aggregate(sourceCatalog, metadata, wcsresids, diaSources=None):
297 """Generate aggregate metrics (e.g. total numbers of false
298 positives) from all the Sources
in the sourceCatalog
"""
299 for source
in sourceCatalog:
300 sourceId = source.getId()
301 if sourceId
in wcsresids:
304 coord, resids = wcsresids[sourceId]
305 key = source.schema[
"RegisterResidualBearing"].asKey()
306 setter = getattr(source,
"set"+key.getTypeString())
307 setter(key, resids[0])
308 key = source.schema[
"RegisterResidualDistance"].asKey()
309 setter = getattr(source,
"set"+key.getTypeString())
310 setter(key, resids[1])
311 key = source.schema[
"RegisterRefPosition"].asKey()
312 setter = getattr(source,
"set"+key.getTypeString())
314 coord.getDec().asRadians()))
316 metadata.add(
"NFalsePositivesTotal", len(diaSources))
320 for source
in diaSources:
321 refId = source.get(
"refMatchId")
322 srcId = source.get(
"srcMatchId")
327 if refId == 0
and srcId == 0:
329 metadata.add(
"NFalsePositivesRefAssociated", nRefMatch)
330 metadata.add(
"NFalsePositivesSrcAssociated", nSrcMatch)
331 metadata.add(
"NFalsePositivesUnassociated", nunmatched)
332 for kType
in (
"LOCAL",
"SPATIAL"):
333 for sName
in (
"KCDiffimMean",
"KCDiffimMedian",
"KCDiffimIQR",
"KCDiffimStDev",
334 "KCDiffimKSProb",
"KCDiffimADSig",
"KCDiffimChiSq",
335 "KCDiffimMseResids",
"KCDiffimMseKernel"):
336 if sName ==
"KCDiffimMseKernel" and kType ==
"LOCAL":
338 kName =
"%s_%s" % (sName, kType)
339 vals = np.array([s.get(kName)
for s
in sourceCatalog])
340 idx = np.isfinite(vals)
341 metadata.add(
"%s_MEAN" % (kName), np.mean(vals[idx]))
342 metadata.add(
"%s_MEDIAN" % (kName), np.median(vals[idx]))
343 metadata.add(
"%s_STDEV" % (kName), np.std(vals[idx]))
_calculateStats(di, dof=0.)
aggregate(sourceCatalog, metadata, wcsresids, diaSources=None)
__init__(self, nKernelSpatial)
apply(cls, candidateList, spatialKernel, spatialBackground, dof=0)
addToSchema(self, inSourceCatalog)