22 from __future__
import absolute_import, division, print_function
24 from builtins
import zip
25 from builtins
import object
33 from .
import diffimLib
34 from .utils
import calcCentroid, calcWidth
38 """Quality Assessment class for Kernel Candidates""" 41 """Class to undertake QA of KernelCandidates after modeling of 42 the Psf-matching kernel. Both directly--fitted diffim (LOCAL) 43 and spatially--interpolated kernel diffim (SPATIAL) metrics 44 are calculated, based on the distribution of residuals in the 45 KernelCandidates stamp. 47 @param nKernelSpatial : Number of terms in the spatial model; needed to initialize per-basis QA arrays 50 self.
fields.append(afwTable.Field[
"PointD"](
"RegisterRefPosition",
51 "Position of reference object for registration (radians)."))
53 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualBearing",
54 "Angle of residual wrt declination parallel in radians"))
56 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualDistance",
57 "Offset of residual in radians"))
60 for kType
in (
"LOCAL",
"SPATIAL"):
62 commentAndUnit = metricMap[k][
'comment']
63 self.
fields.append(afwTable.Field[metricMap[k][
'type']](k%(kType), *commentAndUnit))
65 self.
fields.append(afwTable.Field[
"I"](
"KCKernelStatus_LOCAL",
66 "Status of the KernelCandidate"))
68 self.
fields.append(afwTable.Field[
"ArrayD"](
"KernelCoeffValues_LOCAL",
69 "Original basis coefficients",
72 self.
fields.append(afwTable.Field[
"F"](
"BackgroundValue_LOCAL",
73 "Evaluation of background model at this point"))
75 self.
fields.append(afwTable.Field[
"F"](
"KCDiffimMseKernel_SPATIAL",
76 "Mean squared error of spatial kernel estimate"))
79 nameList = [
'KCDiffimMean_%s',
'KCDiffimMedian_%s',
'KCDiffimIQR_%s',
'KCDiffimStDev_%s',
80 'KCDiffimKSD_%s',
'KCDiffimKSProb_%s',
'KCDiffimADA2_%s',
'KCDiffimADCrit_%s',
81 'KCDiffimADSig_%s',
'KCDiffimChiSq_%s',
'KCDiffimMseResids_%s',
'KCKernelCentX_%s',
82 'KCKernelCentY_%s',
'KCKernelStdX_%s',
'KCKernelStdY_%s',
'KernelCandidateId_%s']
83 typeList = [
'F',
'F',
'F',
'F',
'F',
'F',
'F',
'ArrayD',
'ArrayD',
'F',
'F',
'F',
85 commentList = [(
"Mean of KernelCandidate diffim",
"sigma"),
86 (
"Median of KernelCandidate diffim",
"sigma"),
87 (
"Inner quartile range of KernelCandidate diffim",
"sigma"),
88 (
"Standard deviation of KernelCandidate diffim",
"sigma"),
89 (
"D from K-S test of diffim pixels relative to Normal", ),
90 (
"Prob from K-S test of diffim pixels relative to Normal",
"likelihood"),
91 (
"Anderson-Darling test statistic of diffim pixels relative to Normal", ),
92 (
"Critical values for the significance levels in KCDiffimADSig. If A2 is greater " +
93 "than this number, hypothesis that the distributions are similar can be rejected.", 5),
94 (
"Anderson-Darling significance levels for the Normal distribution", 5),
95 (
"Reduced chi^2 of the residual.",
"likelihood"),
96 (
"Mean squared error in diffim : Variance + Bias**2",),
97 (
"Centroid in X for this Kernel",
"pixel"),
98 (
"Centroid in Y for this Kernel",
"pixel"),
99 (
"Standard deviation in X for this Kernel",
"pixel"),
100 (
"Standard deviation in Y for this Kernel",
"pixel"),
101 (
"Id for this KernelCandidate",)]
103 for name, mtype, comment
in zip(nameList, typeList, commentList):
104 metricMap[name] = {
'type': mtype,
'comment': comment}
109 """Add the to-be-generated QA keys to the Source schema""" 110 schema = inSourceCatalog.getSchema()
112 psfDef = inSourceCatalog.getPsfFluxDefinition()
113 centroidDef = inSourceCatalog.getCentroidDefinition()
114 shapeDef = inSourceCatalog.getShapeDefinition()
115 for n
in schema.getNames():
116 inKeys.append(schema[n].asKey())
119 schema.addField(field)
120 outSourceCatalog = afwTable.SourceCatalog(schema)
121 for source
in inSourceCatalog:
122 rec = outSourceCatalog.addNew()
124 if k.getTypeString() ==
'Coord':
125 rec.setCoord(source.getCoord())
127 setter = getattr(rec,
"set"+k.getTypeString())
128 getter = getattr(source,
"get"+k.getTypeString())
130 outSourceCatalog.definePsfFlux(psfDef)
131 outSourceCatalog.defineCentroid(centroidDef)
132 outSourceCatalog.defineShape(shapeDef)
133 return outSourceCatalog
135 def _calculateStats(self, di, dof=0.):
136 """Calculate the core QA statistics on a difference image""" 138 maskArr = di.getMask().getArray()
141 maskArr &= mask.getPlaneBitMask([
"BAD",
"SAT",
"NO_DATA",
"EDGE"])
144 diArr = ma.array(di.getImage().getArray(), mask=maskArr)
145 varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
148 diArr /= np.sqrt(varArr)
153 median = ma.extras.median(diArr)
156 data = ma.getdata(diArr[~diArr.mask])
157 iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
160 chisq = np.sum(np.power(data, 2.))
166 variance = np.power(data, 2.).mean()
167 mseResids = bias**2 + variance
172 rchisq = chisq/(len(data) - 1 - dof)
175 D, prob = scipy.stats.kstest(data,
'norm')
177 A2, crit, sig = scipy.stats.anderson(data,
'norm')
179 if np.isinf(A2)
or np.isnan(A2):
181 except ZeroDivisionError:
189 return {
"mean": mean,
"stdev": stdev,
"median": median,
"iqr": iqr,
190 "D": D,
"prob": prob,
"A2": A2,
"crit": crit,
"sig": sig,
191 "rchisq": rchisq,
"mseResids": mseResids}
193 def apply(self, candidateList, spatialKernel, spatialBackground, dof=0):
194 """Evaluate the QA metrics for all KernelCandidates in the 195 candidateList; set the values of the metrics in their 196 associated Sources""" 197 for kernelCandidate
in candidateList:
198 source = kernelCandidate.getSource()
199 schema = source.schema
202 if kernelCandidate.getStatus() != afwMath.SpatialCellCandidate.UNKNOWN:
203 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
204 di = kernelCandidate.getDifferenceImage(kType)
205 kernelValues = kernelCandidate.getKernel(kType).getKernelParameters()
206 kernelValues = np.asarray(kernelValues)
208 lkim = kernelCandidate.getKernelImage(kType)
210 stdx, stdy =
calcWidth(lkim.getArray(), centx, centy)
216 metrics = {
"KCDiffimMean_LOCAL": localResults[
"mean"],
217 "KCDiffimMedian_LOCAL": localResults[
"median"],
218 "KCDiffimIQR_LOCAL": localResults[
"iqr"],
219 "KCDiffimStDev_LOCAL": localResults[
"stdev"],
220 "KCDiffimKSD_LOCAL": localResults[
"D"],
221 "KCDiffimKSProb_LOCAL": localResults[
"prob"],
222 "KCDiffimADA2_LOCAL": localResults[
"A2"],
223 "KCDiffimADCrit_LOCAL": localResults[
"crit"],
224 "KCDiffimADSig_LOCAL": localResults[
"sig"],
225 "KCDiffimChiSq_LOCAL": localResults[
"rchisq"],
226 "KCDiffimMseResids_LOCAL": localResults[
"mseResids"],
227 "KCKernelCentX_LOCAL": centx,
228 "KCKernelCentY_LOCAL": centy,
229 "KCKernelStdX_LOCAL": stdx,
230 "KCKernelStdY_LOCAL": stdy,
231 "KernelCandidateId_LOCAL": kernelCandidate.getId(),
232 "KernelCoeffValues_LOCAL": kernelValues}
234 key = schema[k].asKey()
235 setter = getattr(source,
"set"+key.getTypeString())
236 setter(key, metrics[k])
239 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
240 lkim = kernelCandidate.getKernelImage(kType)
246 skim = afwImage.ImageD(spatialKernel.getDimensions())
247 spatialKernel.computeImage(skim,
False, kernelCandidate.getXCenter(),
248 kernelCandidate.getYCenter())
250 stdx, stdy =
calcWidth(skim.getArray(), centx, centy)
253 sbg = spatialBackground(kernelCandidate.getXCenter(), kernelCandidate.getYCenter())
254 di = kernelCandidate.getDifferenceImage(sk, sbg)
260 bias = np.mean(skim.getArray())
261 variance = np.mean(np.power(skim.getArray(), 2.))
262 mseKernel = bias**2 + variance
266 metrics = {
"KCDiffimMean_SPATIAL": spatialResults[
"mean"],
267 "KCDiffimMedian_SPATIAL": spatialResults[
"median"],
268 "KCDiffimIQR_SPATIAL": spatialResults[
"iqr"],
269 "KCDiffimStDev_SPATIAL": spatialResults[
"stdev"],
270 "KCDiffimKSD_SPATIAL": spatialResults[
"D"],
271 "KCDiffimKSProb_SPATIAL": spatialResults[
"prob"],
272 "KCDiffimADA2_SPATIAL": spatialResults[
"A2"],
273 "KCDiffimADCrit_SPATIAL": spatialResults[
"crit"],
274 "KCDiffimADSig_SPATIAL": spatialResults[
"sig"],
275 "KCDiffimChiSq_SPATIAL": spatialResults[
"rchisq"],
276 "KCDiffimMseResids_SPATIAL": spatialResults[
"mseResids"],
277 "KCDiffimMseKernel_SPATIAL": mseKernel,
278 "KCKernelCentX_SPATIAL": centx,
279 "KCKernelCentY_SPATIAL": centy,
280 "KCKernelStdX_SPATIAL": stdx,
281 "KCKernelStdY_SPATIAL": stdy,
282 "KernelCandidateId_SPATIAL": kernelCandidate.getId()}
284 key = schema[k].asKey()
285 setter = getattr(source,
"set"+key.getTypeString())
286 setter(key, metrics[k])
288 def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None):
289 """Generate aggregate metrics (e.g. total numbers of false 290 positives) from all the Sources in the sourceCatalog""" 291 for source
in sourceCatalog:
292 sourceId = source.getId()
293 if sourceId
in wcsresids:
296 coord, resids = wcsresids[sourceId]
297 key = source.schema[
"RegisterResidualBearing"].asKey()
298 setter = getattr(source,
"set"+key.getTypeString())
299 setter(key, resids[0])
300 key = source.schema[
"RegisterResidualDistance"].asKey()
301 setter = getattr(source,
"set"+key.getTypeString())
302 setter(key, resids[1])
303 key = source.schema[
"RegisterRefPosition"].asKey()
304 setter = getattr(source,
"set"+key.getTypeString())
306 coord.getDec().asRadians()))
308 metadata.add(
"NFalsePositivesTotal", len(diaSources))
312 for source
in diaSources:
313 refId = source.get(
"refMatchId")
314 srcId = source.get(
"srcMatchId")
319 if refId == 0
and srcId == 0:
321 metadata.add(
"NFalsePositivesRefAssociated", nRefMatch)
322 metadata.add(
"NFalsePositivesSrcAssociated", nSrcMatch)
323 metadata.add(
"NFalsePositivesUnassociated", nunmatched)
324 for kType
in (
"LOCAL",
"SPATIAL"):
325 for sName
in (
"KCDiffimMean",
"KCDiffimMedian",
"KCDiffimIQR",
"KCDiffimStDev",
326 "KCDiffimKSProb",
"KCDiffimADSig",
"KCDiffimChiSq",
327 "KCDiffimMseResids",
"KCDiffimMseKernel"):
328 if sName ==
"KCDiffimMseKernel" and kType ==
"LOCAL":
330 kName =
"%s_%s" % (sName, kType)
331 vals = np.array([s.get(kName)
for s
in sourceCatalog])
332 idx = np.isfinite(vals)
333 metadata.add(
"%s_MEAN" % (kName), np.mean(vals[idx]))
334 metadata.add(
"%s_MEDIAN" % (kName), np.median(vals[idx]))
335 metadata.add(
"%s_STDEV" % (kName), np.std(vals[idx]))
def addToSchema(self, inSourceCatalog)
def _calculateStats(self, di, dof=0.)
def __init__(self, nKernelSpatial)
def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None)
def apply(self, candidateList, spatialKernel, spatialBackground, dof=0)
def calcWidth(arr, centx, centy)