23 __all__ = [
"KernelCandidateQa"]
32 from .
import diffimLib
33 from .utils
import calcCentroid, calcWidth
37 """Quality Assessment class for Kernel Candidates""" 40 """Class to undertake QA of KernelCandidates after modeling of 41 the Psf-matching kernel. Both directly--fitted diffim (LOCAL) 42 and spatially--interpolated kernel diffim (SPATIAL) metrics 43 are calculated, based on the distribution of residuals in the 44 KernelCandidates stamp. 46 @param nKernelSpatial : Number of terms in the spatial model; needed to initialize per-basis QA arrays 49 self.
fields.append(afwTable.Field[
"PointD"](
50 "RegisterRefPosition",
51 "Position of reference object for registration (radians)."))
53 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualBearing",
54 "Angle of residual wrt declination parallel in radians"))
56 self.
fields.append(afwTable.Field[
"Angle"](
"RegisterResidualDistance",
57 "Offset of residual in radians"))
60 for kType
in (
"LOCAL",
"SPATIAL"):
62 commentAndUnit = metricMap[k][
'comment']
63 self.
fields.append(afwTable.Field[metricMap[k][
'type']](k%(kType), *commentAndUnit))
65 self.
fields.append(afwTable.Field[
"I"](
"KCKernelStatus_LOCAL",
66 "Status of the KernelCandidate"))
68 self.
fields.append(afwTable.Field[
"ArrayD"](
"KernelCoeffValues_LOCAL",
69 "Original basis coefficients",
72 self.
fields.append(afwTable.Field[
"F"](
"BackgroundValue_LOCAL",
73 "Evaluation of background model at this point"))
75 self.
fields.append(afwTable.Field[
"F"](
"KCDiffimMseKernel_SPATIAL",
76 "Mean squared error of spatial kernel estimate"))
79 nameList = [
'KCDiffimMean_%s',
'KCDiffimMedian_%s',
'KCDiffimIQR_%s',
'KCDiffimStDev_%s',
80 'KCDiffimKSD_%s',
'KCDiffimKSProb_%s',
'KCDiffimADA2_%s',
'KCDiffimADCrit_%s',
81 'KCDiffimADSig_%s',
'KCDiffimChiSq_%s',
'KCDiffimMseResids_%s',
'KCKernelCentX_%s',
82 'KCKernelCentY_%s',
'KCKernelStdX_%s',
'KCKernelStdY_%s',
'KernelCandidateId_%s']
83 typeList = [
'F',
'F',
'F',
'F',
'F',
'F',
'F',
'ArrayD',
'ArrayD',
'F',
'F',
'F',
86 (
"Mean of KernelCandidate diffim",
"sigma"),
87 (
"Median of KernelCandidate diffim",
"sigma"),
88 (
"Inner quartile range of KernelCandidate diffim",
"sigma"),
89 (
"Standard deviation of KernelCandidate diffim",
"sigma"),
90 (
"D from K-S test of diffim pixels relative to Normal", ),
91 (
"Prob from K-S test of diffim pixels relative to Normal",
"likelihood"),
92 (
"Anderson-Darling test statistic of diffim pixels relative to Normal", ),
93 (
"Critical values for the significance levels in KCDiffimADSig. If A2 is greater " +
94 "than this number, hypothesis that the distributions are similar can be rejected.", 5),
95 (
"Anderson-Darling significance levels for the Normal distribution", 5),
96 (
"Reduced chi^2 of the residual.",
"likelihood"),
97 (
"Mean squared error in diffim : Variance + Bias**2",),
98 (
"Centroid in X for this Kernel",
"pixel"),
99 (
"Centroid in Y for this Kernel",
"pixel"),
100 (
"Standard deviation in X for this Kernel",
"pixel"),
101 (
"Standard deviation in Y for this Kernel",
"pixel"),
102 (
"Id for this KernelCandidate",),
105 for name, mtype, comment
in zip(nameList, typeList, commentList):
106 metricMap[name] = {
'type': mtype,
'comment': comment}
111 """Add the to-be-generated QA keys to the Source schema""" 112 schema = inSourceCatalog.getSchema()
114 psfDef = inSourceCatalog.getPsfFluxDefinition()
115 centroidDef = inSourceCatalog.getCentroidDefinition()
116 shapeDef = inSourceCatalog.getShapeDefinition()
117 for n
in schema.getNames():
118 inKeys.append(schema[n].asKey())
121 schema.addField(field)
122 outSourceCatalog = afwTable.SourceCatalog(schema)
123 for source
in inSourceCatalog:
124 rec = outSourceCatalog.addNew()
126 if k.getTypeString() ==
'Coord':
127 rec.setCoord(source.getCoord())
129 setter = getattr(rec,
"set"+k.getTypeString())
130 getter = getattr(source,
"get"+k.getTypeString())
132 outSourceCatalog.definePsfFlux(psfDef)
133 outSourceCatalog.defineCentroid(centroidDef)
134 outSourceCatalog.defineShape(shapeDef)
135 return outSourceCatalog
137 def _calculateStats(self, di, dof=0.):
138 """Calculate the core QA statistics on a difference image""" 140 maskArr = di.getMask().getArray()
143 maskArr &= mask.getPlaneBitMask([
"BAD",
"SAT",
"NO_DATA",
"EDGE"])
146 diArr = ma.array(di.getImage().getArray(), mask=maskArr)
147 varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
150 diArr /= np.sqrt(varArr)
155 median = ma.extras.median(diArr)
158 data = ma.getdata(diArr[~diArr.mask])
159 iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
162 chisq = np.sum(np.power(data, 2.))
168 variance = np.power(data, 2.).mean()
169 mseResids = bias**2 + variance
174 rchisq = chisq/(len(data) - 1 - dof)
177 D, prob = scipy.stats.kstest(data,
'norm')
179 A2, crit, sig = scipy.stats.anderson(data,
'norm')
181 if np.isinf(A2)
or np.isnan(A2):
183 except ZeroDivisionError:
191 return {
"mean": mean,
"stdev": stdev,
"median": median,
"iqr": iqr,
192 "D": D,
"prob": prob,
"A2": A2,
"crit": crit,
"sig": sig,
193 "rchisq": rchisq,
"mseResids": mseResids}
195 def apply(self, candidateList, spatialKernel, spatialBackground, dof=0):
196 """Evaluate the QA metrics for all KernelCandidates in the 197 candidateList; set the values of the metrics in their 198 associated Sources""" 199 for kernelCandidate
in candidateList:
200 source = kernelCandidate.getSource()
201 schema = source.schema
204 if kernelCandidate.getStatus() != afwMath.SpatialCellCandidate.UNKNOWN:
205 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
206 di = kernelCandidate.getDifferenceImage(kType)
207 kernelValues = kernelCandidate.getKernel(kType).getKernelParameters()
208 kernelValues = np.asarray(kernelValues)
210 lkim = kernelCandidate.getKernelImage(kType)
212 stdx, stdy =
calcWidth(lkim.getArray(), centx, centy)
218 metrics = {
"KCDiffimMean_LOCAL": localResults[
"mean"],
219 "KCDiffimMedian_LOCAL": localResults[
"median"],
220 "KCDiffimIQR_LOCAL": localResults[
"iqr"],
221 "KCDiffimStDev_LOCAL": localResults[
"stdev"],
222 "KCDiffimKSD_LOCAL": localResults[
"D"],
223 "KCDiffimKSProb_LOCAL": localResults[
"prob"],
224 "KCDiffimADA2_LOCAL": localResults[
"A2"],
225 "KCDiffimADCrit_LOCAL": localResults[
"crit"],
226 "KCDiffimADSig_LOCAL": localResults[
"sig"],
227 "KCDiffimChiSq_LOCAL": localResults[
"rchisq"],
228 "KCDiffimMseResids_LOCAL": localResults[
"mseResids"],
229 "KCKernelCentX_LOCAL": centx,
230 "KCKernelCentY_LOCAL": centy,
231 "KCKernelStdX_LOCAL": stdx,
232 "KCKernelStdY_LOCAL": stdy,
233 "KernelCandidateId_LOCAL": kernelCandidate.getId(),
234 "KernelCoeffValues_LOCAL": kernelValues}
236 key = schema[k].asKey()
237 setter = getattr(source,
"set"+key.getTypeString())
238 setter(key, metrics[k])
241 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
242 lkim = kernelCandidate.getKernelImage(kType)
248 skim = afwImage.ImageD(spatialKernel.getDimensions())
249 spatialKernel.computeImage(skim,
False, kernelCandidate.getXCenter(),
250 kernelCandidate.getYCenter())
252 stdx, stdy =
calcWidth(skim.getArray(), centx, centy)
255 sbg = spatialBackground(kernelCandidate.getXCenter(), kernelCandidate.getYCenter())
256 di = kernelCandidate.getDifferenceImage(sk, sbg)
262 bias = np.mean(skim.getArray())
263 variance = np.mean(np.power(skim.getArray(), 2.))
264 mseKernel = bias**2 + variance
268 metrics = {
"KCDiffimMean_SPATIAL": spatialResults[
"mean"],
269 "KCDiffimMedian_SPATIAL": spatialResults[
"median"],
270 "KCDiffimIQR_SPATIAL": spatialResults[
"iqr"],
271 "KCDiffimStDev_SPATIAL": spatialResults[
"stdev"],
272 "KCDiffimKSD_SPATIAL": spatialResults[
"D"],
273 "KCDiffimKSProb_SPATIAL": spatialResults[
"prob"],
274 "KCDiffimADA2_SPATIAL": spatialResults[
"A2"],
275 "KCDiffimADCrit_SPATIAL": spatialResults[
"crit"],
276 "KCDiffimADSig_SPATIAL": spatialResults[
"sig"],
277 "KCDiffimChiSq_SPATIAL": spatialResults[
"rchisq"],
278 "KCDiffimMseResids_SPATIAL": spatialResults[
"mseResids"],
279 "KCDiffimMseKernel_SPATIAL": mseKernel,
280 "KCKernelCentX_SPATIAL": centx,
281 "KCKernelCentY_SPATIAL": centy,
282 "KCKernelStdX_SPATIAL": stdx,
283 "KCKernelStdY_SPATIAL": stdy,
284 "KernelCandidateId_SPATIAL": kernelCandidate.getId()}
286 key = schema[k].asKey()
287 setter = getattr(source,
"set"+key.getTypeString())
288 setter(key, metrics[k])
290 def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None):
291 """Generate aggregate metrics (e.g. total numbers of false 292 positives) from all the Sources in the sourceCatalog""" 293 for source
in sourceCatalog:
294 sourceId = source.getId()
295 if sourceId
in wcsresids:
298 coord, resids = wcsresids[sourceId]
299 key = source.schema[
"RegisterResidualBearing"].asKey()
300 setter = getattr(source,
"set"+key.getTypeString())
301 setter(key, resids[0])
302 key = source.schema[
"RegisterResidualDistance"].asKey()
303 setter = getattr(source,
"set"+key.getTypeString())
304 setter(key, resids[1])
305 key = source.schema[
"RegisterRefPosition"].asKey()
306 setter = getattr(source,
"set"+key.getTypeString())
307 setter(key, afwGeom.Point2D(coord.getRa().asRadians(),
308 coord.getDec().asRadians()))
310 metadata.add(
"NFalsePositivesTotal", len(diaSources))
314 for source
in diaSources:
315 refId = source.get(
"refMatchId")
316 srcId = source.get(
"srcMatchId")
321 if refId == 0
and srcId == 0:
323 metadata.add(
"NFalsePositivesRefAssociated", nRefMatch)
324 metadata.add(
"NFalsePositivesSrcAssociated", nSrcMatch)
325 metadata.add(
"NFalsePositivesUnassociated", nunmatched)
326 for kType
in (
"LOCAL",
"SPATIAL"):
327 for sName
in (
"KCDiffimMean",
"KCDiffimMedian",
"KCDiffimIQR",
"KCDiffimStDev",
328 "KCDiffimKSProb",
"KCDiffimADSig",
"KCDiffimChiSq",
329 "KCDiffimMseResids",
"KCDiffimMseKernel"):
330 if sName ==
"KCDiffimMseKernel" and kType ==
"LOCAL":
332 kName =
"%s_%s" % (sName, kType)
333 vals = np.array([s.get(kName)
for s
in sourceCatalog])
334 idx = np.isfinite(vals)
335 metadata.add(
"%s_MEAN" % (kName), np.mean(vals[idx]))
336 metadata.add(
"%s_MEDIAN" % (kName), np.median(vals[idx]))
337 metadata.add(
"%s_STDEV" % (kName), np.std(vals[idx]))
def addToSchema(self, inSourceCatalog)
def _calculateStats(self, di, dof=0.)
def __init__(self, nKernelSpatial)
def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None)
def apply(self, candidateList, spatialKernel, spatialBackground, dof=0)
def calcWidth(arr, centx, centy)