22 from __future__
import absolute_import, division, print_function
24 from builtins
import zip
25 from builtins
import object
29 import lsst.afw.geom
as afwGeom
30 import lsst.afw.image
as afwImage
31 import lsst.afw.table
as afwTable
32 import lsst.afw.math
as afwMath
33 from .
import diffimLib
34 from .utils
import calcCentroid, calcWidth
38 """Quality Assessment class for Kernel Candidates"""
41 """Class to undertake QA of KernelCandidates after modeling of
42 the Psf-matching kernel. Both directly--fitted diffim (LOCAL)
43 and spatially--interpolated kernel diffim (SPATIAL) metrics
44 are calculated, based on the distribution of residuals in the
45 KernelCandidates stamp.
47 @param nKernelSpatial : Number of terms in the spatial model; needed to initialize per-basis QA arrays
50 self.fields.append(afwTable.Field[
"PointD"](
"RegisterRefPosition",
51 "Position of reference object for registration (radians)."))
53 self.fields.append(afwTable.Field[
"Angle"](
"RegisterResidualBearing",
54 "Angle of residual wrt declination parallel in radians"))
56 self.fields.append(afwTable.Field[
"Angle"](
"RegisterResidualDistance",
57 "Offset of residual in radians"))
60 for kType
in (
"LOCAL",
"SPATIAL"):
62 commentAndUnit = metricMap[k][
'comment']
63 self.fields.append(afwTable.Field[metricMap[k][
'type']](k%(kType), *commentAndUnit))
65 self.fields.append(afwTable.Field[
"I"](
"KCKernelStatus_LOCAL",
66 "Status of the KernelCandidate"))
68 self.fields.append(afwTable.Field[
"ArrayD"](
"KernelCoeffValues_LOCAL",
69 "Original basis coefficients",
72 self.fields.append(afwTable.Field[
"F"](
"BackgroundValue_LOCAL",
73 "Evaluation of background model at this point"))
75 self.fields.append(afwTable.Field[
"F"](
"KCDiffimMseKernel_SPATIAL",
76 "Mean squared error of spatial kernel estimate"))
79 nameList = [
'KCDiffimMean_%s',
'KCDiffimMedian_%s',
'KCDiffimIQR_%s',
'KCDiffimStDev_%s',
80 'KCDiffimKSD_%s',
'KCDiffimKSProb_%s',
'KCDiffimADA2_%s',
'KCDiffimADCrit_%s',
81 'KCDiffimADSig_%s',
'KCDiffimChiSq_%s',
'KCDiffimMseResids_%s',
'KCKernelCentX_%s',
82 'KCKernelCentY_%s',
'KCKernelStdX_%s',
'KCKernelStdY_%s',
'KernelCandidateId_%s']
83 typeList = [
'F',
'F',
'F',
'F',
'F',
'F',
'F',
'ArrayD',
'ArrayD',
'F',
'F',
'F',
85 commentList = [(
"Mean of KernelCandidate diffim",
"sigma"),
86 (
"Median of KernelCandidate diffim",
"sigma"),
87 (
"Inner quartile range of KernelCandidate diffim",
"sigma"),
88 (
"Standard deviation of KernelCandidate diffim",
"sigma"),
89 (
"D from K-S test of diffim pixels relative to Normal", ),
90 (
"Prob from K-S test of diffim pixels relative to Normal",
"likelihood"),
91 (
"Anderson-Darling test statistic of diffim pixels relative to Normal", ),
92 (
"Critical values for the significance levels in KCDiffimADSig. If A2 is greater " +
93 "than this number, hypothesis that the distributions are similar can be rejected.", 5),
94 (
"Anderson-Darling significance levels for the Normal distribution", 5),
95 (
"Reduced chi^2 of the residual.",
"likelihood"),
96 (
"Mean squared error in diffim : Variance + Bias**2",),
97 (
"Centroid in X for this Kernel",
"pixel"),
98 (
"Centroid in Y for this Kernel",
"pixel"),
99 (
"Standard deviation in X for this Kernel",
"pixel"),
100 (
"Standard deviation in Y for this Kernel",
"pixel"),
101 (
"Id for this KernelCandidate",)]
103 for name, mtype, comment
in zip(nameList, typeList, commentList):
104 metricMap[name] = {
'type': mtype,
'comment': comment}
109 """Add the to-be-generated QA keys to the Source schema"""
110 schema = inSourceCatalog.getSchema()
112 psfDef = inSourceCatalog.getPsfFluxDefinition()
113 centroidDef = inSourceCatalog.getCentroidDefinition()
114 shapeDef = inSourceCatalog.getShapeDefinition()
115 for n
in schema.getNames():
116 inKeys.append(schema[n].asKey())
119 schema.addField(field)
120 outSourceCatalog = afwTable.SourceCatalog(schema)
121 for source
in inSourceCatalog:
122 rec = outSourceCatalog.addNew()
124 if k.getTypeString() ==
'Coord':
125 rec.setCoord(source.getCoord())
127 setter = getattr(rec,
"set"+k.getTypeString())
128 getter = getattr(source,
"get"+k.getTypeString())
130 outSourceCatalog.definePsfFlux(psfDef)
131 outSourceCatalog.defineCentroid(centroidDef)
132 outSourceCatalog.defineShape(shapeDef)
133 return outSourceCatalog
136 """Calculate the core QA statistics on a difference image"""
138 maskArr = di.getMask().getArray()
141 maskArr &= mask.getPlaneBitMask([
"BAD",
"SAT",
"NO_DATA",
"EDGE"])
144 diArr = ma.array(di.getImage().getArray(), mask=maskArr)
145 varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
148 diArr /= np.sqrt(varArr)
153 median = ma.extras.median(diArr)
156 data = ma.getdata(diArr[~diArr.mask])
157 iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
160 chisq = np.sum(np.power(data, 2.))
166 variance = np.power(data, 2.).mean()
167 mseResids = bias**2 + variance
172 rchisq = chisq/(len(data) - 1 - dof)
175 D, prob = scipy.stats.kstest(data,
'norm')
177 A2, crit, sig = scipy.stats.anderson(data,
'norm')
179 if np.isinf(A2)
or np.isnan(A2):
181 except ZeroDivisionError:
189 return {
"mean": mean,
"stdev": stdev,
"median": median,
"iqr": iqr,
190 "D": D,
"prob": prob,
"A2": A2,
"crit": crit,
"sig": sig,
191 "rchisq": rchisq,
"mseResids": mseResids}
193 def apply(self, candidateList, spatialKernel, spatialBackground, dof=0):
194 """Evaluate the QA metrics for all KernelCandidates in the
195 candidateList; set the values of the metrics in their
196 associated Sources"""
197 for kernelCandidate
in candidateList:
198 source = kernelCandidate.getSource()
199 schema = source.schema
202 if kernelCandidate.getStatus() != afwMath.SpatialCellCandidate.UNKNOWN:
203 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
204 di = kernelCandidate.getDifferenceImage(kType)
205 kernelValues = kernelCandidate.getKernel(kType).getKernelParameters()
206 kernelValues = np.asarray(kernelValues)
208 lkim = kernelCandidate.getKernelImage(kType)
210 stdx, stdy =
calcWidth(lkim.getArray(), centx, centy)
216 metrics = {
"KCDiffimMean_LOCAL": localResults[
"mean"],
217 "KCDiffimMedian_LOCAL": localResults[
"median"],
218 "KCDiffimIQR_LOCAL": localResults[
"iqr"],
219 "KCDiffimStDev_LOCAL": localResults[
"stdev"],
220 "KCDiffimKSD_LOCAL": localResults[
"D"],
221 "KCDiffimKSProb_LOCAL": localResults[
"prob"],
222 "KCDiffimADA2_LOCAL": localResults[
"A2"],
223 "KCDiffimADCrit_LOCAL": localResults[
"crit"],
224 "KCDiffimADSig_LOCAL": localResults[
"sig"],
225 "KCDiffimChiSq_LOCAL": localResults[
"rchisq"],
226 "KCDiffimMseResids_LOCAL": localResults[
"mseResids"],
227 "KCKernelCentX_LOCAL": centx,
228 "KCKernelCentY_LOCAL": centy,
229 "KCKernelStdX_LOCAL": stdx,
230 "KCKernelStdY_LOCAL": stdy,
231 "KernelCandidateId_LOCAL": kernelCandidate.getId(),
232 "KernelCoeffValues_LOCAL": kernelValues}
234 key = schema[k].asKey()
235 setter = getattr(source,
"set"+key.getTypeString())
236 setter(key, metrics[k])
239 kType = getattr(diffimLib.KernelCandidateF,
"ORIG")
240 lkim = kernelCandidate.getKernelImage(kType)
246 skim = afwImage.ImageD(spatialKernel.getDimensions())
247 spatialKernel.computeImage(skim,
False, kernelCandidate.getXCenter(),
248 kernelCandidate.getYCenter())
250 stdx, stdy =
calcWidth(skim.getArray(), centx, centy)
252 sk = afwMath.FixedKernel(skim)
253 sbg = spatialBackground(kernelCandidate.getXCenter(), kernelCandidate.getYCenter())
254 di = kernelCandidate.getDifferenceImage(sk, sbg)
260 bias = np.mean(skim.getArray())
261 variance = np.mean(np.power(skim.getArray(), 2.))
262 mseKernel = bias**2 + variance
266 metrics = {
"KCDiffimMean_SPATIAL": spatialResults[
"mean"],
267 "KCDiffimMedian_SPATIAL": spatialResults[
"median"],
268 "KCDiffimIQR_SPATIAL": spatialResults[
"iqr"],
269 "KCDiffimStDev_SPATIAL": spatialResults[
"stdev"],
270 "KCDiffimKSD_SPATIAL": spatialResults[
"D"],
271 "KCDiffimKSProb_SPATIAL": spatialResults[
"prob"],
272 "KCDiffimADA2_SPATIAL": spatialResults[
"A2"],
273 "KCDiffimADCrit_SPATIAL": spatialResults[
"crit"],
274 "KCDiffimADSig_SPATIAL": spatialResults[
"sig"],
275 "KCDiffimChiSq_SPATIAL": spatialResults[
"rchisq"],
276 "KCDiffimMseResids_SPATIAL": spatialResults[
"mseResids"],
277 "KCDiffimMseKernel_SPATIAL": mseKernel,
278 "KCKernelCentX_SPATIAL": centx,
279 "KCKernelCentY_SPATIAL": centy,
280 "KCKernelStdX_SPATIAL": stdx,
281 "KCKernelStdY_SPATIAL": stdy,
282 "KernelCandidateId_SPATIAL": kernelCandidate.getId()}
284 key = schema[k].asKey()
285 setter = getattr(source,
"set"+key.getTypeString())
286 setter(key, metrics[k])
288 def aggregate(self, sourceCatalog, metadata, wcsresids, diaSources=None):
289 """Generate aggregate metrics (e.g. total numbers of false
290 positives) from all the Sources in the sourceCatalog"""
291 for source
in sourceCatalog:
292 sourceId = source.getId()
293 if sourceId
in wcsresids:
296 coord, resids = wcsresids[sourceId]
297 key = source.schema[
"RegisterResidualBearing"].asKey()
298 setter = getattr(source,
"set"+key.getTypeString())
299 setter(key, resids[0])
300 key = source.schema[
"RegisterResidualDistance"].asKey()
301 setter = getattr(source,
"set"+key.getTypeString())
302 setter(key, resids[1])
303 key = source.schema[
"RegisterRefPosition"].asKey()
304 setter = getattr(source,
"set"+key.getTypeString())
305 setter(key, afwGeom.Point2D(coord.getRa().asRadians(),
306 coord.getDec().asRadians()))
308 metadata.add(
"NFalsePositivesTotal", len(diaSources))
312 for source
in diaSources:
313 refId = source.get(
"refMatchId")
314 srcId = source.get(
"srcMatchId")
319 if refId == 0
and srcId == 0:
321 metadata.add(
"NFalsePositivesRefAssociated", nRefMatch)
322 metadata.add(
"NFalsePositivesSrcAssociated", nSrcMatch)
323 metadata.add(
"NFalsePositivesUnassociated", nunmatched)
324 for kType
in (
"LOCAL",
"SPATIAL"):
325 for sName
in (
"KCDiffimMean",
"KCDiffimMedian",
"KCDiffimIQR",
"KCDiffimStDev",
326 "KCDiffimKSProb",
"KCDiffimADSig",
"KCDiffimChiSq",
327 "KCDiffimMseResids",
"KCDiffimMseKernel"):
328 if sName ==
"KCDiffimMseKernel" and kType ==
"LOCAL":
330 kName =
"%s_%s" % (sName, kType)
331 vals = np.array([s.get(kName)
for s
in sourceCatalog])
332 idx = np.isfinite(vals)
333 metadata.add(
"%s_MEAN" % (kName), np.mean(vals[idx]))
334 metadata.add(
"%s_MEDIAN" % (kName), np.median(vals[idx]))
335 metadata.add(
"%s_STDEV" % (kName), np.std(vals[idx]))