lsst.ip.diffim g1186b97052+fe7b7dcd23
dipoleFitTask.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008-2016 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23import logging
24import numpy as np
25import warnings
26
27import lsst.afw.image as afwImage
28import lsst.meas.base as measBase
29import lsst.afw.table as afwTable
30import lsst.afw.detection as afwDet
31import lsst.geom as geom
32import lsst.pex.exceptions as pexExcept
33import lsst.pex.config as pexConfig
34from lsst.pipe.base import Struct
35from lsst.utils.timer import timeMethod
36
37__all__ = ("DipoleFitTask", "DipoleFitPlugin", "DipoleFitTaskConfig", "DipoleFitPluginConfig",
38 "DipoleFitAlgorithm")
39
40
41# Create a new measurement task (`DipoleFitTask`) that can handle all other SFM tasks but can
42# pass a separate pos- and neg- exposure/image to the `DipoleFitPlugin`s `run()` method.
43
44
45class DipoleFitPluginConfig(measBase.SingleFramePluginConfig):
46 """Configuration for DipoleFitPlugin
47 """
48
49 fitAllDiaSources = pexConfig.Field(
50 dtype=float, default=False,
51 doc="""Attempte dipole fit of all diaSources (otherwise just the ones consisting of overlapping
52 positive and negative footprints)""")
53
54 maxSeparation = pexConfig.Field(
55 dtype=float, default=5.,
56 doc="Assume dipole is not separated by more than maxSeparation * psfSigma")
57
58 relWeight = pexConfig.Field(
59 dtype=float, default=0.5,
60 doc="""Relative weighting of pre-subtraction images (higher -> greater influence of pre-sub.
61 images on fit)""")
62
63 tolerance = pexConfig.Field(
64 dtype=float, default=1e-7,
65 doc="Fit tolerance")
66
67 fitBackground = pexConfig.Field(
68 dtype=int, default=1,
69 doc="Set whether and how to fit for linear gradient in pre-sub. images. Possible values:"
70 "0: do not fit background at all"
71 "1 (default): pre-fit the background using linear least squares and then do not fit it as part"
72 "of the dipole fitting optimization"
73 "2: pre-fit the background using linear least squares (as in 1), and use the parameter"
74 "estimates from that fit as starting parameters for an integrated re-fit of the background")
75
76 fitSeparateNegParams = pexConfig.Field(
77 dtype=bool, default=False,
78 doc="Include parameters to fit for negative values (flux, gradient) separately from pos.")
79
80 # Config params for classification of detected diaSources as dipole or not
81 minSn = pexConfig.Field(
82 dtype=float, default=np.sqrt(2) * 5.0,
83 doc="Minimum quadrature sum of positive+negative lobe S/N to be considered a dipole")
84
85 maxFluxRatio = pexConfig.Field(
86 dtype=float, default=0.65,
87 doc="Maximum flux ratio in either lobe to be considered a dipole")
88
89 maxChi2DoF = pexConfig.Field(
90 dtype=float, default=0.05,
91 doc="""Maximum Chi2/DoF significance of fit to be considered a dipole.
92 Default value means \"Choose a chi2DoF corresponding to a significance level of at most 0.05\"
93 (note this is actually a significance, not a chi2 value).""")
94
95
96class DipoleFitTaskConfig(measBase.SingleFrameMeasurementConfig):
97 """Measurement of detected diaSources as dipoles
98
99 Currently we keep the "old" DipoleMeasurement algorithms turned on.
100 """
101
102 def setDefaults(self):
103 measBase.SingleFrameMeasurementConfig.setDefaults(self)
104
105 self.plugins.names = ["base_CircularApertureFlux",
106 "base_PixelFlags",
107 "base_SkyCoord",
108 "base_PsfFlux",
109 "base_SdssCentroid",
110 "base_SdssShape",
111 "base_GaussianFlux",
112 "base_PeakLikelihoodFlux",
113 "base_PeakCentroid",
114 "base_NaiveCentroid",
115 "ip_diffim_NaiveDipoleCentroid",
116 "ip_diffim_NaiveDipoleFlux",
117 "ip_diffim_PsfDipoleFlux",
118 "ip_diffim_ClassificationDipole",
119 ]
120
121 self.slots.calibFlux = None
122 self.slots.modelFlux = None
123 self.slots.gaussianFlux = None
124 self.slots.shape = "base_SdssShape"
125 self.slots.centroid = "ip_diffim_NaiveDipoleCentroid"
126 self.doReplaceWithNoisedoReplaceWithNoise = False
127
128
129class DipoleFitTask(measBase.SingleFrameMeasurementTask):
130 """A task that fits a dipole to a difference image, with an optional separate detection image.
131
132 Because it subclasses SingleFrameMeasurementTask, and calls
133 SingleFrameMeasurementTask.run() from its run() method, it still
134 can be used identically to a standard SingleFrameMeasurementTask.
135 """
136
137 ConfigClass = DipoleFitTaskConfig
138 _DefaultName = "ip_diffim_DipoleFit"
139
140 def __init__(self, schema, algMetadata=None, **kwargs):
141
142 measBase.SingleFrameMeasurementTask.__init__(self, schema, algMetadata, **kwargs)
143
144 dpFitPluginConfig = self.config.plugins['ip_diffim_DipoleFit']
145
146 self.dipoleFitterdipoleFitter = DipoleFitPlugin(dpFitPluginConfig, name=self._DefaultName_DefaultName,
147 schema=schema, metadata=algMetadata,
148 logName=self.log.name)
149
150 @timeMethod
151 def run(self, sources, exposure, posExp=None, negExp=None, **kwargs):
152 """Run dipole measurement and classification
153
154 Parameters
155 ----------
157 ``diaSources`` that will be measured using dipole measurement
158 exposure : `lsst.afw.image.Exposure`
159 The difference exposure on which the ``diaSources`` of the ``sources`` parameter
160 were detected. If neither ``posExp`` nor ``negExp`` are set, then the dipole is also
161 fitted directly to this difference image.
162 posExp : `lsst.afw.image.Exposure`, optional
163 "Positive" exposure, typically a science exposure, or None if unavailable
164 When `posExp` is `None`, will compute `posImage = exposure + negExp`.
165 negExp : `lsst.afw.image.Exposure`, optional
166 "Negative" exposure, typically a template exposure, or None if unavailable
167 When `negExp` is `None`, will compute `negImage = posExp - exposure`.
168 **kwargs
169 Additional keyword arguments for `lsst.meas.base.sfm.SingleFrameMeasurementTask`.
170 """
171
172 measBase.SingleFrameMeasurementTask.run(self, sources, exposure, **kwargs)
173
174 if not sources:
175 return
176
177 for source in sources:
178 self.dipoleFitterdipoleFitter.measure(source, exposure, posExp, negExp)
179
180
181class DipoleModel(object):
182 """Lightweight class containing methods for generating a dipole model for fitting
183 to sources in diffims, used by DipoleFitAlgorithm.
184
185 See also:
186 `DMTN-007: Dipole characterization for image differencing <https://dmtn-007.lsst.io>`_.
187 """
188
189 def __init__(self):
190 import lsstDebug
191 self.debugdebug = lsstDebug.Info(__name__).debug
192 self.loglog = logging.getLogger(__name__)
193
194 def makeBackgroundModel(self, in_x, pars=None):
195 """Generate gradient model (2-d array) with up to 2nd-order polynomial
196
197 Parameters
198 ----------
199 in_x : `numpy.array`
200 (2, w, h)-dimensional `numpy.array`, containing the
201 input x,y meshgrid providing the coordinates upon which to
202 compute the gradient. This will typically be generated via
203 `_generateXYGrid()`. `w` and `h` correspond to the width and
204 height of the desired grid.
205 pars : `list` of `float`, optional
206 Up to 6 floats for up
207 to 6 2nd-order 2-d polynomial gradient parameters, in the
208 following order: (intercept, x, y, xy, x**2, y**2). If `pars`
209 is emtpy or `None`, do nothing and return `None` (for speed).
210
211 Returns
212 -------
213 result : `None` or `numpy.array`
214 return None, or 2-d numpy.array of width/height matching
215 input bbox, containing computed gradient values.
216 """
217
218 # Don't fit for other gradient parameters if the intercept is not included.
219 if (pars is None) or (len(pars) <= 0) or (pars[0] is None):
220 return
221
222 y, x = in_x[0, :], in_x[1, :]
223 gradient = np.full_like(x, pars[0], dtype='float64')
224 if len(pars) > 1 and pars[1] is not None:
225 gradient += pars[1] * x
226 if len(pars) > 2 and pars[2] is not None:
227 gradient += pars[2] * y
228 if len(pars) > 3 and pars[3] is not None:
229 gradient += pars[3] * (x * y)
230 if len(pars) > 4 and pars[4] is not None:
231 gradient += pars[4] * (x * x)
232 if len(pars) > 5 and pars[5] is not None:
233 gradient += pars[5] * (y * y)
234
235 return gradient
236
237 def _generateXYGrid(self, bbox):
238 """Generate a meshgrid covering the x,y coordinates bounded by bbox
239
240 Parameters
241 ----------
242 bbox : `lsst.geom.Box2I`
243 input Bounding Box defining the coordinate limits
244
245 Returns
246 -------
247 in_x : `numpy.array`
248 (2, w, h)-dimensional numpy array containing the grid indexing over x- and
249 y- coordinates
250 """
251
252 x, y = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
253 in_x = np.array([y, x]).astype(np.float64)
254 in_x[0, :] -= np.mean(in_x[0, :])
255 in_x[1, :] -= np.mean(in_x[1, :])
256 return in_x
257
258 def _getHeavyFootprintSubimage(self, fp, badfill=np.nan, grow=0):
259 """Extract the image from a ``~lsst.afw.detection.HeavyFootprint``
260 as an `lsst.afw.image.ImageF`.
261
262 Parameters
263 ----------
265 HeavyFootprint to use to generate the subimage
266 badfill : `float`, optional
267 Value to fill in pixels in extracted image that are outside the footprint
268 grow : `int`
269 Optionally grow the footprint by this amount before extraction
270
271 Returns
272 -------
273 subim2 : `lsst.afw.image.ImageF`
274 An `~lsst.afw.image.ImageF` containing the subimage.
275 """
276 bbox = fp.getBBox()
277 if grow > 0:
278 bbox.grow(grow)
279
280 subim2 = afwImage.ImageF(bbox, badfill)
281 fp.getSpans().unflatten(subim2.getArray(), fp.getImageArray(), bbox.getCorners()[0])
282 return subim2
283
284 def fitFootprintBackground(self, source, posImage, order=1):
285 """Fit a linear (polynomial) model of given order (max 2) to the background of a footprint.
286
287 Only fit the pixels OUTSIDE of the footprint, but within its bounding box.
288
289 Parameters
290 ----------
292 SourceRecord, the footprint of which is to be fit
293 posImage : `lsst.afw.image.Exposure`
294 The exposure from which to extract the footprint subimage
295 order : `int`
296 Polynomial order of background gradient to fit.
297
298 Returns
299 -------
300 pars : `tuple` of `float`
301 `tuple` of length (1 if order==0; 3 if order==1; 6 if order == 2),
302 containing the resulting fit parameters
303 """
304
305 # TODO look into whether to use afwMath background methods -- see
306 # http://lsst-web.ncsa.illinois.edu/doxygen/x_masterDoxyDoc/_background_example.html
307 fp = source.getFootprint()
308 bbox = fp.getBBox()
309 bbox.grow(3)
310 posImg = afwImage.ImageF(posImage.getMaskedImage().getImage(), bbox, afwImage.PARENT)
311
312 # This code constructs the footprint image so that we can identify the pixels that are
313 # outside the footprint (but within the bounding box). These are the pixels used for
314 # fitting the background.
315 posHfp = afwDet.HeavyFootprintF(fp, posImage.getMaskedImage())
316 posFpImg = self._getHeavyFootprintSubimage_getHeavyFootprintSubimage(posHfp, grow=3)
317
318 isBg = np.isnan(posFpImg.getArray()).ravel()
319
320 data = posImg.getArray().ravel()
321 data = data[isBg]
322 B = data
323
324 x, y = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
325 x = x.astype(np.float64).ravel()
326 x -= np.mean(x)
327 x = x[isBg]
328 y = y.astype(np.float64).ravel()
329 y -= np.mean(y)
330 y = y[isBg]
331 b = np.ones_like(x, dtype=np.float64)
332
333 M = np.vstack([b]).T # order = 0
334 if order == 1:
335 M = np.vstack([b, x, y]).T
336 elif order == 2:
337 M = np.vstack([b, x, y, x**2., y**2., x*y]).T
338
339 pars = np.linalg.lstsq(M, B, rcond=-1)[0]
340 return pars
341
342 def makeStarModel(self, bbox, psf, xcen, ycen, flux):
343 """Generate a 2D image model of a single PDF centered at the given coordinates.
344
345 Parameters
346 ----------
347 bbox : `lsst.geom.Box`
348 Bounding box marking pixel coordinates for generated model
349 psf : TODO: DM-17458
350 Psf model used to generate the 'star'
351 xcen : `float`
352 Desired x-centroid of the 'star'
353 ycen : `float`
354 Desired y-centroid of the 'star'
355 flux : `float`
356 Desired flux of the 'star'
357
358 Returns
359 -------
360 p_Im : `lsst.afw.image.Image`
361 2-d stellar image of width/height matching input ``bbox``,
362 containing PSF with given centroid and flux
363 """
364
365 # Generate the psf image, normalize to flux
366 psf_img = psf.computeImage(geom.Point2D(xcen, ycen)).convertF()
367 psf_img_sum = np.nansum(psf_img.getArray())
368 psf_img *= (flux/psf_img_sum)
369
370 # Clip the PSF image bounding box to fall within the footprint bounding box
371 psf_box = psf_img.getBBox()
372 psf_box.clip(bbox)
373 psf_img = afwImage.ImageF(psf_img, psf_box, afwImage.PARENT)
374
375 # Then actually crop the psf image.
376 # Usually not necessary, but if the dipole is near the edge of the image...
377 # Would be nice if we could compare original pos_box with clipped pos_box and
378 # see if it actually was clipped.
379 p_Im = afwImage.ImageF(bbox)
380 tmpSubim = afwImage.ImageF(p_Im, psf_box, afwImage.PARENT)
381 tmpSubim += psf_img
382
383 return p_Im
384
385 def makeModel(self, x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None,
386 b=None, x1=None, y1=None, xy=None, x2=None, y2=None,
387 bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None,
388 **kwargs):
389 """Generate dipole model with given parameters.
390
391 This is the function whose sum-of-squared difference from data
392 is minimized by `lmfit`.
393
394 x : TODO: DM-17458
395 Input independent variable. Used here as the grid on
396 which to compute the background gradient model.
397 flux : `float`
398 Desired flux of the positive lobe of the dipole
399 xcenPos : `float`
400 Desired x-centroid of the positive lobe of the dipole
401 ycenPos : `float`
402 Desired y-centroid of the positive lobe of the dipole
403 xcenNeg : `float`
404 Desired x-centroid of the negative lobe of the dipole
405 ycenNeg : `float`
406 Desired y-centroid of the negative lobe of the dipole
407 fluxNeg : `float`, optional
408 Desired flux of the negative lobe of the dipole, set to 'flux' if None
409 b, x1, y1, xy, x2, y2 : `float`
410 Gradient parameters for positive lobe.
411 bNeg, x1Neg, y1Neg, xyNeg, x2Neg, y2Neg : `float`, optional
412 Gradient parameters for negative lobe.
413 They are set to the corresponding positive values if None.
414
415 **kwargs
416 Keyword arguments passed through ``lmfit`` and
417 used by this function. These must include:
418
419 - ``psf`` Psf model used to generate the 'star'
420 - ``rel_weight`` Used to signify least-squares weighting of posImage/negImage
421 relative to diffim. If ``rel_weight == 0`` then posImage/negImage are ignored.
422 - ``bbox`` Bounding box containing region to be modelled
423
424 Returns
425 -------
426 zout : `numpy.array`
427 Has width and height matching the input bbox, and
428 contains the dipole model with given centroids and flux(es). If
429 ``rel_weight`` = 0, this is a 2-d array with dimensions matching
430 those of bbox; otherwise a stack of three such arrays,
431 representing the dipole (diffim), positive and negative images
432 respectively.
433 """
434
435 psf = kwargs.get('psf')
436 rel_weight = kwargs.get('rel_weight') # if > 0, we're including pre-sub. images
437 fp = kwargs.get('footprint')
438 bbox = fp.getBBox()
439
440 if fluxNeg is None:
441 fluxNeg = flux
442
443 if self.debugdebug:
444 self.loglog.debug('%.2f %.2f %.2f %.2f %.2f %.2f',
445 flux, fluxNeg, xcenPos, ycenPos, xcenNeg, ycenNeg)
446 if x1 is not None:
447 self.loglog.debug(' %.2f %.2f %.2f', b, x1, y1)
448 if xy is not None:
449 self.loglog.debug(' %.2f %.2f %.2f', xy, x2, y2)
450
451 posIm = self.makeStarModelmakeStarModel(bbox, psf, xcenPos, ycenPos, flux)
452 negIm = self.makeStarModelmakeStarModel(bbox, psf, xcenNeg, ycenNeg, fluxNeg)
453
454 in_x = x
455 if in_x is None: # use the footprint to generate the input grid
456 y, x = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
457 in_x = np.array([x, y]) * 1.
458 in_x[0, :] -= in_x[0, :].mean() # center it!
459 in_x[1, :] -= in_x[1, :].mean()
460
461 if b is not None:
462 gradient = self.makeBackgroundModelmakeBackgroundModel(in_x, (b, x1, y1, xy, x2, y2))
463
464 # If bNeg is None, then don't fit the negative background separately
465 if bNeg is not None:
466 gradientNeg = self.makeBackgroundModelmakeBackgroundModel(in_x, (bNeg, x1Neg, y1Neg, xyNeg, x2Neg, y2Neg))
467 else:
468 gradientNeg = gradient
469
470 posIm.getArray()[:, :] += gradient
471 negIm.getArray()[:, :] += gradientNeg
472
473 # Generate the diffIm model
474 diffIm = afwImage.ImageF(bbox)
475 diffIm += posIm
476 diffIm -= negIm
477
478 zout = diffIm.getArray()
479 if rel_weight > 0.:
480 zout = np.append([zout], [posIm.getArray(), negIm.getArray()], axis=0)
481
482 return zout
483
484
485class DipoleFitAlgorithm(object):
486 """Fit a dipole model using an image difference.
487
488 See also:
489 `DMTN-007: Dipole characterization for image differencing <https://dmtn-007.lsst.io>`_.
490 """
491
492 # This is just a private version number to sync with the ipython notebooks that I have been
493 # using for algorithm development.
494 _private_version_ = '0.0.5'
495
496 # Below is a (somewhat incomplete) list of improvements
497 # that would be worth investigating, given the time:
498
499 # todo 1. evaluate necessity for separate parameters for pos- and neg- images
500 # todo 2. only fit background OUTSIDE footprint (DONE) and dipole params INSIDE footprint (NOT DONE)?
501 # todo 3. correct normalization of least-squares weights based on variance planes
502 # todo 4. account for PSFs that vary across the exposures (should be happening by default?)
503 # todo 5. correctly account for NA/masks (i.e., ignore!)
504 # todo 6. better exception handling in the plugin
505 # todo 7. better classification of dipoles (e.g. by comparing chi2 fit vs. monopole?)
506 # todo 8. (DONE) Initial fast estimate of background gradient(s) params -- perhaps using numpy.lstsq
507 # todo 9. (NOT NEEDED - see (2)) Initial fast test whether a background gradient needs to be fit
508 # todo 10. (DONE) better initial estimate for flux when there's a strong gradient
509 # todo 11. (DONE) requires a new package `lmfit` -- investiate others? (astropy/scipy/iminuit?)
510
511 def __init__(self, diffim, posImage=None, negImage=None):
512 """Algorithm to run dipole measurement on a diaSource
513
514 Parameters
515 ----------
516 diffim : `lsst.afw.image.Exposure`
517 Exposure on which the diaSources were detected
518 posImage : `lsst.afw.image.Exposure`
519 "Positive" exposure from which the template was subtracted
520 negImage : `lsst.afw.image.Exposure`
521 "Negative" exposure which was subtracted from the posImage
522 """
523
524 self.diffimdiffim = diffim
525 self.posImageposImage = posImage
526 self.negImagenegImage = negImage
527 self.psfSigmapsfSigma = None
528 if diffim is not None:
529 self.psfSigmapsfSigma = diffim.getPsf().computeShape().getDeterminantRadius()
530
531 self.loglog = logging.getLogger(__name__)
532
533 import lsstDebug
534 self.debugdebug = lsstDebug.Info(__name__).debug
535
536 def fitDipoleImpl(self, source, tol=1e-7, rel_weight=0.5,
537 fitBackground=1, bgGradientOrder=1, maxSepInSigma=5.,
538 separateNegParams=True, verbose=False):
539 """Fit a dipole model to an input difference image.
540
541 Actually, fits the subimage bounded by the input source's
542 footprint) and optionally constrain the fit using the
543 pre-subtraction images posImage and negImage.
544
545 Parameters
546 ----------
547 source : TODO: DM-17458
548 TODO: DM-17458
549 tol : float, optional
550 TODO: DM-17458
551 rel_weight : `float`, optional
552 TODO: DM-17458
553 fitBackground : `int`, optional
554 TODO: DM-17458
555 bgGradientOrder : `int`, optional
556 TODO: DM-17458
557 maxSepInSigma : `float`, optional
558 TODO: DM-17458
559 separateNegParams : `bool`, optional
560 TODO: DM-17458
561 verbose : `bool`, optional
562 TODO: DM-17458
563
564 Returns
565 -------
566 result : `lmfit.MinimizerResult`
567 return `lmfit.MinimizerResult` object containing the fit
568 parameters and other information.
569 """
570
571 # Only import lmfit if someone wants to use the new DipoleFitAlgorithm.
572 import lmfit
573
574 fp = source.getFootprint()
575 bbox = fp.getBBox()
576 subim = afwImage.MaskedImageF(self.diffimdiffim.getMaskedImage(), bbox=bbox, origin=afwImage.PARENT)
577
578 z = diArr = subim.getArrays()[0]
579 weights = 1. / subim.getArrays()[2] # get the weights (=1/variance)
580
581 if rel_weight > 0. and ((self.posImageposImage is not None) or (self.negImagenegImage is not None)):
582 if self.negImagenegImage is not None:
583 negSubim = afwImage.MaskedImageF(self.negImagenegImage.getMaskedImage(), bbox, origin=afwImage.PARENT)
584 if self.posImageposImage is not None:
585 posSubim = afwImage.MaskedImageF(self.posImageposImage.getMaskedImage(), bbox, origin=afwImage.PARENT)
586 if self.posImageposImage is None: # no science image provided; generate it from diffim + negImage
587 posSubim = subim.clone()
588 posSubim += negSubim
589 if self.negImagenegImage is None: # no template provided; generate it from the posImage - diffim
590 negSubim = posSubim.clone()
591 negSubim -= subim
592
593 z = np.append([z], [posSubim.getArrays()[0],
594 negSubim.getArrays()[0]], axis=0)
595 # Weight the pos/neg images by rel_weight relative to the diffim
596 weights = np.append([weights], [1. / posSubim.getArrays()[2] * rel_weight,
597 1. / negSubim.getArrays()[2] * rel_weight], axis=0)
598 else:
599 rel_weight = 0. # a short-cut for "don't include the pre-subtraction data"
600
601 # It seems that `lmfit` requires a static functor as its optimized method, which eliminates
602 # the ability to pass a bound method or other class method. Here we write a wrapper which
603 # makes this possible.
604 def dipoleModelFunctor(x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None,
605 b=None, x1=None, y1=None, xy=None, x2=None, y2=None,
606 bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None,
607 **kwargs):
608 """Generate dipole model with given parameters.
609
610 It simply defers to `modelObj.makeModel()`, where `modelObj` comes
611 out of `kwargs['modelObj']`.
612 """
613 modelObj = kwargs.pop('modelObj')
614 return modelObj.makeModel(x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=fluxNeg,
615 b=b, x1=x1, y1=y1, xy=xy, x2=x2, y2=y2,
616 bNeg=bNeg, x1Neg=x1Neg, y1Neg=y1Neg, xyNeg=xyNeg,
617 x2Neg=x2Neg, y2Neg=y2Neg, **kwargs)
618
619 dipoleModel = DipoleModel()
620
621 modelFunctor = dipoleModelFunctor # dipoleModel.makeModel does not work for now.
622 # Create the lmfit model (lmfit uses scipy 'leastsq' option by default - Levenberg-Marquardt)
623 # Note we can also tell it to drop missing values from the data.
624 gmod = lmfit.Model(modelFunctor, verbose=verbose, missing='drop')
625 # independent_vars=independent_vars) #, param_names=param_names)
626
627 # Add the constraints for centroids, fluxes.
628 # starting constraint - near centroid of footprint
629 fpCentroid = np.array([fp.getCentroid().getX(), fp.getCentroid().getY()])
630 cenNeg = cenPos = fpCentroid
631
632 pks = fp.getPeaks()
633
634 if len(pks) >= 1:
635 cenPos = pks[0].getF() # if individual (merged) peaks were detected, use those
636 if len(pks) >= 2: # peaks are already sorted by centroid flux so take the most negative one
637 cenNeg = pks[-1].getF()
638
639 # For close/faint dipoles the starting locs (min/max) might be way off, let's help them a bit.
640 # First assume dipole is not separated by more than 5*psfSigma.
641 maxSep = self.psfSigmapsfSigma * maxSepInSigma
642
643 # As an initial guess -- assume the dipole is close to the center of the footprint.
644 if np.sum(np.sqrt((np.array(cenPos) - fpCentroid)**2.)) > maxSep:
645 cenPos = fpCentroid
646 if np.sum(np.sqrt((np.array(cenNeg) - fpCentroid)**2.)) > maxSep:
647 cenPos = fpCentroid
648
649 # parameter hints/constraints: https://lmfit.github.io/lmfit-py/model.html#model-param-hints-section
650 # might make sense to not use bounds -- see http://lmfit.github.io/lmfit-py/bounds.html
651 # also see this discussion -- https://github.com/scipy/scipy/issues/3129
652 gmod.set_param_hint('xcenPos', value=cenPos[0],
653 min=cenPos[0]-maxSep, max=cenPos[0]+maxSep)
654 gmod.set_param_hint('ycenPos', value=cenPos[1],
655 min=cenPos[1]-maxSep, max=cenPos[1]+maxSep)
656 gmod.set_param_hint('xcenNeg', value=cenNeg[0],
657 min=cenNeg[0]-maxSep, max=cenNeg[0]+maxSep)
658 gmod.set_param_hint('ycenNeg', value=cenNeg[1],
659 min=cenNeg[1]-maxSep, max=cenNeg[1]+maxSep)
660
661 # Use the (flux under the dipole)*5 for an estimate.
662 # Lots of testing showed that having startingFlux be too high was better than too low.
663 startingFlux = np.nansum(np.abs(diArr) - np.nanmedian(np.abs(diArr))) * 5.
664 posFlux = negFlux = startingFlux
665
666 # TBD: set max. flux limit?
667 gmod.set_param_hint('flux', value=posFlux, min=0.1)
668
669 if separateNegParams:
670 # TBD: set max negative lobe flux limit?
671 gmod.set_param_hint('fluxNeg', value=np.abs(negFlux), min=0.1)
672
673 # Fixed parameters (don't fit for them if there are no pre-sub images or no gradient fit requested):
674 # Right now (fitBackground == 1), we fit a linear model to the background and then subtract
675 # it from the data and then don't fit the background again (this is faster).
676 # A slower alternative (fitBackground == 2) is to use the estimated background parameters as
677 # starting points in the integrated model fit. That is currently not performed by default,
678 # but might be desirable in some cases.
679 bgParsPos = bgParsNeg = (0., 0., 0.)
680 if ((rel_weight > 0.) and (fitBackground != 0) and (bgGradientOrder >= 0)):
681 pbg = 0.
682 bgFitImage = self.posImageposImage if self.posImageposImage is not None else self.negImagenegImage
683 # Fit the gradient to the background (linear model)
684 bgParsPos = bgParsNeg = dipoleModel.fitFootprintBackground(source, bgFitImage,
685 order=bgGradientOrder)
686
687 # Generate the gradient and subtract it from the pre-subtraction image data
688 if fitBackground == 1:
689 in_x = dipoleModel._generateXYGrid(bbox)
690 pbg = dipoleModel.makeBackgroundModel(in_x, tuple(bgParsPos))
691 z[1, :] -= pbg
692 z[1, :] -= np.nanmedian(z[1, :])
693 posFlux = np.nansum(z[1, :])
694 gmod.set_param_hint('flux', value=posFlux*1.5, min=0.1)
695
696 if separateNegParams and self.negImagenegImage is not None:
697 bgParsNeg = dipoleModel.fitFootprintBackground(source, self.negImagenegImage,
698 order=bgGradientOrder)
699 pbg = dipoleModel.makeBackgroundModel(in_x, tuple(bgParsNeg))
700 z[2, :] -= pbg
701 z[2, :] -= np.nanmedian(z[2, :])
702 if separateNegParams:
703 negFlux = np.nansum(z[2, :])
704 gmod.set_param_hint('fluxNeg', value=negFlux*1.5, min=0.1)
705
706 # Do not subtract the background from the images but include the background parameters in the fit
707 if fitBackground == 2:
708 if bgGradientOrder >= 0:
709 gmod.set_param_hint('b', value=bgParsPos[0])
710 if separateNegParams:
711 gmod.set_param_hint('bNeg', value=bgParsNeg[0])
712 if bgGradientOrder >= 1:
713 gmod.set_param_hint('x1', value=bgParsPos[1])
714 gmod.set_param_hint('y1', value=bgParsPos[2])
715 if separateNegParams:
716 gmod.set_param_hint('x1Neg', value=bgParsNeg[1])
717 gmod.set_param_hint('y1Neg', value=bgParsNeg[2])
718 if bgGradientOrder >= 2:
719 gmod.set_param_hint('xy', value=bgParsPos[3])
720 gmod.set_param_hint('x2', value=bgParsPos[4])
721 gmod.set_param_hint('y2', value=bgParsPos[5])
722 if separateNegParams:
723 gmod.set_param_hint('xyNeg', value=bgParsNeg[3])
724 gmod.set_param_hint('x2Neg', value=bgParsNeg[4])
725 gmod.set_param_hint('y2Neg', value=bgParsNeg[5])
726
727 y, x = np.mgrid[bbox.getBeginY():bbox.getEndY(), bbox.getBeginX():bbox.getEndX()]
728 in_x = np.array([x, y]).astype(np.float)
729 in_x[0, :] -= in_x[0, :].mean() # center it!
730 in_x[1, :] -= in_x[1, :].mean()
731
732 # Instead of explicitly using a mask to ignore flagged pixels, just set the ignored pixels'
733 # weights to 0 in the fit. TBD: need to inspect mask planes to set this mask.
734 mask = np.ones_like(z, dtype=bool) # TBD: set mask values to False if the pixels are to be ignored
735
736 # I'm not sure about the variance planes in the diffim (or convolved pre-sub. images
737 # for that matter) so for now, let's just do an un-weighted least-squares fit
738 # (override weights computed above).
739 weights = mask.astype(np.float64)
740 if self.posImageposImage is not None and rel_weight > 0.:
741 weights = np.array([np.ones_like(diArr), np.ones_like(diArr)*rel_weight,
742 np.ones_like(diArr)*rel_weight])
743
744 # Set the weights to zero if mask is False
745 if np.any(~mask):
746 weights[~mask] = 0.
747
748 # Note that although we can, we're not required to set initial values for params here,
749 # since we set their param_hint's above.
750 # Can add "method" param to not use 'leastsq' (==levenberg-marquardt), e.g. "method='nelder'"
751 with warnings.catch_warnings():
752 warnings.simplefilter("ignore") # temporarily turn off silly lmfit warnings
753 result = gmod.fit(z, weights=weights, x=in_x,
754 verbose=verbose,
755 fit_kws={'ftol': tol, 'xtol': tol, 'gtol': tol,
756 'maxfev': 250}, # see scipy docs
757 psf=self.diffimdiffim.getPsf(), # hereon: kwargs that get passed to genDipoleModel()
758 rel_weight=rel_weight,
759 footprint=fp,
760 modelObj=dipoleModel)
761
762 if verbose: # the ci_report() seems to fail if neg params are constrained -- TBD why.
763 # Never wanted in production - this takes a long time (longer than the fit!)
764 # This is how to get confidence intervals out:
765 # https://lmfit.github.io/lmfit-py/confidence.html and
766 # http://cars9.uchicago.edu/software/python/lmfit/model.html
767 print(result.fit_report(show_correl=False))
768 if separateNegParams:
769 print(result.ci_report())
770
771 return result
772
773 def fitDipole(self, source, tol=1e-7, rel_weight=0.1,
774 fitBackground=1, maxSepInSigma=5., separateNegParams=True,
775 bgGradientOrder=1, verbose=False, display=False):
776 """Fit a dipole model to an input ``diaSource`` (wraps `fitDipoleImpl`).
777
778 Actually, fits the subimage bounded by the input source's
779 footprint) and optionally constrain the fit using the
780 pre-subtraction images self.posImageposImage (science) and
781 self.negImagenegImage (template). Wraps the output into a
782 `pipeBase.Struct` named tuple after computing additional
783 statistics such as orientation and SNR.
784
785 Parameters
786 ----------
788 Record containing the (merged) dipole source footprint detected on the diffim
789 tol : `float`, optional
790 Tolerance parameter for scipy.leastsq() optimization
791 rel_weight : `float`, optional
792 Weighting of posImage/negImage relative to the diffim in the fit
793 fitBackground : `int`, {0, 1, 2}, optional
794 How to fit linear background gradient in posImage/negImage
795
796 - 0: do not fit background at all
797 - 1 (default): pre-fit the background using linear least squares and then do not fit it
798 as part of the dipole fitting optimization
799 - 2: pre-fit the background using linear least squares (as in 1), and use the parameter
800 estimates from that fit as starting parameters for an integrated "re-fit" of the
801 background as part of the overall dipole fitting optimization.
802 maxSepInSigma : `float`, optional
803 Allowed window of centroid parameters relative to peak in input source footprint
804 separateNegParams : `bool`, optional
805 Fit separate parameters to the flux and background gradient in
806 bgGradientOrder : `int`, {0, 1, 2}, optional
807 Desired polynomial order of background gradient
808 verbose: `bool`, optional
809 Be verbose
810 display
811 Display input data, best fit model(s) and residuals in a matplotlib window.
812
813 Returns
814 -------
815 result : `struct`
816 `pipeBase.Struct` object containing the fit parameters and other information.
817
818 result : `callable`
819 `lmfit.MinimizerResult` object for debugging and error estimation, etc.
820
821 Notes
822 -----
823 Parameter `fitBackground` has three options, thus it is an integer:
824
825 """
826
827 fitResult = self.fitDipoleImplfitDipoleImpl(
828 source, tol=tol, rel_weight=rel_weight, fitBackground=fitBackground,
829 maxSepInSigma=maxSepInSigma, separateNegParams=separateNegParams,
830 bgGradientOrder=bgGradientOrder, verbose=verbose)
831
832 # Display images, model fits and residuals (currently uses matplotlib display functions)
833 if display:
834 fp = source.getFootprint()
835 self.displayFitResultsdisplayFitResults(fp, fitResult)
836
837 fitParams = fitResult.best_values
838 if fitParams['flux'] <= 1.: # usually around 0.1 -- the minimum flux allowed -- i.e. bad fit.
839 out = Struct(posCentroidX=np.nan, posCentroidY=np.nan,
840 negCentroidX=np.nan, negCentroidY=np.nan,
841 posFlux=np.nan, negFlux=np.nan, posFluxErr=np.nan, negFluxErr=np.nan,
842 centroidX=np.nan, centroidY=np.nan, orientation=np.nan,
843 signalToNoise=np.nan, chi2=np.nan, redChi2=np.nan)
844 return out, fitResult
845
846 centroid = ((fitParams['xcenPos'] + fitParams['xcenNeg']) / 2.,
847 (fitParams['ycenPos'] + fitParams['ycenNeg']) / 2.)
848 dx, dy = fitParams['xcenPos'] - fitParams['xcenNeg'], fitParams['ycenPos'] - fitParams['ycenNeg']
849 angle = np.arctan2(dy, dx) / np.pi * 180. # convert to degrees (should keep as rad?)
850
851 # Exctract flux value, compute signalToNoise from flux/variance_within_footprint
852 # Also extract the stderr of flux estimate.
853 def computeSumVariance(exposure, footprint):
854 box = footprint.getBBox()
855 subim = afwImage.MaskedImageF(exposure.getMaskedImage(), box, origin=afwImage.PARENT)
856 return np.sqrt(np.nansum(subim.getArrays()[1][:, :]))
857
858 fluxVal = fluxVar = fitParams['flux']
859 fluxErr = fluxErrNeg = fitResult.params['flux'].stderr
860 if self.posImageposImage is not None:
861 fluxVar = computeSumVariance(self.posImageposImage, source.getFootprint())
862 else:
863 fluxVar = computeSumVariance(self.diffimdiffim, source.getFootprint())
864
865 fluxValNeg, fluxVarNeg = fluxVal, fluxVar
866 if separateNegParams:
867 fluxValNeg = fitParams['fluxNeg']
868 fluxErrNeg = fitResult.params['fluxNeg'].stderr
869 if self.negImagenegImage is not None:
870 fluxVarNeg = computeSumVariance(self.negImagenegImage, source.getFootprint())
871
872 try:
873 signalToNoise = np.sqrt((fluxVal/fluxVar)**2 + (fluxValNeg/fluxVarNeg)**2)
874 except ZeroDivisionError: # catch divide by zero - should never happen.
875 signalToNoise = np.nan
876
877 out = Struct(posCentroidX=fitParams['xcenPos'], posCentroidY=fitParams['ycenPos'],
878 negCentroidX=fitParams['xcenNeg'], negCentroidY=fitParams['ycenNeg'],
879 posFlux=fluxVal, negFlux=-fluxValNeg, posFluxErr=fluxErr, negFluxErr=fluxErrNeg,
880 centroidX=centroid[0], centroidY=centroid[1], orientation=angle,
881 signalToNoise=signalToNoise, chi2=fitResult.chisqr, redChi2=fitResult.redchi)
882
883 # fitResult may be returned for debugging
884 return out, fitResult
885
886 def displayFitResults(self, footprint, result):
887 """Display data, model fits and residuals (currently uses matplotlib display functions).
888
889 Parameters
890 ----------
891 footprint : TODO: DM-17458
892 Footprint containing the dipole that was fit
893 result : `lmfit.MinimizerResult`
894 `lmfit.MinimizerResult` object returned by `lmfit` optimizer
895
896 Returns
897 -------
898 fig : `matplotlib.pyplot.plot`
899 """
900 try:
901 import matplotlib.pyplot as plt
902 except ImportError as err:
903 self.loglog.warning('Unable to import matplotlib: %s', err)
904 raise err
905
906 def display2dArray(arr, title='Data', extent=None):
907 """Use `matplotlib.pyplot.imshow` to display a 2-D array with a given coordinate range.
908 """
909 fig = plt.imshow(arr, origin='lower', interpolation='none', cmap='gray', extent=extent)
910 plt.title(title)
911 plt.colorbar(fig, cmap='gray')
912 return fig
913
914 z = result.data
915 fit = result.best_fit
916 bbox = footprint.getBBox()
917 extent = (bbox.getBeginX(), bbox.getEndX(), bbox.getBeginY(), bbox.getEndY())
918 if z.shape[0] == 3:
919 fig = plt.figure(figsize=(8, 8))
920 for i in range(3):
921 plt.subplot(3, 3, i*3+1)
922 display2dArray(z[i, :], 'Data', extent=extent)
923 plt.subplot(3, 3, i*3+2)
924 display2dArray(fit[i, :], 'Model', extent=extent)
925 plt.subplot(3, 3, i*3+3)
926 display2dArray(z[i, :] - fit[i, :], 'Residual', extent=extent)
927 return fig
928 else:
929 fig = plt.figure(figsize=(8, 2.5))
930 plt.subplot(1, 3, 1)
931 display2dArray(z, 'Data', extent=extent)
932 plt.subplot(1, 3, 2)
933 display2dArray(fit, 'Model', extent=extent)
934 plt.subplot(1, 3, 3)
935 display2dArray(z - fit, 'Residual', extent=extent)
936 return fig
937
938 plt.show()
939
940
941@measBase.register("ip_diffim_DipoleFit")
942class DipoleFitPlugin(measBase.SingleFramePlugin):
943 """A single frame measurement plugin that fits dipoles to all merged (two-peak) ``diaSources``.
944
945 This measurement plugin accepts up to three input images in
946 its `measure` method. If these are provided, it includes data
947 from the pre-subtraction posImage (science image) and optionally
948 negImage (template image) to constrain the fit. The meat of the
949 fitting routines are in the class `~lsst.module.name.DipoleFitAlgorithm`.
950
951 Notes
952 -----
953 The motivation behind this plugin and the necessity for including more than
954 one exposure are documented in DMTN-007 (http://dmtn-007.lsst.io).
955
956 This class is named `ip_diffim_DipoleFit` so that it may be used alongside
957 the existing `ip_diffim_DipoleMeasurement` classes until such a time as those
958 are deemed to be replaceable by this.
959 """
960
961 ConfigClass = DipoleFitPluginConfig
962 DipoleFitAlgorithmClass = DipoleFitAlgorithm # Pointer to the class that performs the fit
963
964 FAILURE_EDGE = 1 # too close to the edge
965 FAILURE_FIT = 2 # failure in the fitting
966 FAILURE_NOT_DIPOLE = 4 # input source is not a putative dipole to begin with
967
968 @classmethod
970 """Set execution order to `FLUX_ORDER`.
971
972 This includes algorithms that require both `getShape()` and `getCentroid()`,
973 in addition to a Footprint and its Peaks.
974 """
975 return cls.FLUX_ORDER
976
977 def __init__(self, config, name, schema, metadata, logName=None):
978 if logName is None:
979 logName = name
980 measBase.SingleFramePlugin.__init__(self, config, name, schema, metadata, logName=logName)
981
982 self.loglog = logging.getLogger(logName)
983
984 self._setupSchema_setupSchema(config, name, schema, metadata)
985
986 def _setupSchema(self, config, name, schema, metadata):
987 # Get a FunctorKey that can quickly look up the "blessed" centroid value.
988 self.centroidKeycentroidKey = afwTable.Point2DKey(schema["slot_Centroid"])
989
990 # Add some fields for our outputs, and save their Keys.
991 # Use setattr() to programmatically set the pos/neg named attributes to values, e.g.
992 # self.posCentroidKeyX = 'ip_diffim_DipoleFit_pos_centroid_x'
993
994 for pos_neg in ['pos', 'neg']:
995
996 key = schema.addField(
997 schema.join(name, pos_neg, "instFlux"), type=float, units="count",
998 doc="Dipole {0} lobe flux".format(pos_neg))
999 setattr(self, ''.join((pos_neg, 'FluxKey')), key)
1000
1001 key = schema.addField(
1002 schema.join(name, pos_neg, "instFluxErr"), type=float, units="count",
1003 doc="1-sigma uncertainty for {0} dipole flux".format(pos_neg))
1004 setattr(self, ''.join((pos_neg, 'FluxErrKey')), key)
1005
1006 for x_y in ['x', 'y']:
1007 key = schema.addField(
1008 schema.join(name, pos_neg, "centroid", x_y), type=float, units="pixel",
1009 doc="Dipole {0} lobe centroid".format(pos_neg))
1010 setattr(self, ''.join((pos_neg, 'CentroidKey', x_y.upper())), key)
1011
1012 for x_y in ['x', 'y']:
1013 key = schema.addField(
1014 schema.join(name, "centroid", x_y), type=float, units="pixel",
1015 doc="Dipole centroid")
1016 setattr(self, ''.join(('centroidKey', x_y.upper())), key)
1017
1018 self.fluxKeyfluxKey = schema.addField(
1019 schema.join(name, "instFlux"), type=float, units="count",
1020 doc="Dipole overall flux")
1021
1022 self.orientationKeyorientationKey = schema.addField(
1023 schema.join(name, "orientation"), type=float, units="deg",
1024 doc="Dipole orientation")
1025
1026 self.separationKeyseparationKey = schema.addField(
1027 schema.join(name, "separation"), type=float, units="pixel",
1028 doc="Pixel separation between positive and negative lobes of dipole")
1029
1030 self.chi2dofKeychi2dofKey = schema.addField(
1031 schema.join(name, "chi2dof"), type=float,
1032 doc="Chi2 per degree of freedom of dipole fit")
1033
1034 self.signalToNoiseKeysignalToNoiseKey = schema.addField(
1035 schema.join(name, "signalToNoise"), type=float,
1036 doc="Estimated signal-to-noise of dipole fit")
1037
1038 self.classificationFlagKeyclassificationFlagKey = schema.addField(
1039 schema.join(name, "flag", "classification"), type="Flag",
1040 doc="Flag indicating diaSource is classified as a dipole")
1041
1042 self.classificationAttemptedFlagKeyclassificationAttemptedFlagKey = schema.addField(
1043 schema.join(name, "flag", "classificationAttempted"), type="Flag",
1044 doc="Flag indicating diaSource was attempted to be classified as a dipole")
1045
1046 self.flagKeyflagKey = schema.addField(
1047 schema.join(name, "flag"), type="Flag",
1048 doc="General failure flag for dipole fit")
1049
1050 self.edgeFlagKeyedgeFlagKey = schema.addField(
1051 schema.join(name, "flag", "edge"), type="Flag",
1052 doc="Flag set when dipole is too close to edge of image")
1053
1054 def measure(self, measRecord, exposure, posExp=None, negExp=None):
1055 """Perform the non-linear least squares minimization on the putative dipole source.
1056
1057 Parameters
1058 ----------
1059 measRecord : `lsst.afw.table.SourceRecord`
1060 diaSources that will be measured using dipole measurement
1061 exposure : `lsst.afw.image.Exposure`
1062 Difference exposure on which the diaSources were detected; `exposure = posExp-negExp`
1063 If both `posExp` and `negExp` are `None`, will attempt to fit the
1064 dipole to just the `exposure` with no constraint.
1065 posExp : `lsst.afw.image.Exposure`, optional
1066 "Positive" exposure, typically a science exposure, or None if unavailable
1067 When `posExp` is `None`, will compute `posImage = exposure + negExp`.
1068 negExp : `lsst.afw.image.Exposure`, optional
1069 "Negative" exposure, typically a template exposure, or None if unavailable
1070 When `negExp` is `None`, will compute `negImage = posExp - exposure`.
1071
1072 Notes
1073 -----
1074 The main functionality of this routine was placed outside of
1075 this plugin (into `DipoleFitAlgorithm.fitDipole()`) so that
1076 `DipoleFitAlgorithm.fitDipole()` can be called separately for
1077 testing (@see `tests/testDipoleFitter.py`)
1078
1079 Returns
1080 -------
1081 result : TODO: DM-17458
1082 TODO: DM-17458
1083 """
1084
1085 result = None
1086 pks = measRecord.getFootprint().getPeaks()
1087
1088 # Check if the footprint consists of a putative dipole - else don't fit it.
1089 if (
1090 (len(pks) <= 1) # one peak in the footprint - not a dipole
1091 or (len(pks) > 1 and (np.sign(pks[0].getPeakValue())
1092 == np.sign(pks[-1].getPeakValue()))) # peaks are same sign - not a dipole
1093 ):
1094 measRecord.set(self.classificationFlagKeyclassificationFlagKey, False)
1095 measRecord.set(self.classificationAttemptedFlagKeyclassificationAttemptedFlagKey, False)
1096 self.failfail(measRecord, measBase.MeasurementError('not a dipole', self.FAILURE_NOT_DIPOLEFAILURE_NOT_DIPOLE))
1097 if not self.config.fitAllDiaSources:
1098 return result
1099
1100 try:
1101 alg = self.DipoleFitAlgorithmClassDipoleFitAlgorithmClass(exposure, posImage=posExp, negImage=negExp)
1102 result, _ = alg.fitDipole(
1103 measRecord, rel_weight=self.config.relWeight,
1104 tol=self.config.tolerance,
1105 maxSepInSigma=self.config.maxSeparation,
1106 fitBackground=self.config.fitBackground,
1107 separateNegParams=self.config.fitSeparateNegParams,
1108 verbose=False, display=False)
1109 except pexExcept.LengthError:
1110 self.failfail(measRecord, measBase.MeasurementError('edge failure', self.FAILURE_EDGEFAILURE_EDGE))
1111 except Exception:
1112 self.failfail(measRecord, measBase.MeasurementError('dipole fit failure', self.FAILURE_FITFAILURE_FIT))
1113
1114 if result is None:
1115 measRecord.set(self.classificationFlagKeyclassificationFlagKey, False)
1116 measRecord.set(self.classificationAttemptedFlagKeyclassificationAttemptedFlagKey, False)
1117 return result
1118
1119 self.loglog.debug("Dipole fit result: %d %s", measRecord.getId(), str(result))
1120
1121 if result.posFlux <= 1.: # usually around 0.1 -- the minimum flux allowed -- i.e. bad fit.
1122 self.failfail(measRecord, measBase.MeasurementError('dipole fit failure', self.FAILURE_FITFAILURE_FIT))
1123
1124 # add chi2, coord/flux uncertainties (TBD), dipole classification
1125 # Add the relevant values to the measRecord
1126 measRecord[self.posFluxKey] = result.posFlux
1127 measRecord[self.posFluxErrKey] = result.signalToNoise # to be changed to actual sigma!
1128 measRecord[self.posCentroidKeyX] = result.posCentroidX
1129 measRecord[self.posCentroidKeyY] = result.posCentroidY
1130
1131 measRecord[self.negFluxKey] = result.negFlux
1132 measRecord[self.negFluxErrKey] = result.signalToNoise # to be changed to actual sigma!
1133 measRecord[self.negCentroidKeyX] = result.negCentroidX
1134 measRecord[self.negCentroidKeyY] = result.negCentroidY
1135
1136 # Dia source flux: average of pos+neg
1137 measRecord[self.fluxKeyfluxKey] = (abs(result.posFlux) + abs(result.negFlux))/2.
1138 measRecord[self.orientationKeyorientationKey] = result.orientation
1139 measRecord[self.separationKeyseparationKey] = np.sqrt((result.posCentroidX - result.negCentroidX)**2.
1140 + (result.posCentroidY - result.negCentroidY)**2.)
1141 measRecord[self.centroidKeyX] = result.centroidX
1142 measRecord[self.centroidKeyY] = result.centroidY
1143
1144 measRecord[self.signalToNoiseKeysignalToNoiseKey] = result.signalToNoise
1145 measRecord[self.chi2dofKeychi2dofKey] = result.redChi2
1146
1147 self.doClassifydoClassify(measRecord, result.chi2)
1148
1149 def doClassify(self, measRecord, chi2val):
1150 """Classify a source as a dipole.
1151
1152 Parameters
1153 ----------
1154 measRecord : TODO: DM-17458
1155 TODO: DM-17458
1156 chi2val : TODO: DM-17458
1157 TODO: DM-17458
1158
1159 Notes
1160 -----
1161 Sources are classified as dipoles, or not, according to three criteria:
1162
1163 1. Does the total signal-to-noise surpass the ``minSn``?
1164 2. Are the pos/neg fluxes greater than 1.0 and no more than 0.65 (``maxFluxRatio``)
1165 of the total flux? By default this will never happen since ``posFlux == negFlux``.
1166 3. Is it a good fit (``chi2dof`` < 1)? (Currently not used.)
1167 """
1168
1169 # First, does the total signal-to-noise surpass the minSn?
1170 passesSn = measRecord[self.signalToNoiseKeysignalToNoiseKey] > self.config.minSn
1171
1172 # Second, are the pos/neg fluxes greater than 1.0 and no more than 0.65 (param maxFluxRatio)
1173 # of the total flux? By default this will never happen since posFlux = negFlux.
1174 passesFluxPos = (abs(measRecord[self.posFluxKey])
1175 / (measRecord[self.fluxKeyfluxKey]*2.)) < self.config.maxFluxRatio
1176 passesFluxPos &= (abs(measRecord[self.posFluxKey]) >= 1.0)
1177 passesFluxNeg = (abs(measRecord[self.negFluxKey])
1178 / (measRecord[self.fluxKeyfluxKey]*2.)) < self.config.maxFluxRatio
1179 passesFluxNeg &= (abs(measRecord[self.negFluxKey]) >= 1.0)
1180 allPass = (passesSn and passesFluxPos and passesFluxNeg) # and passesChi2)
1181
1182 # Third, is it a good fit (chi2dof < 1)?
1183 # Use scipy's chi2 cumulative distrib to estimate significance
1184 # This doesn't really work since I don't trust the values in the variance plane (which
1185 # affects the least-sq weights, which affects the resulting chi2).
1186 # But I'm going to keep this here for future use.
1187 if False:
1188 from scipy.stats import chi2
1189 ndof = chi2val / measRecord[self.chi2dofKeychi2dofKey]
1190 significance = chi2.cdf(chi2val, ndof)
1191 passesChi2 = significance < self.config.maxChi2DoF
1192 allPass = allPass and passesChi2
1193
1194 measRecord.set(self.classificationAttemptedFlagKeyclassificationAttemptedFlagKey, True)
1195
1196 if allPass: # Note cannot pass `allPass` into the `measRecord.set()` call below...?
1197 measRecord.set(self.classificationFlagKeyclassificationFlagKey, True)
1198 else:
1199 measRecord.set(self.classificationFlagKeyclassificationFlagKey, False)
1200
1201 def fail(self, measRecord, error=None):
1202 """Catch failures and set the correct flags.
1203 """
1204
1205 measRecord.set(self.flagKeyflagKey, True)
1206 if error is not None:
1207 if error.getFlagBit() == self.FAILURE_EDGEFAILURE_EDGE:
1208 self.loglog.warning('DipoleFitPlugin not run on record %d: %s', measRecord.getId(), str(error))
1209 measRecord.set(self.edgeFlagKeyedgeFlagKey, True)
1210 if error.getFlagBit() == self.FAILURE_FITFAILURE_FIT:
1211 self.loglog.warning('DipoleFitPlugin failed on record %d: %s', measRecord.getId(), str(error))
1212 measRecord.set(self.flagKeyflagKey, True)
1213 if error.getFlagBit() == self.FAILURE_NOT_DIPOLEFAILURE_NOT_DIPOLE:
1214 self.loglog.debug('DipoleFitPlugin not run on record %d: %s',
1215 measRecord.getId(), str(error))
1216 measRecord.set(self.classificationAttemptedFlagKeyclassificationAttemptedFlagKey, False)
1217 measRecord.set(self.flagKeyflagKey, True)
1218 else:
1219 self.loglog.warning('DipoleFitPlugin failed on record %d', measRecord.getId())
def fitDipoleImpl(self, source, tol=1e-7, rel_weight=0.5, fitBackground=1, bgGradientOrder=1, maxSepInSigma=5., separateNegParams=True, verbose=False)
def fitDipole(self, source, tol=1e-7, rel_weight=0.1, fitBackground=1, maxSepInSigma=5., separateNegParams=True, bgGradientOrder=1, verbose=False, display=False)
def displayFitResults(self, footprint, result)
def __init__(self, diffim, posImage=None, negImage=None)
def fail(self, measRecord, error=None)
def measure(self, measRecord, exposure, posExp=None, negExp=None)
def doClassify(self, measRecord, chi2val)
def _setupSchema(self, config, name, schema, metadata)
def __init__(self, config, name, schema, metadata, logName=None)
def __init__(self, schema, algMetadata=None, **kwargs)
def run(self, sources, exposure, posExp=None, negExp=None, **kwargs)
def _getHeavyFootprintSubimage(self, fp, badfill=np.nan, grow=0)
def makeModel(self, x, flux, xcenPos, ycenPos, xcenNeg, ycenNeg, fluxNeg=None, b=None, x1=None, y1=None, xy=None, x2=None, y2=None, bNeg=None, x1Neg=None, y1Neg=None, xyNeg=None, x2Neg=None, y2Neg=None, **kwargs)
def makeBackgroundModel(self, in_x, pars=None)
def fitFootprintBackground(self, source, posImage, order=1)
def makeStarModel(self, bbox, psf, xcen, ycen, flux)