lsst.ip.diffim  14.0-6-gf4bc96c+4
imageMapReduce.py
Go to the documentation of this file.
1 from __future__ import absolute_import, division, print_function
2 from future import standard_library
3 standard_library.install_aliases()
4 #
5 # LSST Data Management System
6 # Copyright 2016 AURA/LSST.
7 #
8 # This product includes software developed by the
9 # LSST Project (http://www.lsst.org/).
10 #
11 # This program is free software: you can redistribute it and/or modify
12 # it under the terms of the GNU General Public License as published by
13 # the Free Software Foundation, either version 3 of the License, or
14 # (at your option) any later version.
15 #
16 # This program is distributed in the hope that it will be useful,
17 # but WITHOUT ANY WARRANTY; without even the implied warranty of
18 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 # GNU General Public License for more details.
20 #
21 # You should have received a copy of the LSST License Statement and
22 # the GNU General Public License along with this program. If not,
23 # see <https://www.lsstcorp.org/LegalNotices/>.
24 #
25 
26 import numpy as np
27 import abc
28 from future.utils import with_metaclass
29 
30 import lsst.afw.image as afwImage
31 import lsst.afw.geom as afwGeom
32 import lsst.afw.table as afwTable
33 import lsst.meas.algorithms as measAlg
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 
37 __all__ = ("ImageMapReduceTask", "ImageMapReduceConfig",
38  "ImageMapper", "ImageMapperConfig",
39  "ImageReducer", "ImageReducerConfig")
40 
41 
42 """Tasks for processing an exposure via processing on
43 multiple sub-exposures and then collecting the results
44 to either re-stitch the sub-exposures back into a new
45 exposure, or return summary results for each sub-exposure.
46 
47 This provides a framework for arbitrary mapper-reducer
48 operations on an exposure by implementing simple operations in
49 subTasks. It currently is not parallelized, although it could be in
50 the future. It does enable operations such as spatially-mapped
51 processing on a grid across an image, processing regions surrounding
52 centroids (such as for PSF processing), etc.
53 
54 It is implemented as primary Task, `ImageMapReduceTask` which contains
55 two subtasks, `ImageMapper` and `ImageReducer`.
56 `ImageMapReduceTask` configures the centroids and sub-exposure
57 dimensions to be processed, and then calls the `run` methods of the
58 `ImageMapper` and `ImageReducer` on those sub-exposures.
59 `ImageMapReduceTask` may be configured with a list of sub-exposure
60 centroids (`config.cellCentroidsX` and `config.cellCentroidsY`) and a
61 single pair of bounding boxes defining their dimensions, or a set of
62 parameters defining a regular grid of centroids (`config.gridStepX`
63 and `config.gridStepY`).
64 
65 `ImageMapper` is an abstract class and must be subclassed with
66 an implemented `run` method to provide the desired operation for
67 processing individual sub-exposures. It is called from
68 `ImageMapReduceTask.run`, and may return a new, processed sub-exposure
69 which is to be "stitched" back into a new resulting larger exposure
70 (depending on the configured `ImageMapReduceTask.mapper`);
71 otherwise if it does not return an lsst.afw.image.Exposure, then the results are
72 passed back directly to the caller.
73 
74 `ImageReducer` will either stitch the `mapperResults` list of
75 results generated by the `ImageMapper` together into a new
76 Exposure (by default) or pass it through to the
77 caller. `ImageReducer` has an implemented `run` method for
78 basic reducing operations (`reduceOperation`) such as `average` (which
79 will average all overlapping pixels from sub-exposures produced by the
80 `ImageMapper` into the new exposure). Another notable
81 implemented `reduceOperation` is 'none', in which case the
82 `mapperResults` list is simply returned directly.
83 """
84 
85 class ImageMapperConfig(pexConfig.Config):
86  """Configuration parameters for ImageMapper
87  """
88  pass
89 
90 
91 class ImageMapper(with_metaclass(abc.ABCMeta, pipeBase.Task)):
92  """Abstract base class for any task that is to be
93  used as `ImageMapReduceConfig.mapper`.
94 
95  An `ImageMapper` is responsible for processing individual
96  sub-exposures in its `run` method, which is called from
97  `ImageMapReduceTask.run`. `run` may return a processed new
98  sub-exposure which can be be "stitched" back into a new resulting
99  larger exposure (depending on the configured
100  `ImageReducer`); otherwise if it does not return an
101  lsst.afw.image.Exposure, then the
102  `ImageReducer.config.reducer.reduceOperation`
103  should be set to 'none' and the result will be propagated
104  as-is.
105  """
106  ConfigClass = ImageMapperConfig
107  _DefaultName = "ip_diffim_ImageMapper"
108 
109  @abc.abstractmethod
110  def run(self, subExposure, expandedSubExposure, fullBBox, **kwargs):
111  """Perform operation on `subExposure`.
112 
113  To be implemented by subclasses. See class docstring for more
114  details. This method is given the `subExposure` which
115  is to be operated upon, and an `expandedSubExposure` which
116  will contain `subExposure` with additional surrounding
117  pixels. This allows for, for example, convolutions (which
118  should be performed on `expandedSubExposure`), to prevent the
119  returned sub-exposure from containing invalid pixels.
120 
121  This method may return a new, processed sub-exposure which can
122  be be "stitched" back into a new resulting larger exposure
123  (depending on the paired, configured `ImageReducer`);
124  otherwise if it does not return an lsst.afw.image.Exposure, then the
125  `ImageReducer.config.mapper.reduceOperation`
126  should be set to 'none' and the result will be propagated
127  as-is.
128 
129  Parameters
130  ----------
131  subExposure : lsst.afw.image.Exposure
132  the sub-exposure upon which to operate
133  expandedSubExposure : lsst.afw.image.Exposure
134  the expanded sub-exposure upon which to operate
135  fullBBox : afwGeom.BoundingBox
136  the bounding box of the original exposure
137  kwargs :
138  additional keyword arguments propagated from
139  `ImageMapReduceTask.run`.
140 
141  Returns
142  -------
143  A `pipeBase.Struct containing the result of the `subExposure` processing,
144  which may itself be of any type. See above for details. If it is an
145  lsst.afw.image.Exposure (processed sub-exposure), then the name in the Struct
146  should be 'subExposure'. This is implemented here as a pass-through
147  example only.
148  """
149  return pipeBase.Struct(subExposure=subExposure)
150 
151 
152 class ImageReducerConfig(pexConfig.Config):
153  """Configuration parameters for the ImageReducer
154  """
155  reduceOperation = pexConfig.ChoiceField(
156  dtype=str,
157  doc="""Operation to use for reducing subimages into new image.""",
158  default="average",
159  allowed={
160  "none": """simply return a list of values and don't re-map results into
161  a new image (noop operation)""",
162  "copy": """copy pixels directly from subimage into correct location in
163  new exposure (potentially non-deterministic for overlaps)""",
164  "sum": """add pixels from overlaps (probably never wanted; used for testing)
165  into correct location in new exposure""",
166  "average": """same as copy, but also average pixels from overlapped regions
167  (NaNs ignored)""",
168  "coaddPsf": """Instead of constructing an Exposure, take a list of returned
169  PSFs and use CoaddPsf to construct a single PSF that covers the
170  entire input exposure""",
171  }
172  )
173  badMaskPlanes = pexConfig.ListField(
174  dtype=str,
175  doc="""Mask planes to set for invalid pixels""",
176  default=('INVALID_MAPREDUCE', 'BAD', 'NO_DATA')
177  )
178 
179 
180 class ImageReducer(pipeBase.Task):
181  """Base class for any 'reduce' task that is to be
182  used as `ImageMapReduceConfig.reducer`.
183 
184  Basic reduce operations are provided by the `run` method
185  of this class, to be selected by its config.
186  """
187  ConfigClass = ImageReducerConfig
188  _DefaultName = "ip_diffim_ImageReducer"
189 
190  def run(self, mapperResults, exposure, **kwargs):
191  """Reduce a list of items produced by `ImageMapper`.
192 
193  Either stitch the passed `mapperResults` list
194  together into a new Exposure (default) or pass it through
195  (if `self.config.reduceOperation` is 'none').
196 
197  If `self.config.reduceOperation` is not 'none', then expect
198  that the `pipeBase.Struct`s in the `mapperResults` list
199  contain sub-exposures named 'subExposure', to be stitched back
200  into a single Exposure with the same dimensions, PSF, and mask
201  as the input `exposure`. Otherwise, the `mapperResults` list
202  is simply returned directly.
203 
204  Parameters
205  ----------
206  mapperResults : list
207  list of `pipeBase.Struct` returned by `ImageMapper.run`.
208  exposure : lsst.afw.image.Exposure
209  the original exposure which is cloned to use as the
210  basis for the resulting exposure (if
211  self.config.mapper.reduceOperation is not 'none')
212  kwargs :
213  additional keyword arguments propagated from
214  `ImageMapReduceTask.run`.
215 
216  Returns
217  -------
218  A `pipeBase.Struct` containing either an `lsst.afw.image.Exposure` (named 'exposure')
219  or a list (named 'result'), depending on `config.reduceOperation`.
220 
221  Notes
222  -----
223  1. This currently correctly handles overlapping sub-exposures.
224  For overlapping sub-exposures, use `config.reduceOperation='average'`.
225  2. This correctly handles varying PSFs, constructing the resulting
226  exposure's PSF via CoaddPsf (DM-9629).
227 
228  Known issues
229  ------------
230  1. To be done: correct handling of masks (nearly there)
231  2. This logic currently makes *two* copies of the original exposure
232  (one here and one in `mapper.run()`). Possibly of concern
233  for large images on memory-constrained systems.
234  """
235  # No-op; simply pass mapperResults directly to ImageMapReduceTask.run
236  if self.config.reduceOperation == 'none':
237  return pipeBase.Struct(result=mapperResults)
238 
239  if self.config.reduceOperation == 'coaddPsf':
240  # Each element of `mapperResults` should contain 'psf' and 'bbox'
241  coaddPsf = self._constructPsf(mapperResults, exposure)
242  return pipeBase.Struct(result=coaddPsf)
243 
244  newExp = exposure.clone()
245  newMI = newExp.getMaskedImage()
246 
247  reduceOp = self.config.reduceOperation
248  if reduceOp == 'copy':
249  weights = None
250  newMI.getImage()[:, :] = np.nan
251  newMI.getVariance()[:, :] = np.nan
252  else:
253  newMI.getImage()[:, :] = 0.
254  newMI.getVariance()[:, :] = 0.
255  if reduceOp == 'average': # make an array to keep track of weights
256  weights = afwImage.ImageI(newMI.getBBox())
257 
258  for item in mapperResults:
259  item = item.subExposure # Expected named value in the pipeBase.Struct
260  if not (isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureI) or
261  isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureD)):
262  raise TypeError("""Expecting an Exposure type, got %s.
263  Consider using `reduceOperation="none".""" % str(type(item)))
264  subExp = newExp.Factory(newExp, item.getBBox())
265  subMI = subExp.getMaskedImage()
266  patchMI = item.getMaskedImage()
267  isValid = ~np.isnan(patchMI.getImage().getArray() * patchMI.getVariance().getArray())
268 
269  if reduceOp == 'copy':
270  subMI.getImage().getArray()[isValid] = patchMI.getImage().getArray()[isValid]
271  subMI.getVariance().getArray()[isValid] = patchMI.getVariance().getArray()[isValid]
272  subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray()
273 
274  if reduceOp == 'sum' or reduceOp == 'average': # much of these two options is the same
275  subMI.getImage().getArray()[isValid] += patchMI.getImage().getArray()[isValid]
276  subMI.getVariance().getArray()[isValid] += patchMI.getVariance().getArray()[isValid]
277  subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray()
278  if reduceOp == 'average':
279  # wtsView is a view into the `weights` Image
280  wtsView = afwImage.ImageI(weights, item.getBBox())
281  wtsView.getArray()[isValid] += 1
282 
283  # New mask plane - for debugging map-reduced images
284  mask = newMI.getMask()
285  for m in self.config.badMaskPlanes:
286  mask.addMaskPlane(m)
287  bad = mask.getPlaneBitMask(self.config.badMaskPlanes)
288 
289  isNan = np.where(np.isnan(newMI.getImage().getArray() * newMI.getVariance().getArray()))
290  if len(isNan[0]) > 0:
291  # set mask to INVALID for pixels where produced exposure is NaN
292  mask.getArray()[isNan[0], isNan[1]] |= bad
293 
294  if reduceOp == 'average':
295  wts = weights.getArray().astype(np.float)
296  self.log.info('AVERAGE: Maximum overlap: %f', np.nanmax(wts))
297  self.log.info('AVERAGE: Average overlap: %f', np.nanmean(wts))
298  self.log.info('AVERAGE: Minimum overlap: %f', np.nanmin(wts))
299  wtsZero = np.equal(wts, 0.)
300  wtsZeroInds = np.where(wtsZero)
301  wtsZeroSum = len(wtsZeroInds[0])
302  self.log.info('AVERAGE: Number of zero pixels: %f (%f%%)', wtsZeroSum,
303  wtsZeroSum * 100. / wtsZero.size)
304  notWtsZero = ~wtsZero
305  tmp = newMI.getImage().getArray()
306  np.divide(tmp, wts, out=tmp, where=notWtsZero)
307  tmp = newMI.getVariance().getArray()
308  np.divide(tmp, wts, out=tmp, where=notWtsZero)
309  if len(wtsZeroInds[0]) > 0:
310  newMI.getImage().getArray()[wtsZeroInds] = np.nan
311  newMI.getVariance().getArray()[wtsZeroInds] = np.nan
312  # set mask to something for pixels where wts == 0.
313  # happens sometimes if operation failed on a certain subexposure
314  mask.getArray()[wtsZeroInds] |= bad
315 
316  # Not sure how to construct a PSF when reduceOp=='copy'...
317  if reduceOp == 'sum' or reduceOp == 'average':
318  psf = self._constructPsf(mapperResults, exposure)
319  newExp.setPsf(psf)
320 
321  return pipeBase.Struct(exposure=newExp)
322 
323  def _constructPsf(self, mapperResults, exposure):
324  """Construct a CoaddPsf based on PSFs from individual subExposures
325 
326  Currently uses (and returns) a CoaddPsf. TBD if we want to
327  create a custom subclass of CoaddPsf to differentiate it.
328 
329  Parameters
330  ----------
331  mapperResults : list
332  list of `pipeBase.Struct` returned by `ImageMapper.run`.
333  For this to work, each element of `mapperResults` must contain
334  a `subExposure` element, from which the component Psfs are
335  extracted (thus the reducerTask cannot have
336  `reduceOperation = 'none'`.
337  exposure : lsst.afw.image.Exposure
338  the original exposure which is used here solely for its
339  bounding-box and WCS.
340 
341  Returns
342  -------
343  A `measAlg.CoaddPsf` constructed from the PSFs of the individual
344  subExposures.
345  """
346  schema = afwTable.ExposureTable.makeMinimalSchema()
347  schema.addField("weight", type="D", doc="Coadd weight")
348  mycatalog = afwTable.ExposureCatalog(schema)
349 
350  # We're just using the exposure's WCS (assuming that the subExposures'
351  # WCSs are the same, which they better be!).
352  wcsref = exposure.getWcs()
353  for i, res in enumerate(mapperResults):
354  record = mycatalog.getTable().makeRecord()
355  if 'subExposure' in res.getDict():
356  subExp = res.subExposure
357  if subExp.getWcs() != wcsref:
358  raise ValueError('Wcs of subExposure is different from exposure')
359  record.setPsf(subExp.getPsf())
360  record.setWcs(subExp.getWcs())
361  record.setBBox(subExp.getBBox())
362  elif 'psf' in res.getDict():
363  record.setPsf(res.psf)
364  record.setWcs(wcsref)
365  record.setBBox(res.bbox)
366  record['weight'] = 1.0
367  record['id'] = i
368  mycatalog.append(record)
369 
370  # create the coaddpsf
371  psf = measAlg.CoaddPsf(mycatalog, wcsref, 'weight')
372  return psf
373 
374 
375 class ImageMapReduceConfig(pexConfig.Config):
376  """Configuration parameters for the ImageMapReduceTask
377  """
378  mapper = pexConfig.ConfigurableField(
379  doc="Task to run on each subimage",
380  target=ImageMapper,
381  )
382 
383  reducer = pexConfig.ConfigurableField(
384  doc="Task to combine results of mapper task",
385  target=ImageReducer,
386  )
387 
388  # Separate cellCentroidsX and cellCentroidsY since pexConfig.ListField accepts limited dtypes
389  # (i.e., no Point2D). The resulting set of centroids is the "vertical stack" of
390  # `cellCentroidsX` and `cellCentroidsY`, i.e. for (1,2), (3,4) respectively, the
391  # resulting centroids are ((1,3), (2,4)).
392  cellCentroidsX = pexConfig.ListField(
393  dtype=float,
394  doc="""Input X centroids around which to place subimages.
395  If None, use grid config options below.""",
396  optional=True,
397  default=None
398  )
399 
400  cellCentroidsY = pexConfig.ListField(
401  dtype=float,
402  doc="""Input Y centroids around which to place subimages.
403  If None, use grid config options below.""",
404  optional=True,
405  default=None
406  )
407 
408  cellSizeX = pexConfig.Field(
409  dtype=float,
410  doc="""Dimensions of each grid cell in x direction""",
411  default=10.,
412  check=lambda x: x > 0.
413  )
414 
415  cellSizeY = pexConfig.Field(
416  dtype=float,
417  doc="""Dimensions of each grid cell in y direction""",
418  default=10.,
419  check=lambda x: x > 0.
420  )
421 
422  gridStepX = pexConfig.Field(
423  dtype=float,
424  doc="""Spacing between subsequent grid cells in x direction. If equal to
425  cellSizeX, then there is no overlap in the x direction.""",
426  default=10.,
427  check=lambda x: x > 0.
428  )
429 
430  gridStepY = pexConfig.Field(
431  dtype=float,
432  doc="""Spacing between subsequent grid cells in y direction. If equal to
433  cellSizeY, then there is no overlap in the y direction.""",
434  default=10.,
435  check=lambda x: x > 0.
436  )
437 
438  borderSizeX = pexConfig.Field(
439  dtype=float,
440  doc="""Dimensions of grid cell border in +/- x direction, to be used
441  for generating `expandedSubExposure`.""",
442  default=5.,
443  check=lambda x: x > 0.
444  )
445 
446  borderSizeY = pexConfig.Field(
447  dtype=float,
448  doc="""Dimensions of grid cell border in +/- y direction, to be used
449  for generating `expandedSubExposure`.""",
450  default=5.,
451  check=lambda x: x > 0.
452  )
453 
454  adjustGridOption = pexConfig.ChoiceField(
455  dtype=str,
456  doc="""Whether and how to adjust grid to fit evenly within, and cover entire
457  image""",
458  default="spacing",
459  allowed={
460  "spacing": "adjust spacing between centers of grid cells (allowing overlaps)",
461  "size": "adjust the sizes of the grid cells (disallowing overlaps)",
462  "none": "do not adjust the grid sizes or spacing"
463  }
464  )
465 
466  scaleByFwhm = pexConfig.Field(
467  dtype=bool,
468  doc="""Scale cellSize/gridStep/borderSize/overlapSize by PSF FWHM rather
469  than pixels?""",
470  default=True
471  )
472 
473  returnSubImages = pexConfig.Field(
474  dtype=bool,
475  doc="""Return the input subExposures alongside the processed ones (for debugging)""",
476  default=False
477  )
478 
479  ignoreMaskPlanes = pexConfig.ListField(
480  dtype=str,
481  doc="""Mask planes to ignore for sigma-clipped statistics""",
482  default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
483  )
484 
485 
486 
492 
493 
494 class ImageMapReduceTask(pipeBase.Task):
495  """Split an Exposure into subExposures (optionally on a grid) and
496  perform the same operation on each.
497 
498  Perform 'simple' operations on a gridded set of subExposures of a
499  larger Exposure, and then (by default) have those subExposures
500  stitched back together into a new, full-sized image.
501 
502  Contrary to the expectation given by its name, this task does not
503  perform these operations in parallel, although it could be updatd
504  to provide such functionality.
505 
506  The actual operations are performed by two subTasks passed to the
507  config. The exposure passed to this task's `run` method will be
508  divided, and those subExposures will be passed to the subTasks,
509  along with the original exposure. The reducing operation is
510  performed by the second subtask.
511  """
512  ConfigClass = ImageMapReduceConfig
513  _DefaultName = "ip_diffim_imageMapReduce"
514 
515  def __init__(self, *args, **kwargs):
516  """Create the image map-reduce task
517 
518  Parameters
519  ----------
520  args :
521  arguments to be passed to
522  `lsst.pipe.base.task.Task.__init__`
523  kwargs :
524  additional keyword arguments to be passed to
525  `lsst.pipe.base.task.Task.__init__`
526  """
527  pipeBase.Task.__init__(self, *args, **kwargs)
528 
529  self.boxes0 = self.boxes1 = None
530  self.makeSubtask("mapper")
531  self.makeSubtask("reducer")
532 
533  @pipeBase.timeMethod
534  def run(self, exposure, **kwargs):
535  """Perform a map-reduce operation on the given exposure.
536 
537  Split the exposure into sub-expposures on a grid (parameters
538  given by `ImageMapReduceConfig`) and perform
539  `config.mapper.run()` on each. Reduce the resulting
540  sub-exposures by running `config.reducer.run()`.
541 
542  Parameters
543  ----------
544  exposure : lsst.afw.image.Exposure
545  the full exposure to process
546  kwargs :
547  additional keyword arguments to be passed to
548  subtask `run` methods
549 
550  Returns
551  -------
552  output of `reducer.run()`
553 
554  """
555  self.log.info("Mapper sub-task: %s", self.mapper._DefaultName)
556  mapperResults = self._runMapper(exposure, **kwargs)
557  self.log.info("Reducer sub-task: %s", self.reducer._DefaultName)
558  result = self._reduceImage(mapperResults, exposure, **kwargs)
559  return result
560 
561  def _runMapper(self, exposure, doClone=False, **kwargs):
562  """Perform `mapper.run` on each sub-exposure
563 
564  Perform `mapper.run` on each sub-exposure across a
565  grid on `exposure` generated by `_generateGrid`. Also pass to
566  `mapper.run` an 'expanded sub-exposure' containing the
567  same region as the sub-exposure but with an expanded bounding box.
568 
569  Parameters
570  ----------
571  exposure : lsst.afw.image.Exposure
572  the original exposure which is used as the template
573  doClone : boolean
574  if True, clone the subimages before passing to subtask;
575  in that case, the sub-exps do not have to be considered as read-only
576  kwargs :
577  additional keyword arguments to be passed to
578  `mapper.run` and `self._generateGrid`, including `forceEvenSized`.
579 
580  Returns
581  -------
582  a list of `pipeBase.Struct`s as returned by `mapper.run`.
583  """
584  if self.boxes0 is None:
585  self._generateGrid(exposure, **kwargs) # possibly pass `forceEvenSized`
586  if len(self.boxes0) != len(self.boxes1):
587  raise ValueError('Bounding boxes list and expanded bounding boxes list are of different lengths')
588 
589  self.log.info("Processing %d sub-exposures", len(self.boxes0))
590  mapperResults = []
591  for box0, box1 in zip(self.boxes0, self.boxes1):
592  subExp = exposure.Factory(exposure, box0)
593  expandedSubExp = exposure.Factory(exposure, box1)
594  if doClone:
595  subExp = subExp.clone()
596  expandedSubExp = expandedSubExp.clone()
597  result = self.mapper.run(subExp, expandedSubExp, exposure.getBBox(), **kwargs)
598  if self.config.returnSubImages:
599  toAdd = pipeBase.Struct(inputSubExposure=subExp,
600  inputExpandedSubExposure=expandedSubExp)
601  result.mergeItems(toAdd, 'inputSubExposure', 'inputExpandedSubExposure')
602  mapperResults.append(result)
603 
604  return mapperResults
605 
606  def _reduceImage(self, mapperResults, exposure, **kwargs):
607  """Reduce/merge a set of sub-exposures into a final result
608 
609  Return an exposure of the same dimensions as `exposure`.
610  `mapperResults` is expected to have been produced by `runMapper`.
611 
612  Parameters
613  ----------
614  mapperResults : list
615  list of `pipeBase.Struct`, each of which was produced by
616  `config.mapper`
617  exposure : lsst.afw.image.Exposure
618  the original exposure
619  **kwargs :
620  additional keyword arguments
621 
622  Returns
623  -------
624  Output of `reducer.run` which is a `pipeBase.Struct`.
625  """
626  result = self.reducer.run(mapperResults, exposure, **kwargs)
627  return result
628 
629  def _generateGrid(self, exposure, forceEvenSized=False, **kwargs):
630  """Generate two lists of bounding boxes that evenly grid `exposure`
631 
632  Unless the config was provided with `cellCentroidsX` and
633  `cellCentroidsY`, grid (subimage) centers are spaced evenly
634  by gridStepX/Y. Then the grid is adjusted as little as
635  possible to evenly cover the input exposure (if
636  adjustGridOption is not 'none'). Then the second set of
637  bounding boxes is expanded by borderSizeX/Y. The expanded
638  bounding boxes are adjusted to ensure that they intersect the
639  exposure's bounding box. The resulting lists of bounding boxes
640  and corresponding expanded bounding boxes are set to
641  `self.boxes0`, `self.boxes1`.
642 
643  Parameters
644  ----------
645  exposure : lsst.afw.image.Exposure
646  input exposure whose full bounding box is to be evenly gridded.
647  forceEvenSized : boolean
648  force grid elements to have even-valued x- and y- dimensions?
649  (Potentially useful if doing Fourier transform of subExposures.)
650  """
651  # kwargs are ignored, but necessary to enable optional passing of
652  # `forceEvenSized` from `_runMapper`.
653  bbox = exposure.getBBox()
654 
655  # Extract the config parameters for conciseness.
656  cellCentroidsX = self.config.cellCentroidsX
657  cellCentroidsY = self.config.cellCentroidsY
658  cellSizeX = self.config.cellSizeX
659  cellSizeY = self.config.cellSizeY
660  gridStepX = self.config.gridStepX
661  gridStepY = self.config.gridStepY
662  borderSizeX = self.config.borderSizeX
663  borderSizeY = self.config.borderSizeY
664  adjustGridOption = self.config.adjustGridOption
665  scaleByFwhm = self.config.scaleByFwhm
666 
667  if cellCentroidsX is None or len(cellCentroidsX) <= 0:
668  # Not given centroids; construct them from cellSize/gridStep
669 
670  psfFwhm = (exposure.getPsf().computeShape().getDeterminantRadius() *
671  2.*np.sqrt(2.*np.log(2.)))
672  if scaleByFwhm:
673  self.log.info("Scaling grid parameters by %f" % psfFwhm)
674 
675  def rescaleValue(val):
676  if scaleByFwhm:
677  return np.rint(val*psfFwhm).astype(int)
678  else:
679  return np.rint(val).astype(int)
680 
681  cellSizeX = rescaleValue(cellSizeX)
682  cellSizeY = rescaleValue(cellSizeY)
683  gridStepX = rescaleValue(gridStepX)
684  gridStepY = rescaleValue(gridStepY)
685  borderSizeX = rescaleValue(borderSizeX)
686  borderSizeY = rescaleValue(borderSizeY)
687 
688  nGridX = bbox.getWidth()//gridStepX
689  nGridY = bbox.getHeight()//gridStepY
690 
691  if adjustGridOption == 'spacing':
692  # Readjust spacings so that they fit perfectly in the image.
693  nGridX = bbox.getWidth()//cellSizeX + 1
694  nGridY = bbox.getHeight()//cellSizeY + 1
695  xLinSpace = np.linspace(cellSizeX//2, bbox.getWidth() - cellSizeX//2, nGridX)
696  yLinSpace = np.linspace(cellSizeY//2, bbox.getHeight() - cellSizeY//2, nGridY)
697 
698  elif adjustGridOption == 'size':
699  cellSizeX = gridStepX
700  cellSizeY = gridStepY
701  xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, cellSizeX)
702  yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, cellSizeY)
703  cellSizeX += 1 # add 1 to make sure there are no gaps
704  cellSizeY += 1
705 
706  else:
707  xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, gridStepX)
708  yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, gridStepY)
709 
710  cellCentroids = [(x, y) for x in xLinSpace for y in yLinSpace]
711 
712  else:
713  # in py3 zip returns an iterator, but want to test length below, so use this instead:
714  cellCentroids = [(cellCentroidsX[i], cellCentroidsY[i]) for i in range(len(cellCentroidsX))]
715 
716  # first "main" box at 0,0
717  bbox0 = afwGeom.Box2I(afwGeom.Point2I(bbox.getBegin()), afwGeom.Extent2I(cellSizeX, cellSizeY))
718  # first expanded box
719  bbox1 = afwGeom.Box2I(bbox0)
720  bbox1.grow(afwGeom.Extent2I(borderSizeX, borderSizeY))
721 
722  self.boxes0 = [] # "main" boxes; store in task so can be extracted if needed
723  self.boxes1 = [] # "expanded" boxes
724 
725  def _makeBoxEvenSized(bb):
726  """Force a bounding-box to have dimensions that are modulo 2."""
727 
728  if bb.getWidth() % 2 == 1: # grow to the right
729  bb.include(afwGeom.Point2I(bb.getMaxX()+1, bb.getMaxY())) # Expand by 1 pixel!
730  bb.clip(bbox)
731  if bb.getWidth() % 2 == 1: # clipped at right -- so grow to the left
732  bb.include(afwGeom.Point2I(bb.getMinX()-1, bb.getMaxY()))
733  bb.clip(bbox)
734  if bb.getHeight() % 2 == 1: # grow upwards
735  bb.include(afwGeom.Point2I(bb.getMaxX(), bb.getMaxY()+1)) # Expand by 1 pixel!
736  bb.clip(bbox)
737  if bb.getHeight() % 2 == 1: # clipped upwards -- so grow down
738  bb.include(afwGeom.Point2I(bb.getMaxX(), bb.getMinY()-1))
739  bb.clip(bbox)
740  if bb.getWidth() % 2 == 1 or bb.getHeight() % 2 == 1: # Box is probably too big
741  raise RuntimeError('Cannot make bounding box even-sized. Probably too big.')
742 
743  return bb
744 
745  # Use given or grid-parameterized centroids as centers for bounding boxes
746  if cellCentroids is not None and len(cellCentroids) > 0:
747  for x, y in cellCentroids:
748  centroid = afwGeom.Point2D(x, y)
749  bb0 = afwGeom.Box2I(bbox0)
750  xoff = int(np.floor(centroid.getX())) - bb0.getWidth()//2
751  yoff = int(np.floor(centroid.getY())) - bb0.getHeight()//2
752  bb0.shift(afwGeom.Extent2I(xoff, yoff))
753  bb0.clip(bbox)
754  if forceEvenSized:
755  bb0 = _makeBoxEvenSized(bb0)
756  bb1 = afwGeom.Box2I(bbox1)
757  bb1.shift(afwGeom.Extent2I(xoff, yoff))
758  bb1.clip(bbox)
759  if forceEvenSized:
760  bb1 = _makeBoxEvenSized(bb1)
761 
762  if bb0.getArea() > 1 and bb1.getArea() > 1:
763  self.boxes0.append(bb0)
764  self.boxes1.append(bb1)
765 
766  return self.boxes0, self.boxes1
767 
768  def plotBoxes(self, fullBBox, skip=3):
769  """Plot both grids of boxes using matplotlib.
770 
771  Will compute the grid via `_generateGrid` if
772  `self.boxes0` and `self.boxes1` have not already been set.
773 
774  Parameters
775  ----------
776  exposure : lsst.afw.image.Exposure
777  Exposure whose bounding box is gridded by this task.
778  skip : int
779  Plot every skip-ped box (help make plots less confusing)
780  """
781  import matplotlib.pyplot as plt
782 
783  if self.boxes0 is None:
784  raise RuntimeError('Cannot plot boxes. Run _generateGrid first.')
785  self._plotBoxGrid(self.boxes0[::skip], fullBBox, ls='--')
786  # reset the color cycle -- see
787  # http://stackoverflow.com/questions/24193174/reset-color-cycle-in-matplotlib
788  plt.gca().set_prop_cycle(None)
789  self._plotBoxGrid(self.boxes1[::skip], fullBBox, ls=':')
790 
791  def _plotBoxGrid(self, boxes, bbox, **kwargs):
792  """Plot a grid of boxes using matplotlib.
793 
794  Parameters
795  ----------
796  boxes : list
797  a list of `afwGeom.BoundingBox`es
798  bbox : afwGeom.BoundingBox
799  an overall bounding box
800  **kwargs :
801  additional keyword arguments for matplotlib
802  """
803  import matplotlib.pyplot as plt
804 
805  def plotBox(box):
806  corners = np.array([np.array([pt.getX(), pt.getY()]) for pt in box.getCorners()])
807  corners = np.vstack([corners, corners[0, :]])
808  plt.plot(corners[:, 0], corners[:, 1], **kwargs)
809 
810  for b in boxes:
811  plotBox(b)
812  plt.xlim(bbox.getBeginX(), bbox.getEndX())
813  plt.ylim(bbox.getBeginY(), bbox.getEndY())
def _generateGrid(self, exposure, forceEvenSized=False, kwargs)
def _reduceImage(self, mapperResults, exposure, kwargs)
def _constructPsf(self, mapperResults, exposure)
def _runMapper(self, exposure, doClone=False, kwargs)
def run(self, subExposure, expandedSubExposure, fullBBox, kwargs)
def run(self, mapperResults, exposure, kwargs)