Coverage for python/lsst/ip/diffim/imageMapReduce.py: 15%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

264 statements  

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24import abc 

25 

26import lsst.afw.image as afwImage 

27import lsst.afw.table as afwTable 

28import lsst.geom as geom 

29import lsst.meas.algorithms as measAlg 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32 

33__all__ = ("ImageMapReduceTask", "ImageMapReduceConfig", 

34 "ImageMapper", "ImageMapperConfig", 

35 "ImageReducer", "ImageReducerConfig") 

36 

37 

38"""Tasks for processing an exposure via processing on 

39multiple sub-exposures and then collecting the results 

40to either re-stitch the sub-exposures back into a new 

41exposure, or return summary results for each sub-exposure. 

42 

43This provides a framework for arbitrary mapper-reducer 

44operations on an exposure by implementing simple operations in 

45subTasks. It currently is not parallelized, although it could be in 

46the future. It does enable operations such as spatially-mapped 

47processing on a grid across an image, processing regions surrounding 

48centroids (such as for PSF processing), etc. 

49 

50It is implemented as primary Task, `ImageMapReduceTask` which contains 

51two subtasks, `ImageMapper` and `ImageReducer`. 

52`ImageMapReduceTask` configures the centroids and sub-exposure 

53dimensions to be processed, and then calls the `run` methods of the 

54`ImageMapper` and `ImageReducer` on those sub-exposures. 

55`ImageMapReduceTask` may be configured with a list of sub-exposure 

56centroids (`config.cellCentroidsX` and `config.cellCentroidsY`) and a 

57single pair of bounding boxes defining their dimensions, or a set of 

58parameters defining a regular grid of centroids (`config.gridStepX` 

59and `config.gridStepY`). 

60 

61`ImageMapper` is an abstract class and must be subclassed with 

62an implemented `run` method to provide the desired operation for 

63processing individual sub-exposures. It is called from 

64`ImageMapReduceTask.run`, and may return a new, processed sub-exposure 

65which is to be "stitched" back into a new resulting larger exposure 

66(depending on the configured `ImageMapReduceTask.mapper`); 

67otherwise if it does not return an lsst.afw.image.Exposure, then the results are 

68passed back directly to the caller. 

69 

70`ImageReducer` will either stitch the `mapperResults` list of 

71results generated by the `ImageMapper` together into a new 

72Exposure (by default) or pass it through to the 

73caller. `ImageReducer` has an implemented `run` method for 

74basic reducing operations (`reduceOperation`) such as `average` (which 

75will average all overlapping pixels from sub-exposures produced by the 

76`ImageMapper` into the new exposure). Another notable 

77implemented `reduceOperation` is 'none', in which case the 

78`mapperResults` list is simply returned directly. 

79""" 

80 

81 

82class ImageMapperConfig(pexConfig.Config): 

83 """Configuration parameters for ImageMapper 

84 """ 

85 pass 

86 

87 

88class ImageMapper(pipeBase.Task, metaclass=abc.ABCMeta): 

89 """Abstract base class for any task that is to be 

90 used as `ImageMapReduceConfig.mapper`. 

91 

92 Notes 

93 ----- 

94 An `ImageMapper` is responsible for processing individual 

95 sub-exposures in its `run` method, which is called from 

96 `ImageMapReduceTask.run`. `run` may return a processed new 

97 sub-exposure which can be be "stitched" back into a new resulting 

98 larger exposure (depending on the configured 

99 `ImageReducer`); otherwise if it does not return an 

100 lsst.afw.image.Exposure, then the 

101 `ImageReducer.config.reducer.reduceOperation` 

102 should be set to 'none' and the result will be propagated 

103 as-is. 

104 """ 

105 ConfigClass = ImageMapperConfig 

106 _DefaultName = "ip_diffim_ImageMapper" 

107 

108 @abc.abstractmethod 

109 def run(self, subExposure, expandedSubExposure, fullBBox, **kwargs): 

110 """Perform operation on `subExposure`. 

111 

112 To be implemented by subclasses. See class docstring for more 

113 details. This method is given the `subExposure` which 

114 is to be operated upon, and an `expandedSubExposure` which 

115 will contain `subExposure` with additional surrounding 

116 pixels. This allows for, for example, convolutions (which 

117 should be performed on `expandedSubExposure`), to prevent the 

118 returned sub-exposure from containing invalid pixels. 

119 

120 This method may return a new, processed sub-exposure which can 

121 be be "stitched" back into a new resulting larger exposure 

122 (depending on the paired, configured `ImageReducer`); 

123 otherwise if it does not return an lsst.afw.image.Exposure, then the 

124 `ImageReducer.config.mapper.reduceOperation` 

125 should be set to 'none' and the result will be propagated 

126 as-is. 

127 

128 Parameters 

129 ---------- 

130 subExposure : `lsst.afw.image.Exposure` 

131 the sub-exposure upon which to operate 

132 expandedSubExposure : `lsst.afw.image.Exposure` 

133 the expanded sub-exposure upon which to operate 

134 fullBBox : `lsst.geom.Box2I` 

135 the bounding box of the original exposure 

136 kwargs : 

137 additional keyword arguments propagated from 

138 `ImageMapReduceTask.run`. 

139 

140 Returns 

141 ------- 

142 result : `lsst.pipe.base.Struct` 

143 A structure containing the result of the `subExposure` processing, 

144 which may itself be of any type. See above for details. If it is an 

145 `lsst.afw.image.Exposure` (processed sub-exposure), then the name in 

146 the Struct should be 'subExposure'. This is implemented here as a 

147 pass-through example only. 

148 """ 

149 return pipeBase.Struct(subExposure=subExposure) 

150 

151 

152class ImageReducerConfig(pexConfig.Config): 

153 """Configuration parameters for the ImageReducer 

154 """ 

155 reduceOperation = pexConfig.ChoiceField( 

156 dtype=str, 

157 doc="""Operation to use for reducing subimages into new image.""", 

158 default="average", 

159 allowed={ 

160 "none": """simply return a list of values and don't re-map results into 

161 a new image (noop operation)""", 

162 "copy": """copy pixels directly from subimage into correct location in 

163 new exposure (potentially non-deterministic for overlaps)""", 

164 "sum": """add pixels from overlaps (probably never wanted; used for testing) 

165 into correct location in new exposure""", 

166 "average": """same as copy, but also average pixels from overlapped regions 

167 (NaNs ignored)""", 

168 "coaddPsf": """Instead of constructing an Exposure, take a list of returned 

169 PSFs and use CoaddPsf to construct a single PSF that covers the 

170 entire input exposure""", 

171 } 

172 ) 

173 badMaskPlanes = pexConfig.ListField( 

174 dtype=str, 

175 doc="""Mask planes to set for invalid pixels""", 

176 default=('INVALID_MAPREDUCE', 'BAD', 'NO_DATA') 

177 ) 

178 

179 

180class ImageReducer(pipeBase.Task): 

181 """Base class for any 'reduce' task that is to be 

182 used as `ImageMapReduceConfig.reducer`. 

183 

184 Basic reduce operations are provided by the `run` method 

185 of this class, to be selected by its config. 

186 """ 

187 ConfigClass = ImageReducerConfig 

188 _DefaultName = "ip_diffim_ImageReducer" 

189 

190 def run(self, mapperResults, exposure, **kwargs): 

191 """Reduce a list of items produced by `ImageMapper`. 

192 

193 Either stitch the passed `mapperResults` list 

194 together into a new Exposure (default) or pass it through 

195 (if `self.config.reduceOperation` is 'none'). 

196 

197 If `self.config.reduceOperation` is not 'none', then expect 

198 that the `pipeBase.Struct`s in the `mapperResults` list 

199 contain sub-exposures named 'subExposure', to be stitched back 

200 into a single Exposure with the same dimensions, PSF, and mask 

201 as the input `exposure`. Otherwise, the `mapperResults` list 

202 is simply returned directly. 

203 

204 Parameters 

205 ---------- 

206 mapperResults : `list` 

207 list of `lsst.pipe.base.Struct` returned by `ImageMapper.run`. 

208 exposure : `lsst.afw.image.Exposure` 

209 the original exposure which is cloned to use as the 

210 basis for the resulting exposure (if 

211 ``self.config.mapper.reduceOperation`` is not 'None') 

212 kwargs : 

213 additional keyword arguments propagated from 

214 `ImageMapReduceTask.run`. 

215 

216 Returns 

217 ------- 

218 A `lsst.pipe.base.Struct` containing either an `lsst.afw.image.Exposure` 

219 (named 'exposure') or a list (named 'result'), 

220 depending on `config.reduceOperation`. 

221 

222 Notes 

223 ----- 

224 1. This currently correctly handles overlapping sub-exposures. 

225 For overlapping sub-exposures, use `config.reduceOperation='average'`. 

226 2. This correctly handles varying PSFs, constructing the resulting 

227 exposure's PSF via CoaddPsf (DM-9629). 

228 

229 Known issues 

230 

231 1. To be done: correct handling of masks (nearly there) 

232 2. This logic currently makes *two* copies of the original exposure 

233 (one here and one in `mapper.run()`). Possibly of concern 

234 for large images on memory-constrained systems. 

235 """ 

236 # No-op; simply pass mapperResults directly to ImageMapReduceTask.run 

237 if self.config.reduceOperation == 'none': 

238 return pipeBase.Struct(result=mapperResults) 

239 

240 if self.config.reduceOperation == 'coaddPsf': 

241 # Each element of `mapperResults` should contain 'psf' and 'bbox' 

242 coaddPsf = self._constructPsf(mapperResults, exposure) 

243 return pipeBase.Struct(result=coaddPsf) 

244 

245 newExp = exposure.clone() 

246 newMI = newExp.getMaskedImage() 

247 

248 reduceOp = self.config.reduceOperation 

249 if reduceOp == 'copy': 

250 weights = None 

251 newMI.getImage()[:, :] = np.nan 

252 newMI.getVariance()[:, :] = np.nan 

253 else: 

254 newMI.getImage()[:, :] = 0. 

255 newMI.getVariance()[:, :] = 0. 

256 if reduceOp == 'average': # make an array to keep track of weights 

257 weights = afwImage.ImageI(newMI.getBBox()) 

258 

259 for item in mapperResults: 

260 item = item.subExposure # Expected named value in the pipeBase.Struct 

261 if not (isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureI) 

262 or isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureD)): 

263 raise TypeError("""Expecting an Exposure type, got %s. 

264 Consider using `reduceOperation="none".""" % str(type(item))) 

265 subExp = newExp.Factory(newExp, item.getBBox()) 

266 subMI = subExp.getMaskedImage() 

267 patchMI = item.getMaskedImage() 

268 isValid = ~np.isnan(patchMI.getImage().getArray() * patchMI.getVariance().getArray()) 

269 

270 if reduceOp == 'copy': 

271 subMI.getImage().getArray()[isValid] = patchMI.getImage().getArray()[isValid] 

272 subMI.getVariance().getArray()[isValid] = patchMI.getVariance().getArray()[isValid] 

273 subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray() 

274 

275 if reduceOp == 'sum' or reduceOp == 'average': # much of these two options is the same 

276 subMI.getImage().getArray()[isValid] += patchMI.getImage().getArray()[isValid] 

277 subMI.getVariance().getArray()[isValid] += patchMI.getVariance().getArray()[isValid] 

278 subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray() 

279 if reduceOp == 'average': 

280 # wtsView is a view into the `weights` Image 

281 wtsView = afwImage.ImageI(weights, item.getBBox()) 

282 wtsView.getArray()[isValid] += 1 

283 

284 # New mask plane - for debugging map-reduced images 

285 mask = newMI.getMask() 

286 for m in self.config.badMaskPlanes: 

287 mask.addMaskPlane(m) 

288 bad = mask.getPlaneBitMask(self.config.badMaskPlanes) 

289 

290 isNan = np.where(np.isnan(newMI.getImage().getArray() * newMI.getVariance().getArray())) 

291 if len(isNan[0]) > 0: 

292 # set mask to INVALID for pixels where produced exposure is NaN 

293 mask.getArray()[isNan[0], isNan[1]] |= bad 

294 

295 if reduceOp == 'average': 

296 wts = weights.getArray().astype(np.float) 

297 self.log.info('AVERAGE: Maximum overlap: %f', np.nanmax(wts)) 

298 self.log.info('AVERAGE: Average overlap: %f', np.nanmean(wts)) 

299 self.log.info('AVERAGE: Minimum overlap: %f', np.nanmin(wts)) 

300 wtsZero = np.equal(wts, 0.) 

301 wtsZeroInds = np.where(wtsZero) 

302 wtsZeroSum = len(wtsZeroInds[0]) 

303 self.log.info('AVERAGE: Number of zero pixels: %f (%f%%)', wtsZeroSum, 

304 wtsZeroSum * 100. / wtsZero.size) 

305 notWtsZero = ~wtsZero 

306 tmp = newMI.getImage().getArray() 

307 np.divide(tmp, wts, out=tmp, where=notWtsZero) 

308 tmp = newMI.getVariance().getArray() 

309 np.divide(tmp, wts, out=tmp, where=notWtsZero) 

310 if len(wtsZeroInds[0]) > 0: 

311 newMI.getImage().getArray()[wtsZeroInds] = np.nan 

312 newMI.getVariance().getArray()[wtsZeroInds] = np.nan 

313 # set mask to something for pixels where wts == 0. 

314 # happens sometimes if operation failed on a certain subexposure 

315 mask.getArray()[wtsZeroInds] |= bad 

316 

317 # Not sure how to construct a PSF when reduceOp=='copy'... 

318 if reduceOp == 'sum' or reduceOp == 'average': 

319 psf = self._constructPsf(mapperResults, exposure) 

320 newExp.setPsf(psf) 

321 

322 return pipeBase.Struct(exposure=newExp) 

323 

324 def _constructPsf(self, mapperResults, exposure): 

325 """Construct a CoaddPsf based on PSFs from individual subExposures 

326 

327 Currently uses (and returns) a CoaddPsf. TBD if we want to 

328 create a custom subclass of CoaddPsf to differentiate it. 

329 

330 Parameters 

331 ---------- 

332 mapperResults : `list` 

333 list of `pipeBase.Struct` returned by `ImageMapper.run`. 

334 For this to work, each element of `mapperResults` must contain 

335 a `subExposure` element, from which the component Psfs are 

336 extracted (thus the reducerTask cannot have 

337 `reduceOperation = 'none'`. 

338 exposure : `lsst.afw.image.Exposure` 

339 the original exposure which is used here solely for its 

340 bounding-box and WCS. 

341 

342 Returns 

343 ------- 

344 psf : `lsst.meas.algorithms.CoaddPsf` 

345 A psf constructed from the PSFs of the individual subExposures. 

346 """ 

347 schema = afwTable.ExposureTable.makeMinimalSchema() 

348 schema.addField("weight", type="D", doc="Coadd weight") 

349 mycatalog = afwTable.ExposureCatalog(schema) 

350 

351 # We're just using the exposure's WCS (assuming that the subExposures' 

352 # WCSs are the same, which they better be!). 

353 wcsref = exposure.getWcs() 

354 for i, res in enumerate(mapperResults): 

355 record = mycatalog.getTable().makeRecord() 

356 if 'subExposure' in res.getDict(): 

357 subExp = res.subExposure 

358 if subExp.getWcs() != wcsref: 

359 raise ValueError('Wcs of subExposure is different from exposure') 

360 record.setPsf(subExp.getPsf()) 

361 record.setWcs(subExp.getWcs()) 

362 record.setBBox(subExp.getBBox()) 

363 elif 'psf' in res.getDict(): 

364 record.setPsf(res.psf) 

365 record.setWcs(wcsref) 

366 record.setBBox(res.bbox) 

367 record['weight'] = 1.0 

368 record['id'] = i 

369 mycatalog.append(record) 

370 

371 # create the coaddpsf 

372 psf = measAlg.CoaddPsf(mycatalog, wcsref, 'weight') 

373 return psf 

374 

375 

376class ImageMapReduceConfig(pexConfig.Config): 

377 """Configuration parameters for the ImageMapReduceTask 

378 """ 

379 mapper = pexConfig.ConfigurableField( 

380 doc="Task to run on each subimage", 

381 target=ImageMapper, 

382 ) 

383 

384 reducer = pexConfig.ConfigurableField( 

385 doc="Task to combine results of mapper task", 

386 target=ImageReducer, 

387 ) 

388 

389 # Separate cellCentroidsX and cellCentroidsY since pexConfig.ListField accepts limited dtypes 

390 # (i.e., no Point2D). The resulting set of centroids is the "vertical stack" of 

391 # `cellCentroidsX` and `cellCentroidsY`, i.e. for (1,2), (3,4) respectively, the 

392 # resulting centroids are ((1,3), (2,4)). 

393 cellCentroidsX = pexConfig.ListField( 

394 dtype=float, 

395 doc="""Input X centroids around which to place subimages. 

396 If None, use grid config options below.""", 

397 optional=True, 

398 default=None 

399 ) 

400 

401 cellCentroidsY = pexConfig.ListField( 

402 dtype=float, 

403 doc="""Input Y centroids around which to place subimages. 

404 If None, use grid config options below.""", 

405 optional=True, 

406 default=None 

407 ) 

408 

409 cellSizeX = pexConfig.Field( 409 ↛ exitline 409 didn't jump to the function exit

410 dtype=float, 

411 doc="""Dimensions of each grid cell in x direction""", 

412 default=10., 

413 check=lambda x: x > 0. 

414 ) 

415 

416 cellSizeY = pexConfig.Field( 416 ↛ exitline 416 didn't jump to the function exit

417 dtype=float, 

418 doc="""Dimensions of each grid cell in y direction""", 

419 default=10., 

420 check=lambda x: x > 0. 

421 ) 

422 

423 gridStepX = pexConfig.Field( 423 ↛ exitline 423 didn't jump to the function exit

424 dtype=float, 

425 doc="""Spacing between subsequent grid cells in x direction. If equal to 

426 cellSizeX, then there is no overlap in the x direction.""", 

427 default=10., 

428 check=lambda x: x > 0. 

429 ) 

430 

431 gridStepY = pexConfig.Field( 431 ↛ exitline 431 didn't jump to the function exit

432 dtype=float, 

433 doc="""Spacing between subsequent grid cells in y direction. If equal to 

434 cellSizeY, then there is no overlap in the y direction.""", 

435 default=10., 

436 check=lambda x: x > 0. 

437 ) 

438 

439 borderSizeX = pexConfig.Field( 439 ↛ exitline 439 didn't jump to the function exit

440 dtype=float, 

441 doc="""Dimensions of grid cell border in +/- x direction, to be used 

442 for generating `expandedSubExposure`.""", 

443 default=5., 

444 check=lambda x: x > 0. 

445 ) 

446 

447 borderSizeY = pexConfig.Field( 447 ↛ exitline 447 didn't jump to the function exit

448 dtype=float, 

449 doc="""Dimensions of grid cell border in +/- y direction, to be used 

450 for generating `expandedSubExposure`.""", 

451 default=5., 

452 check=lambda x: x > 0. 

453 ) 

454 

455 adjustGridOption = pexConfig.ChoiceField( 

456 dtype=str, 

457 doc="""Whether and how to adjust grid to fit evenly within, and cover entire 

458 image""", 

459 default="spacing", 

460 allowed={ 

461 "spacing": "adjust spacing between centers of grid cells (allowing overlaps)", 

462 "size": "adjust the sizes of the grid cells (disallowing overlaps)", 

463 "none": "do not adjust the grid sizes or spacing" 

464 } 

465 ) 

466 

467 scaleByFwhm = pexConfig.Field( 

468 dtype=bool, 

469 doc="""Scale cellSize/gridStep/borderSize/overlapSize by PSF FWHM rather 

470 than pixels?""", 

471 default=True 

472 ) 

473 

474 returnSubImages = pexConfig.Field( 

475 dtype=bool, 

476 doc="""Return the input subExposures alongside the processed ones (for debugging)""", 

477 default=False 

478 ) 

479 

480 ignoreMaskPlanes = pexConfig.ListField( 

481 dtype=str, 

482 doc="""Mask planes to ignore for sigma-clipped statistics""", 

483 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

484 ) 

485 

486 

487class ImageMapReduceTask(pipeBase.Task): 

488 """Split an Exposure into subExposures (optionally on a grid) and 

489 perform the same operation on each. 

490 

491 Perform 'simple' operations on a gridded set of subExposures of a 

492 larger Exposure, and then (by default) have those subExposures 

493 stitched back together into a new, full-sized image. 

494 

495 Contrary to the expectation given by its name, this task does not 

496 perform these operations in parallel, although it could be updatd 

497 to provide such functionality. 

498 

499 The actual operations are performed by two subTasks passed to the 

500 config. The exposure passed to this task's `run` method will be 

501 divided, and those subExposures will be passed to the subTasks, 

502 along with the original exposure. The reducing operation is 

503 performed by the second subtask. 

504 """ 

505 ConfigClass = ImageMapReduceConfig 

506 _DefaultName = "ip_diffim_imageMapReduce" 

507 

508 def __init__(self, *args, **kwargs): 

509 """Create the image map-reduce task 

510 

511 Parameters 

512 ---------- 

513 args : 

514 arguments to be passed to 

515 `lsst.pipe.base.task.Task.__init__` 

516 kwargs : 

517 additional keyword arguments to be passed to 

518 `lsst.pipe.base.task.Task.__init__` 

519 """ 

520 pipeBase.Task.__init__(self, *args, **kwargs) 

521 

522 self.boxes0 = self.boxes1 = None 

523 self.makeSubtask("mapper") 

524 self.makeSubtask("reducer") 

525 

526 @pipeBase.timeMethod 

527 def run(self, exposure, **kwargs): 

528 """Perform a map-reduce operation on the given exposure. 

529 

530 Split the exposure into sub-expposures on a grid (parameters 

531 given by `ImageMapReduceConfig`) and perform 

532 `config.mapper.run()` on each. Reduce the resulting 

533 sub-exposures by running `config.reducer.run()`. 

534 

535 Parameters 

536 ---------- 

537 exposure : `lsst.afw.image.Exposure` 

538 the full exposure to process 

539 kwargs : 

540 additional keyword arguments to be passed to 

541 subtask `run` methods 

542 

543 Returns 

544 ------- 

545 output of `reducer.run()` 

546 

547 """ 

548 self.log.info("Mapper sub-task: %s", self.mapper._DefaultName) 

549 mapperResults = self._runMapper(exposure, **kwargs) 

550 self.log.info("Reducer sub-task: %s", self.reducer._DefaultName) 

551 result = self._reduceImage(mapperResults, exposure, **kwargs) 

552 return result 

553 

554 def _runMapper(self, exposure, doClone=False, **kwargs): 

555 """Perform `mapper.run` on each sub-exposure 

556 

557 Perform `mapper.run` on each sub-exposure across a 

558 grid on `exposure` generated by `_generateGrid`. Also pass to 

559 `mapper.run` an 'expanded sub-exposure' containing the 

560 same region as the sub-exposure but with an expanded bounding box. 

561 

562 Parameters 

563 ---------- 

564 exposure : `lsst.afw.image.Exposure` 

565 the original exposure which is used as the template 

566 doClone : `bool` 

567 if True, clone the subimages before passing to subtask; 

568 in that case, the sub-exps do not have to be considered as read-only 

569 kwargs : 

570 additional keyword arguments to be passed to 

571 `mapper.run` and `self._generateGrid`, including `forceEvenSized`. 

572 

573 Returns 

574 ------- 

575 a list of `pipeBase.Struct`s as returned by `mapper.run`. 

576 """ 

577 if self.boxes0 is None: 

578 self._generateGrid(exposure, **kwargs) # possibly pass `forceEvenSized` 

579 if len(self.boxes0) != len(self.boxes1): 

580 raise ValueError('Bounding boxes list and expanded bounding boxes list are of different lengths') 

581 

582 self.log.info("Processing %d sub-exposures", len(self.boxes0)) 

583 mapperResults = [] 

584 for box0, box1 in zip(self.boxes0, self.boxes1): 

585 subExp = exposure.Factory(exposure, box0) 

586 expandedSubExp = exposure.Factory(exposure, box1) 

587 if doClone: 

588 subExp = subExp.clone() 

589 expandedSubExp = expandedSubExp.clone() 

590 result = self.mapper.run(subExp, expandedSubExp, exposure.getBBox(), **kwargs) 

591 if self.config.returnSubImages: 

592 toAdd = pipeBase.Struct(inputSubExposure=subExp, 

593 inputExpandedSubExposure=expandedSubExp) 

594 result.mergeItems(toAdd, 'inputSubExposure', 'inputExpandedSubExposure') 

595 mapperResults.append(result) 

596 

597 return mapperResults 

598 

599 def _reduceImage(self, mapperResults, exposure, **kwargs): 

600 """Reduce/merge a set of sub-exposures into a final result 

601 

602 Return an exposure of the same dimensions as `exposure`. 

603 `mapperResults` is expected to have been produced by `runMapper`. 

604 

605 Parameters 

606 ---------- 

607 mapperResults : `list` 

608 `list` of `lsst.pipe.base.Struct`, each of which was produced by 

609 `config.mapper` 

610 exposure : `lsst.afw.image.Exposure` 

611 the original exposure 

612 **kwargs 

613 additional keyword arguments 

614 

615 Returns 

616 ------- 

617 Output of `reducer.run` which is a `pipeBase.Struct`. 

618 """ 

619 result = self.reducer.run(mapperResults, exposure, **kwargs) 

620 return result 

621 

622 def _generateGrid(self, exposure, forceEvenSized=False, **kwargs): 

623 """Generate two lists of bounding boxes that evenly grid `exposure` 

624 

625 Unless the config was provided with `cellCentroidsX` and 

626 `cellCentroidsY`, grid (subimage) centers are spaced evenly 

627 by gridStepX/Y. Then the grid is adjusted as little as 

628 possible to evenly cover the input exposure (if 

629 adjustGridOption is not 'none'). Then the second set of 

630 bounding boxes is expanded by borderSizeX/Y. The expanded 

631 bounding boxes are adjusted to ensure that they intersect the 

632 exposure's bounding box. The resulting lists of bounding boxes 

633 and corresponding expanded bounding boxes are set to 

634 `self.boxes0`, `self.boxes1`. 

635 

636 Parameters 

637 ---------- 

638 exposure : `lsst.afw.image.Exposure` 

639 input exposure whose full bounding box is to be evenly gridded. 

640 forceEvenSized : `bool` 

641 force grid elements to have even-valued x- and y- dimensions? 

642 (Potentially useful if doing Fourier transform of subExposures.) 

643 """ 

644 # kwargs are ignored, but necessary to enable optional passing of 

645 # `forceEvenSized` from `_runMapper`. 

646 bbox = exposure.getBBox() 

647 

648 # Extract the config parameters for conciseness. 

649 cellCentroidsX = self.config.cellCentroidsX 

650 cellCentroidsY = self.config.cellCentroidsY 

651 cellSizeX = self.config.cellSizeX 

652 cellSizeY = self.config.cellSizeY 

653 gridStepX = self.config.gridStepX 

654 gridStepY = self.config.gridStepY 

655 borderSizeX = self.config.borderSizeX 

656 borderSizeY = self.config.borderSizeY 

657 adjustGridOption = self.config.adjustGridOption 

658 scaleByFwhm = self.config.scaleByFwhm 

659 

660 if cellCentroidsX is None or len(cellCentroidsX) <= 0: 

661 # Not given centroids; construct them from cellSize/gridStep 

662 

663 psfFwhm = (exposure.getPsf().computeShape().getDeterminantRadius() 

664 * 2.*np.sqrt(2.*np.log(2.))) 

665 if scaleByFwhm: 

666 self.log.info("Scaling grid parameters by %f", psfFwhm) 

667 

668 def rescaleValue(val): 

669 if scaleByFwhm: 

670 return np.rint(val*psfFwhm).astype(int) 

671 else: 

672 return np.rint(val).astype(int) 

673 

674 cellSizeX = rescaleValue(cellSizeX) 

675 cellSizeY = rescaleValue(cellSizeY) 

676 gridStepX = rescaleValue(gridStepX) 

677 gridStepY = rescaleValue(gridStepY) 

678 borderSizeX = rescaleValue(borderSizeX) 

679 borderSizeY = rescaleValue(borderSizeY) 

680 

681 nGridX = bbox.getWidth()//gridStepX 

682 nGridY = bbox.getHeight()//gridStepY 

683 

684 if adjustGridOption == 'spacing': 

685 # Readjust spacings so that they fit perfectly in the image. 

686 nGridX = bbox.getWidth()//cellSizeX + 1 

687 nGridY = bbox.getHeight()//cellSizeY + 1 

688 xLinSpace = np.linspace(cellSizeX//2, bbox.getWidth() - cellSizeX//2, nGridX) 

689 yLinSpace = np.linspace(cellSizeY//2, bbox.getHeight() - cellSizeY//2, nGridY) 

690 

691 elif adjustGridOption == 'size': 

692 cellSizeX = gridStepX 

693 cellSizeY = gridStepY 

694 xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, cellSizeX) 

695 yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, cellSizeY) 

696 cellSizeX += 1 # add 1 to make sure there are no gaps 

697 cellSizeY += 1 

698 

699 else: 

700 xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, gridStepX) 

701 yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, gridStepY) 

702 

703 cellCentroids = [(x, y) for x in xLinSpace for y in yLinSpace] 

704 

705 else: 

706 # in py3 zip returns an iterator, but want to test length below, so use this instead: 

707 cellCentroids = [(cellCentroidsX[i], cellCentroidsY[i]) for i in range(len(cellCentroidsX))] 

708 

709 # first "main" box at 0,0 

710 bbox0 = geom.Box2I(geom.Point2I(bbox.getBegin()), geom.Extent2I(cellSizeX, cellSizeY)) 

711 # first expanded box 

712 bbox1 = geom.Box2I(bbox0) 

713 bbox1.grow(geom.Extent2I(borderSizeX, borderSizeY)) 

714 

715 self.boxes0 = [] # "main" boxes; store in task so can be extracted if needed 

716 self.boxes1 = [] # "expanded" boxes 

717 

718 def _makeBoxEvenSized(bb): 

719 """Force a bounding-box to have dimensions that are modulo 2.""" 

720 

721 if bb.getWidth() % 2 == 1: # grow to the right 

722 bb.include(geom.Point2I(bb.getMaxX()+1, bb.getMaxY())) # Expand by 1 pixel! 

723 bb.clip(bbox) 

724 if bb.getWidth() % 2 == 1: # clipped at right -- so grow to the left 

725 bb.include(geom.Point2I(bb.getMinX()-1, bb.getMaxY())) 

726 bb.clip(bbox) 

727 if bb.getHeight() % 2 == 1: # grow upwards 

728 bb.include(geom.Point2I(bb.getMaxX(), bb.getMaxY()+1)) # Expand by 1 pixel! 

729 bb.clip(bbox) 

730 if bb.getHeight() % 2 == 1: # clipped upwards -- so grow down 

731 bb.include(geom.Point2I(bb.getMaxX(), bb.getMinY()-1)) 

732 bb.clip(bbox) 

733 if bb.getWidth() % 2 == 1 or bb.getHeight() % 2 == 1: # Box is probably too big 

734 raise RuntimeError('Cannot make bounding box even-sized. Probably too big.') 

735 

736 return bb 

737 

738 # Use given or grid-parameterized centroids as centers for bounding boxes 

739 if cellCentroids is not None and len(cellCentroids) > 0: 

740 for x, y in cellCentroids: 

741 centroid = geom.Point2D(x, y) 

742 bb0 = geom.Box2I(bbox0) 

743 xoff = int(np.floor(centroid.getX())) - bb0.getWidth()//2 

744 yoff = int(np.floor(centroid.getY())) - bb0.getHeight()//2 

745 bb0.shift(geom.Extent2I(xoff, yoff)) 

746 bb0.clip(bbox) 

747 if forceEvenSized: 

748 bb0 = _makeBoxEvenSized(bb0) 

749 bb1 = geom.Box2I(bbox1) 

750 bb1.shift(geom.Extent2I(xoff, yoff)) 

751 bb1.clip(bbox) 

752 if forceEvenSized: 

753 bb1 = _makeBoxEvenSized(bb1) 

754 

755 if bb0.getArea() > 1 and bb1.getArea() > 1: 

756 self.boxes0.append(bb0) 

757 self.boxes1.append(bb1) 

758 

759 return self.boxes0, self.boxes1 

760 

761 def plotBoxes(self, fullBBox, skip=3): 

762 """Plot both grids of boxes using matplotlib. 

763 

764 Will compute the grid via `_generateGrid` if 

765 `self.boxes0` and `self.boxes1` have not already been set. 

766 

767 Parameters 

768 ---------- 

769 exposure : `lsst.afw.image.Exposure` 

770 Exposure whose bounding box is gridded by this task. 

771 skip : `int` 

772 Plot every skip-ped box (help make plots less confusing) 

773 """ 

774 import matplotlib.pyplot as plt 

775 

776 if self.boxes0 is None: 

777 raise RuntimeError('Cannot plot boxes. Run _generateGrid first.') 

778 self._plotBoxGrid(self.boxes0[::skip], fullBBox, ls='--') 

779 # reset the color cycle -- see 

780 # http://stackoverflow.com/questions/24193174/reset-color-cycle-in-matplotlib 

781 plt.gca().set_prop_cycle(None) 

782 self._plotBoxGrid(self.boxes1[::skip], fullBBox, ls=':') 

783 

784 def _plotBoxGrid(self, boxes, bbox, **kwargs): 

785 """Plot a grid of boxes using matplotlib. 

786 

787 Parameters 

788 ---------- 

789 boxes : `list` of `lsst.geom.Box2I` 

790 a list of bounding boxes. 

791 bbox : `lsst.geom.Box2I` 

792 an overall bounding box 

793 **kwargs 

794 additional keyword arguments for matplotlib 

795 """ 

796 import matplotlib.pyplot as plt 

797 

798 def plotBox(box): 

799 corners = np.array([np.array([pt.getX(), pt.getY()]) for pt in box.getCorners()]) 

800 corners = np.vstack([corners, corners[0, :]]) 

801 plt.plot(corners[:, 0], corners[:, 1], **kwargs) 

802 

803 for b in boxes: 

804 plotBox(b) 

805 plt.xlim(bbox.getBeginX(), bbox.getEndX()) 

806 plt.ylim(bbox.getBeginY(), bbox.getEndY())