Coverage for python/lsst/ip/diffim/imageMapReduce.py: 16%

267 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-11 03:16 -0800

1# 

2# LSST Data Management System 

3# Copyright 2016 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24import abc 

25 

26import lsst.afw.image as afwImage 

27import lsst.afw.table as afwTable 

28import lsst.geom as geom 

29import lsst.meas.algorithms as measAlg 

30import lsst.pex.config as pexConfig 

31import lsst.pipe.base as pipeBase 

32from lsst.utils.timer import timeMethod 

33 

34__all__ = ("ImageMapReduceTask", "ImageMapReduceConfig", 

35 "ImageMapper", "ImageMapperConfig", 

36 "ImageReducer", "ImageReducerConfig") 

37 

38 

39"""Tasks for processing an exposure via processing on 

40multiple sub-exposures and then collecting the results 

41to either re-stitch the sub-exposures back into a new 

42exposure, or return summary results for each sub-exposure. 

43 

44This provides a framework for arbitrary mapper-reducer 

45operations on an exposure by implementing simple operations in 

46subTasks. It currently is not parallelized, although it could be in 

47the future. It does enable operations such as spatially-mapped 

48processing on a grid across an image, processing regions surrounding 

49centroids (such as for PSF processing), etc. 

50 

51It is implemented as primary Task, `ImageMapReduceTask` which contains 

52two subtasks, `ImageMapper` and `ImageReducer`. 

53`ImageMapReduceTask` configures the centroids and sub-exposure 

54dimensions to be processed, and then calls the `run` methods of the 

55`ImageMapper` and `ImageReducer` on those sub-exposures. 

56`ImageMapReduceTask` may be configured with a list of sub-exposure 

57centroids (`config.cellCentroidsX` and `config.cellCentroidsY`) and a 

58single pair of bounding boxes defining their dimensions, or a set of 

59parameters defining a regular grid of centroids (`config.gridStepX` 

60and `config.gridStepY`). 

61 

62`ImageMapper` is an abstract class and must be subclassed with 

63an implemented `run` method to provide the desired operation for 

64processing individual sub-exposures. It is called from 

65`ImageMapReduceTask.run`, and may return a new, processed sub-exposure 

66which is to be "stitched" back into a new resulting larger exposure 

67(depending on the configured `ImageMapReduceTask.mapper`); 

68otherwise if it does not return an lsst.afw.image.Exposure, then the results are 

69passed back directly to the caller. 

70 

71`ImageReducer` will either stitch the `mapperResults` list of 

72results generated by the `ImageMapper` together into a new 

73Exposure (by default) or pass it through to the 

74caller. `ImageReducer` has an implemented `run` method for 

75basic reducing operations (`reduceOperation`) such as `average` (which 

76will average all overlapping pixels from sub-exposures produced by the 

77`ImageMapper` into the new exposure). Another notable 

78implemented `reduceOperation` is 'none', in which case the 

79`mapperResults` list is simply returned directly. 

80""" 

81 

82 

83class ImageMapperConfig(pexConfig.Config): 

84 """Configuration parameters for ImageMapper 

85 """ 

86 pass 

87 

88 

89class ImageMapper(pipeBase.Task, metaclass=abc.ABCMeta): 

90 """Abstract base class for any task that is to be 

91 used as `ImageMapReduceConfig.mapper`. 

92 

93 Notes 

94 ----- 

95 An `ImageMapper` is responsible for processing individual 

96 sub-exposures in its `run` method, which is called from 

97 `ImageMapReduceTask.run`. `run` may return a processed new 

98 sub-exposure which can be be "stitched" back into a new resulting 

99 larger exposure (depending on the configured 

100 `ImageReducer`); otherwise if it does not return an 

101 lsst.afw.image.Exposure, then the 

102 `ImageReducer.config.reducer.reduceOperation` 

103 should be set to 'none' and the result will be propagated 

104 as-is. 

105 """ 

106 ConfigClass = ImageMapperConfig 

107 _DefaultName = "ip_diffim_ImageMapper" 

108 

109 @abc.abstractmethod 

110 def run(self, subExposure, expandedSubExposure, fullBBox, **kwargs): 

111 """Perform operation on `subExposure`. 

112 

113 To be implemented by subclasses. See class docstring for more 

114 details. This method is given the `subExposure` which 

115 is to be operated upon, and an `expandedSubExposure` which 

116 will contain `subExposure` with additional surrounding 

117 pixels. This allows for, for example, convolutions (which 

118 should be performed on `expandedSubExposure`), to prevent the 

119 returned sub-exposure from containing invalid pixels. 

120 

121 This method may return a new, processed sub-exposure which can 

122 be be "stitched" back into a new resulting larger exposure 

123 (depending on the paired, configured `ImageReducer`); 

124 otherwise if it does not return an lsst.afw.image.Exposure, then the 

125 `ImageReducer.config.mapper.reduceOperation` 

126 should be set to 'none' and the result will be propagated 

127 as-is. 

128 

129 Parameters 

130 ---------- 

131 subExposure : `lsst.afw.image.Exposure` 

132 the sub-exposure upon which to operate 

133 expandedSubExposure : `lsst.afw.image.Exposure` 

134 the expanded sub-exposure upon which to operate 

135 fullBBox : `lsst.geom.Box2I` 

136 the bounding box of the original exposure 

137 kwargs : 

138 additional keyword arguments propagated from 

139 `ImageMapReduceTask.run`. 

140 

141 Returns 

142 ------- 

143 result : `lsst.pipe.base.Struct` 

144 A structure containing the result of the `subExposure` processing, 

145 which may itself be of any type. See above for details. If it is an 

146 `lsst.afw.image.Exposure` (processed sub-exposure), then the name in 

147 the Struct should be 'subExposure'. This is implemented here as a 

148 pass-through example only. 

149 """ 

150 return pipeBase.Struct(subExposure=subExposure) 

151 

152 

153class ImageReducerConfig(pexConfig.Config): 

154 """Configuration parameters for the ImageReducer 

155 """ 

156 reduceOperation = pexConfig.ChoiceField( 

157 dtype=str, 

158 doc="""Operation to use for reducing subimages into new image.""", 

159 default="average", 

160 allowed={ 

161 "none": """simply return a list of values and don't re-map results into 

162 a new image (noop operation)""", 

163 "copy": """copy pixels directly from subimage into correct location in 

164 new exposure (potentially non-deterministic for overlaps)""", 

165 "sum": """add pixels from overlaps (probably never wanted; used for testing) 

166 into correct location in new exposure""", 

167 "average": """same as copy, but also average pixels from overlapped regions 

168 (NaNs ignored)""", 

169 "coaddPsf": """Instead of constructing an Exposure, take a list of returned 

170 PSFs and use CoaddPsf to construct a single PSF that covers the 

171 entire input exposure""", 

172 } 

173 ) 

174 badMaskPlanes = pexConfig.ListField( 

175 dtype=str, 

176 doc="""Mask planes to set for invalid pixels""", 

177 default=('INVALID_MAPREDUCE', 'BAD', 'NO_DATA') 

178 ) 

179 

180 

181class ImageReducer(pipeBase.Task): 

182 """Base class for any 'reduce' task that is to be 

183 used as `ImageMapReduceConfig.reducer`. 

184 

185 Basic reduce operations are provided by the `run` method 

186 of this class, to be selected by its config. 

187 """ 

188 ConfigClass = ImageReducerConfig 

189 _DefaultName = "ip_diffim_ImageReducer" 

190 

191 def run(self, mapperResults, exposure, **kwargs): 

192 """Reduce a list of items produced by `ImageMapper`. 

193 

194 Either stitch the passed `mapperResults` list 

195 together into a new Exposure (default) or pass it through 

196 (if `self.config.reduceOperation` is 'none'). 

197 

198 If `self.config.reduceOperation` is not 'none', then expect 

199 that the `pipeBase.Struct`s in the `mapperResults` list 

200 contain sub-exposures named 'subExposure', to be stitched back 

201 into a single Exposure with the same dimensions, PSF, and mask 

202 as the input `exposure`. Otherwise, the `mapperResults` list 

203 is simply returned directly. 

204 

205 Parameters 

206 ---------- 

207 mapperResults : `list` 

208 list of `lsst.pipe.base.Struct` returned by `ImageMapper.run`. 

209 exposure : `lsst.afw.image.Exposure` 

210 the original exposure which is cloned to use as the 

211 basis for the resulting exposure (if 

212 ``self.config.mapper.reduceOperation`` is not 'None') 

213 kwargs : 

214 additional keyword arguments propagated from 

215 `ImageMapReduceTask.run`. 

216 

217 Returns 

218 ------- 

219 A `lsst.pipe.base.Struct` containing either an `lsst.afw.image.Exposure` 

220 (named 'exposure') or a list (named 'result'), 

221 depending on `config.reduceOperation`. 

222 

223 Notes 

224 ----- 

225 1. This currently correctly handles overlapping sub-exposures. 

226 For overlapping sub-exposures, use `config.reduceOperation='average'`. 

227 2. This correctly handles varying PSFs, constructing the resulting 

228 exposure's PSF via CoaddPsf (DM-9629). 

229 

230 Known issues 

231 

232 1. To be done: correct handling of masks (nearly there) 

233 2. This logic currently makes *two* copies of the original exposure 

234 (one here and one in `mapper.run()`). Possibly of concern 

235 for large images on memory-constrained systems. 

236 """ 

237 # No-op; simply pass mapperResults directly to ImageMapReduceTask.run 

238 if self.config.reduceOperation == 'none': 

239 return pipeBase.Struct(result=mapperResults) 

240 

241 if self.config.reduceOperation == 'coaddPsf': 

242 # Each element of `mapperResults` should contain 'psf' and 'bbox' 

243 coaddPsf = self._constructPsf(mapperResults, exposure) 

244 return pipeBase.Struct(result=coaddPsf) 

245 

246 newExp = exposure.clone() 

247 newMI = newExp.getMaskedImage() 

248 

249 reduceOp = self.config.reduceOperation 

250 if reduceOp == 'copy': 

251 weights = None 

252 newMI.getImage()[:, :] = np.nan 

253 newMI.getVariance()[:, :] = np.nan 

254 else: 

255 newMI.getImage()[:, :] = 0. 

256 newMI.getVariance()[:, :] = 0. 

257 if reduceOp == 'average': # make an array to keep track of weights 

258 weights = afwImage.ImageI(newMI.getBBox()) 

259 

260 for item in mapperResults: 

261 item = item.subExposure # Expected named value in the pipeBase.Struct 

262 if not (isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureI) 

263 or isinstance(item, afwImage.ExposureU) or isinstance(item, afwImage.ExposureD)): 

264 raise TypeError("""Expecting an Exposure type, got %s. 

265 Consider using `reduceOperation="none".""" % str(type(item))) 

266 subExp = newExp.Factory(newExp, item.getBBox()) 

267 subMI = subExp.getMaskedImage() 

268 patchMI = item.getMaskedImage() 

269 isValid = ~np.isnan(patchMI.getImage().getArray() * patchMI.getVariance().getArray()) 

270 

271 if reduceOp == 'copy': 

272 subMI.getImage().getArray()[isValid] = patchMI.getImage().getArray()[isValid] 

273 subMI.getVariance().getArray()[isValid] = patchMI.getVariance().getArray()[isValid] 

274 subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray() 

275 

276 if reduceOp == 'sum' or reduceOp == 'average': # much of these two options is the same 

277 subMI.getImage().getArray()[isValid] += patchMI.getImage().getArray()[isValid] 

278 subMI.getVariance().getArray()[isValid] += patchMI.getVariance().getArray()[isValid] 

279 subMI.getMask().getArray()[:, :] |= patchMI.getMask().getArray() 

280 if reduceOp == 'average': 

281 # wtsView is a view into the `weights` Image 

282 wtsView = afwImage.ImageI(weights, item.getBBox()) 

283 wtsView.getArray()[isValid] += 1 

284 

285 # New mask plane - for debugging map-reduced images 

286 mask = newMI.getMask() 

287 for m in self.config.badMaskPlanes: 

288 mask.addMaskPlane(m) 

289 bad = mask.getPlaneBitMask(self.config.badMaskPlanes) 

290 

291 isNan = np.where(np.isnan(newMI.getImage().getArray() * newMI.getVariance().getArray())) 

292 if len(isNan[0]) > 0: 

293 # set mask to INVALID for pixels where produced exposure is NaN 

294 mask.getArray()[isNan[0], isNan[1]] |= bad 

295 

296 if reduceOp == 'average': 

297 wts = weights.getArray().astype(float) 

298 self.log.info('AVERAGE: Maximum overlap: %f', np.nanmax(wts)) 

299 self.log.info('AVERAGE: Average overlap: %f', np.nanmean(wts)) 

300 self.log.info('AVERAGE: Minimum overlap: %f', np.nanmin(wts)) 

301 wtsZero = np.equal(wts, 0.) 

302 wtsZeroInds = np.where(wtsZero) 

303 wtsZeroSum = len(wtsZeroInds[0]) 

304 self.log.info('AVERAGE: Number of zero pixels: %f (%f%%)', wtsZeroSum, 

305 wtsZeroSum * 100. / wtsZero.size) 

306 notWtsZero = ~wtsZero 

307 tmp = newMI.getImage().getArray() 

308 np.divide(tmp, wts, out=tmp, where=notWtsZero) 

309 tmp = newMI.getVariance().getArray() 

310 np.divide(tmp, wts, out=tmp, where=notWtsZero) 

311 if len(wtsZeroInds[0]) > 0: 

312 newMI.getImage().getArray()[wtsZeroInds] = np.nan 

313 newMI.getVariance().getArray()[wtsZeroInds] = np.nan 

314 # set mask to something for pixels where wts == 0. 

315 # happens sometimes if operation failed on a certain subexposure 

316 mask.getArray()[wtsZeroInds] |= bad 

317 

318 # Not sure how to construct a PSF when reduceOp=='copy'... 

319 if reduceOp == 'sum' or reduceOp == 'average': 

320 psf = self._constructPsf(mapperResults, exposure) 

321 newExp.setPsf(psf) 

322 

323 return pipeBase.Struct(exposure=newExp) 

324 

325 def _constructPsf(self, mapperResults, exposure): 

326 """Construct a CoaddPsf based on PSFs from individual subExposures 

327 

328 Currently uses (and returns) a CoaddPsf. TBD if we want to 

329 create a custom subclass of CoaddPsf to differentiate it. 

330 

331 Parameters 

332 ---------- 

333 mapperResults : `list` 

334 list of `pipeBase.Struct` returned by `ImageMapper.run`. 

335 For this to work, each element of `mapperResults` must contain 

336 a `subExposure` element, from which the component Psfs are 

337 extracted (thus the reducerTask cannot have 

338 `reduceOperation = 'none'`. 

339 exposure : `lsst.afw.image.Exposure` 

340 the original exposure which is used here solely for its 

341 bounding-box and WCS. 

342 

343 Returns 

344 ------- 

345 psf : `lsst.meas.algorithms.CoaddPsf` 

346 A psf constructed from the PSFs of the individual subExposures. 

347 """ 

348 schema = afwTable.ExposureTable.makeMinimalSchema() 

349 schema.addField("weight", type="D", doc="Coadd weight") 

350 mycatalog = afwTable.ExposureCatalog(schema) 

351 

352 # We're just using the exposure's WCS (assuming that the subExposures' 

353 # WCSs are the same, which they better be!). 

354 wcsref = exposure.getWcs() 

355 for i, res in enumerate(mapperResults): 

356 record = mycatalog.getTable().makeRecord() 

357 if 'subExposure' in res.getDict(): 

358 subExp = res.subExposure 

359 if subExp.getWcs() != wcsref: 

360 raise ValueError('Wcs of subExposure is different from exposure') 

361 record.setPsf(subExp.getPsf()) 

362 record.setWcs(subExp.getWcs()) 

363 record.setBBox(subExp.getBBox()) 

364 elif 'psf' in res.getDict(): 

365 record.setPsf(res.psf) 

366 record.setWcs(wcsref) 

367 record.setBBox(res.bbox) 

368 record['weight'] = 1.0 

369 record['id'] = i 

370 mycatalog.append(record) 

371 

372 # create the coaddpsf 

373 psf = measAlg.CoaddPsf(mycatalog, wcsref, 'weight') 

374 return psf 

375 

376 

377class ImageMapReduceConfig(pexConfig.Config): 

378 """Configuration parameters for the ImageMapReduceTask 

379 """ 

380 mapper = pexConfig.ConfigurableField( 

381 doc="Task to run on each subimage", 

382 target=ImageMapper, 

383 ) 

384 

385 reducer = pexConfig.ConfigurableField( 

386 doc="Task to combine results of mapper task", 

387 target=ImageReducer, 

388 ) 

389 

390 # Separate cellCentroidsX and cellCentroidsY since pexConfig.ListField accepts limited dtypes 

391 # (i.e., no Point2D). The resulting set of centroids is the "vertical stack" of 

392 # `cellCentroidsX` and `cellCentroidsY`, i.e. for (1,2), (3,4) respectively, the 

393 # resulting centroids are ((1,3), (2,4)). 

394 cellCentroidsX = pexConfig.ListField( 

395 dtype=float, 

396 doc="""Input X centroids around which to place subimages. 

397 If None, use grid config options below.""", 

398 optional=True, 

399 default=None 

400 ) 

401 

402 cellCentroidsY = pexConfig.ListField( 

403 dtype=float, 

404 doc="""Input Y centroids around which to place subimages. 

405 If None, use grid config options below.""", 

406 optional=True, 

407 default=None 

408 ) 

409 

410 cellSizeX = pexConfig.Field( 410 ↛ exitline 410 didn't jump to the function exit

411 dtype=float, 

412 doc="""Dimensions of each grid cell in x direction""", 

413 default=10., 

414 check=lambda x: x > 0. 

415 ) 

416 

417 cellSizeY = pexConfig.Field( 417 ↛ exitline 417 didn't jump to the function exit

418 dtype=float, 

419 doc="""Dimensions of each grid cell in y direction""", 

420 default=10., 

421 check=lambda x: x > 0. 

422 ) 

423 

424 gridStepX = pexConfig.Field( 424 ↛ exitline 424 didn't jump to the function exit

425 dtype=float, 

426 doc="""Spacing between subsequent grid cells in x direction. If equal to 

427 cellSizeX, then there is no overlap in the x direction.""", 

428 default=10., 

429 check=lambda x: x > 0. 

430 ) 

431 

432 gridStepY = pexConfig.Field( 432 ↛ exitline 432 didn't jump to the function exit

433 dtype=float, 

434 doc="""Spacing between subsequent grid cells in y direction. If equal to 

435 cellSizeY, then there is no overlap in the y direction.""", 

436 default=10., 

437 check=lambda x: x > 0. 

438 ) 

439 

440 borderSizeX = pexConfig.Field( 440 ↛ exitline 440 didn't jump to the function exit

441 dtype=float, 

442 doc="""Dimensions of grid cell border in +/- x direction, to be used 

443 for generating `expandedSubExposure`.""", 

444 default=5., 

445 check=lambda x: x > 0. 

446 ) 

447 

448 borderSizeY = pexConfig.Field( 448 ↛ exitline 448 didn't jump to the function exit

449 dtype=float, 

450 doc="""Dimensions of grid cell border in +/- y direction, to be used 

451 for generating `expandedSubExposure`.""", 

452 default=5., 

453 check=lambda x: x > 0. 

454 ) 

455 

456 adjustGridOption = pexConfig.ChoiceField( 

457 dtype=str, 

458 doc="""Whether and how to adjust grid to fit evenly within, and cover entire 

459 image""", 

460 default="spacing", 

461 allowed={ 

462 "spacing": "adjust spacing between centers of grid cells (allowing overlaps)", 

463 "size": "adjust the sizes of the grid cells (disallowing overlaps)", 

464 "none": "do not adjust the grid sizes or spacing" 

465 } 

466 ) 

467 

468 scaleByFwhm = pexConfig.Field( 

469 dtype=bool, 

470 doc="""Scale cellSize/gridStep/borderSize/overlapSize by PSF FWHM rather 

471 than pixels?""", 

472 default=True 

473 ) 

474 

475 returnSubImages = pexConfig.Field( 

476 dtype=bool, 

477 doc="""Return the input subExposures alongside the processed ones (for debugging)""", 

478 default=False 

479 ) 

480 

481 ignoreMaskPlanes = pexConfig.ListField( 

482 dtype=str, 

483 doc="""Mask planes to ignore for sigma-clipped statistics""", 

484 default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE") 

485 ) 

486 

487 

488class ImageMapReduceTask(pipeBase.Task): 

489 """Split an Exposure into subExposures (optionally on a grid) and 

490 perform the same operation on each. 

491 

492 Perform 'simple' operations on a gridded set of subExposures of a 

493 larger Exposure, and then (by default) have those subExposures 

494 stitched back together into a new, full-sized image. 

495 

496 Contrary to the expectation given by its name, this task does not 

497 perform these operations in parallel, although it could be updatd 

498 to provide such functionality. 

499 

500 The actual operations are performed by two subTasks passed to the 

501 config. The exposure passed to this task's `run` method will be 

502 divided, and those subExposures will be passed to the subTasks, 

503 along with the original exposure. The reducing operation is 

504 performed by the second subtask. 

505 """ 

506 ConfigClass = ImageMapReduceConfig 

507 _DefaultName = "ip_diffim_imageMapReduce" 

508 

509 def __init__(self, *args, **kwargs): 

510 """Create the image map-reduce task 

511 

512 Parameters 

513 ---------- 

514 args : 

515 arguments to be passed to 

516 `lsst.pipe.base.task.Task.__init__` 

517 kwargs : 

518 additional keyword arguments to be passed to 

519 `lsst.pipe.base.task.Task.__init__` 

520 """ 

521 pipeBase.Task.__init__(self, *args, **kwargs) 

522 

523 self.boxes0 = self.boxes1 = None 

524 self.makeSubtask("mapper") 

525 self.makeSubtask("reducer") 

526 

527 @timeMethod 

528 def run(self, exposure, **kwargs): 

529 """Perform a map-reduce operation on the given exposure. 

530 

531 Split the exposure into sub-expposures on a grid (parameters 

532 given by `ImageMapReduceConfig`) and perform 

533 `config.mapper.run()` on each. Reduce the resulting 

534 sub-exposures by running `config.reducer.run()`. 

535 

536 Parameters 

537 ---------- 

538 exposure : `lsst.afw.image.Exposure` 

539 the full exposure to process 

540 kwargs : 

541 additional keyword arguments to be passed to 

542 subtask `run` methods 

543 

544 Returns 

545 ------- 

546 output of `reducer.run()` 

547 

548 """ 

549 self.log.info("Mapper sub-task: %s", self.mapper._DefaultName) 

550 mapperResults = self._runMapper(exposure, **kwargs) 

551 self.log.info("Reducer sub-task: %s", self.reducer._DefaultName) 

552 result = self._reduceImage(mapperResults, exposure, **kwargs) 

553 return result 

554 

555 def _runMapper(self, exposure, doClone=False, **kwargs): 

556 """Perform `mapper.run` on each sub-exposure 

557 

558 Perform `mapper.run` on each sub-exposure across a 

559 grid on `exposure` generated by `_generateGrid`. Also pass to 

560 `mapper.run` an 'expanded sub-exposure' containing the 

561 same region as the sub-exposure but with an expanded bounding box. 

562 

563 Parameters 

564 ---------- 

565 exposure : `lsst.afw.image.Exposure` 

566 the original exposure which is used as the template 

567 doClone : `bool` 

568 if True, clone the subimages before passing to subtask; 

569 in that case, the sub-exps do not have to be considered as read-only 

570 kwargs : 

571 additional keyword arguments to be passed to 

572 `mapper.run` and `self._generateGrid`, including `forceEvenSized`. 

573 

574 Returns 

575 ------- 

576 a list of `pipeBase.Struct`s as returned by `mapper.run`. 

577 """ 

578 if self.boxes0 is None: 

579 self._generateGrid(exposure, **kwargs) # possibly pass `forceEvenSized` 

580 if len(self.boxes0) != len(self.boxes1): 

581 raise ValueError('Bounding boxes list and expanded bounding boxes list are of different lengths') 

582 

583 self.log.info("Processing %d sub-exposures", len(self.boxes0)) 

584 mapperResults = [] 

585 for box0, box1 in zip(self.boxes0, self.boxes1): 

586 subExp = exposure.Factory(exposure, box0) 

587 expandedSubExp = exposure.Factory(exposure, box1) 

588 if doClone: 

589 subExp = subExp.clone() 

590 expandedSubExp = expandedSubExp.clone() 

591 result = self.mapper.run(subExp, expandedSubExp, exposure.getBBox(), **kwargs) 

592 if self.config.returnSubImages: 

593 toAdd = pipeBase.Struct(inputSubExposure=subExp, 

594 inputExpandedSubExposure=expandedSubExp) 

595 result.mergeItems(toAdd, 'inputSubExposure', 'inputExpandedSubExposure') 

596 mapperResults.append(result) 

597 

598 return mapperResults 

599 

600 def _reduceImage(self, mapperResults, exposure, **kwargs): 

601 """Reduce/merge a set of sub-exposures into a final result 

602 

603 Return an exposure of the same dimensions as `exposure`. 

604 `mapperResults` is expected to have been produced by `runMapper`. 

605 

606 Parameters 

607 ---------- 

608 mapperResults : `list` 

609 `list` of `lsst.pipe.base.Struct`, each of which was produced by 

610 `config.mapper` 

611 exposure : `lsst.afw.image.Exposure` 

612 the original exposure 

613 **kwargs 

614 additional keyword arguments 

615 

616 Returns 

617 ------- 

618 Output of `reducer.run` which is a `pipeBase.Struct`. 

619 """ 

620 result = self.reducer.run(mapperResults, exposure, **kwargs) 

621 return result 

622 

623 def _generateGrid(self, exposure, forceEvenSized=False, **kwargs): 

624 """Generate two lists of bounding boxes that evenly grid `exposure` 

625 

626 Unless the config was provided with `cellCentroidsX` and 

627 `cellCentroidsY`, grid (subimage) centers are spaced evenly 

628 by gridStepX/Y. Then the grid is adjusted as little as 

629 possible to evenly cover the input exposure (if 

630 adjustGridOption is not 'none'). Then the second set of 

631 bounding boxes is expanded by borderSizeX/Y. The expanded 

632 bounding boxes are adjusted to ensure that they intersect the 

633 exposure's bounding box. The resulting lists of bounding boxes 

634 and corresponding expanded bounding boxes are set to 

635 `self.boxes0`, `self.boxes1`. 

636 

637 Parameters 

638 ---------- 

639 exposure : `lsst.afw.image.Exposure` 

640 input exposure whose full bounding box is to be evenly gridded. 

641 forceEvenSized : `bool` 

642 force grid elements to have even-valued x- and y- dimensions? 

643 (Potentially useful if doing Fourier transform of subExposures.) 

644 """ 

645 # kwargs are ignored, but necessary to enable optional passing of 

646 # `forceEvenSized` from `_runMapper`. 

647 bbox = exposure.getBBox() 

648 

649 # Extract the config parameters for conciseness. 

650 cellCentroidsX = self.config.cellCentroidsX 

651 cellCentroidsY = self.config.cellCentroidsY 

652 cellSizeX = self.config.cellSizeX 

653 cellSizeY = self.config.cellSizeY 

654 gridStepX = self.config.gridStepX 

655 gridStepY = self.config.gridStepY 

656 borderSizeX = self.config.borderSizeX 

657 borderSizeY = self.config.borderSizeY 

658 adjustGridOption = self.config.adjustGridOption 

659 scaleByFwhm = self.config.scaleByFwhm 

660 

661 if cellCentroidsX is None or len(cellCentroidsX) <= 0: 

662 # Not given centroids; construct them from cellSize/gridStep 

663 psf = exposure.getPsf() 

664 psfFwhm = (psf.computeShape(psf.getAveragePosition()).getDeterminantRadius() 

665 * 2.*np.sqrt(2.*np.log(2.))) 

666 if scaleByFwhm: 

667 self.log.info("Scaling grid parameters by %f", psfFwhm) 

668 

669 def rescaleValue(val): 

670 if scaleByFwhm: 

671 return np.rint(val*psfFwhm).astype(int) 

672 else: 

673 return np.rint(val).astype(int) 

674 

675 cellSizeX = rescaleValue(cellSizeX) 

676 cellSizeY = rescaleValue(cellSizeY) 

677 gridStepX = rescaleValue(gridStepX) 

678 gridStepY = rescaleValue(gridStepY) 

679 borderSizeX = rescaleValue(borderSizeX) 

680 borderSizeY = rescaleValue(borderSizeY) 

681 

682 nGridX = bbox.getWidth()//gridStepX 

683 nGridY = bbox.getHeight()//gridStepY 

684 

685 if adjustGridOption == 'spacing': 

686 # Readjust spacings so that they fit perfectly in the image. 

687 nGridX = bbox.getWidth()//cellSizeX + 1 

688 nGridY = bbox.getHeight()//cellSizeY + 1 

689 xLinSpace = np.linspace(cellSizeX//2, bbox.getWidth() - cellSizeX//2, nGridX) 

690 yLinSpace = np.linspace(cellSizeY//2, bbox.getHeight() - cellSizeY//2, nGridY) 

691 

692 elif adjustGridOption == 'size': 

693 cellSizeX = gridStepX 

694 cellSizeY = gridStepY 

695 xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, cellSizeX) 

696 yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, cellSizeY) 

697 cellSizeX += 1 # add 1 to make sure there are no gaps 

698 cellSizeY += 1 

699 

700 else: 

701 xLinSpace = np.arange(cellSizeX//2, bbox.getWidth() + cellSizeX//2, gridStepX) 

702 yLinSpace = np.arange(cellSizeY//2, bbox.getHeight() + cellSizeY//2, gridStepY) 

703 

704 cellCentroids = [(x, y) for x in xLinSpace for y in yLinSpace] 

705 

706 else: 

707 # in py3 zip returns an iterator, but want to test length below, so use this instead: 

708 cellCentroids = [(cellCentroidsX[i], cellCentroidsY[i]) for i in range(len(cellCentroidsX))] 

709 

710 # first "main" box at 0,0 

711 bbox0 = geom.Box2I(geom.Point2I(bbox.getBegin()), geom.Extent2I(cellSizeX, cellSizeY)) 

712 # first expanded box 

713 bbox1 = geom.Box2I(bbox0) 

714 bbox1.grow(geom.Extent2I(borderSizeX, borderSizeY)) 

715 

716 self.boxes0 = [] # "main" boxes; store in task so can be extracted if needed 

717 self.boxes1 = [] # "expanded" boxes 

718 

719 def _makeBoxEvenSized(bb): 

720 """Force a bounding-box to have dimensions that are modulo 2.""" 

721 

722 if bb.getWidth() % 2 == 1: # grow to the right 

723 bb.include(geom.Point2I(bb.getMaxX()+1, bb.getMaxY())) # Expand by 1 pixel! 

724 bb.clip(bbox) 

725 if bb.getWidth() % 2 == 1: # clipped at right -- so grow to the left 

726 bb.include(geom.Point2I(bb.getMinX()-1, bb.getMaxY())) 

727 bb.clip(bbox) 

728 if bb.getHeight() % 2 == 1: # grow upwards 

729 bb.include(geom.Point2I(bb.getMaxX(), bb.getMaxY()+1)) # Expand by 1 pixel! 

730 bb.clip(bbox) 

731 if bb.getHeight() % 2 == 1: # clipped upwards -- so grow down 

732 bb.include(geom.Point2I(bb.getMaxX(), bb.getMinY()-1)) 

733 bb.clip(bbox) 

734 if bb.getWidth() % 2 == 1 or bb.getHeight() % 2 == 1: # Box is probably too big 

735 raise RuntimeError('Cannot make bounding box even-sized. Probably too big.') 

736 

737 return bb 

738 

739 # Use given or grid-parameterized centroids as centers for bounding boxes 

740 if cellCentroids is not None and len(cellCentroids) > 0: 

741 for x, y in cellCentroids: 

742 centroid = geom.Point2D(x, y) 

743 bb0 = geom.Box2I(bbox0) 

744 xoff = int(np.floor(centroid.getX())) - bb0.getWidth()//2 

745 yoff = int(np.floor(centroid.getY())) - bb0.getHeight()//2 

746 bb0.shift(geom.Extent2I(xoff, yoff)) 

747 bb0.clip(bbox) 

748 if forceEvenSized: 

749 bb0 = _makeBoxEvenSized(bb0) 

750 bb1 = geom.Box2I(bbox1) 

751 bb1.shift(geom.Extent2I(xoff, yoff)) 

752 bb1.clip(bbox) 

753 if forceEvenSized: 

754 bb1 = _makeBoxEvenSized(bb1) 

755 

756 if bb0.getArea() > 1 and bb1.getArea() > 1: 

757 self.boxes0.append(bb0) 

758 self.boxes1.append(bb1) 

759 

760 return self.boxes0, self.boxes1 

761 

762 def plotBoxes(self, fullBBox, skip=3): 

763 """Plot both grids of boxes using matplotlib. 

764 

765 Will compute the grid via `_generateGrid` if 

766 `self.boxes0` and `self.boxes1` have not already been set. 

767 

768 Parameters 

769 ---------- 

770 exposure : `lsst.afw.image.Exposure` 

771 Exposure whose bounding box is gridded by this task. 

772 skip : `int` 

773 Plot every skip-ped box (help make plots less confusing) 

774 """ 

775 import matplotlib.pyplot as plt 

776 

777 if self.boxes0 is None: 

778 raise RuntimeError('Cannot plot boxes. Run _generateGrid first.') 

779 self._plotBoxGrid(self.boxes0[::skip], fullBBox, ls='--') 

780 # reset the color cycle -- see 

781 # http://stackoverflow.com/questions/24193174/reset-color-cycle-in-matplotlib 

782 plt.gca().set_prop_cycle(None) 

783 self._plotBoxGrid(self.boxes1[::skip], fullBBox, ls=':') 

784 

785 def _plotBoxGrid(self, boxes, bbox, **kwargs): 

786 """Plot a grid of boxes using matplotlib. 

787 

788 Parameters 

789 ---------- 

790 boxes : `list` of `lsst.geom.Box2I` 

791 a list of bounding boxes. 

792 bbox : `lsst.geom.Box2I` 

793 an overall bounding box 

794 **kwargs 

795 additional keyword arguments for matplotlib 

796 """ 

797 import matplotlib.pyplot as plt 

798 

799 def plotBox(box): 

800 corners = np.array([np.array([pt.getX(), pt.getY()]) for pt in box.getCorners()]) 

801 corners = np.vstack([corners, corners[0, :]]) 

802 plt.plot(corners[:, 0], corners[:, 1], **kwargs) 

803 

804 for b in boxes: 

805 plotBox(b) 

806 plt.xlim(bbox.getBeginX(), bbox.getEndX()) 

807 plt.ylim(bbox.getBeginY(), bbox.getEndY())