Coverage for python/lsst/cp/pipe/cpCombine.py: 25%

212 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-03 09:58 +0000

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import time 

23 

24import lsst.geom as geom 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.math as afwMath 

29import lsst.afw.image as afwImage 

30 

31from lsst.ip.isr.vignette import maskVignettedRegion 

32 

33from astro_metadata_translator import merge_headers, ObservationGroup 

34from astro_metadata_translator.serialize import dates_to_fits 

35 

36 

37__all__ = ["CalibStatsConfig", "CalibStatsTask", 

38 "CalibCombineConfig", "CalibCombineConnections", "CalibCombineTask", 

39 "CalibCombineByFilterConfig", "CalibCombineByFilterConnections", "CalibCombineByFilterTask"] 

40 

41 

42# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py 

43class CalibStatsConfig(pexConfig.Config): 

44 """Parameters controlling the measurement of background 

45 statistics. 

46 """ 

47 

48 stat = pexConfig.Field( 

49 dtype=str, 

50 default="MEANCLIP", 

51 doc="Statistic name to use to estimate background (from `~lsst.afw.math.Property`)", 

52 ) 

53 clip = pexConfig.Field( 

54 dtype=float, 

55 default=3.0, 

56 doc="Clipping threshold for background", 

57 ) 

58 nIter = pexConfig.Field( 

59 dtype=int, 

60 default=3, 

61 doc="Clipping iterations for background", 

62 ) 

63 mask = pexConfig.ListField( 

64 dtype=str, 

65 default=["DETECTED", "BAD", "NO_DATA"], 

66 doc="Mask planes to reject", 

67 ) 

68 

69 

70class CalibStatsTask(pipeBase.Task): 

71 """Measure statistics on the background 

72 

73 This can be useful for scaling the background, e.g., for flats and 

74 fringe frames. 

75 """ 

76 

77 ConfigClass = CalibStatsConfig 

78 

79 def run(self, exposureOrImage): 

80 """Measure a particular statistic on an image (of some sort). 

81 

82 Parameters 

83 ---------- 

84 exposureOrImage : `lsst.afw.image.Exposure`, 

85 `lsst.afw.image.MaskedImage`, or 

86 `lsst.afw.image.Image` 

87 Exposure or image to calculate statistics on. 

88 

89 Returns 

90 ------- 

91 results : float 

92 Resulting statistic value. 

93 """ 

94 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

95 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

96 try: 

97 image = exposureOrImage.getMaskedImage() 

98 except Exception: 

99 try: 

100 image = exposureOrImage.getImage() 

101 except Exception: 

102 image = exposureOrImage 

103 statType = afwMath.stringToStatisticsProperty(self.config.stat) 

104 return afwMath.makeStatistics(image, statType, stats).getValue() 

105 

106 

107class CalibCombineConnections(pipeBase.PipelineTaskConnections, 

108 dimensions=("instrument", "detector")): 

109 inputExpHandles = cT.Input( 

110 name="cpInputs", 

111 doc="Input pre-processed exposures to combine.", 

112 storageClass="Exposure", 

113 dimensions=("instrument", "detector", "exposure"), 

114 multiple=True, 

115 deferLoad=True, 

116 ) 

117 inputScales = cT.Input( 

118 name="cpScales", 

119 doc="Input scale factors to use.", 

120 storageClass="StructuredDataDict", 

121 dimensions=("instrument", ), 

122 multiple=False, 

123 ) 

124 

125 outputData = cT.Output( 

126 name="cpProposal", 

127 doc="Output combined proposed calibration to be validated and certified..", 

128 storageClass="ExposureF", 

129 dimensions=("instrument", "detector"), 

130 isCalibration=True, 

131 ) 

132 

133 def __init__(self, *, config=None): 

134 super().__init__(config=config) 

135 

136 if config and config.exposureScaling != "InputList": 

137 self.inputs.discard("inputScales") 

138 

139 

140# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py 

141class CalibCombineConfig(pipeBase.PipelineTaskConfig, 

142 pipelineConnections=CalibCombineConnections): 

143 """Configuration for combining calib exposures. 

144 """ 

145 

146 calibrationType = pexConfig.Field( 

147 dtype=str, 

148 default="calibration", 

149 doc="Name of calibration to be generated.", 

150 ) 

151 

152 exposureScaling = pexConfig.ChoiceField( 

153 dtype=str, 

154 allowed={ 

155 "Unity": "Do not scale inputs. Scale factor is 1.0.", 

156 "ExposureTime": "Scale inputs by their exposure time.", 

157 "DarkTime": "Scale inputs by their dark time.", 

158 "MeanStats": "Scale inputs based on their mean values.", 

159 "InputList": "Scale inputs based on a list of values.", 

160 }, 

161 default="Unity", 

162 doc="Scaling to be applied to each input exposure.", 

163 ) 

164 scalingLevel = pexConfig.ChoiceField( 

165 dtype=str, 

166 allowed={ 

167 "DETECTOR": "Scale by detector.", 

168 "AMP": "Scale by amplifier.", 

169 }, 

170 default="DETECTOR", 

171 doc="Region to scale.", 

172 ) 

173 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field( 

174 dtype=int, 

175 default=5, 

176 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread", 

177 ) 

178 subregionSize = pexConfig.ListField( 

179 dtype=int, 

180 doc="Width, height of subregion size.", 

181 length=2, 

182 # This is 200 rows for all detectors smaller than 10k in width. 

183 default=(10000, 200), 

184 ) 

185 

186 doVignette = pexConfig.Field( 

187 dtype=bool, 

188 default=False, 

189 doc="Copy vignette polygon to output and censor vignetted pixels?" 

190 ) 

191 

192 mask = pexConfig.ListField( 

193 dtype=str, 

194 default=["SAT", "DETECTED", "INTRP"], 

195 doc="Mask planes to respect", 

196 ) 

197 combine = pexConfig.Field( 

198 dtype=str, 

199 default="MEANCLIP", 

200 doc="Statistic name to use for combination (from `~lsst.afw.math.Property`)", 

201 ) 

202 clip = pexConfig.Field( 

203 dtype=float, 

204 default=3.0, 

205 doc="Clipping threshold for combination", 

206 ) 

207 nIter = pexConfig.Field( 

208 dtype=int, 

209 default=3, 

210 doc="Clipping iterations for combination", 

211 ) 

212 stats = pexConfig.ConfigurableField( 

213 target=CalibStatsTask, 

214 doc="Background statistics configuration", 

215 ) 

216 

217 

218class CalibCombineTask(pipeBase.PipelineTask, 

219 pipeBase.CmdLineTask): 

220 """Task to combine calib exposures.""" 

221 

222 ConfigClass = CalibCombineConfig 

223 _DefaultName = "cpCombine" 

224 

225 def __init__(self, **kwargs): 

226 super().__init__(**kwargs) 

227 self.makeSubtask("stats") 

228 

229 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

230 inputs = butlerQC.get(inputRefs) 

231 

232 dimensions = [expHandle.dataId.byName() for expHandle in inputRefs.inputExpHandles] 

233 inputs["inputDims"] = dimensions 

234 

235 outputs = self.run(**inputs) 

236 butlerQC.put(outputs, outputRefs) 

237 

238 def run(self, inputExpHandles, inputScales=None, inputDims=None): 

239 """Combine calib exposures for a single detector. 

240 

241 Parameters 

242 ---------- 

243 inputExpHandles : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

244 Input list of exposure handles to combine. 

245 inputScales : `dict` [`dict` [`dict` [`float`]]], optional 

246 Dictionary of scales, indexed by detector (`int`), 

247 amplifier (`int`), and exposure (`int`). Used for 

248 'inputExps' scaling. 

249 inputDims : `list` [`dict`] 

250 List of dictionaries of input data dimensions/values. 

251 Each list entry should contain: 

252 

253 ``"exposure"`` 

254 exposure id value (`int`) 

255 ``"detector"`` 

256 detector id value (`int`) 

257 

258 Returns 

259 ------- 

260 results : `lsst.pipe.base.Struct` 

261 The results struct containing: 

262 

263 ``outputData`` 

264 Final combined exposure generated from the inputs 

265 (`lsst.afw.image.Exposure`). 

266 

267 Raises 

268 ------ 

269 RuntimeError 

270 Raised if no input data is found. Also raised if 

271 config.exposureScaling == InputList, and a necessary scale 

272 was not found. 

273 """ 

274 width, height = self.getDimensions(inputExpHandles) 

275 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

276 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

277 numExps = len(inputExpHandles) 

278 if numExps < 1: 

279 raise RuntimeError("No valid input data") 

280 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance: 

281 stats.setCalcErrorFromInputVariance(True) 

282 

283 inputDetector = inputExpHandles[0].get(component="detector") 

284 

285 # Create output exposure for combined data. 

286 combined = afwImage.MaskedImageF(width, height) 

287 combinedExp = afwImage.makeExposure(combined) 

288 

289 # Apply scaling: 

290 expScales = [] 

291 if inputDims is None: 

292 inputDims = [dict() for i in inputExpHandles] 

293 

294 for index, (expHandle, dims) in enumerate(zip(inputExpHandles, inputDims)): 

295 scale = 1.0 

296 visitInfo = expHandle.get(component="visitInfo") 

297 if self.config.exposureScaling == "ExposureTime": 

298 scale = visitInfo.getExposureTime() 

299 elif self.config.exposureScaling == "DarkTime": 

300 scale = visitInfo.getDarkTime() 

301 elif self.config.exposureScaling == "MeanStats": 

302 # Note: there may a bug freeing memory here. TBD. 

303 exp = expHandle.get() 

304 scale = self.stats.run(exp) 

305 del exp 

306 elif self.config.exposureScaling == "InputList": 

307 visitId = dims.get("exposure", None) 

308 detectorId = dims.get("detector", None) 

309 if visitId is None or detectorId is None: 

310 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})") 

311 if detectorId not in inputScales["expScale"]: 

312 raise RuntimeError(f"Could not identify a scaling for input {index}" 

313 f" detector {detectorId}") 

314 

315 if self.config.scalingLevel == "DETECTOR": 

316 if visitId not in inputScales["expScale"][detectorId]: 

317 raise RuntimeError(f"Could not identify a scaling for input {index}" 

318 f"detector {detectorId} visit {visitId}") 

319 scale = inputScales["expScale"][detectorId][visitId] 

320 elif self.config.scalingLevel == "AMP": 

321 scale = [inputScales["expScale"][detectorId][amp.getName()][visitId] 

322 for amp in inputDetector] 

323 else: 

324 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}") 

325 elif self.config.exposureScaling == "Unity": 

326 scale = 1.0 

327 else: 

328 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.") 

329 

330 expScales.append(scale) 

331 self.log.info("Scaling input %d by %s", index, scale) 

332 

333 self.combine(combinedExp, inputExpHandles, expScales, stats) 

334 

335 self.interpolateNans(combined) 

336 

337 if self.config.doVignette: 

338 polygon = inputExpHandles[0].get(component="validPolygon") 

339 maskVignettedRegion(combined, polygon=polygon, vignetteValue=0.0) 

340 

341 # Combine headers 

342 self.combineHeaders(inputExpHandles, combinedExp, 

343 calibType=self.config.calibrationType, scales=expScales) 

344 

345 # Set the detector 

346 combinedExp.setDetector(inputDetector) 

347 

348 # Return 

349 return pipeBase.Struct( 

350 outputData=combinedExp, 

351 ) 

352 

353 def getDimensions(self, expHandleList): 

354 """Get dimensions of the inputs. 

355 

356 Parameters 

357 ---------- 

358 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

359 Exposure handles to check the sizes of. 

360 

361 Returns 

362 ------- 

363 width, height : `int` 

364 Unique set of input dimensions. 

365 """ 

366 dimList = [expHandle.get(component="bbox").getDimensions() for expHandle in expHandleList] 

367 

368 return self.getSize(dimList) 

369 

370 def getSize(self, dimList): 

371 """Determine a consistent size, given a list of image sizes. 

372 

373 Parameters 

374 ----------- 

375 dimList : `list` [`tuple` [`int`, `int`]] 

376 List of dimensions. 

377 

378 Raises 

379 ------ 

380 RuntimeError 

381 If input dimensions are inconsistent. 

382 

383 Returns 

384 -------- 

385 width, height : `int` 

386 Common dimensions. 

387 """ 

388 dim = set((w, h) for w, h in dimList) 

389 if len(dim) != 1: 

390 raise RuntimeError("Inconsistent dimensions: %s" % dim) 

391 return dim.pop() 

392 

393 def applyScale(self, exposure, scale=None): 

394 """Apply scale to input exposure. 

395 

396 This implementation applies a flux scaling: the input exposure is 

397 divided by the provided scale. 

398 

399 Parameters 

400 ---------- 

401 exposure : `lsst.afw.image.Exposure` 

402 Exposure to scale. 

403 scale : `float` or `list` [`float`], optional 

404 Constant scale to divide the exposure by. 

405 """ 

406 if scale is not None: 

407 mi = exposure.getMaskedImage() 

408 if isinstance(scale, list): 

409 for amp, ampScale in zip(exposure.getDetector(), scale): 

410 ampIm = mi[amp.getBBox()] 

411 ampIm /= ampScale 

412 else: 

413 mi /= scale 

414 

415 @staticmethod 

416 def _subBBoxIter(bbox, subregionSize): 

417 """Iterate over subregions of a bbox. 

418 

419 Parameters 

420 ---------- 

421 bbox : `lsst.geom.Box2I` 

422 Bounding box over which to iterate. 

423 subregionSize: `lsst.geom.Extent2I` 

424 Size of sub-bboxes. 

425 

426 Yields 

427 ------ 

428 subBBox : `lsst.geom.Box2I` 

429 Next sub-bounding box of size ``subregionSize`` or 

430 smaller; each ``subBBox`` is contained within ``bbox``, so 

431 it may be smaller than ``subregionSize`` at the edges of 

432 ``bbox``, but it will never be empty. 

433 """ 

434 if bbox.isEmpty(): 

435 raise RuntimeError("bbox %s is empty" % (bbox,)) 

436 if subregionSize[0] < 1 or subregionSize[1] < 1: 

437 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,)) 

438 

439 for rowShift in range(0, bbox.getHeight(), subregionSize[1]): 

440 for colShift in range(0, bbox.getWidth(), subregionSize[0]): 

441 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize) 

442 subBBox.clip(bbox) 

443 if subBBox.isEmpty(): 

444 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, " 

445 "colShift=%s, rowShift=%s" % 

446 (bbox, subregionSize, colShift, rowShift)) 

447 yield subBBox 

448 

449 def combine(self, target, expHandleList, expScaleList, stats): 

450 """Combine multiple images. 

451 

452 Parameters 

453 ---------- 

454 target : `lsst.afw.image.Exposure` 

455 Output exposure to construct. 

456 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

457 Input exposure handles to combine. 

458 expScaleList : `list` [`float`] 

459 List of scales to apply to each input image. 

460 stats : `lsst.afw.math.StatisticsControl` 

461 Control explaining how to combine the input images. 

462 """ 

463 combineType = afwMath.stringToStatisticsProperty(self.config.combine) 

464 

465 subregionSizeArr = self.config.subregionSize 

466 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1]) 

467 for subBbox in self._subBBoxIter(target.getBBox(), subregionSize): 

468 images = [] 

469 for expHandle, expScale in zip(expHandleList, expScaleList): 

470 inputExp = expHandle.get(parameters={"bbox": subBbox}) 

471 self.applyScale(inputExp, expScale) 

472 images.append(inputExp.getMaskedImage()) 

473 

474 combinedSubregion = afwMath.statisticsStack(images, combineType, stats) 

475 target.maskedImage.assign(combinedSubregion, subBbox) 

476 

477 def combineHeaders(self, expHandleList, calib, calibType="CALIB", scales=None): 

478 """Combine input headers to determine the set of common headers, 

479 supplemented by calibration inputs. 

480 

481 Parameters 

482 ---------- 

483 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

484 Input list of exposure handles to combine. 

485 calib : `lsst.afw.image.Exposure` 

486 Output calibration to construct headers for. 

487 calibType : `str`, optional 

488 OBSTYPE the output should claim. 

489 scales : `list` [`float`], optional 

490 Scale values applied to each input to record. 

491 

492 Returns 

493 ------- 

494 header : `lsst.daf.base.PropertyList` 

495 Constructed header. 

496 """ 

497 # Header 

498 header = calib.getMetadata() 

499 header.set("OBSTYPE", calibType) 

500 

501 # Keywords we care about 

502 comments = {"TIMESYS": "Time scale for all dates", 

503 "DATE-OBS": "Start date of earliest input observation", 

504 "MJD-OBS": "[d] Start MJD of earliest input observation", 

505 "DATE-END": "End date of oldest input observation", 

506 "MJD-END": "[d] End MJD of oldest input observation", 

507 "MJD-AVG": "[d] MJD midpoint of all input observations", 

508 "DATE-AVG": "Midpoint date of all input observations"} 

509 

510 # Creation date 

511 now = time.localtime() 

512 calibDate = time.strftime("%Y-%m-%d", now) 

513 calibTime = time.strftime("%X %Z", now) 

514 header.set("CALIB_CREATE_DATE", calibDate) 

515 header.set("CALIB_CREATE_TIME", calibTime) 

516 

517 # Merge input headers 

518 inputHeaders = [expHandle.get(component="metadata") for expHandle in expHandleList] 

519 merged = merge_headers(inputHeaders, mode="drop") 

520 

521 # Scan the first header for items that were dropped due to 

522 # conflict, and replace them. 

523 for k, v in merged.items(): 

524 if k not in header: 

525 md = inputHeaders[0] 

526 comment = md.getComment(k) if k in md else None 

527 header.set(k, v, comment=comment) 

528 

529 # Construct list of visits 

530 visitInfoList = [expHandle.get(component="visitInfo") for expHandle in expHandleList] 

531 for i, visit in enumerate(visitInfoList): 

532 if visit is None: 

533 continue 

534 header.set("CPP_INPUT_%d" % (i,), visit.id) 

535 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate())) 

536 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime()) 

537 if scales is not None: 

538 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i]) 

539 

540 # Not yet working: DM-22302 

541 # Create an observation group so we can add some standard headers 

542 # independent of the form in the input files. 

543 # Use try block in case we are dealing with unexpected data headers 

544 try: 

545 group = ObservationGroup(visitInfoList, pedantic=False) 

546 except Exception: 

547 self.log.warning("Exception making an obs group for headers. Continuing.") 

548 # Fall back to setting a DATE-OBS from the calibDate 

549 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)} 

550 comments["DATE-OBS"] = "Date of start of day of calibration midpoint" 

551 else: 

552 oldest, newest = group.extremes() 

553 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end) 

554 

555 for k, v in dateCards.items(): 

556 header.set(k, v, comment=comments.get(k, None)) 

557 

558 return header 

559 

560 def interpolateNans(self, exp): 

561 """Interpolate over NANs in the combined image. 

562 

563 NANs can result from masked areas on the CCD. We don't want 

564 them getting into our science images, so we replace them with 

565 the median of the image. 

566 

567 Parameters 

568 ---------- 

569 exp : `lsst.afw.image.Exposure` 

570 Exp to check for NaNs. 

571 """ 

572 array = exp.getImage().getArray() 

573 bad = np.isnan(array) 

574 if np.any(bad): 

575 median = np.median(array[np.logical_not(bad)]) 

576 count = np.sum(bad) 

577 array[bad] = median 

578 self.log.warning("Found and fixed %s NAN pixels", count) 

579 

580 

581# Create versions of the Connections, Config, and Task that support 

582# filter constraints. 

583class CalibCombineByFilterConnections(CalibCombineConnections, 

584 dimensions=("instrument", "detector", "physical_filter")): 

585 inputScales = cT.Input( 

586 name="cpFilterScales", 

587 doc="Input scale factors to use.", 

588 storageClass="StructuredDataDict", 

589 dimensions=("instrument", "physical_filter"), 

590 multiple=False, 

591 ) 

592 

593 outputData = cT.Output( 

594 name="cpFilterProposal", 

595 doc="Output combined proposed calibration to be validated and certified.", 

596 storageClass="ExposureF", 

597 dimensions=("instrument", "detector", "physical_filter"), 

598 isCalibration=True, 

599 ) 

600 

601 def __init__(self, *, config=None): 

602 super().__init__(config=config) 

603 

604 if config and config.exposureScaling != "InputList": 

605 self.inputs.discard("inputScales") 

606 

607 

608class CalibCombineByFilterConfig(CalibCombineConfig, 

609 pipelineConnections=CalibCombineByFilterConnections): 

610 pass 

611 

612 

613class CalibCombineByFilterTask(CalibCombineTask): 

614 """Task to combine calib exposures.""" 

615 

616 ConfigClass = CalibCombineByFilterConfig 

617 _DefaultName = "cpFilterCombine" 

618 pass