Coverage for python/lsst/cp/pipe/cpCombine.py: 18%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

225 statements  

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import time 

23import logging 

24 

25import lsst.pex.config as pexConfig 

26import lsst.pipe.base as pipeBase 

27import lsst.pipe.base.connectionTypes as cT 

28import lsst.afw.math as afwMath 

29import lsst.afw.image as afwImage 

30 

31from lsst.geom import Point2D 

32from astro_metadata_translator import merge_headers, ObservationGroup 

33from astro_metadata_translator.serialize import dates_to_fits 

34 

35 

36__all__ = ['CalibStatsConfig', 'CalibStatsTask', 'vignetteExposure', 

37 'CalibCombineConfig', 'CalibCombineConnections', 'CalibCombineTask', 

38 'CalibCombineByFilterConfig', 'CalibCombineByFilterConnections', 'CalibCombineByFilterTask'] 

39 

40 

41# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py 

42class CalibStatsConfig(pexConfig.Config): 

43 """Parameters controlling the measurement of background 

44 statistics. 

45 """ 

46 

47 stat = pexConfig.Field( 

48 dtype=str, 

49 default='MEANCLIP', 

50 doc="Statistic name to use to estimate background (from `~lsst.afw.math.Property`)", 

51 ) 

52 clip = pexConfig.Field( 

53 dtype=float, 

54 default=3.0, 

55 doc="Clipping threshold for background", 

56 ) 

57 nIter = pexConfig.Field( 

58 dtype=int, 

59 default=3, 

60 doc="Clipping iterations for background", 

61 ) 

62 mask = pexConfig.ListField( 

63 dtype=str, 

64 default=["DETECTED", "BAD", "NO_DATA"], 

65 doc="Mask planes to reject", 

66 ) 

67 

68 

69class CalibStatsTask(pipeBase.Task): 

70 """Measure statistics on the background 

71 

72 This can be useful for scaling the background, e.g., for flats and 

73 fringe frames. 

74 """ 

75 

76 ConfigClass = CalibStatsConfig 

77 

78 def run(self, exposureOrImage): 

79 """Measure a particular statistic on an image (of some sort). 

80 

81 Parameters 

82 ---------- 

83 exposureOrImage : `lsst.afw.image.Exposure`, 

84 `lsst.afw.image.MaskedImage`, or 

85 `lsst.afw.image.Image` 

86 Exposure or image to calculate statistics on. 

87 

88 Returns 

89 ------- 

90 results : float 

91 Resulting statistic value. 

92 """ 

93 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

94 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

95 try: 

96 image = exposureOrImage.getMaskedImage() 

97 except Exception: 

98 try: 

99 image = exposureOrImage.getImage() 

100 except Exception: 

101 image = exposureOrImage 

102 statType = afwMath.stringToStatisticsProperty(self.config.stat) 

103 return afwMath.makeStatistics(image, statType, stats).getValue() 

104 

105 

106class CalibCombineConnections(pipeBase.PipelineTaskConnections, 

107 dimensions=("instrument", "detector")): 

108 inputExps = cT.Input( 

109 name="cpInputs", 

110 doc="Input pre-processed exposures to combine.", 

111 storageClass="Exposure", 

112 dimensions=("instrument", "detector", "exposure"), 

113 multiple=True, 

114 ) 

115 inputScales = cT.Input( 

116 name="cpScales", 

117 doc="Input scale factors to use.", 

118 storageClass="StructuredDataDict", 

119 dimensions=("instrument", ), 

120 multiple=False, 

121 ) 

122 

123 outputData = cT.Output( 

124 name="cpProposal", 

125 doc="Output combined proposed calibration to be validated and certified..", 

126 storageClass="ExposureF", 

127 dimensions=("instrument", "detector"), 

128 isCalibration=True, 

129 ) 

130 

131 def __init__(self, *, config=None): 

132 super().__init__(config=config) 

133 

134 if config and config.exposureScaling != 'InputList': 

135 self.inputs.discard("inputScales") 

136 

137 

138# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py 

139class CalibCombineConfig(pipeBase.PipelineTaskConfig, 

140 pipelineConnections=CalibCombineConnections): 

141 """Configuration for combining calib exposures. 

142 """ 

143 

144 calibrationType = pexConfig.Field( 

145 dtype=str, 

146 default="calibration", 

147 doc="Name of calibration to be generated.", 

148 ) 

149 

150 exposureScaling = pexConfig.ChoiceField( 

151 dtype=str, 

152 allowed={ 

153 "Unity": "Do not scale inputs. Scale factor is 1.0.", 

154 "ExposureTime": "Scale inputs by their exposure time.", 

155 "DarkTime": "Scale inputs by their dark time.", 

156 "MeanStats": "Scale inputs based on their mean values.", 

157 "InputList": "Scale inputs based on a list of values.", 

158 }, 

159 default="Unity", 

160 doc="Scaling to be applied to each input exposure.", 

161 ) 

162 scalingLevel = pexConfig.ChoiceField( 

163 dtype=str, 

164 allowed={ 

165 "DETECTOR": "Scale by detector.", 

166 "AMP": "Scale by amplifier.", 

167 }, 

168 default="DETECTOR", 

169 doc="Region to scale.", 

170 ) 

171 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field( 

172 dtype=int, 

173 default=5, 

174 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread", 

175 ) 

176 

177 doVignette = pexConfig.Field( 

178 dtype=bool, 

179 default=False, 

180 doc="Copy vignette polygon to output and censor vignetted pixels?" 

181 ) 

182 

183 mask = pexConfig.ListField( 

184 dtype=str, 

185 default=["SAT", "DETECTED", "INTRP"], 

186 doc="Mask planes to respect", 

187 ) 

188 combine = pexConfig.Field( 

189 dtype=str, 

190 default='MEANCLIP', 

191 doc="Statistic name to use for combination (from `~lsst.afw.math.Property`)", 

192 ) 

193 clip = pexConfig.Field( 

194 dtype=float, 

195 default=3.0, 

196 doc="Clipping threshold for combination", 

197 ) 

198 nIter = pexConfig.Field( 

199 dtype=int, 

200 default=3, 

201 doc="Clipping iterations for combination", 

202 ) 

203 stats = pexConfig.ConfigurableField( 

204 target=CalibStatsTask, 

205 doc="Background statistics configuration", 

206 ) 

207 

208 

209class CalibCombineTask(pipeBase.PipelineTask, 

210 pipeBase.CmdLineTask): 

211 """Task to combine calib exposures.""" 

212 

213 ConfigClass = CalibCombineConfig 

214 _DefaultName = 'cpCombine' 

215 

216 def __init__(self, **kwargs): 

217 super().__init__(**kwargs) 

218 self.makeSubtask("stats") 

219 

220 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

221 inputs = butlerQC.get(inputRefs) 

222 

223 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps] 

224 inputs['inputDims'] = dimensions 

225 

226 outputs = self.run(**inputs) 

227 butlerQC.put(outputs, outputRefs) 

228 

229 def run(self, inputExps, inputScales=None, inputDims=None): 

230 """Combine calib exposures for a single detector. 

231 

232 Parameters 

233 ---------- 

234 inputExps : `list` [`lsst.afw.image.Exposure`] 

235 Input list of exposures to combine. 

236 inputScales : `dict` [`dict` [`dict` [`float`]]], optional 

237 Dictionary of scales, indexed by detector (`int`), 

238 amplifier (`int`), and exposure (`int`). Used for 

239 'inputExps' scaling. 

240 inputDims : `list` [`dict`] 

241 List of dictionaries of input data dimensions/values. 

242 Each list entry should contain: 

243 

244 ``"exposure"`` 

245 exposure id value (`int`) 

246 ``"detector"`` 

247 detector id value (`int`) 

248 

249 Returns 

250 ------- 

251 results : `lsst.pipe.base.Struct` 

252 The results struct containing: 

253 

254 ``combinedExp`` 

255 Final combined exposure generated from the inputs 

256 (`lsst.afw.image.Exposure`). 

257 

258 Raises 

259 ------ 

260 RuntimeError 

261 Raised if no input data is found. Also raised if 

262 config.exposureScaling == InputList, and a necessary scale 

263 was not found. 

264 """ 

265 width, height = self.getDimensions(inputExps) 

266 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

267 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

268 numExps = len(inputExps) 

269 if numExps < 1: 

270 raise RuntimeError("No valid input data") 

271 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance: 

272 stats.setCalcErrorFromInputVariance(True) 

273 

274 # Check that all inputs either share the same detector (based 

275 # on detId), or that no inputs have any detector. 

276 detectorList = [exp.getDetector() for exp in inputExps] 

277 if None in detectorList: 

278 self.log.warning("Not all input detectors defined.") 

279 detectorIds = [det.getId() if det is not None else None for det in detectorList] 

280 detectorSerials = [det.getId() if det is not None else None for det in detectorList] 

281 numDetectorIds = len(set(detectorIds)) 

282 numDetectorSerials = len(set(detectorSerials)) 

283 numDetectors = len(set([numDetectorIds, numDetectorSerials])) 

284 if numDetectors != 1: 

285 raise RuntimeError("Input data contains multiple detectors.") 

286 

287 inputDetector = inputExps[0].getDetector() 

288 

289 # Create output exposure for combined data. 

290 combined = afwImage.MaskedImageF(width, height) 

291 combinedExp = afwImage.makeExposure(combined) 

292 

293 # Apply scaling: 

294 expScales = [] 

295 if inputDims is None: 

296 inputDims = [dict() for i in inputExps] 

297 

298 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)): 

299 scale = 1.0 

300 if exp is None: 

301 self.log.warning("Input %d is None (%s); unable to scale exp.", index, dims) 

302 continue 

303 

304 if self.config.exposureScaling == "ExposureTime": 

305 scale = exp.getInfo().getVisitInfo().getExposureTime() 

306 elif self.config.exposureScaling == "DarkTime": 

307 scale = exp.getInfo().getVisitInfo().getDarkTime() 

308 elif self.config.exposureScaling == "MeanStats": 

309 scale = self.stats.run(exp) 

310 elif self.config.exposureScaling == "InputList": 

311 visitId = dims.get('exposure', None) 

312 detectorId = dims.get('detector', None) 

313 if visitId is None or detectorId is None: 

314 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})") 

315 if detectorId not in inputScales['expScale']: 

316 raise RuntimeError(f"Could not identify a scaling for input {index}" 

317 f" detector {detectorId}") 

318 

319 if self.config.scalingLevel == "DETECTOR": 

320 if visitId not in inputScales['expScale'][detectorId]: 

321 raise RuntimeError(f"Could not identify a scaling for input {index}" 

322 f"detector {detectorId} visit {visitId}") 

323 scale = inputScales['expScale'][detectorId][visitId] 

324 elif self.config.scalingLevel == 'AMP': 

325 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId] 

326 for amp in exp.getDetector()] 

327 else: 

328 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}") 

329 elif self.config.exposureScaling == 'Unity': 

330 scale = 1.0 

331 else: 

332 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.") 

333 

334 expScales.append(scale) 

335 self.log.info("Scaling input %d by %s", index, scale) 

336 self.applyScale(exp, scale) 

337 

338 self.combine(combined, inputExps, stats) 

339 

340 self.interpolateNans(combined) 

341 

342 if self.config.doVignette: 

343 polygon = inputExps[0].getInfo().getValidPolygon() 

344 vignetteExposure(combined, polygon=polygon, doUpdateMask=True, 

345 doSetValue=True, vignetteValue=0.0) 

346 

347 # Combine headers 

348 self.combineHeaders(inputExps, combinedExp, 

349 calibType=self.config.calibrationType, scales=expScales) 

350 

351 # Set the detector 

352 combinedExp.setDetector(inputDetector) 

353 

354 # Return 

355 return pipeBase.Struct( 

356 outputData=combinedExp, 

357 ) 

358 

359 def getDimensions(self, expList): 

360 """Get dimensions of the inputs. 

361 

362 Parameters 

363 ---------- 

364 expList : `list` [`lsst.afw.image.Exposure`] 

365 Exps to check the sizes of. 

366 

367 Returns 

368 ------- 

369 width, height : `int` 

370 Unique set of input dimensions. 

371 """ 

372 dimList = [exp.getDimensions() for exp in expList if exp is not None] 

373 return self.getSize(dimList) 

374 

375 def getSize(self, dimList): 

376 """Determine a consistent size, given a list of image sizes. 

377 

378 Parameters 

379 ----------- 

380 dimList : `list` [`tuple` [`int`, `int`]] 

381 List of dimensions. 

382 

383 Raises 

384 ------ 

385 RuntimeError 

386 If input dimensions are inconsistent. 

387 

388 Returns 

389 -------- 

390 width, height : `int` 

391 Common dimensions. 

392 """ 

393 dim = set((w, h) for w, h in dimList) 

394 if len(dim) != 1: 

395 raise RuntimeError("Inconsistent dimensions: %s" % dim) 

396 return dim.pop() 

397 

398 def applyScale(self, exposure, scale=None): 

399 """Apply scale to input exposure. 

400 

401 This implementation applies a flux scaling: the input exposure is 

402 divided by the provided scale. 

403 

404 Parameters 

405 ---------- 

406 exposure : `lsst.afw.image.Exposure` 

407 Exposure to scale. 

408 scale : `float` or `list` [`float`], optional 

409 Constant scale to divide the exposure by. 

410 """ 

411 if scale is not None: 

412 mi = exposure.getMaskedImage() 

413 if isinstance(scale, list): 

414 for amp, ampScale in zip(exposure.getDetector(), scale): 

415 ampIm = mi[amp.getBBox()] 

416 ampIm /= ampScale 

417 else: 

418 mi /= scale 

419 

420 def combine(self, target, expList, stats): 

421 """Combine multiple images. 

422 

423 Parameters 

424 ---------- 

425 target : `lsst.afw.image.Exposure` 

426 Output exposure to construct. 

427 expList : `list` [`lsst.afw.image.Exposure`] 

428 Input exposures to combine. 

429 stats : `lsst.afw.math.StatisticsControl` 

430 Control explaining how to combine the input images. 

431 """ 

432 images = [img.getMaskedImage() for img in expList if img is not None] 

433 combineType = afwMath.stringToStatisticsProperty(self.config.combine) 

434 afwMath.statisticsStack(target, images, combineType, stats) 

435 

436 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None): 

437 """Combine input headers to determine the set of common headers, 

438 supplemented by calibration inputs. 

439 

440 Parameters 

441 ---------- 

442 expList : `list` [`lsst.afw.image.Exposure`] 

443 Input list of exposures to combine. 

444 calib : `lsst.afw.image.Exposure` 

445 Output calibration to construct headers for. 

446 calibType : `str`, optional 

447 OBSTYPE the output should claim. 

448 scales : `list` [`float`], optional 

449 Scale values applied to each input to record. 

450 

451 Returns 

452 ------- 

453 header : `lsst.daf.base.PropertyList` 

454 Constructed header. 

455 """ 

456 # Header 

457 header = calib.getMetadata() 

458 header.set("OBSTYPE", calibType) 

459 

460 # Keywords we care about 

461 comments = {"TIMESYS": "Time scale for all dates", 

462 "DATE-OBS": "Start date of earliest input observation", 

463 "MJD-OBS": "[d] Start MJD of earliest input observation", 

464 "DATE-END": "End date of oldest input observation", 

465 "MJD-END": "[d] End MJD of oldest input observation", 

466 "MJD-AVG": "[d] MJD midpoint of all input observations", 

467 "DATE-AVG": "Midpoint date of all input observations"} 

468 

469 # Creation date 

470 now = time.localtime() 

471 calibDate = time.strftime("%Y-%m-%d", now) 

472 calibTime = time.strftime("%X %Z", now) 

473 header.set("CALIB_CREATE_DATE", calibDate) 

474 header.set("CALIB_CREATE_TIME", calibTime) 

475 

476 # Merge input headers 

477 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None] 

478 merged = merge_headers(inputHeaders, mode='drop') 

479 for k, v in merged.items(): 

480 if k not in header: 

481 md = expList[0].getMetadata() 

482 comment = md.getComment(k) if k in md else None 

483 header.set(k, v, comment=comment) 

484 

485 # Construct list of visits 

486 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None] 

487 for i, visit in enumerate(visitInfoList): 

488 if visit is None: 

489 continue 

490 header.set("CPP_INPUT_%d" % (i,), visit.id) 

491 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate())) 

492 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime()) 

493 if scales is not None: 

494 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i]) 

495 

496 # Not yet working: DM-22302 

497 # Create an observation group so we can add some standard headers 

498 # independent of the form in the input files. 

499 # Use try block in case we are dealing with unexpected data headers 

500 try: 

501 group = ObservationGroup(visitInfoList, pedantic=False) 

502 except Exception: 

503 self.log.warning("Exception making an obs group for headers. Continuing.") 

504 # Fall back to setting a DATE-OBS from the calibDate 

505 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)} 

506 comments["DATE-OBS"] = "Date of start of day of calibration midpoint" 

507 else: 

508 oldest, newest = group.extremes() 

509 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end) 

510 

511 for k, v in dateCards.items(): 

512 header.set(k, v, comment=comments.get(k, None)) 

513 

514 return header 

515 

516 def interpolateNans(self, exp): 

517 """Interpolate over NANs in the combined image. 

518 

519 NANs can result from masked areas on the CCD. We don't want 

520 them getting into our science images, so we replace them with 

521 the median of the image. 

522 

523 Parameters 

524 ---------- 

525 exp : `lsst.afw.image.Exposure` 

526 Exp to check for NaNs. 

527 """ 

528 array = exp.getImage().getArray() 

529 bad = np.isnan(array) 

530 

531 median = np.median(array[np.logical_not(bad)]) 

532 count = np.sum(np.logical_not(bad)) 

533 array[bad] = median 

534 if count > 0: 

535 self.log.warning("Found %s NAN pixels", count) 

536 

537 

538# Create versions of the Connections, Config, and Task that support 

539# filter constraints. 

540class CalibCombineByFilterConnections(CalibCombineConnections, 

541 dimensions=("instrument", "detector", "physical_filter")): 

542 inputScales = cT.Input( 

543 name="cpFilterScales", 

544 doc="Input scale factors to use.", 

545 storageClass="StructuredDataDict", 

546 dimensions=("instrument", "physical_filter"), 

547 multiple=False, 

548 ) 

549 

550 outputData = cT.Output( 

551 name="cpFilterProposal", 

552 doc="Output combined proposed calibration to be validated and certified.", 

553 storageClass="ExposureF", 

554 dimensions=("instrument", "detector", "physical_filter"), 

555 isCalibration=True, 

556 ) 

557 

558 def __init__(self, *, config=None): 

559 super().__init__(config=config) 

560 

561 if config and config.exposureScaling != 'InputList': 

562 self.inputs.discard("inputScales") 

563 

564 

565class CalibCombineByFilterConfig(CalibCombineConfig, 

566 pipelineConnections=CalibCombineByFilterConnections): 

567 pass 

568 

569 

570class CalibCombineByFilterTask(CalibCombineTask): 

571 """Task to combine calib exposures.""" 

572 

573 ConfigClass = CalibCombineByFilterConfig 

574 _DefaultName = 'cpFilterCombine' 

575 pass 

576 

577 

578def vignetteExposure(exposure, polygon=None, 

579 doUpdateMask=True, maskPlane="NO_DATA", 

580 doSetValue=False, vignetteValue=0.0, 

581 log=None): 

582 """Apply vignetted polygon to image pixels. 

583 

584 Parameters 

585 ---------- 

586 exposure : `lsst.afw.image.Exposure` 

587 Image to be updated. 

588 doUpdateMask : `bool`, optional 

589 Update the exposure mask for vignetted area? 

590 maskPlane : `str`, optional 

591 Mask plane to assign. 

592 doSetValue : `bool`, optional 

593 Set image value for vignetted area? 

594 vignetteValue : `float`, optional 

595 Value to assign. 

596 log : `logging.Logger`, optional 

597 Log to write to. 

598 

599 Raises 

600 ------ 

601 RuntimeError 

602 Raised if no valid polygon exists. 

603 """ 

604 polygon = polygon if polygon else exposure.getInfo().getValidPolygon() 

605 if not polygon: 

606 raise RuntimeError("Could not find valid polygon!") 

607 log = log if log else logging.getLogger(__name__) 

608 

609 fullyIlluminated = True 

610 for corner in exposure.getBBox().getCorners(): 

611 if not polygon.contains(Point2D(corner)): 

612 fullyIlluminated = False 

613 

614 log.info("Exposure is fully illuminated? %s", fullyIlluminated) 

615 

616 if not fullyIlluminated: 

617 # Scan pixels. 

618 mask = exposure.getMask() 

619 numPixels = mask.getBBox().getArea() 

620 

621 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int), 

622 np.arange(0, mask.getHeight(), dtype=int)) 

623 

624 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in 

625 zip(xx.reshape(numPixels), yy.reshape(numPixels))]) 

626 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth()) 

627 

628 if doUpdateMask: 

629 bitMask = mask.getPlaneBitMask(maskPlane) 

630 maskArray = mask.getArray() 

631 maskArray[vignMask] |= bitMask 

632 if doSetValue: 

633 imageArray = exposure.getImage().getArray() 

634 imageArray[vignMask] = vignetteValue 

635 log.info("Exposure contains %d vignetted pixels.", 

636 np.count_nonzero(vignMask))