Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import time 

23 

24import lsst.pex.config as pexConfig 

25import lsst.pipe.base as pipeBase 

26import lsst.pipe.base.connectionTypes as cT 

27import lsst.afw.math as afwMath 

28import lsst.afw.image as afwImage 

29 

30from lsst.geom import Point2D 

31from lsst.log import Log 

32from astro_metadata_translator import merge_headers, ObservationGroup 

33from astro_metadata_translator.serialize import dates_to_fits 

34 

35 

36# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py 

37class CalibStatsConfig(pexConfig.Config): 

38 """Parameters controlling the measurement of background statistics. 

39 """ 

40 stat = pexConfig.Field( 

41 dtype=str, 

42 default='MEANCLIP', 

43 doc="Statistic name to use to estimate background (from lsst.afw.math)", 

44 ) 

45 clip = pexConfig.Field( 

46 dtype=float, 

47 default=3.0, 

48 doc="Clipping threshold for background", 

49 ) 

50 nIter = pexConfig.Field( 

51 dtype=int, 

52 default=3, 

53 doc="Clipping iterations for background", 

54 ) 

55 mask = pexConfig.ListField( 

56 dtype=str, 

57 default=["DETECTED", "BAD", "NO_DATA"], 

58 doc="Mask planes to reject", 

59 ) 

60 

61 

62class CalibStatsTask(pipeBase.Task): 

63 """Measure statistics on the background 

64 

65 This can be useful for scaling the background, e.g., for flats and fringe frames. 

66 """ 

67 ConfigClass = CalibStatsConfig 

68 

69 def run(self, exposureOrImage): 

70 """Measure a particular statistic on an image (of some sort). 

71 

72 Parameters 

73 ---------- 

74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image` 

75 Exposure or image to calculate statistics on. 

76 

77 Returns 

78 ------- 

79 results : float 

80 Resulting statistic value. 

81 """ 

82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

83 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

84 try: 

85 image = exposureOrImage.getMaskedImage() 

86 except Exception: 

87 try: 

88 image = exposureOrImage.getImage() 

89 except Exception: 

90 image = exposureOrImage 

91 statType = afwMath.stringToStatisticsProperty(self.config.stat) 

92 return afwMath.makeStatistics(image, statType, stats).getValue() 

93 

94 

95class CalibCombineConnections(pipeBase.PipelineTaskConnections, 

96 dimensions=("instrument", "detector")): 

97 inputExps = cT.Input( 

98 name="cpInputs", 

99 doc="Input pre-processed exposures to combine.", 

100 storageClass="Exposure", 

101 dimensions=("instrument", "detector", "exposure"), 

102 multiple=True, 

103 ) 

104 inputScales = cT.Input( 

105 name="cpScales", 

106 doc="Input scale factors to use.", 

107 storageClass="StructuredDataDict", 

108 dimensions=("instrument", ), 

109 multiple=False, 

110 ) 

111 

112 outputData = cT.Output( 

113 name="cpProposal", 

114 doc="Output combined proposed calibration to be validated and certified..", 

115 storageClass="ExposureF", 

116 dimensions=("instrument", "detector"), 

117 isCalibration=True, 

118 ) 

119 

120 def __init__(self, *, config=None): 

121 super().__init__(config=config) 

122 

123 if config and config.exposureScaling != 'InputList': 

124 self.inputs.discard("inputScales") 

125 

126 

127# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py 

128class CalibCombineConfig(pipeBase.PipelineTaskConfig, 

129 pipelineConnections=CalibCombineConnections): 

130 """Configuration for combining calib exposures. 

131 """ 

132 calibrationType = pexConfig.Field( 

133 dtype=str, 

134 default="calibration", 

135 doc="Name of calibration to be generated.", 

136 ) 

137 

138 exposureScaling = pexConfig.ChoiceField( 

139 dtype=str, 

140 allowed={ 

141 "Unity": "Do not scale inputs. Scale factor is 1.0.", 

142 "ExposureTime": "Scale inputs by their exposure time.", 

143 "DarkTime": "Scale inputs by their dark time.", 

144 "MeanStats": "Scale inputs based on their mean values.", 

145 "InputList": "Scale inputs based on a list of values.", 

146 }, 

147 default="Unity", 

148 doc="Scaling to be applied to each input exposure.", 

149 ) 

150 scalingLevel = pexConfig.ChoiceField( 

151 dtype=str, 

152 allowed={ 

153 "DETECTOR": "Scale by detector.", 

154 "AMP": "Scale by amplifier.", 

155 }, 

156 default="DETECTOR", 

157 doc="Region to scale.", 

158 ) 

159 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field( 

160 dtype=int, 

161 default=5, 

162 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread", 

163 ) 

164 

165 doVignette = pexConfig.Field( 

166 dtype=bool, 

167 default=False, 

168 doc="Copy vignette polygon to output and censor vignetted pixels?" 

169 ) 

170 

171 mask = pexConfig.ListField( 

172 dtype=str, 

173 default=["SAT", "DETECTED", "INTRP"], 

174 doc="Mask planes to respect", 

175 ) 

176 combine = pexConfig.Field( 

177 dtype=str, 

178 default='MEANCLIP', 

179 doc="Statistic name to use for combination (from lsst.afw.math)", 

180 ) 

181 clip = pexConfig.Field( 

182 dtype=float, 

183 default=3.0, 

184 doc="Clipping threshold for combination", 

185 ) 

186 nIter = pexConfig.Field( 

187 dtype=int, 

188 default=3, 

189 doc="Clipping iterations for combination", 

190 ) 

191 stats = pexConfig.ConfigurableField( 

192 target=CalibStatsTask, 

193 doc="Background statistics configuration", 

194 ) 

195 

196 

197class CalibCombineTask(pipeBase.PipelineTask, 

198 pipeBase.CmdLineTask): 

199 """Task to combine calib exposures.""" 

200 ConfigClass = CalibCombineConfig 

201 _DefaultName = 'cpCombine' 

202 

203 def __init__(self, **kwargs): 

204 super().__init__(**kwargs) 

205 self.makeSubtask("stats") 

206 

207 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

208 inputs = butlerQC.get(inputRefs) 

209 

210 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps] 

211 inputs['inputDims'] = dimensions 

212 

213 outputs = self.run(**inputs) 

214 butlerQC.put(outputs, outputRefs) 

215 

216 def run(self, inputExps, inputScales=None, inputDims=None): 

217 """Combine calib exposures for a single detector. 

218 

219 Parameters 

220 ---------- 

221 inputExps : `list` [`lsst.afw.image.Exposure`] 

222 Input list of exposures to combine. 

223 inputScales : `dict` [`dict` [`dict` [`float`]]], optional 

224 Dictionary of scales, indexed by detector (`int`), 

225 amplifier (`int`), and exposure (`int`). Used for 

226 'inputList' scaling. 

227 inputDims : `list` [`dict`] 

228 List of dictionaries of input data dimensions/values. 

229 Each list entry should contain: 

230 

231 ``"exposure"`` 

232 exposure id value (`int`) 

233 ``"detector"`` 

234 detector id value (`int`) 

235 

236 Returns 

237 ------- 

238 combinedExp : `lsst.afw.image.Exposure` 

239 Final combined exposure generated from the inputs. 

240 

241 Raises 

242 ------ 

243 RuntimeError 

244 Raised if no input data is found. Also raised if 

245 config.exposureScaling == InputList, and a necessary scale 

246 was not found. 

247 """ 

248 width, height = self.getDimensions(inputExps) 

249 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

250 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

251 numExps = len(inputExps) 

252 if numExps < 1: 

253 raise RuntimeError("No valid input data") 

254 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance: 

255 stats.setCalcErrorFromInputVariance(True) 

256 

257 # Create output exposure for combined data. 

258 combined = afwImage.MaskedImageF(width, height) 

259 combinedExp = afwImage.makeExposure(combined) 

260 

261 # Apply scaling: 

262 expScales = [] 

263 if inputDims is None: 

264 inputDims = [dict() for i in inputExps] 

265 

266 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)): 

267 scale = 1.0 

268 if exp is None: 

269 self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims) 

270 continue 

271 

272 if self.config.exposureScaling == "ExposureTime": 

273 scale = exp.getInfo().getVisitInfo().getExposureTime() 

274 elif self.config.exposureScaling == "DarkTime": 

275 scale = exp.getInfo().getVisitInfo().getDarkTime() 

276 elif self.config.exposureScaling == "MeanStats": 

277 scale = self.stats.run(exp) 

278 elif self.config.exposureScaling == "InputList": 

279 visitId = dims.get('exposure', None) 

280 detectorId = dims.get('detector', None) 

281 if visitId is None or detectorId is None: 

282 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})") 

283 if detectorId not in inputScales['expScale']: 

284 raise RuntimeError(f"Could not identify a scaling for input {index}" 

285 f" detector {detectorId}") 

286 

287 if self.config.scalingLevel == "DETECTOR": 

288 if visitId not in inputScales['expScale'][detectorId]: 

289 raise RuntimeError(f"Could not identify a scaling for input {index}" 

290 f"detector {detectorId} visit {visitId}") 

291 scale = inputScales['expScale'][detectorId][visitId] 

292 elif self.config.scalingLevel == 'AMP': 

293 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId] 

294 for amp in exp.getDetector()] 

295 else: 

296 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}") 

297 elif self.config.exposureScaling == 'Unity': 

298 scale = 1.0 

299 else: 

300 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.") 

301 

302 expScales.append(scale) 

303 self.log.info("Scaling input %d by %s", index, scale) 

304 self.applyScale(exp, scale) 

305 

306 self.combine(combined, inputExps, stats) 

307 

308 self.interpolateNans(combined) 

309 

310 if self.config.doVignette: 

311 polygon = inputExps[0].getInfo().getValidPolygon() 

312 VignetteExposure(combined, polygon=polygon, doUpdateMask=True, 

313 doSetValue=True, vignetteValue=0.0) 

314 

315 # Combine headers 

316 self.combineHeaders(inputExps, combinedExp, 

317 calibType=self.config.calibrationType, scales=expScales) 

318 

319 # Return 

320 return pipeBase.Struct( 

321 outputData=combinedExp, 

322 ) 

323 

324 def getDimensions(self, expList): 

325 """Get dimensions of the inputs. 

326 

327 Parameters 

328 ---------- 

329 expList : `list` [`lsst.afw.image.Exposure`] 

330 Exps to check the sizes of. 

331 

332 Returns 

333 ------- 

334 width, height : `int` 

335 Unique set of input dimensions. 

336 """ 

337 dimList = [exp.getDimensions() for exp in expList if exp is not None] 

338 return self.getSize(dimList) 

339 

340 def getSize(self, dimList): 

341 """Determine a consistent size, given a list of image sizes. 

342 

343 Parameters 

344 ----------- 

345 dimList : iterable of `tuple` (`int`, `int`) 

346 List of dimensions. 

347 

348 Raises 

349 ------ 

350 RuntimeError 

351 If input dimensions are inconsistent. 

352 

353 Returns 

354 -------- 

355 width, height : `int` 

356 Common dimensions. 

357 """ 

358 dim = set((w, h) for w, h in dimList) 

359 if len(dim) != 1: 

360 raise RuntimeError("Inconsistent dimensions: %s" % dim) 

361 return dim.pop() 

362 

363 def applyScale(self, exposure, scale=None): 

364 """Apply scale to input exposure. 

365 

366 This implementation applies a flux scaling: the input exposure is 

367 divided by the provided scale. 

368 

369 Parameters 

370 ---------- 

371 exposure : `lsst.afw.image.Exposure` 

372 Exposure to scale. 

373 scale : `float` or `list` [`float`], optional 

374 Constant scale to divide the exposure by. 

375 """ 

376 if scale is not None: 

377 mi = exposure.getMaskedImage() 

378 if isinstance(scale, list): 

379 for amp, ampScale in zip(exposure.getDetector(), scale): 

380 ampIm = mi[amp.getBBox()] 

381 ampIm /= ampScale 

382 else: 

383 mi /= scale 

384 

385 def combine(self, target, expList, stats): 

386 """Combine multiple images. 

387 

388 Parameters 

389 ---------- 

390 target : `lsst.afw.image.Exposure` 

391 Output exposure to construct. 

392 expList : `list` [`lsst.afw.image.Exposure`] 

393 Input exposures to combine. 

394 stats : `lsst.afw.math.StatisticsControl` 

395 Control explaining how to combine the input images. 

396 """ 

397 images = [img.getMaskedImage() for img in expList if img is not None] 

398 combineType = afwMath.stringToStatisticsProperty(self.config.combine) 

399 afwMath.statisticsStack(target, images, combineType, stats) 

400 

401 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None): 

402 """Combine input headers to determine the set of common headers, 

403 supplemented by calibration inputs. 

404 

405 Parameters 

406 ---------- 

407 expList : `list` of `lsst.afw.image.Exposure` 

408 Input list of exposures to combine. 

409 calib : `lsst.afw.image.Exposure` 

410 Output calibration to construct headers for. 

411 calibType: `str`, optional 

412 OBSTYPE the output should claim. 

413 scales: `list` of `float`, optional 

414 Scale values applied to each input to record. 

415 

416 Returns 

417 ------- 

418 header : `lsst.daf.base.PropertyList` 

419 Constructed header. 

420 """ 

421 # Header 

422 header = calib.getMetadata() 

423 header.set("OBSTYPE", calibType) 

424 

425 # Keywords we care about 

426 comments = {"TIMESYS": "Time scale for all dates", 

427 "DATE-OBS": "Start date of earliest input observation", 

428 "MJD-OBS": "[d] Start MJD of earliest input observation", 

429 "DATE-END": "End date of oldest input observation", 

430 "MJD-END": "[d] End MJD of oldest input observation", 

431 "MJD-AVG": "[d] MJD midpoint of all input observations", 

432 "DATE-AVG": "Midpoint date of all input observations"} 

433 

434 # Creation date 

435 now = time.localtime() 

436 calibDate = time.strftime("%Y-%m-%d", now) 

437 calibTime = time.strftime("%X %Z", now) 

438 header.set("CALIB_CREATE_DATE", calibDate) 

439 header.set("CALIB_CREATE_TIME", calibTime) 

440 

441 # Merge input headers 

442 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None] 

443 merged = merge_headers(inputHeaders, mode='drop') 

444 for k, v in merged.items(): 

445 if k not in header: 

446 md = expList[0].getMetadata() 

447 comment = md.getComment(k) if k in md else None 

448 header.set(k, v, comment=comment) 

449 

450 # Construct list of visits 

451 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None] 

452 for i, visit in enumerate(visitInfoList): 

453 if visit is None: 

454 continue 

455 header.set("CPP_INPUT_%d" % (i,), visit.getExposureId()) 

456 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate())) 

457 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime()) 

458 if scales is not None: 

459 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i]) 

460 

461 # Not yet working: DM-22302 

462 # Create an observation group so we can add some standard headers 

463 # independent of the form in the input files. 

464 # Use try block in case we are dealing with unexpected data headers 

465 try: 

466 group = ObservationGroup(visitInfoList, pedantic=False) 

467 except Exception: 

468 self.log.warn("Exception making an obs group for headers. Continuing.") 

469 # Fall back to setting a DATE-OBS from the calibDate 

470 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)} 

471 comments["DATE-OBS"] = "Date of start of day of calibration midpoint" 

472 else: 

473 oldest, newest = group.extremes() 

474 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end) 

475 

476 for k, v in dateCards.items(): 

477 header.set(k, v, comment=comments.get(k, None)) 

478 

479 return header 

480 

481 def interpolateNans(self, exp): 

482 """Interpolate over NANs in the combined image. 

483 

484 NANs can result from masked areas on the CCD. We don't want them getting 

485 into our science images, so we replace them with the median of the image. 

486 

487 Parameters 

488 ---------- 

489 exp : `lsst.afw.image.Exposure` 

490 Exp to check for NaNs. 

491 """ 

492 array = exp.getImage().getArray() 

493 bad = np.isnan(array) 

494 

495 median = np.median(array[np.logical_not(bad)]) 

496 count = np.sum(np.logical_not(bad)) 

497 array[bad] = median 

498 if count > 0: 

499 self.log.warn("Found %s NAN pixels", count) 

500 

501 

502# Create versions of the Connections, Config, and Task that support filter constraints. 

503class CalibCombineByFilterConnections(CalibCombineConnections, 

504 dimensions=("instrument", "detector", "physical_filter")): 

505 inputScales = cT.Input( 

506 name="cpFilterScales", 

507 doc="Input scale factors to use.", 

508 storageClass="StructuredDataDict", 

509 dimensions=("instrument", "physical_filter"), 

510 multiple=False, 

511 ) 

512 

513 outputData = cT.Output( 

514 name="cpFilterProposal", 

515 doc="Output combined proposed calibration to be validated and certified.", 

516 storageClass="ExposureF", 

517 dimensions=("instrument", "detector", "physical_filter"), 

518 isCalibration=True, 

519 ) 

520 

521 def __init__(self, *, config=None): 

522 super().__init__(config=config) 

523 

524 if config and config.exposureScaling != 'InputList': 

525 self.inputs.discard("inputScales") 

526 

527 

528class CalibCombineByFilterConfig(CalibCombineConfig, 

529 pipelineConnections=CalibCombineByFilterConnections): 

530 pass 

531 

532 

533class CalibCombineByFilterTask(CalibCombineTask): 

534 """Task to combine calib exposures.""" 

535 ConfigClass = CalibCombineByFilterConfig 

536 _DefaultName = 'cpFilterCombine' 

537 pass 

538 

539 

540def VignetteExposure(exposure, polygon=None, 

541 doUpdateMask=True, maskPlane='BAD', 

542 doSetValue=False, vignetteValue=0.0, 

543 log=None): 

544 """Apply vignetted polygon to image pixels. 

545 

546 Parameters 

547 ---------- 

548 exposure : `lsst.afw.image.Exposure` 

549 Image to be updated. 

550 doUpdateMask : `bool`, optional 

551 Update the exposure mask for vignetted area? 

552 maskPlane : `str`, optional, 

553 Mask plane to assign. 

554 doSetValue : `bool`, optional 

555 Set image value for vignetted area? 

556 vignetteValue : `float`, optional 

557 Value to assign. 

558 log : `lsst.log.Log`, optional 

559 Log to write to. 

560 

561 Raises 

562 ------ 

563 RuntimeError 

564 Raised if no valid polygon exists. 

565 """ 

566 polygon = polygon if polygon else exposure.getInfo().getValidPolygon() 

567 if not polygon: 

568 raise RuntimeError("Could not find valid polygon!") 

569 log = log if log else Log.getLogger(__name__.partition(".")[2]) 

570 

571 fullyIlluminated = True 

572 for corner in exposure.getBBox().getCorners(): 

573 if not polygon.contains(Point2D(corner)): 

574 fullyIlluminated = False 

575 

576 log.info("Exposure is fully illuminated? %s", fullyIlluminated) 

577 

578 if not fullyIlluminated: 

579 # Scan pixels. 

580 mask = exposure.getMask() 

581 numPixels = mask.getBBox().getArea() 

582 

583 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int), 

584 np.arange(0, mask.getHeight(), dtype=int)) 

585 

586 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in 

587 zip(xx.reshape(numPixels), yy.reshape(numPixels))]) 

588 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth()) 

589 

590 if doUpdateMask: 

591 bitMask = mask.getPlaneBitMask(maskPlane) 

592 maskArray = mask.getArray() 

593 maskArray[vignMask] |= bitMask 

594 if doSetValue: 

595 imageArray = exposure.getImage().getArray() 

596 imageArray[vignMask] = vignetteValue 

597 log.info("Exposure contains %d vignetted pixels.", 

598 np.count_nonzero(vignMask))