Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import time 

23 

24import lsst.pex.config as pexConfig 

25import lsst.pipe.base as pipeBase 

26import lsst.pipe.base.connectionTypes as cT 

27import lsst.afw.math as afwMath 

28import lsst.afw.image as afwImage 

29 

30from lsst.geom import Point2D 

31from lsst.log import Log 

32from astro_metadata_translator import merge_headers, ObservationGroup 

33from astro_metadata_translator.serialize import dates_to_fits 

34 

35 

36# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py 

37class CalibStatsConfig(pexConfig.Config): 

38 """Parameters controlling the measurement of background statistics. 

39 """ 

40 stat = pexConfig.Field( 

41 dtype=int, 

42 default=int(afwMath.MEANCLIP), 

43 doc="Statistic to use to estimate background (from lsst.afw.math)", 

44 ) 

45 clip = pexConfig.Field( 

46 dtype=float, 

47 default=3.0, 

48 doc="Clipping threshold for background", 

49 ) 

50 nIter = pexConfig.Field( 

51 dtype=int, 

52 default=3, 

53 doc="Clipping iterations for background", 

54 ) 

55 mask = pexConfig.ListField( 

56 dtype=str, 

57 default=["DETECTED", "BAD", "NO_DATA"], 

58 doc="Mask planes to reject", 

59 ) 

60 

61 

62class CalibStatsTask(pipeBase.Task): 

63 """Measure statistics on the background 

64 

65 This can be useful for scaling the background, e.g., for flats and fringe frames. 

66 """ 

67 ConfigClass = CalibStatsConfig 

68 

69 def run(self, exposureOrImage): 

70 """Measure a particular statistic on an image (of some sort). 

71 

72 Parameters 

73 ---------- 

74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image` 

75 Exposure or image to calculate statistics on. 

76 

77 Returns 

78 ------- 

79 results : float 

80 Resulting statistic value. 

81 """ 

82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

83 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

84 try: 

85 image = exposureOrImage.getMaskedImage() 

86 except Exception: 

87 try: 

88 image = exposureOrImage.getImage() 

89 except Exception: 

90 image = exposureOrImage 

91 

92 return afwMath.makeStatistics(image, self.config.stat, stats).getValue() 

93 

94 

95class CalibCombineConnections(pipeBase.PipelineTaskConnections, 

96 dimensions=("instrument", "detector")): 

97 inputExps = cT.Input( 

98 name="cpInputs", 

99 doc="Input pre-processed exposures to combine.", 

100 storageClass="ExposureF", 

101 dimensions=("instrument", "detector", "exposure"), 

102 multiple=True, 

103 ) 

104 inputScales = cT.Input( 

105 name="cpScales", 

106 doc="Input scale factors to use.", 

107 storageClass="StructuredDataDict", 

108 dimensions=("instrument", ), 

109 multiple=False, 

110 ) 

111 

112 outputData = cT.Output( 

113 name="cpProposal", 

114 doc="Output combined proposed calibration.", 

115 storageClass="ExposureF", 

116 dimensions=("instrument", "detector"), 

117 ) 

118 

119 def __init__(self, *, config=None): 

120 super().__init__(config=config) 

121 

122 if config and len(config.calibrationDimensions) != 0: 

123 newDimensions = tuple(config.calibrationDimensions) 

124 newOutputData = cT.Output( 

125 name=self.outputData.name, 

126 doc=self.outputData.doc, 

127 storageClass=self.outputData.storageClass, 

128 dimensions=self.allConnections['outputData'].dimensions + newDimensions 

129 ) 

130 self.dimensions.update(config.calibrationDimensions) 

131 self.outputData = newOutputData 

132 

133 if config.exposureScaling == 'InputList': 

134 newInputScales = cT.PrerequisiteInput( 

135 name=self.inputScales.name, 

136 doc=self.inputScales.doc, 

137 storageClass=self.inputScales.storageClass, 

138 dimensions=self.allConnections['inputScales'].dimensions + newDimensions 

139 ) 

140 self.dimensions.update(config.calibrationDimensions) 

141 self.inputScales = newInputScales 

142 

143 

144# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py 

145class CalibCombineConfig(pipeBase.PipelineTaskConfig, 

146 pipelineConnections=CalibCombineConnections): 

147 """Configuration for combining calib exposures. 

148 """ 

149 calibrationType = pexConfig.Field( 

150 dtype=str, 

151 default="calibration", 

152 doc="Name of calibration to be generated.", 

153 ) 

154 calibrationDimensions = pexConfig.ListField( 

155 dtype=str, 

156 default=[], 

157 doc="List of updated dimensions to append to output." 

158 ) 

159 

160 exposureScaling = pexConfig.ChoiceField( 

161 dtype=str, 

162 allowed={ 

163 "None": "No scaling used.", 

164 "ExposureTime": "Scale inputs by their exposure time.", 

165 "DarkTime": "Scale inputs by their dark time.", 

166 "MeanStats": "Scale inputs based on their mean values.", 

167 "InputList": "Scale inputs based on a list of values.", 

168 }, 

169 default=None, 

170 doc="Scaling to be applied to each input exposure.", 

171 ) 

172 scalingLevel = pexConfig.ChoiceField( 

173 dtype=str, 

174 allowed={ 

175 "DETECTOR": "Scale by detector.", 

176 "AMP": "Scale by amplifier.", 

177 }, 

178 default="DETECTOR", 

179 doc="Region to scale.", 

180 ) 

181 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field( 

182 dtype=int, 

183 default=5, 

184 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread", 

185 ) 

186 

187 doVignette = pexConfig.Field( 

188 dtype=bool, 

189 default=False, 

190 doc="Copy vignette polygon to output and censor vignetted pixels?" 

191 ) 

192 

193 mask = pexConfig.ListField( 

194 dtype=str, 

195 default=["SAT", "DETECTED", "INTRP"], 

196 doc="Mask planes to respect", 

197 ) 

198 combine = pexConfig.Field( 

199 dtype=int, 

200 default=int(afwMath.MEANCLIP), 

201 doc="Statistic to use for combination (from lsst.afw.math)", 

202 ) 

203 clip = pexConfig.Field( 

204 dtype=float, 

205 default=3.0, 

206 doc="Clipping threshold for combination", 

207 ) 

208 nIter = pexConfig.Field( 

209 dtype=int, 

210 default=3, 

211 doc="Clipping iterations for combination", 

212 ) 

213 stats = pexConfig.ConfigurableField( 

214 target=CalibStatsTask, 

215 doc="Background statistics configuration", 

216 ) 

217 

218 

219class CalibCombineTask(pipeBase.PipelineTask, 

220 pipeBase.CmdLineTask): 

221 """Task to combine calib exposures.""" 

222 ConfigClass = CalibCombineConfig 

223 _DefaultName = 'cpCombine' 

224 

225 def __init__(self, **kwargs): 

226 super().__init__(**kwargs) 

227 self.makeSubtask("stats") 

228 

229 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

230 inputs = butlerQC.get(inputRefs) 

231 

232 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps] 

233 inputs['inputDims'] = dimensions 

234 

235 outputs = self.run(**inputs) 

236 butlerQC.put(outputs, outputRefs) 

237 

238 def run(self, inputExps, inputScales=None, inputDims=None): 

239 """Combine calib exposures for a single detector. 

240 

241 Parameters 

242 ---------- 

243 inputExps : `list` [`lsst.afw.image.Exposure`] 

244 Input list of exposures to combine. 

245 inputScales : `dict` [`dict` [`dict` [`float`]]], optional 

246 Dictionary of scales, indexed by detector (`int`), 

247 amplifier (`int`), and exposure (`int`). Used for 

248 'inputList' scaling. 

249 inputDims : `list` [`dict`] 

250 List of dictionaries of input data dimensions/values. 

251 Each list entry should contain: 

252 

253 ``"exposure"`` 

254 exposure id value (`int`) 

255 ``"detector"`` 

256 detector id value (`int`) 

257 

258 Returns 

259 ------- 

260 combinedExp : `lsst.afw.image.Exposure` 

261 Final combined exposure generated from the inputs. 

262 

263 Raises 

264 ------ 

265 RuntimeError 

266 Raised if no input data is found. Also raised if 

267 config.exposureScaling == InputList, and a necessary scale 

268 was not found. 

269 """ 

270 width, height = self.getDimensions(inputExps) 

271 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

272 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

273 numExps = len(inputExps) 

274 if numExps < 1: 

275 raise RuntimeError("No valid input data") 

276 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance: 

277 stats.setCalcErrorFromInputVariance(True) 

278 

279 # Create output exposure for combined data. 

280 combined = afwImage.MaskedImageF(width, height) 

281 combinedExp = afwImage.makeExposure(combined) 

282 

283 # Apply scaling: 

284 expScales = [] 

285 if inputDims is None: 

286 inputDims = [dict() for i in inputExps] 

287 

288 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)): 

289 scale = 1.0 

290 if exp is None: 

291 self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims) 

292 continue 

293 

294 if self.config.exposureScaling == "ExposureTime": 

295 scale = exp.getInfo().getVisitInfo().getExposureTime() 

296 elif self.config.exposureScaling == "DarkTime": 

297 scale = exp.getInfo().getVisitInfo().getDarkTime() 

298 elif self.config.exposureScaling == "MeanStats": 

299 scale = self.stats.run(exp) 

300 elif self.config.exposureScaling == "InputList": 

301 visitId = dims.get('exposure', None) 

302 detectorId = dims.get('detector', None) 

303 if visitId is None or detectorId is None: 

304 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})") 

305 if detectorId not in inputScales['expScale']: 

306 raise RuntimeError(f"Could not identify a scaling for input {index}" 

307 f" detector {detectorId}") 

308 

309 if self.config.scalingLevel == "DETECTOR": 

310 if visitId not in inputScales['expScale'][detectorId]: 

311 raise RuntimeError(f"Could not identify a scaling for input {index}" 

312 f"detector {detectorId} visit {visitId}") 

313 scale = inputScales['expScale'][detectorId][visitId] 

314 elif self.config.scalingLevel == 'AMP': 

315 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId] 

316 for amp in exp.getDetector()] 

317 else: 

318 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}") 

319 elif self.config.exposureScaling == 'None': 

320 scale = 1.0 

321 else: 

322 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.") 

323 

324 expScales.append(scale) 

325 self.log.info("Scaling input %d by %s", index, scale) 

326 self.applyScale(exp, scale) 

327 

328 self.combine(combined, inputExps, stats) 

329 

330 self.interpolateNans(combined) 

331 

332 if self.config.doVignette: 

333 polygon = inputExps[0].getInfo().getValidPolygon() 

334 VignetteExposure(combined, polygon=polygon, doUpdateMask=True, 

335 doSetValue=True, vignetteValue=0.0) 

336 

337 # Combine headers 

338 self.combineHeaders(inputExps, combinedExp, 

339 calibType=self.config.calibrationType, scales=expScales) 

340 

341 # Return 

342 return pipeBase.Struct( 

343 outputData=combinedExp, 

344 ) 

345 

346 def getDimensions(self, expList): 

347 """Get dimensions of the inputs. 

348 

349 Parameters 

350 ---------- 

351 expList : `list` [`lsst.afw.image.Exposure`] 

352 Exps to check the sizes of. 

353 

354 Returns 

355 ------- 

356 width, height : `int` 

357 Unique set of input dimensions. 

358 """ 

359 dimList = [exp.getDimensions() for exp in expList if exp is not None] 

360 return self.getSize(dimList) 

361 

362 def getSize(self, dimList): 

363 """Determine a consistent size, given a list of image sizes. 

364 

365 Parameters 

366 ----------- 

367 dimList : iterable of `tuple` (`int`, `int`) 

368 List of dimensions. 

369 

370 Raises 

371 ------ 

372 RuntimeError 

373 If input dimensions are inconsistent. 

374 

375 Returns 

376 -------- 

377 width, height : `int` 

378 Common dimensions. 

379 """ 

380 dim = set((w, h) for w, h in dimList) 

381 if len(dim) != 1: 

382 raise RuntimeError("Inconsistent dimensions: %s" % dim) 

383 return dim.pop() 

384 

385 def applyScale(self, exposure, scale=None): 

386 """Apply scale to input exposure. 

387 

388 This implementation applies a flux scaling: the input exposure is 

389 divided by the provided scale. 

390 

391 Parameters 

392 ---------- 

393 exposure : `lsst.afw.image.Exposure` 

394 Exposure to scale. 

395 scale : `float` or `list` [`float`], optional 

396 Constant scale to divide the exposure by. 

397 """ 

398 if scale is not None: 

399 mi = exposure.getMaskedImage() 

400 if isinstance(scale, list): 

401 for amp, ampScale in zip(exposure.getDetector(), scale): 

402 ampIm = mi[amp.getBBox()] 

403 ampIm /= ampScale 

404 else: 

405 mi /= scale 

406 

407 def combine(self, target, expList, stats): 

408 """Combine multiple images. 

409 

410 Parameters 

411 ---------- 

412 target : `lsst.afw.image.Exposure` 

413 Output exposure to construct. 

414 expList : `list` [`lsst.afw.image.Exposure`] 

415 Input exposures to combine. 

416 stats : `lsst.afw.math.StatisticsControl` 

417 Control explaining how to combine the input images. 

418 """ 

419 images = [img.getMaskedImage() for img in expList if img is not None] 

420 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats) 

421 

422 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None): 

423 """Combine input headers to determine the set of common headers, 

424 supplemented by calibration inputs. 

425 

426 Parameters 

427 ---------- 

428 expList : `list` of `lsst.afw.image.Exposure` 

429 Input list of exposures to combine. 

430 calib : `lsst.afw.image.Exposure` 

431 Output calibration to construct headers for. 

432 calibType: `str`, optional 

433 OBSTYPE the output should claim. 

434 scales: `list` of `float`, optional 

435 Scale values applied to each input to record. 

436 

437 Returns 

438 ------- 

439 header : `lsst.daf.base.PropertyList` 

440 Constructed header. 

441 """ 

442 # Header 

443 header = calib.getMetadata() 

444 header.set("OBSTYPE", calibType) 

445 

446 # Keywords we care about 

447 comments = {"TIMESYS": "Time scale for all dates", 

448 "DATE-OBS": "Start date of earliest input observation", 

449 "MJD-OBS": "[d] Start MJD of earliest input observation", 

450 "DATE-END": "End date of oldest input observation", 

451 "MJD-END": "[d] End MJD of oldest input observation", 

452 "MJD-AVG": "[d] MJD midpoint of all input observations", 

453 "DATE-AVG": "Midpoint date of all input observations"} 

454 

455 # Creation date 

456 now = time.localtime() 

457 calibDate = time.strftime("%Y-%m-%d", now) 

458 calibTime = time.strftime("%X %Z", now) 

459 header.set("CALIB_CREATE_DATE", calibDate) 

460 header.set("CALIB_CREATE_TIME", calibTime) 

461 

462 # Merge input headers 

463 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None] 

464 merged = merge_headers(inputHeaders, mode='drop') 

465 for k, v in merged.items(): 

466 if k not in header: 

467 md = expList[0].getMetadata() 

468 comment = md.getComment(k) if k in md else None 

469 header.set(k, v, comment=comment) 

470 

471 # Construct list of visits 

472 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None] 

473 for i, visit in enumerate(visitInfoList): 

474 if visit is None: 

475 continue 

476 header.set("CPP_INPUT_%d" % (i,), visit.getExposureId()) 

477 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate())) 

478 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime()) 

479 if scales is not None: 

480 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i]) 

481 

482 # Not yet working: DM-22302 

483 # Create an observation group so we can add some standard headers 

484 # independent of the form in the input files. 

485 # Use try block in case we are dealing with unexpected data headers 

486 try: 

487 group = ObservationGroup(visitInfoList, pedantic=False) 

488 except Exception: 

489 self.log.warn("Exception making an obs group for headers. Continuing.") 

490 # Fall back to setting a DATE-OBS from the calibDate 

491 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)} 

492 comments["DATE-OBS"] = "Date of start of day of calibration midpoint" 

493 else: 

494 oldest, newest = group.extremes() 

495 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end) 

496 

497 for k, v in dateCards.items(): 

498 header.set(k, v, comment=comments.get(k, None)) 

499 

500 return header 

501 

502 def interpolateNans(self, exp): 

503 """Interpolate over NANs in the combined image. 

504 

505 NANs can result from masked areas on the CCD. We don't want them getting 

506 into our science images, so we replace them with the median of the image. 

507 

508 Parameters 

509 ---------- 

510 exp : `lsst.afw.image.Exposure` 

511 Exp to check for NaNs. 

512 """ 

513 array = exp.getImage().getArray() 

514 bad = np.isnan(array) 

515 

516 median = np.median(array[np.logical_not(bad)]) 

517 count = np.sum(np.logical_not(bad)) 

518 array[bad] = median 

519 if count > 0: 

520 self.log.warn("Found %s NAN pixels", count) 

521 

522 

523def VignetteExposure(exposure, polygon=None, 

524 doUpdateMask=True, maskPlane='BAD', 

525 doSetValue=False, vignetteValue=0.0, 

526 log=None): 

527 """Apply vignetted polygon to image pixels. 

528 

529 Parameters 

530 ---------- 

531 exposure : `lsst.afw.image.Exposure` 

532 Image to be updated. 

533 doUpdateMask : `bool`, optional 

534 Update the exposure mask for vignetted area? 

535 maskPlane : `str`, optional, 

536 Mask plane to assign. 

537 doSetValue : `bool`, optional 

538 Set image value for vignetted area? 

539 vignetteValue : `float`, optional 

540 Value to assign. 

541 log : `lsst.log.Log`, optional 

542 Log to write to. 

543 

544 Raises 

545 ------ 

546 RuntimeError 

547 Raised if no valid polygon exists. 

548 """ 

549 polygon = polygon if polygon else exposure.getInfo().getValidPolygon() 

550 if not polygon: 

551 raise RuntimeError("Could not find valid polygon!") 

552 log = log if log else Log.getLogger(__name__.partition(".")[2]) 

553 

554 fullyIlluminated = True 

555 for corner in exposure.getBBox().getCorners(): 

556 if not polygon.contains(Point2D(corner)): 

557 fullyIlluminated = False 

558 

559 log.info("Exposure is fully illuminated? %s", fullyIlluminated) 

560 

561 if not fullyIlluminated: 

562 # Scan pixels. 

563 mask = exposure.getMask() 

564 numPixels = mask.getBBox().getArea() 

565 

566 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int), 

567 np.arange(0, mask.getHeight(), dtype=int)) 

568 

569 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in 

570 zip(xx.reshape(numPixels), yy.reshape(numPixels))]) 

571 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth()) 

572 

573 if doUpdateMask: 

574 bitMask = mask.getPlaneBitMask(maskPlane) 

575 maskArray = mask.getArray() 

576 maskArray[vignMask] |= bitMask 

577 if doSetValue: 

578 imageArray = exposure.getImage().getArray() 

579 imageArray[vignMask] = vignetteValue 

580 log.info("Exposure contains %d vignetted pixels.", 

581 np.count_nonzero(vignMask))