Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21import numpy as np 

22import time 

23 

24import lsst.pex.config as pexConfig 

25import lsst.pipe.base as pipeBase 

26import lsst.pipe.base.connectionTypes as cT 

27import lsst.afw.math as afwMath 

28import lsst.afw.image as afwImage 

29 

30from lsst.geom import Point2D 

31from lsst.log import Log 

32from astro_metadata_translator import merge_headers, ObservationGroup 

33from astro_metadata_translator.serialize import dates_to_fits 

34 

35 

36# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py 

37class CalibStatsConfig(pexConfig.Config): 

38 """Parameters controlling the measurement of background statistics. 

39 """ 

40 stat = pexConfig.Field( 

41 dtype=int, 

42 default=int(afwMath.MEANCLIP), 

43 doc="Statistic to use to estimate background (from lsst.afw.math)", 

44 ) 

45 clip = pexConfig.Field( 

46 dtype=float, 

47 default=3.0, 

48 doc="Clipping threshold for background", 

49 ) 

50 nIter = pexConfig.Field( 

51 dtype=int, 

52 default=3, 

53 doc="Clipping iterations for background", 

54 ) 

55 mask = pexConfig.ListField( 

56 dtype=str, 

57 default=["DETECTED", "BAD", "NO_DATA"], 

58 doc="Mask planes to reject", 

59 ) 

60 

61 

62class CalibStatsTask(pipeBase.Task): 

63 """Measure statistics on the background 

64 

65 This can be useful for scaling the background, e.g., for flats and fringe frames. 

66 """ 

67 ConfigClass = CalibStatsConfig 

68 

69 def run(self, exposureOrImage): 

70 """Measure a particular statistic on an image (of some sort). 

71 

72 Parameters 

73 ---------- 

74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image` 

75 Exposure or image to calculate statistics on. 

76 

77 Returns 

78 ------- 

79 results : float 

80 Resulting statistic value. 

81 """ 

82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

83 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

84 try: 

85 image = exposureOrImage.getMaskedImage() 

86 except Exception: 

87 try: 

88 image = exposureOrImage.getImage() 

89 except Exception: 

90 image = exposureOrImage 

91 

92 return afwMath.makeStatistics(image, self.config.stat, stats).getValue() 

93 

94 

95class CalibCombineConnections(pipeBase.PipelineTaskConnections, 

96 dimensions=("instrument", "detector")): 

97 inputExps = cT.Input( 

98 name="cpInputs", 

99 doc="Input pre-processed exposures to combine.", 

100 storageClass="Exposure", 

101 dimensions=("instrument", "detector", "exposure"), 

102 multiple=True, 

103 ) 

104 inputScales = cT.Input( 

105 name="cpScales", 

106 doc="Input scale factors to use.", 

107 storageClass="StructuredDataDict", 

108 dimensions=("instrument", ), 

109 multiple=False, 

110 ) 

111 

112 outputData = cT.Output( 

113 name="cpProposal", 

114 doc="Output combined proposed calibration.", 

115 storageClass="ExposureF", 

116 dimensions=("instrument", "detector"), 

117 ) 

118 

119 def __init__(self, *, config=None): 

120 super().__init__(config=config) 

121 

122 if config and config.exposureScaling != 'InputList': 

123 self.inputs.discard("inputScales") 

124 

125 if config and len(config.calibrationDimensions) != 0: 

126 newDimensions = tuple(config.calibrationDimensions) 

127 newOutputData = cT.Output( 

128 name=self.outputData.name, 

129 doc=self.outputData.doc, 

130 storageClass=self.outputData.storageClass, 

131 dimensions=self.allConnections['outputData'].dimensions + newDimensions 

132 ) 

133 self.dimensions.update(config.calibrationDimensions) 

134 self.outputData = newOutputData 

135 

136 if config.exposureScaling == 'InputList': 

137 newInputScales = cT.PrerequisiteInput( 

138 name=self.inputScales.name, 

139 doc=self.inputScales.doc, 

140 storageClass=self.inputScales.storageClass, 

141 dimensions=self.allConnections['inputScales'].dimensions + newDimensions 

142 ) 

143 self.dimensions.update(config.calibrationDimensions) 

144 self.inputScales = newInputScales 

145 

146 

147# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py 

148class CalibCombineConfig(pipeBase.PipelineTaskConfig, 

149 pipelineConnections=CalibCombineConnections): 

150 """Configuration for combining calib exposures. 

151 """ 

152 calibrationType = pexConfig.Field( 

153 dtype=str, 

154 default="calibration", 

155 doc="Name of calibration to be generated.", 

156 ) 

157 calibrationDimensions = pexConfig.ListField( 

158 dtype=str, 

159 default=[], 

160 doc="List of updated dimensions to append to output." 

161 ) 

162 

163 exposureScaling = pexConfig.ChoiceField( 

164 dtype=str, 

165 allowed={ 

166 "None": "No scaling used.", 

167 "ExposureTime": "Scale inputs by their exposure time.", 

168 "DarkTime": "Scale inputs by their dark time.", 

169 "MeanStats": "Scale inputs based on their mean values.", 

170 "InputList": "Scale inputs based on a list of values.", 

171 }, 

172 default=None, 

173 doc="Scaling to be applied to each input exposure.", 

174 ) 

175 scalingLevel = pexConfig.ChoiceField( 

176 dtype=str, 

177 allowed={ 

178 "DETECTOR": "Scale by detector.", 

179 "AMP": "Scale by amplifier.", 

180 }, 

181 default="DETECTOR", 

182 doc="Region to scale.", 

183 ) 

184 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field( 

185 dtype=int, 

186 default=5, 

187 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread", 

188 ) 

189 

190 doVignette = pexConfig.Field( 

191 dtype=bool, 

192 default=False, 

193 doc="Copy vignette polygon to output and censor vignetted pixels?" 

194 ) 

195 

196 mask = pexConfig.ListField( 

197 dtype=str, 

198 default=["SAT", "DETECTED", "INTRP"], 

199 doc="Mask planes to respect", 

200 ) 

201 combine = pexConfig.Field( 

202 dtype=int, 

203 default=int(afwMath.MEANCLIP), 

204 doc="Statistic to use for combination (from lsst.afw.math)", 

205 ) 

206 clip = pexConfig.Field( 

207 dtype=float, 

208 default=3.0, 

209 doc="Clipping threshold for combination", 

210 ) 

211 nIter = pexConfig.Field( 

212 dtype=int, 

213 default=3, 

214 doc="Clipping iterations for combination", 

215 ) 

216 stats = pexConfig.ConfigurableField( 

217 target=CalibStatsTask, 

218 doc="Background statistics configuration", 

219 ) 

220 

221 

222class CalibCombineTask(pipeBase.PipelineTask, 

223 pipeBase.CmdLineTask): 

224 """Task to combine calib exposures.""" 

225 ConfigClass = CalibCombineConfig 

226 _DefaultName = 'cpCombine' 

227 

228 def __init__(self, **kwargs): 

229 super().__init__(**kwargs) 

230 self.makeSubtask("stats") 

231 

232 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

233 inputs = butlerQC.get(inputRefs) 

234 

235 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps] 

236 inputs['inputDims'] = dimensions 

237 

238 outputs = self.run(**inputs) 

239 butlerQC.put(outputs, outputRefs) 

240 

241 def run(self, inputExps, inputScales=None, inputDims=None): 

242 """Combine calib exposures for a single detector. 

243 

244 Parameters 

245 ---------- 

246 inputExps : `list` [`lsst.afw.image.Exposure`] 

247 Input list of exposures to combine. 

248 inputScales : `dict` [`dict` [`dict` [`float`]]], optional 

249 Dictionary of scales, indexed by detector (`int`), 

250 amplifier (`int`), and exposure (`int`). Used for 

251 'inputList' scaling. 

252 inputDims : `list` [`dict`] 

253 List of dictionaries of input data dimensions/values. 

254 Each list entry should contain: 

255 

256 ``"exposure"`` 

257 exposure id value (`int`) 

258 ``"detector"`` 

259 detector id value (`int`) 

260 

261 Returns 

262 ------- 

263 combinedExp : `lsst.afw.image.Exposure` 

264 Final combined exposure generated from the inputs. 

265 

266 Raises 

267 ------ 

268 RuntimeError 

269 Raised if no input data is found. Also raised if 

270 config.exposureScaling == InputList, and a necessary scale 

271 was not found. 

272 """ 

273 width, height = self.getDimensions(inputExps) 

274 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter, 

275 afwImage.Mask.getPlaneBitMask(self.config.mask)) 

276 numExps = len(inputExps) 

277 if numExps < 1: 

278 raise RuntimeError("No valid input data") 

279 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance: 

280 stats.setCalcErrorFromInputVariance(True) 

281 

282 # Create output exposure for combined data. 

283 combined = afwImage.MaskedImageF(width, height) 

284 combinedExp = afwImage.makeExposure(combined) 

285 

286 # Apply scaling: 

287 expScales = [] 

288 if inputDims is None: 

289 inputDims = [dict() for i in inputExps] 

290 

291 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)): 

292 scale = 1.0 

293 if exp is None: 

294 self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims) 

295 continue 

296 

297 if self.config.exposureScaling == "ExposureTime": 

298 scale = exp.getInfo().getVisitInfo().getExposureTime() 

299 elif self.config.exposureScaling == "DarkTime": 

300 scale = exp.getInfo().getVisitInfo().getDarkTime() 

301 elif self.config.exposureScaling == "MeanStats": 

302 scale = self.stats.run(exp) 

303 elif self.config.exposureScaling == "InputList": 

304 visitId = dims.get('exposure', None) 

305 detectorId = dims.get('detector', None) 

306 if visitId is None or detectorId is None: 

307 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})") 

308 if detectorId not in inputScales['expScale']: 

309 raise RuntimeError(f"Could not identify a scaling for input {index}" 

310 f" detector {detectorId}") 

311 

312 if self.config.scalingLevel == "DETECTOR": 

313 if visitId not in inputScales['expScale'][detectorId]: 

314 raise RuntimeError(f"Could not identify a scaling for input {index}" 

315 f"detector {detectorId} visit {visitId}") 

316 scale = inputScales['expScale'][detectorId][visitId] 

317 elif self.config.scalingLevel == 'AMP': 

318 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId] 

319 for amp in exp.getDetector()] 

320 else: 

321 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}") 

322 elif self.config.exposureScaling == 'None': 

323 scale = 1.0 

324 else: 

325 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.") 

326 

327 expScales.append(scale) 

328 self.log.info("Scaling input %d by %s", index, scale) 

329 self.applyScale(exp, scale) 

330 

331 self.combine(combined, inputExps, stats) 

332 

333 self.interpolateNans(combined) 

334 

335 if self.config.doVignette: 

336 polygon = inputExps[0].getInfo().getValidPolygon() 

337 VignetteExposure(combined, polygon=polygon, doUpdateMask=True, 

338 doSetValue=True, vignetteValue=0.0) 

339 

340 # Combine headers 

341 self.combineHeaders(inputExps, combinedExp, 

342 calibType=self.config.calibrationType, scales=expScales) 

343 

344 # Return 

345 return pipeBase.Struct( 

346 outputData=combinedExp, 

347 ) 

348 

349 def getDimensions(self, expList): 

350 """Get dimensions of the inputs. 

351 

352 Parameters 

353 ---------- 

354 expList : `list` [`lsst.afw.image.Exposure`] 

355 Exps to check the sizes of. 

356 

357 Returns 

358 ------- 

359 width, height : `int` 

360 Unique set of input dimensions. 

361 """ 

362 dimList = [exp.getDimensions() for exp in expList if exp is not None] 

363 return self.getSize(dimList) 

364 

365 def getSize(self, dimList): 

366 """Determine a consistent size, given a list of image sizes. 

367 

368 Parameters 

369 ----------- 

370 dimList : iterable of `tuple` (`int`, `int`) 

371 List of dimensions. 

372 

373 Raises 

374 ------ 

375 RuntimeError 

376 If input dimensions are inconsistent. 

377 

378 Returns 

379 -------- 

380 width, height : `int` 

381 Common dimensions. 

382 """ 

383 dim = set((w, h) for w, h in dimList) 

384 if len(dim) != 1: 

385 raise RuntimeError("Inconsistent dimensions: %s" % dim) 

386 return dim.pop() 

387 

388 def applyScale(self, exposure, scale=None): 

389 """Apply scale to input exposure. 

390 

391 This implementation applies a flux scaling: the input exposure is 

392 divided by the provided scale. 

393 

394 Parameters 

395 ---------- 

396 exposure : `lsst.afw.image.Exposure` 

397 Exposure to scale. 

398 scale : `float` or `list` [`float`], optional 

399 Constant scale to divide the exposure by. 

400 """ 

401 if scale is not None: 

402 mi = exposure.getMaskedImage() 

403 if isinstance(scale, list): 

404 for amp, ampScale in zip(exposure.getDetector(), scale): 

405 ampIm = mi[amp.getBBox()] 

406 ampIm /= ampScale 

407 else: 

408 mi /= scale 

409 

410 def combine(self, target, expList, stats): 

411 """Combine multiple images. 

412 

413 Parameters 

414 ---------- 

415 target : `lsst.afw.image.Exposure` 

416 Output exposure to construct. 

417 expList : `list` [`lsst.afw.image.Exposure`] 

418 Input exposures to combine. 

419 stats : `lsst.afw.math.StatisticsControl` 

420 Control explaining how to combine the input images. 

421 """ 

422 images = [img.getMaskedImage() for img in expList if img is not None] 

423 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats) 

424 

425 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None): 

426 """Combine input headers to determine the set of common headers, 

427 supplemented by calibration inputs. 

428 

429 Parameters 

430 ---------- 

431 expList : `list` of `lsst.afw.image.Exposure` 

432 Input list of exposures to combine. 

433 calib : `lsst.afw.image.Exposure` 

434 Output calibration to construct headers for. 

435 calibType: `str`, optional 

436 OBSTYPE the output should claim. 

437 scales: `list` of `float`, optional 

438 Scale values applied to each input to record. 

439 

440 Returns 

441 ------- 

442 header : `lsst.daf.base.PropertyList` 

443 Constructed header. 

444 """ 

445 # Header 

446 header = calib.getMetadata() 

447 header.set("OBSTYPE", calibType) 

448 

449 # Keywords we care about 

450 comments = {"TIMESYS": "Time scale for all dates", 

451 "DATE-OBS": "Start date of earliest input observation", 

452 "MJD-OBS": "[d] Start MJD of earliest input observation", 

453 "DATE-END": "End date of oldest input observation", 

454 "MJD-END": "[d] End MJD of oldest input observation", 

455 "MJD-AVG": "[d] MJD midpoint of all input observations", 

456 "DATE-AVG": "Midpoint date of all input observations"} 

457 

458 # Creation date 

459 now = time.localtime() 

460 calibDate = time.strftime("%Y-%m-%d", now) 

461 calibTime = time.strftime("%X %Z", now) 

462 header.set("CALIB_CREATE_DATE", calibDate) 

463 header.set("CALIB_CREATE_TIME", calibTime) 

464 

465 # Merge input headers 

466 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None] 

467 merged = merge_headers(inputHeaders, mode='drop') 

468 for k, v in merged.items(): 

469 if k not in header: 

470 md = expList[0].getMetadata() 

471 comment = md.getComment(k) if k in md else None 

472 header.set(k, v, comment=comment) 

473 

474 # Construct list of visits 

475 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None] 

476 for i, visit in enumerate(visitInfoList): 

477 if visit is None: 

478 continue 

479 header.set("CPP_INPUT_%d" % (i,), visit.getExposureId()) 

480 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate())) 

481 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime()) 

482 if scales is not None: 

483 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i]) 

484 

485 # Not yet working: DM-22302 

486 # Create an observation group so we can add some standard headers 

487 # independent of the form in the input files. 

488 # Use try block in case we are dealing with unexpected data headers 

489 try: 

490 group = ObservationGroup(visitInfoList, pedantic=False) 

491 except Exception: 

492 self.log.warn("Exception making an obs group for headers. Continuing.") 

493 # Fall back to setting a DATE-OBS from the calibDate 

494 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)} 

495 comments["DATE-OBS"] = "Date of start of day of calibration midpoint" 

496 else: 

497 oldest, newest = group.extremes() 

498 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end) 

499 

500 for k, v in dateCards.items(): 

501 header.set(k, v, comment=comments.get(k, None)) 

502 

503 return header 

504 

505 def interpolateNans(self, exp): 

506 """Interpolate over NANs in the combined image. 

507 

508 NANs can result from masked areas on the CCD. We don't want them getting 

509 into our science images, so we replace them with the median of the image. 

510 

511 Parameters 

512 ---------- 

513 exp : `lsst.afw.image.Exposure` 

514 Exp to check for NaNs. 

515 """ 

516 array = exp.getImage().getArray() 

517 bad = np.isnan(array) 

518 

519 median = np.median(array[np.logical_not(bad)]) 

520 count = np.sum(np.logical_not(bad)) 

521 array[bad] = median 

522 if count > 0: 

523 self.log.warn("Found %s NAN pixels", count) 

524 

525 

526def VignetteExposure(exposure, polygon=None, 

527 doUpdateMask=True, maskPlane='BAD', 

528 doSetValue=False, vignetteValue=0.0, 

529 log=None): 

530 """Apply vignetted polygon to image pixels. 

531 

532 Parameters 

533 ---------- 

534 exposure : `lsst.afw.image.Exposure` 

535 Image to be updated. 

536 doUpdateMask : `bool`, optional 

537 Update the exposure mask for vignetted area? 

538 maskPlane : `str`, optional, 

539 Mask plane to assign. 

540 doSetValue : `bool`, optional 

541 Set image value for vignetted area? 

542 vignetteValue : `float`, optional 

543 Value to assign. 

544 log : `lsst.log.Log`, optional 

545 Log to write to. 

546 

547 Raises 

548 ------ 

549 RuntimeError 

550 Raised if no valid polygon exists. 

551 """ 

552 polygon = polygon if polygon else exposure.getInfo().getValidPolygon() 

553 if not polygon: 

554 raise RuntimeError("Could not find valid polygon!") 

555 log = log if log else Log.getLogger(__name__.partition(".")[2]) 

556 

557 fullyIlluminated = True 

558 for corner in exposure.getBBox().getCorners(): 

559 if not polygon.contains(Point2D(corner)): 

560 fullyIlluminated = False 

561 

562 log.info("Exposure is fully illuminated? %s", fullyIlluminated) 

563 

564 if not fullyIlluminated: 

565 # Scan pixels. 

566 mask = exposure.getMask() 

567 numPixels = mask.getBBox().getArea() 

568 

569 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int), 

570 np.arange(0, mask.getHeight(), dtype=int)) 

571 

572 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in 

573 zip(xx.reshape(numPixels), yy.reshape(numPixels))]) 

574 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth()) 

575 

576 if doUpdateMask: 

577 bitMask = mask.getPlaneBitMask(maskPlane) 

578 maskArray = mask.getArray() 

579 maskArray[vignMask] |= bitMask 

580 if doSetValue: 

581 imageArray = exposure.getImage().getArray() 

582 imageArray[vignMask] = vignetteValue 

583 log.info("Exposure contains %d vignetted pixels.", 

584 np.count_nonzero(vignMask))