Coverage for tests/test_isrTaskLSST.py: 4%

2# LSST Data Management System

5# This product includes software developed by the

6# LSST Project (http://www.lsst.org/).

8# This program is free software: you can redistribute it and/or modify

9# it under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# This program is distributed in the hope that it will be useful,

14# but WITHOUT ANY WARRANTY; without even the implied warranty of

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

16# GNU General Public License for more details.

17#

18# You should have received a copy of the LSST License Statement and

19# the GNU General Public License along with this program. If not,

20# see <https://www.lsstcorp.org/LegalNotices/>.

21#

23import copy

24import unittest

25import numpy as np

26import logging

27import galsim

28from scipy.stats import median_abs_deviation

30import lsst.afw.geom as afwGeom

31import lsst.geom as geom

32from lsst.pipe.base import UnprocessableDataError

33import lsst.ip.isr.isrMockLSST as isrMockLSST

34import lsst.utils.tests

35from lsst.ip.isr.isrTaskLSST import (IsrTaskLSST, IsrTaskLSSTConfig)

36from lsst.ip.isr.crosstalk import CrosstalkCalib

37from lsst.ip.isr import PhotonTransferCurveDataset

38from lsst.ip.isr.vignette import maskVignettedRegion

39from lsst.ip.isr.gainCorrection import GainCorrection

42class IsrTaskLSSTTestCase(lsst.utils.tests.TestCase):

43 """Test IsrTaskLSST"""

44 def setUp(self):

45 mock = isrMockLSST.IsrMockLSST()

46 self.camera = mock.getCamera()

47 self.detector = self.camera[mock.config.detectorIndex]

48 self.namp = len(self.detector)

50 # Create adu (bootstrap) calibration frames

51 self.bias_adu = isrMockLSST.BiasMockLSST(adu=True).run()

52 self.dark_adu = isrMockLSST.DarkMockLSST(adu=True).run()

53 self.flat_adu = isrMockLSST.FlatMockLSST(adu=True).run()

54 self.flat_adu.metadata["FLATSRC"] = "DOME"

56 # Create calibration frames

57 self.bias = isrMockLSST.BiasMockLSST().run()

58 self.dark = isrMockLSST.DarkMockLSST().run()

59 self.flat = isrMockLSST.FlatMockLSST().run()

60 self.flat.metadata["FLATSRC"] = "DOME"

61 self.bf_kernel = isrMockLSST.BfKernelMockLSST().run()

62 self.electroBfDistortionMatrix = isrMockLSST.ElectrostaticBfMockLSST().run()

63 self.cti = isrMockLSST.DeferredChargeMockLSST().run()

65 # The crosstalk ratios in isrMockLSST are in electrons.

66 self.crosstalk = CrosstalkCalib(nAmp=self.namp)

67 self.crosstalk.hasCrosstalk = True

68 self.crosstalk.coeffs = isrMockLSST.CrosstalkCoeffMockLSST().run()

69 for i, amp in enumerate(self.detector):

70 self.crosstalk.fitGains[i] = mock.config.gainDict[amp.getName()]

71 self.crosstalk.crosstalkRatiosUnits = "electron"

73 self.defects = isrMockLSST.DefectMockLSST().run()

75 amp_names = [x.getName() for x in self.detector.getAmplifiers()]

76 self.ptc = PhotonTransferCurveDataset(amp_names,

77 ptcFitType='DUMMY_PTC',

78 covMatrixSide=1)

80 self.saturation_adu = 100_000.0

82 # PTC records noise units in electron, same as the

83 # configuration parameter.

84 for amp_name in amp_names:

85 self.ptc.gain[amp_name] = mock.config.gainDict.get(amp_name, mock.config.gain)

86 self.ptc.noise[amp_name] = mock.config.readNoise

87 self.ptc.ptcTurnoff[amp_name] = self.saturation_adu

88 pre_level = mock.config.clockInjectedOffsetLevel

89 overscan_level = mock.config.biasLevel + (pre_level / self.ptc.gain[amp_name])

90 self.ptc.overscanMedian[amp_name] = overscan_level

91 # We set this sigma level very large because the

92 # noise characteristics of the mock image with

93 # a large serial gradient and small overscan region

94 # amplifies the noise vs LSSTCam.

95 self.ptc.overscanMedianSigma[amp_name] = 10.0

97 # TODO:

98 # self.cti = isrMockLSST.DeferredChargeMockLSST().run()

100 self.linearizer = isrMockLSST.LinearizerMockLSST().run()

101 # We currently only have high-signal non-linearity.

102 mock_config = self.get_mock_config_no_signal()

103 for amp_name in amp_names:

104 coeffs = self.linearizer.linearityCoeffs[amp_name]

105 centers, values = np.split(coeffs, 2)

106 values[centers < mock_config.highSignalNonlinearityThreshold] = 0.0

107 self.linearizer.linearityCoeffs[amp_name] = np.concatenate((centers, values))

108

109 def _check_applied_keys(self, metadata, isr_config, expected_gain_correction=False):

110 """Check if the APPLIED keys have been set properly.

111

112 Parameters

113 ----------

114 metadata : `lsst.daf.base.PropertyList`

115 isr_config : `lsst.ip.isr.IsrTaskLSSTConfig`

116 expected_gain_correction : `bool`, optional

117 Did we expect gain correction to be applied?

118 """

119 key = "LSST ISR GAINCORRECTION APPLIED"

120 self.assertIn(key, metadata)

121 self.assertEqual(metadata[key], expected_gain_correction)

122

123 key = "LSST ISR CROSSTALK APPLIED"

124 self.assertIn(key, metadata)

125 self.assertEqual(metadata[key], isr_config.doCrosstalk)

126

127 key = "LSST ISR OVERSCANLEVEL CHECKED"

128 self.assertIn(key, metadata)

129 self.assertEqual(metadata[key], np.isfinite(isr_config.serialOverscanMedianShiftSigmaThreshold))

130

131 key = "LSST ISR NOISE CHECKED"

132 self.assertIn(key, metadata)

133 self.assertEqual(metadata[key], np.isfinite(isr_config.ampNoiseThreshold))

134

135 key = "LSST ISR LINEARIZER APPLIED"

136 self.assertIn(key, metadata)

137 self.assertEqual(metadata[key], isr_config.doLinearize)

138

139 key = "LSST ISR CTI APPLIED"

140 self.assertIn(key, metadata)

141 self.assertEqual(metadata[key], isr_config.doDeferredCharge)

142

143 key = "LSST ISR BIAS APPLIED"

144 self.assertIn(key, metadata)

145 self.assertEqual(metadata[key], isr_config.doBias)

146

147 key = "LSST ISR DARK APPLIED"

148 self.assertIn(key, metadata)

149 self.assertEqual(metadata[key], isr_config.doDark)

150

151 key = "LSST ISR BF APPLIED"

152 self.assertIn(key, metadata)

153 self.assertEqual(metadata[key], isr_config.doBrighterFatter)

154

155 if isr_config.doBrighterFatter:

156 key = "LSST ISR BF CORR METHOD"

157 self.assertIn(key, metadata)

158 self.assertEqual(

159 metadata[key],

160 isr_config.brighterFatterCorrectionMethod,

161 )

162

163 key = "LSST ISR FLAT APPLIED"

164 self.assertIn(key, metadata)

165 self.assertEqual(metadata[key], isr_config.doFlat)

166

167 if metadata[key]:

168 key2 = "LSST ISR FLAT SOURCE"

169 self.assertIn(key2, metadata)

170 self.assertEqual(metadata[key2], "DOME")

171

172 key = "LSST ISR DEFECTS APPLIED"

173 self.assertIn(key, metadata)

174 self.assertEqual(metadata[key], isr_config.doDefect)

175

176 def test_isrBootstrapBias(self):

177 """Test processing of a ``bootstrap`` bias frame.

178

179 This will be output with ADU units.

180 """

181 mock_config = self.get_mock_config_no_signal()

182

183 mock = isrMockLSST.IsrMockLSST(config=mock_config)

184 input_exp = mock.run()

185

186 isr_config = self.get_isr_config_minimal_corrections()

187 isr_config.doBootstrap = True

188 isr_config.doApplyGains = False

189 isr_config.doBias = True

190 isr_config.doCrosstalk = True

191

192 # Need to make sure we are not masking the negative variance

193 # pixels when directly comparing calibration images and

194 # calibration-corrected calibrations.

195 isr_config.maskNegativeVariance = False

196

197 isr_task = IsrTaskLSST(config=isr_config)

198 with self.assertLogs(level=logging.WARNING) as cm:

199 result = isr_task.run(

200 input_exp.clone(),

201 bias=self.bias_adu,

202 ptc=self.ptc,

203 crosstalk=self.crosstalk,

204 )

205 self.assertIn("Ignoring provided PTC", cm.output[0])

206 self._check_applied_keys(result.exposure.metadata, isr_config)

207

208 # Rerun without doing the bias correction.

209 isr_config.doBias = False

210 isr_task2 = IsrTaskLSST(config=isr_config)

211 with self.assertNoLogs(level=logging.WARNING):

212 result2 = isr_task2.run(input_exp.clone(), crosstalk=self.crosstalk)

213

214 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

215

216 self.assertLess(

217 np.mean(result.exposure.image.array[good_pixels]),

218 np.mean(result2.exposure.image.array[good_pixels]),

219 )

220 self.assertLess(

221 np.std(result.exposure.image.array[good_pixels]),

222 np.std(result2.exposure.image.array[good_pixels]),

223 )

224

225 delta = result2.exposure.image.array - result.exposure.image.array

226 self.assertFloatsAlmostEqual(delta[good_pixels], self.bias_adu.image.array[good_pixels], atol=1e-5)

227

228 metadata = result.exposure.metadata

229

230 key = "LSST ISR BOOTSTRAP"

231 self.assertIn(key, metadata)

232 self.assertEqual(metadata[key], True)

233

234 key = "LSST ISR UNITS"

235 self.assertIn(key, metadata)

236 self.assertEqual(metadata[key], "adu")

237

238 key = "LSST ISR READNOISE UNITS"

239 self.assertIn(key, metadata)

240 self.assertEqual(metadata[key], "electron")

241

242 for amp in self.detector:

243 amp_name = amp.getName()

244 key = f"LSST ISR GAIN {amp_name}"

245 self.assertIn(key, metadata)

246 self.assertEqual(metadata[key], 1.0)

247

248 self._check_bad_column_crosstalk_correction(result.exposure)

249

250 def test_isrBootstrapDark(self):

251 """Test processing of a ``bootstrap`` dark frame.

252

253 This will be output with ADU units.

254 """

255 mock_config = self.get_mock_config_no_signal()

256 mock_config.doAddDark = True

257

258 mock = isrMockLSST.IsrMockLSST(config=mock_config)

259 input_exp = mock.run()

260

261 isr_config = self.get_isr_config_minimal_corrections()

262 isr_config.doBootstrap = True

263 isr_config.doApplyGains = False

264 isr_config.doBias = True

265 isr_config.doDark = True

266 isr_config.maskNegativeVariance = False

267 isr_config.doCrosstalk = True

268

269 isr_task = IsrTaskLSST(config=isr_config)

270 with self.assertLogs(level=logging.WARNING) as cm:

271 result = isr_task.run(

272 input_exp.clone(),

273 bias=self.bias_adu,

274 dark=self.dark_adu,

275 ptc=self.ptc,

276 crosstalk=self.crosstalk,

277 )

278 self.assertIn("Ignoring provided PTC", cm.output[0])

279 self._check_applied_keys(result.exposure.metadata, isr_config)

280

281 # Rerun without doing the dark correction.

282 isr_config.doDark = False

283 isr_task2 = IsrTaskLSST(config=isr_config)

284 with self.assertNoLogs(level=logging.WARNING):

285 result2 = isr_task2.run(input_exp.clone(), bias=self.bias_adu, crosstalk=self.crosstalk)

286

287 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

288

289 self.assertLess(

290 np.mean(result.exposure.image.array[good_pixels]),

291 np.mean(result2.exposure.image.array[good_pixels]),

292 )

293

294 delta = result2.exposure.image.array - result.exposure.image.array

295 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

296 self.assertFloatsAlmostEqual(

297 delta[good_pixels],

298 self.dark_adu.image.array[good_pixels] * exp_time,

299 atol=1e-5,

300 )

301

302 metadata = result.exposure.metadata

303

304 key = "LSST ISR BOOTSTRAP"

305 self.assertIn(key, metadata)

306 self.assertEqual(metadata[key], True)

307

308 key = "LSST ISR UNITS"

309 self.assertIn(key, metadata)

310 self.assertEqual(metadata[key], "adu")

311

312 self._check_bad_column_crosstalk_correction(result.exposure)

313

314 def test_isrBootstrapFlat(self):

315 """Test processing of a ``bootstrap`` flat frame.

316

317 This will be output with ADU units.

318 """

319 mock_config = self.get_mock_config_no_signal()

320 mock_config.doAddDark = True

321 mock_config.doAddFlat = True

322 # The doAddSky option adds the equivalent of flat-field flux.

323 mock_config.doAddSky = True

324

325 mock = isrMockLSST.IsrMockLSST(config=mock_config)

326 input_exp = mock.run()

327

328 isr_config = self.get_isr_config_minimal_corrections()

329 isr_config.doBootstrap = True

330 isr_config.doApplyGains = False

331 isr_config.doBias = True

332 isr_config.doDark = True

333 isr_config.doFlat = True

334 isr_config.maskNegativeVariance = False

335 isr_config.doCrosstalk = True

336

337 isr_task = IsrTaskLSST(config=isr_config)

338 with self.assertLogs(level=logging.WARNING) as cm:

339 result = isr_task.run(

340 input_exp.clone(),

341 bias=self.bias_adu,

342 dark=self.dark_adu,

343 flat=self.flat_adu,

344 ptc=self.ptc,

345 crosstalk=self.crosstalk,

346 )

347 self.assertIn("Ignoring provided PTC", cm.output[0])

348 self._check_applied_keys(result.exposure.metadata, isr_config)

349

350 # Rerun without doing the flat correction.

351 isr_config.doFlat = False

352 isr_task2 = IsrTaskLSST(config=isr_config)

353 with self.assertNoLogs(level=logging.WARNING):

354 result2 = isr_task2.run(

355 input_exp.clone(),

356 bias=self.bias_adu,

357 dark=self.dark_adu,

358 crosstalk=self.crosstalk,

359 )

360

361 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

362

363 # Applying the flat will increase the counts.

364 self.assertGreater(

365 np.mean(result.exposure.image.array[good_pixels]),

366 np.mean(result2.exposure.image.array[good_pixels]),

367 )

368 # And will decrease the sigma.

369 self.assertLess(

370 np.std(result.exposure.image.array[good_pixels]),

371 np.std(result2.exposure.image.array[good_pixels]),

372 )

373

374 ratio = result2.exposure.image.array / result.exposure.image.array

375 self.assertFloatsAlmostEqual(ratio[good_pixels], self.flat_adu.image.array[good_pixels], atol=1e-5)

376

377 # Test the variance plane in the case of adu units.

378 # The expected variance starts with the image array.

379 expected_variance = result.exposure.image.clone()

380 # We have to remove the flat-fielding from the image pixels.

381 expected_variance.array *= self.flat_adu.image.array

382 # And add in the bias variance.

383 expected_variance.array += self.bias_adu.variance.array

384 # And add in the scaled dark variance.

385 scale = result.exposure.visitInfo.darkTime / self.dark_adu.visitInfo.darkTime

386 expected_variance.array += scale**2. * self.dark_adu.variance.array

387 # And add the gain and read noise (in electron) per amp.

388 for amp in self.detector:

389 # We need to use the gain and read noise from the header

390 # because these are bootstraps.

391 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

392 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

393

394 expected_variance[amp.getBBox()].array /= gain

395 # Read noise is always in electron units, but since this is a

396 # bootstrap, the gain is 1.0.

397 expected_variance[amp.getBBox()].array += (read_noise/gain)**2.

398

399 # And apply the full formula for dividing by the flat with variance.

400 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

401 unflat_image_array = result.exposure.image.array * self.flat_adu.image.array

402 expected_variance.array = ((unflat_image_array**2. * self.flat_adu.variance.array

403 + self.flat_adu.image.array**2. * expected_variance.array)

404 / self.flat_adu.image.array**4.)

405

406 self.assertFloatsAlmostEqual(

407 result.exposure.variance.array[good_pixels],

408 expected_variance.array[good_pixels],

409 rtol=1e-6,

410 )

411

412 metadata = result.exposure.metadata

413

414 key = "LSST ISR BOOTSTRAP"

415 self.assertIn(key, metadata)

416 self.assertEqual(metadata[key], True)

417

418 key = "LSST ISR UNITS"

419 self.assertIn(key, metadata)

420 self.assertEqual(metadata[key], "adu")

421

422 self._check_bad_column_crosstalk_correction(result.exposure)

423

424 def test_isrBootstrapAndRegularFlat(self):

425 """Test that bootstrap and "regular" flat processing are equivalent."""

426 # This is a test for DM-52684, for the linearizer units.

427

428 mock_config = self.get_mock_config_no_signal()

429 mock_config.doAddDark = True

430 mock_config.doAddFlat = True

431 # The doAddSky option adds the equivalent of flat-field flux.

432 mock_config.doAddSky = True

433 # We set the sky/flat level to a range where the "high signal"

434 # non-linearity has kicked in.

435 mock_config.skyLevel = 40000.0

436

437 mock = isrMockLSST.IsrMockLSST(config=mock_config)

438 input_exp = mock.run()

439

440 # First config is with linearizer in bootstrap mode.

441

442 isr_config_bootstrap = self.get_isr_config_minimal_corrections()

443 isr_config_bootstrap.doBootstrap = True

444 isr_config_bootstrap.doApplyGains = False

445 isr_config_bootstrap.doLinearize = True

446 isr_config_bootstrap.doBias = False

447 isr_config_bootstrap.doDark = False

448 isr_config_bootstrap.doFlat = False

449 isr_config_bootstrap.maskNegativeVariance = False

450 isr_config_bootstrap.doCrosstalk = True

451

452 isr_task_bootstrap = IsrTaskLSST(config=isr_config_bootstrap)

453 with self.assertNoLogs(level=logging.WARNING):

454 result = isr_task_bootstrap.run(

455 input_exp.clone(),

456 crosstalk=self.crosstalk,

457 linearizer=self.linearizer,

458 )

459

460 exp_bootstrap = result.exposure

461

462 # Apply the gains (after the linearization); this is

463 # similar to processing in PTC building.

464 for amp in self.detector:

465 exp_bootstrap[amp.getBBox()].image.array *= self.ptc.gain[amp.getName()]

466

467 # Run again with non-bootstrap mode.

468 isr_config = self.get_isr_config_minimal_corrections()

469 isr_config.doBootstrap = False

470 isr_config.doApplyGains = True

471 isr_config.doLinearize = True

472 isr_config.doBias = False

473 isr_config.doDark = False

474 isr_config.doFlat = False

475 isr_config.maskNegativeVariance = False

476 isr_config.doCrosstalk = True

477

478 isr_task = IsrTaskLSST(config=isr_config)

479 with self.assertNoLogs(level=logging.WARNING):

480 result = isr_task.run(

481 input_exp.clone(),

482 crosstalk=self.crosstalk,

483 linearizer=self.linearizer,

484 ptc=self.ptc,

485 )

486

487 exp = result.exposure

488

489 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

490

491 delta = exp.image.array - exp_bootstrap.image.array

492

493 self.assertFloatsAlmostEqual(delta[good_pixels], 0.0, atol=1e-2)

494

495 def test_isrBias(self):

496 """Test processing of a bias frame."""

497 mock_config = self.get_mock_config_no_signal()

498

499 mock = isrMockLSST.IsrMockLSST(config=mock_config)

500 input_exp = mock.run()

501

502 isr_config = self.get_isr_config_electronic_corrections()

503 isr_config.doBias = True

504 # We do not do defect correction when processing biases.

505 isr_config.doDefect = False

506 isr_config.maskNegativeVariance = False

507

508 isr_task = IsrTaskLSST(config=isr_config)

509 with self.assertNoLogs(level=logging.WARNING):

510 result = isr_task.run(

511 input_exp.clone(),

512 bias=self.bias,

513 crosstalk=self.crosstalk,

514 ptc=self.ptc,

515 linearizer=self.linearizer,

516 deferredChargeCalib=self.cti,

517 )

518 self._check_applied_keys(result.exposure.metadata, isr_config)

519

520 # Rerun without doing the bias correction.

521 isr_config.doBias = False

522 isr_task2 = IsrTaskLSST(config=isr_config)

523 with self.assertNoLogs(level=logging.WARNING):

524 result2 = isr_task2.run(

525 input_exp.clone(),

526 crosstalk=self.crosstalk,

527 ptc=self.ptc,

528 linearizer=self.linearizer,

529 deferredChargeCalib=self.cti,

530 )

531

532 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

533

534 self.assertLess(

535 np.mean(result.exposure.image.array[good_pixels]),

536 np.mean(result2.exposure.image.array[good_pixels]),

537 )

538

539 self.assertLess(

540 np.std(result.exposure.image.array[good_pixels]),

541 np.std(result2.exposure.image.array[good_pixels]),

542 )

543

544 # Confirm that it is flat with an arbitrary cutoff that depends

545 # on the read noise.

546 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

547

548 delta = result2.exposure.image.array - result.exposure.image.array

549

550 # Note that the bias is made with bias noise + read noise, and

551 # the image contains read noise.

552 self.assertFloatsAlmostEqual(

553 delta[good_pixels],

554 self.bias.image.array[good_pixels],

555 atol=1e-5,

556 )

557

558 self._check_bad_column_crosstalk_correction(result.exposure)

559

560 def test_isrBiasNoParallelOscanCorrection(self):

561 """Test processing of a bias frame with parallel

562 overscan correction turned off."""

563 mock_config = self.get_mock_config_no_signal()

564 mock_config.doAddParallelOverscanRamp = False

565

566 mock = isrMockLSST.IsrMockLSST(config=mock_config)

567 input_exp = mock.run()

568

569 isr_config = self.get_isr_config_electronic_corrections()

570

571 # Turn off the parallel overscan correction

572 amp_oscan_config = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

573 amp_oscan_config.doParallelOverscan = False

574 isr_config.doBias = True

575

576 # We do not do defect correction when processing biases.

577 isr_config.doDefect = False

578 isr_config.maskNegativeVariance = False

579

580 isr_task = IsrTaskLSST(config=isr_config)

581 with self.assertNoLogs(level=logging.WARNING):

582 result = isr_task.run(

583 input_exp.clone(),

584 bias=self.bias,

585 crosstalk=self.crosstalk,

586 ptc=self.ptc,

587 linearizer=self.linearizer,

588 deferredChargeCalib=self.cti,

589 )

590 self._check_applied_keys(result.exposure.metadata, isr_config)

591

592 # Rerun without doing the bias correction.

593 isr_config.doBias = False

594 isr_task2 = IsrTaskLSST(config=isr_config)

595 with self.assertNoLogs(level=logging.WARNING):

596 result2 = isr_task2.run(

597 input_exp.clone(),

598 crosstalk=self.crosstalk,

599 ptc=self.ptc,

600 linearizer=self.linearizer,

601 deferredChargeCalib=self.cti,

602 )

603

604 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

605

606 self.assertLess(

607 np.mean(result.exposure.image.array[good_pixels]),

608 np.mean(result2.exposure.image.array[good_pixels]),

609 )

610

611 self.assertLess(

612 np.std(result.exposure.image.array[good_pixels]),

613 np.std(result2.exposure.image.array[good_pixels]),

614 )

615

616 # Confirm that it is flat with an arbitrary cutoff that depends

617 # on the read noise.

618 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

619

620 delta = result2.exposure.image.array - result.exposure.image.array

621

622 # Note that the bias is made with bias noise + read noise, and

623 # the image contains read noise.

624 self.assertFloatsAlmostEqual(

625 delta[good_pixels],

626 self.bias.image.array[good_pixels],

627 atol=1e-5,

628 )

629

630 self._check_bad_column_crosstalk_correction(result.exposure)

631

632 def test_isrBiasCti(self):

633 """Test over-correction of bias amp edges from prescan."""

634 mock_config = self.get_mock_config_no_signal()

635 mock_config.doAddBrightDefects = False

636

637 mock = isrMockLSST.IsrMockLSST(config=mock_config)

638 input_exp = mock.run()

639

640 isr_config = self.get_isr_config_electronic_corrections()

641 # We do not do defect correction when processing biases.

642 isr_config.doDefect = False

643 isr_config.maskNegativeVariance = False

644

645 isr_task = IsrTaskLSST(config=isr_config)

646 with self.assertNoLogs(level=logging.WARNING):

647 result = isr_task.run(

648 input_exp.clone(),

649 crosstalk=self.crosstalk,

650 ptc=self.ptc,

651 linearizer=self.linearizer,

652 deferredChargeCalib=self.cti,

653 )

654 self._check_applied_keys(result.exposure.metadata, isr_config)

655

656 # Rerun without doing the CTI correction

657 isr_config.doDeferredCharge = False

658

659 isr_task2 = IsrTaskLSST(config=isr_config)

660 with self.assertNoLogs(level=logging.WARNING):

661 result2 = isr_task2.run(

662 input_exp.clone(),

663 crosstalk=self.crosstalk,

664 ptc=self.ptc,

665 linearizer=self.linearizer,

666 )

667

668 # This confirms that things are *close* to equal. Unfortunately,

669 # the unusual camera geometry in the test camera doesn't completely

670 # zero out the prescan pixels, so we need a higher threshold.

671 std_delta = np.std(result2.exposure.image.array - result.exposure.image.array)

672 self.assertLess(std_delta, 0.15)

673

674 def test_isrDark(self):

675 """Test processing of a dark frame."""

676 mock_config = self.get_mock_config_no_signal()

677 mock_config.doAddDark = True

678 # We turn off the bad parallel overscan column because it does

679 # add more noise to that region.

680 mock_config.doAddBadParallelOverscanColumn = False

681

682 mock = isrMockLSST.IsrMockLSST(config=mock_config)

683 input_exp = mock.run()

684

685 isr_config = self.get_isr_config_electronic_corrections()

686 isr_config.doBias = True

687 isr_config.doDark = True

688 # We do not do defect correction when processing darks.

689 isr_config.doDefect = False

690 isr_config.maskNegativeVariance = False

691

692 isr_task = IsrTaskLSST(config=isr_config)

693 with self.assertNoLogs(level=logging.WARNING):

694 result = isr_task.run(

695 input_exp.clone(),

696 bias=self.bias,

697 dark=self.dark,

698 crosstalk=self.crosstalk,

699 ptc=self.ptc,

700 linearizer=self.linearizer,

701 deferredChargeCalib=self.cti,

702 )

703 self._check_applied_keys(result.exposure.metadata, isr_config)

704

705 # Rerun without doing the dark correction.

706 isr_config.doDark = False

707 isr_task2 = IsrTaskLSST(config=isr_config)

708 with self.assertNoLogs(level=logging.WARNING):

709 result2 = isr_task2.run(

710 input_exp.clone(),

711 bias=self.bias,

712 crosstalk=self.crosstalk,

713 ptc=self.ptc,

714 linearizer=self.linearizer,

715 deferredChargeCalib=self.cti,

716 )

717

718 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

719

720 self.assertLess(

721 np.mean(result.exposure.image.array[good_pixels]),

722 np.mean(result2.exposure.image.array[good_pixels]),

723 )

724 # The mock dark has no noise, so these should be equal.

725 self.assertFloatsAlmostEqual(

726 np.std(result.exposure.image.array[good_pixels]),

727 np.std(result2.exposure.image.array[good_pixels]),

728 atol=1e-12,

729 )

730

731 # This is a somewhat arbitrary comparison that includes a fudge

732 # factor for the extra noise from the overscan subtraction.

733 self.assertLess(

734 np.std(result.exposure.image.array[good_pixels]),

735 1.6*np.sqrt(mock_config.darkRate*mock_config.expTime + mock_config.readNoise),

736 )

737

738 delta = result2.exposure.image.array - result.exposure.image.array

739 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

740

741 # Allow <3 pixels to fail this test due to rounding error

742 # if doRoundAdu=True

743 diff = np.abs(delta[good_pixels] - self.dark.image.array[good_pixels] * exp_time)

744 self.assertLess(np.count_nonzero(diff >= 1e-12), 3)

745

746 self._check_bad_column_crosstalk_correction(result.exposure)

747

748 def test_isrFlat(self):

749 """Test processing of a flat frame."""

750 mock_config = self.get_mock_config_no_signal()

751 mock_config.doAddDark = True

752 mock_config.doAddFlat = True

753 # The doAddSky option adds the equivalent of flat-field flux.

754 mock_config.doAddSky = True

755

756 mock = isrMockLSST.IsrMockLSST(config=mock_config)

757 input_exp = mock.run()

758

759 isr_config = self.get_isr_config_electronic_corrections()

760 isr_config.doBias = True

761 isr_config.doDark = True

762 isr_config.doFlat = True

763 # Although we usually do not do defect interpolation when

764 # processing flats, this is a good test of the interpolation.

765 isr_config.doDefect = True

766 isr_config.maskNegativeVariance = False

767

768 isr_task = IsrTaskLSST(config=isr_config)

769 with self.assertNoLogs(level=logging.WARNING):

770 result = isr_task.run(

771 input_exp.clone(),

772 bias=self.bias,

773 dark=self.dark,

774 flat=self.flat,

775 crosstalk=self.crosstalk,

776 defects=self.defects,

777 ptc=self.ptc,

778 linearizer=self.linearizer,

779 deferredChargeCalib=self.cti,

780 )

781 self._check_applied_keys(result.exposure.metadata, isr_config)

782

783 # Rerun without doing the bias correction.

784 isr_config.doFlat = False

785 isr_task2 = IsrTaskLSST(config=isr_config)

786 with self.assertNoLogs(level=logging.WARNING):

787 result2 = isr_task2.run(

788 input_exp.clone(),

789 bias=self.bias,

790 dark=self.dark,

791 crosstalk=self.crosstalk,

792 defects=self.defects,

793 ptc=self.ptc,

794 linearizer=self.linearizer,

795 deferredChargeCalib=self.cti,

796 )

797

798 # With defect correction, we should not need to filter out bad

799 # pixels.

800

801 # Applying the flat will increase the counts.

802 self.assertGreater(

803 np.mean(result.exposure.image.array),

804 np.mean(result2.exposure.image.array),

805 )

806 # And will decrease the sigma.

807 self.assertLess(

808 np.std(result.exposure.image.array),

809 np.std(result2.exposure.image.array),

810 )

811

812 # Check that the resulting image is approximately flat.

813 # In particular that the noise is consistent with sky + margin.

814 self.assertLess(np.std(result.exposure.image.array), np.sqrt(mock_config.skyLevel) + 3.0)

815

816 # Generate a flat without any defects for comparison

817 # (including interpolation)

818 flat_nodefect_config = isrMockLSST.FlatMockLSST.ConfigClass()

819 flat_nodefect_config.doAddBrightDefects = False

820 flat_nodefects = isrMockLSST.FlatMockLSST(config=flat_nodefect_config).run()

821

822 ratio = result2.exposure.image.array / result.exposure.image.array

823 self.assertFloatsAlmostEqual(ratio, flat_nodefects.image.array, atol=1e-4)

824

825 self._check_bad_column_crosstalk_correction(result.exposure)

826

827 def test_isrNoise(self):

828 """Test the recorded noise and gain in the metadata."""

829 mock_config = self.get_mock_config_no_signal()

830 # Remove the overscan scale so that the only variation

831 # in the overscan is from the read noise.

832 mock_config.overscanScale = 0.0

833

834 mock = isrMockLSST.IsrMockLSST(config=mock_config)

835 input_exp = mock.run()

836

837 isr_config = self.get_isr_config_electronic_corrections()

838 isr_config.doBias = True

839 # We do not do defect correction when processing biases.

840 isr_config.doDefect = False

841 isr_config.maskNegativeVariance = False

842

843 isr_task = IsrTaskLSST(config=isr_config)

844 with self.assertNoLogs(level=logging.WARNING):

845 result = isr_task.run(

846 input_exp.clone(),

847 bias=self.bias,

848 crosstalk=self.crosstalk,

849 ptc=self.ptc,

850 deferredChargeCalib=self.cti,

851 linearizer=self.linearizer,

852 )

853 self._check_applied_keys(result.exposure.metadata, isr_config)

854

855 metadata = result.exposure.metadata

856

857 for amp in self.detector:

858 # The overscan noise is always in adu and the readnoise is always

859 # in electron.

860 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

861 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

862

863 # Check that the gain and read noise are consistent with the

864 # values stored in the PTC.

865 self.assertEqual(gain, self.ptc.gain[amp.getName()])

866 self.assertEqual(read_noise, self.ptc.noise[amp.getName()])

867

868 key = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {amp.getName()}"

869 self.assertIn(key, metadata)

870

871 # Determine if the residual serial overscan stddev is consistent

872 # with the PTC readnoise within 3xstandard error.

873 serial_overscan_area = amp.getRawHorizontalOverscanBBox().area

874 self.assertFloatsAlmostEqual(

875 metadata[key] * gain,

876 read_noise,

877 atol=3*read_noise / np.sqrt(serial_overscan_area),

878 )

879

880 def test_isrBrighterFatterKernel(self):

881 """Test processing of a flat frame."""

882 # Image with brighter-fatter correction

883 mock_config = self.get_mock_config_no_signal()

884 mock_config.isTrimmed = False

885 mock_config.doAddDark = True

886 mock_config.doAddFlat = True

887 mock_config.doAddSky = True

888 mock_config.doAddSource = True

889 mock_config.sourceFlux = [75000.0]

890 mock_config.doAddBrighterFatter = True

891

892 mock = isrMockLSST.IsrMockLSST(config=mock_config)

893 input_exp = mock.run()

894

895 isr_config = self.get_isr_config_electronic_corrections()

896 isr_config.doBias = True

897 isr_config.doDark = True

898 isr_config.doFlat = True

899 isr_config.doBrighterFatter = True

900

901 isr_task = IsrTaskLSST(config=isr_config)

902 with self.assertNoLogs(level=logging.WARNING):

903 result = isr_task.run(

904 input_exp.clone(),

905 bias=self.bias,

906 dark=self.dark,

907 flat=self.flat,

908 deferredChargeCalib=self.cti,

909 crosstalk=self.crosstalk,

910 defects=self.defects,

911 ptc=self.ptc,

912 linearizer=self.linearizer,

913 bfKernel=self.bf_kernel,

914 )

915 self._check_applied_keys(result.exposure.metadata, isr_config)

916

917 mock_config = self.get_mock_config_no_signal()

918 mock_config.isTrimmed = False

919 mock_config.doAddDark = True

920 mock_config.doAddFlat = True

921 mock_config.doAddSky = True

922 mock_config.doAddSource = True

923 mock_config.sourceFlux = [75000.0]

924 mock_config.doAddBrighterFatter = False

925

926 mock = isrMockLSST.IsrMockLSST(config=mock_config)

927 input_truth = mock.run()

928

929 isr_config = self.get_isr_config_electronic_corrections()

930 isr_config.doBias = True

931 isr_config.doDark = True

932 isr_config.doFlat = True

933 isr_config.doBrighterFatter = False

934 isr_config.brighterFatterCorrectionMethod = "COULTON18"

935

936 isr_task = IsrTaskLSST(config=isr_config)

937 with self.assertNoLogs(level=logging.WARNING):

938 result_truth = isr_task.run(

939 input_truth.clone(),

940 bias=self.bias,

941 dark=self.dark,

942 flat=self.flat,

943 deferredChargeCalib=self.cti,

944 crosstalk=self.crosstalk,

945 defects=self.defects,

946 ptc=self.ptc,

947 linearizer=self.linearizer,

948 bfKernel=self.bf_kernel,

949 )

950

951 # Measure the source size in the BF-corrected image.

952 # The injected source is a Gaussian with 3.0px

953 image = galsim.ImageF(result.exposure.image.array)

954 image_truth = galsim.ImageF(result_truth.exposure.image.array)

955 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

956 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

957 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

958 strict=False)

959 measured_sigma = hsm_result.moments_sigma

960 true_sigma = hsm_result_truth.moments_sigma

961 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

962

963 # Check that the variance in an amp far away from the

964 # source is expected. The source is in amp 0; this will

965 # check the variation in neighboring amp 1

966 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

967 n_pixels = test_amp_bbox.getArea()

968 stdev = np.std(result.exposure[test_amp_bbox].image.array)

969 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

970 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

971

972 # Check that the variance in the amp with a defect is

973 # unchanged as a result of applying the BF correction after

974 # interpolating. The defect was added to amplifier 2.

975 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

976 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

977 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

978 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

979 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

980

981 # Check that BF has converged in the expected number of iterations.

982 metadata = result.exposure.metadata

983 key = "LSST ISR BF ITERS"

984 self.assertIn(key, metadata)

985 self.assertEqual(metadata[key], 2)

986

987 def test_isrElectrostaticBrighterFatter(self):

988 """Test processing of a flat frame."""

989 # Image with brighter-fatter correction

990 mock_config = self.get_mock_config_no_signal()

991 mock_config.isTrimmed = False

992 mock_config.doAddDark = True

993 mock_config.doAddFlat = True

994 mock_config.doAddSky = True

995 mock_config.doAddSource = True

996 mock_config.sourceFlux = [75000.0]

997 mock_config.doAddBrighterFatter = True

998

999 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1000 input_exp = mock.run()

1001

1002 isr_config = self.get_isr_config_electronic_corrections()

1003 isr_config.doBias = True

1004 isr_config.doDark = True

1005 isr_config.doFlat = True

1006 isr_config.doBrighterFatter = True

1007 isr_config.brighterFatterCorrectionMethod = "ASTIER23"

1008

1009 isr_task = IsrTaskLSST(config=isr_config)

1010 with self.assertNoLogs(level=logging.WARNING):

1011 result = isr_task.run(

1012 input_exp.clone(),

1013 bias=self.bias,

1014 dark=self.dark,

1015 flat=self.flat,

1016 deferredChargeCalib=self.cti,

1017 crosstalk=self.crosstalk,

1018 defects=self.defects,

1019 ptc=self.ptc,

1020 linearizer=self.linearizer,

1021 electroBfDistortionMatrix=self.electroBfDistortionMatrix,

1022 )

1023 self._check_applied_keys(result.exposure.metadata, isr_config)

1024

1025 mock_config = self.get_mock_config_no_signal()

1026 mock_config.isTrimmed = False

1027 mock_config.doAddDark = True

1028 mock_config.doAddFlat = True

1029 mock_config.doAddSky = True

1030 mock_config.doAddSource = True

1031 mock_config.sourceFlux = [75000.0]

1032 mock_config.doAddBrighterFatter = False

1033

1034 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1035 input_truth = mock.run()

1036

1037 isr_config = self.get_isr_config_electronic_corrections()

1038 isr_config.doBias = True

1039 isr_config.doDark = True

1040 isr_config.doFlat = True

1041 isr_config.doBrighterFatter = False

1042

1043 isr_task = IsrTaskLSST(config=isr_config)

1044 with self.assertNoLogs(level=logging.WARNING):

1045 result_truth = isr_task.run(

1046 input_truth.clone(),

1047 bias=self.bias,

1048 dark=self.dark,

1049 flat=self.flat,

1050 deferredChargeCalib=self.cti,

1051 crosstalk=self.crosstalk,

1052 defects=self.defects,

1053 ptc=self.ptc,

1054 linearizer=self.linearizer,

1055 electroBfDistortionMatrix=self.electroBfDistortionMatrix,

1056 )

1057

1058 # Measure the source size in the BF-corrected image.

1059 # The injected source is a Gaussian with 3.0px

1060 image = galsim.ImageF(result.exposure.image.array)

1061 image_truth = galsim.ImageF(result_truth.exposure.image.array)

1062 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

1063 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

1064 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

1065 strict=False)

1066 measured_sigma = hsm_result.moments_sigma

1067 true_sigma = hsm_result_truth.moments_sigma

1068 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

1069

1070 # Check that the variance in an amp far away from the

1071 # source is expected. The source is in amp 0; this will

1072 # check the variation in neighboring amp 1

1073 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

1074 n_pixels = test_amp_bbox.getArea()

1075 stdev = np.std(result.exposure[test_amp_bbox].image.array)

1076 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

1077 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

1078

1079 # Check that the variance in the amp with a defect is

1080 # unchanged as a result of applying the BF correction after

1081 # interpolating. The defect was added to amplifier 2.

1082 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

1083 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

1084 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

1085 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

1086 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

1087

1088 def test_isrSkyImage(self):

1089 """Test processing of a sky image."""

1090 mock_config = self.get_mock_config_no_signal()

1091 mock_config.doAddDark = True

1092 mock_config.doAddFlat = True

1093 # Set this to False until we have fringe correction.

1094 mock_config.doAddFringe = False

1095 mock_config.doAddSky = True

1096 mock_config.doAddSource = True

1097

1098 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1099 input_exp = mock.run()

1100

1101 isr_config = self.get_isr_config_electronic_corrections()

1102 isr_config.doCorrectGains = True

1103 isr_config.doBias = True

1104 isr_config.doDark = True

1105 isr_config.doFlat = True

1106

1107 ptc = copy.copy(self.ptc)

1108 ptc.gain[ptc.ampNames[0]] *= 0.95

1109

1110 adjustments = np.ones(len(ptc.ampNames))

1111 adjustments[0] /= 0.95

1112 gainCorrection = GainCorrection(ampNames=ptc.ampNames, gainAdjustments=adjustments)

1113

1114 isr_task = IsrTaskLSST(config=isr_config)

1115 with self.assertNoLogs(level=logging.WARNING):

1116 result = isr_task.run(

1117 input_exp.clone(),

1118 bias=self.bias,

1119 dark=self.dark,

1120 flat=self.flat,

1121 crosstalk=self.crosstalk,

1122 defects=self.defects,

1123 ptc=self.ptc,

1124 gainCorrection=gainCorrection,

1125 linearizer=self.linearizer,

1126 deferredChargeCalib=self.cti,

1127 )

1128 self._check_applied_keys(result.exposure.metadata, isr_config, expected_gain_correction=True)

1129

1130 # Confirm that the output has the defect line as bad.

1131 sat_val = 2**result.exposure.mask.getMaskPlane("BAD")

1132 for defect in self.defects:

1133 np.testing.assert_array_equal(

1134 result.exposure.mask[defect.getBBox()].array & sat_val,

1135 sat_val,

1136 )

1137

1138 clean_mock_config = self.get_mock_config_clean()

1139 # We want the dark noise for more direct comparison.

1140 clean_mock_config.doAddDarkNoiseOnly = True

1141 clean_mock_config.doAddSky = True

1142 clean_mock_config.doAddSource = True

1143

1144 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1145 clean_exp = clean_mock.run()

1146

1147 delta = result.exposure.image.array - clean_exp.image.array

1148

1149 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

1150

1151 # We compare the good pixels in the entirety.

1152 self.assertLess(np.std(delta[good_pixels]), 5.0)

1153 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1154

1155 # Make sure the corrected image is overall consistent with the

1156 # straight image.

1157 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1158

1159 # And overall where the interpolation is a bit worse but

1160 # the statistics are still fine.

1161 self.assertLess(np.std(delta), 5.5)

1162

1163 metadata = result.exposure.metadata

1164

1165 key = "LSST ISR BOOTSTRAP"

1166 self.assertIn(key, metadata)

1167 self.assertEqual(metadata[key], False)

1168

1169 key = "LSST ISR UNITS"

1170 self.assertIn(key, metadata)

1171 self.assertEqual(metadata[key], "electron")

1172

1173 for amp in self.detector:

1174 amp_name = amp.getName()

1175 key = f"LSST ISR GAIN {amp_name}"

1176 self.assertIn(key, metadata)

1177 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1178 key = f"LSST ISR READNOISE {amp_name}"

1179 self.assertIn(key, metadata)

1180 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1181 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1182 self.assertIn(key, metadata)

1183 self.assertEqual(metadata[key], self.saturation_adu * gain)

1184 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1185 self.assertIn(key, metadata)

1186 self.assertEqual(metadata[key], self.saturation_adu * gain)

1187

1188 # Test the variance plane in the case of electron units.

1189 # The expected variance starts with the image array.

1190 expected_variance = result.exposure.image.clone()

1191 # We have to remove the flat-fielding from the image pixels.

1192 expected_variance.array *= self.flat.image.array

1193 # And add in the bias variance.

1194 expected_variance.array += self.bias.variance.array

1195 # And add in the scaled dark variance.

1196 scale = result.exposure.visitInfo.darkTime / self.dark.visitInfo.darkTime

1197 expected_variance.array += scale**2. * self.dark.variance.array

1198 # And add the read noise (in electrons) per amp.

1199 for amp in self.detector:

1200 gain = self.ptc.gain[amp.getName()]

1201 read_noise = self.ptc.noise[amp.getName()]

1202

1203 # The image, read noise, and variance plane should all have

1204 # units of electrons, electrons, and electrons^2.

1205 expected_variance[amp.getBBox()].array += read_noise**2.

1206

1207 # And apply the full formula for dividing by the flat with variance.

1208 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

1209 unflat_image_array = result.exposure.image.array * self.flat.image.array

1210 expected_variance.array = ((unflat_image_array**2. * self.flat.variance.array

1211 + self.flat.image.array**2. * expected_variance.array)

1212 / self.flat.image.array**4.)

1213

1214 self.assertFloatsAlmostEqual(

1215 result.exposure.variance.array[good_pixels],

1216 expected_variance.array[good_pixels],

1217 rtol=1e-6,

1218 )

1219

1220 def test_isrSkyImageSaturated(self):

1221 """Test processing of a sky image.

1222

1223 This variation uses saturated pixels instead of defects.

1224

1225 This additionally tests the gain config override.

1226 """

1227 mock_config = self.get_mock_config_no_signal()

1228 mock_config.doAddDark = True

1229 mock_config.doAddFlat = True

1230 # Set this to False until we have fringe correction.

1231 mock_config.doAddFringe = False

1232 mock_config.doAddSky = True

1233 mock_config.doAddSource = True

1234 mock_config.brightDefectLevel = 170_000.0 # Above saturation.

1235

1236 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1237 input_exp = mock.run()

1238

1239 isr_config = self.get_isr_config_electronic_corrections()

1240 isr_config.doBias = True

1241 isr_config.doDark = True

1242 isr_config.doFlat = True

1243 # We turn off defect masking to test the saturation code.

1244 # However, the same pixels below should be masked/interpolated.

1245 isr_config.doDefect = False

1246

1247 # Use a config override saturation value, confirm it is picked up.

1248 saturation_level = self.saturation_adu * 1.05

1249

1250 # This code will set the gain of one amp to the same as the ptc

1251 # value, and we will check that it is logged and used but the

1252 # results should be the same.

1253 detectorConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector)

1254 detectorConfig.defaultAmpConfig.saturation = saturation_level

1255 overscanAmpConfig = copy.copy(detectorConfig.defaultAmpConfig)

1256 overscanAmpConfig.gain = self.ptc.gain[self.detector[1].getName()]

1257 detectorConfig.ampRules[self.detector[1].getName()] = overscanAmpConfig

1258

1259 isr_task = IsrTaskLSST(config=isr_config)

1260 with self.assertLogs(level=logging.WARNING) as cm:

1261 result = isr_task.run(

1262 input_exp.clone(),

1263 bias=self.bias,

1264 dark=self.dark,

1265 flat=self.flat,

1266 deferredChargeCalib=self.cti,

1267 crosstalk=self.crosstalk,

1268 defects=self.defects,

1269 ptc=self.ptc,

1270 linearizer=self.linearizer,

1271 )

1272 self.assertIn("Overriding gain", cm.output[0])

1273 self._check_applied_keys(result.exposure.metadata, isr_config)

1274

1275 # Confirm that the output has the defect line as saturated.

1276 sat_val = 2**result.exposure.mask.getMaskPlane("SAT")

1277 for defect in self.defects:

1278 np.testing.assert_array_equal(

1279 result.exposure.mask[defect.getBBox()].array & sat_val,

1280 sat_val,

1281 )

1282

1283 clean_mock_config = self.get_mock_config_clean()

1284 # We want the dark noise for more direct comparison.

1285 clean_mock_config.doAddDarkNoiseOnly = True

1286 clean_mock_config.doAddSky = True

1287 clean_mock_config.doAddSource = True

1288

1289 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1290 clean_exp = clean_mock.run()

1291

1292 delta = result.exposure.image.array - clean_exp.image.array

1293

1294 bad_val = 2**result.exposure.mask.getMaskPlane("BAD")

1295 good_pixels = np.where((result.exposure.mask.array & (sat_val | bad_val)) == 0)

1296

1297 # We compare the good pixels in the entirety.

1298 self.assertLess(np.std(delta[good_pixels]), 5.0)

1299 # This is sensitive to parallel overscan masking.

1300 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1301

1302 # Make sure the corrected image is overall consistent with the

1303 # straight image.

1304 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1305

1306 # And overall where the interpolation is a bit worse but

1307 # the statistics are still fine. Note that this is worse than

1308 # the defect case because of the widening of the saturation

1309 # trail.

1310 self.assertLess(np.std(delta), 7.0)

1311

1312 metadata = result.exposure.metadata

1313

1314 for amp in self.detector:

1315 amp_name = amp.getName()

1316 key = f"LSST ISR GAIN {amp_name}"

1317 self.assertIn(key, metadata)

1318 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1319 key = f"LSST ISR READNOISE {amp_name}"

1320 self.assertIn(key, metadata)

1321 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1322 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1323 self.assertIn(key, metadata)

1324 self.assertEqual(metadata[key], saturation_level * gain)

1325

1326 def test_isrFlatVignette(self):

1327 """Test ISR when the flat has a validPolygon and vignetted region."""

1328

1329 # We use a flat frame for this test for convenience.

1330 mock_config = self.get_mock_config_no_signal()

1331 mock_config.doAddDark = True

1332 mock_config.doAddFlat = True

1333 # The doAddSky option adds the equivalent of flat-field flux.

1334 mock_config.doAddSky = True

1335

1336 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1337 input_exp = mock.run()

1338

1339 isr_config = self.get_isr_config_electronic_corrections()

1340 isr_config.doBias = True

1341 isr_config.doDark = True

1342 isr_config.doFlat = True

1343 isr_config.doDefect = True

1344

1345 flat = self.flat.clone()

1346 bbox = geom.Box2D(

1347 corner=geom.Point2D(0, 0),

1348 dimensions=geom.Extent2D(50, 50),

1349 )

1350 polygon = afwGeom.Polygon(bbox)

1351 flat.info.setValidPolygon(polygon)

1352 maskVignettedRegion(flat, polygon, vignetteValue=0.0)

1353

1354 isr_task = IsrTaskLSST(config=isr_config)

1355 with self.assertNoLogs(level=logging.WARNING):

1356 result = isr_task.run(

1357 input_exp.clone(),

1358 bias=self.bias,

1359 dark=self.dark,

1360 flat=flat,

1361 crosstalk=self.crosstalk,

1362 ptc=self.ptc,

1363 linearizer=self.linearizer,

1364 defects=self.defects,

1365 deferredChargeCalib=self.cti,

1366 )

1367

1368 self.assertEqual(result.exposure.info.getValidPolygon(), polygon)

1369

1370 noDataFlat = (flat.mask.array & flat.mask.getPlaneBitMask("NO_DATA")) > 0

1371 noDataExp = (result.exposure.mask.array & result.exposure.mask.getPlaneBitMask("NO_DATA")) > 0

1372 np.testing.assert_array_equal(noDataExp, noDataFlat)

1373 np.testing.assert_array_equal(result.exposure.image.array[noDataExp], 0.0)

1374 np.testing.assert_array_equal(result.exposure.variance.array[noDataExp], 0.0)

1375 self.assertFalse(np.any(~np.isfinite(result.exposure.image.array)))

1376 self.assertFalse(np.any(~np.isfinite(result.exposure.variance.array)))

1377

1378 def test_isrFloodedSaturatedE2V(self):

1379 """Test ISR when the amps are completely saturated.

1380

1381 This version tests what happens when the parallel overscan

1382 region is flooded like E2V detectors, where the saturation

1383 spreads evenly, but at a greater level than the saturation

1384 value.

1385 """

1386 # We are simulating a flat field.

1387 # Note that these aren't very important because we are replacing

1388 # the flux, but we may as well.

1389 mock_config = self.get_mock_config_no_signal()

1390 mock_config.doAddDark = True

1391 mock_config.doAddFlat = True

1392 # The doAddSky option adds the equivalent of flat-field flux.

1393 mock_config.doAddSky = True

1394

1395 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1396 input_exp = mock.run()

1397

1398 isr_config = self.get_isr_config_minimal_corrections()

1399 isr_config.doBootstrap = True

1400 isr_config.doApplyGains = False

1401 isr_config.doBias = True

1402 isr_config.doDark = True

1403 isr_config.doFlat = False

1404 # Tun off saturation masking to simulate a PTC flat.

1405 isr_config.doSaturation = False

1406 isr_config.doE2VEdgeBleedMask = False

1407 isr_config.doITLEdgeBleedMask = False

1408

1409 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1410 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1411

1412 detector = input_exp.getDetector()

1413 for i, amp in enumerate(detector):

1414 # For half of the amps we are testing what happens when the

1415 # parallel overscan region is above the configured saturation

1416 # level; for the other half we are testing the other branch

1417 # when it saturates below this level (which is a priori

1418 # unknown).

1419 if i < len(detector) // 2:

1420 data_level = (parallel_overscan_saturation * 1.05

1421 + mock_config.biasLevel

1422 + mock_config.clockInjectedOffsetLevel)

1423 parallel_overscan_level = (parallel_overscan_saturation * 1.1

1424 + mock_config.biasLevel

1425 + mock_config.clockInjectedOffsetLevel)

1426 else:

1427 data_level = (parallel_overscan_saturation * 0.7

1428 + mock_config.biasLevel

1429 + mock_config.clockInjectedOffsetLevel)

1430 parallel_overscan_level = (parallel_overscan_saturation * 0.75

1431 + mock_config.biasLevel

1432 + mock_config.clockInjectedOffsetLevel)

1433

1434 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1435 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1436

1437 isr_task = IsrTaskLSST(config=isr_config)

1438 with self.assertLogs(level=logging.WARNING) as cm:

1439 result = isr_task.run(

1440 input_exp.clone(),

1441 bias=self.bias_adu,

1442 dark=self.dark_adu,

1443 )

1444 self.assertEqual(len(cm.records), len(detector))

1445

1446 n_all = 0

1447 n_level = 0

1448 for record in cm.records:

1449 if "All overscan pixels masked" in record.message:

1450 n_all += 1

1451 if "The level in the overscan region" in record.message:

1452 n_level += 1

1453

1454 self.assertEqual(n_all, len(detector) // 2)

1455 self.assertEqual(n_level, len(detector) // 2)

1456

1457 # And confirm that the post-ISR levels are high for each amp.

1458 for amp in detector:

1459 med = np.median(result.exposure[amp.getBBox()].image.array)

1460 self.assertGreater(med, parallel_overscan_saturation*0.8)

1461

1462 def test_isrFloodedSaturatedITL(self):

1463 """Test ISR when the amps are completely saturated.

1464

1465 This version tests what happens when the parallel overscan

1466 region is flooded like ITL detectors, where the saturation

1467 is at a lower level than the imaging region, and also

1468 spreads partly into the serial/parallel region.

1469 """

1470 # We are simulating a flat field.

1471 # Note that these aren't very important because we are replacing

1472 # the flux, but we may as well.

1473 mock_config = self.get_mock_config_no_signal()

1474 mock_config.doAddDark = True

1475 mock_config.doAddFlat = True

1476 # The doAddSky option adds the equivalent of flat-field flux.

1477 mock_config.doAddSky = True

1478

1479 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1480 input_exp = mock.run()

1481

1482 isr_config = self.get_isr_config_minimal_corrections()

1483 isr_config.doBootstrap = True

1484 isr_config.doApplyGains = False

1485 isr_config.doBias = True

1486 isr_config.doDark = True

1487 isr_config.doFlat = False

1488 # Tun off saturation masking to simulate a PTC flat.

1489 isr_config.doSaturation = False

1490 isr_config.doE2VEdgeBleedMask = False

1491 isr_config.doITLEdgeBleedMask = False

1492

1493 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1494 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1495

1496 detector = input_exp.getDetector()

1497 for i, amp in enumerate(detector):

1498 # For half of the amps we are testing what happens when the

1499 # parallel overscan region is above the configured saturation

1500 # level; for the other half we are testing the other branch

1501 # when it saturates below this level (which is a priori

1502 # unknown).

1503 if i < len(detector) // 2:

1504 data_level = (parallel_overscan_saturation * 1.1

1505 + mock_config.biasLevel

1506 + mock_config.clockInjectedOffsetLevel)

1507 parallel_overscan_level = (parallel_overscan_saturation * 1.05

1508 + mock_config.biasLevel

1509 + mock_config.clockInjectedOffsetLevel)

1510 else:

1511 data_level = (parallel_overscan_saturation * 0.75

1512 + mock_config.biasLevel

1513 + mock_config.clockInjectedOffsetLevel)

1514 parallel_overscan_level = (parallel_overscan_saturation * 0.7

1515 + mock_config.biasLevel

1516 + mock_config.clockInjectedOffsetLevel)

1517

1518 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1519 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1520 # The serial/parallel region for the test camera looks like this:

1521 serial_overscan_bbox = amp.getRawSerialOverscanBBox()

1522 parallel_overscan_bbox = amp.getRawParallelOverscanBBox()

1523

1524 overscan_corner_bbox = geom.Box2I(

1525 geom.Point2I(

1526 serial_overscan_bbox.getMinX(),

1527 parallel_overscan_bbox.getMinY(),

1528 ),

1529 geom.Extent2I(

1530 serial_overscan_bbox.getWidth(),

1531 parallel_overscan_bbox.getHeight(),

1532 ),

1533 )

1534 input_exp[overscan_corner_bbox].image.array[-2:, :] = parallel_overscan_level

1535

1536 isr_task = IsrTaskLSST(config=isr_config)

1537 with self.assertLogs(level=logging.WARNING) as cm:

1538 result = isr_task.run(

1539 input_exp.clone(),

1540 bias=self.bias_adu,

1541 dark=self.dark_adu,

1542 )

1543 self.assertEqual(len(cm.records), len(detector))

1544

1545 n_all = 0

1546 n_level = 0

1547 for record in cm.records:

1548 if "All overscan pixels masked" in record.message:

1549 n_all += 1

1550 if "The level in the overscan region" in record.message:

1551 n_level += 1

1552

1553 self.assertEqual(n_all, len(detector) // 2)

1554 self.assertEqual(n_level, len(detector) // 2)

1555

1556 # And confirm that the post-ISR levels are high for each amp.

1557 for amp in detector:

1558 med = np.median(result.exposure[amp.getBBox()].image.array)

1559 self.assertGreater(med, parallel_overscan_saturation*0.8)

1560

1561 def test_isrBadParallelOverscanColumnsBootstrap(self):

1562 """Test processing a bias when we have a bad parallel overscan column.

1563

1564 This tests in bootstrap mode.

1565 """

1566 # We base this on the bootstrap bias, and make sure

1567 # that the bad column remains.

1568 mock_config = self.get_mock_config_no_signal()

1569 isr_config = self.get_isr_config_minimal_corrections()

1570 isr_config.doSaturation = False

1571 isr_config.doE2VEdgeBleedMask = False

1572 isr_config.doITLEdgeBleedMask = False

1573 isr_config.doBootstrap = True

1574 isr_config.doApplyGains = False

1575

1576 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1577 overscan_sat_level_adu = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1578 # The defect is in amp 2.

1579 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1580 overscan_sat_level = amp_gain * overscan_sat_level_adu

1581 # The expected defect level is in adu for the bootstrap bias.

1582 expected_defect_level = mock_config.brightDefectLevel / amp_gain

1583

1584 # The levels are set in electron units.

1585 # We test 3 levels:

1586 # * 10.0, a very low outlier, to test median smoothing detection

1587 # code. This value is given by gain*threshold + cushion.

1588 # * 575.0, a lowish but outlier level, given by gain*threshold +

1589 # 100.0 (average of the parallel overscan offset) + 10.0

1590 # (an additional cushion).

1591 # * 1.05*saturation.

1592 # Note that the default parallel overscan saturation level for

1593 # bootstrap (pre-saturation-measure) analysis is very low, in

1594 # order to capture all types of amps, even with low saturation.

1595 # Therefore, we only need to test above this saturation level.

1596 # (c.f. test_isrBadParallelOverscanColumns).

1597 levels = np.array([10.0, 575.0, 1.05*overscan_sat_level])

1598

1599 for level in levels:

1600 mock_config.badParallelOverscanColumnLevel = level

1601 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1602 input_exp = mock.run()

1603

1604 isr_task = IsrTaskLSST(config=isr_config)

1605 with self.assertNoLogs(level=logging.WARNING):

1606 result = isr_task.run(input_exp.clone())

1607

1608 for defect in self.defects:

1609 bbox = defect.getBBox()

1610 defect_image = result.exposure[bbox].image.array

1611

1612 # Check that the defect is the correct level

1613 # (not subtracted away).

1614 defect_median = np.median(defect_image)

1615 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1616

1617 # Check that the neighbors aren't over-subtracted.

1618 for neighbor in [-1, 1]:

1619 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1620 neighbor_image = result.exposure[bbox_neighbor].image.array

1621

1622 neighbor_median = np.median(neighbor_image)

1623 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1624

1625 def test_isrBadParallelOverscanColumns(self):

1626 """Test processing a bias when we have a bad parallel overscan column.

1627

1628 This test uses regular non-bootstrap processing.

1629 """

1630 mock_config = self.get_mock_config_no_signal()

1631 isr_config = self.get_isr_config_electronic_corrections()

1632 # We do not do defect correction when processing biases.

1633 isr_config.doDefect = False

1634

1635 # The defect is in amp 2.

1636 sat_level_adu = self.ptc.ptcTurnoff[self.detector[2].getName()]

1637 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1638 sat_level = amp_gain * sat_level_adu

1639 # The expected defect level is in electron for the full bias.

1640 expected_defect_level = mock_config.brightDefectLevel

1641

1642 # The levels are set in electron units.

1643 # We test 4 levels:

1644 # * 10.0, a very low outlier, to test median smoothing detection

1645 # code. This value is given by gain*threshold + cushion.

1646 # * 575.0, a lowish but outlier level, given by gain*threshold +

1647 # 100.0 (average of the parallel overscan offset) + 10.0

1648 # (an additional cushion).

1649 # * 0.9*saturation, following ITL-style parallel overscan bleeds.

1650 # * 1.05*saturation, following E2V-style parallel overscan bleeds.

1651 levels = np.array([10.0, 575.0, 0.9*sat_level, 1.1*sat_level])

1652

1653 for level in levels:

1654 mock_config.badParallelOverscanColumnLevel = level

1655 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1656 input_exp = mock.run()

1657

1658 isr_task = IsrTaskLSST(config=isr_config)

1659 with self.assertNoLogs(level=logging.WARNING):

1660 result = isr_task.run(

1661 input_exp.clone(),

1662 crosstalk=self.crosstalk,

1663 ptc=self.ptc,

1664 linearizer=self.linearizer,

1665 deferredChargeCalib=self.cti,

1666 )

1667

1668 for defect in self.defects:

1669 bbox = defect.getBBox()

1670 defect_image = result.exposure[bbox].image.array

1671

1672 # Check that the defect is the correct level

1673 # (not subtracted away).

1674 defect_median = np.median(defect_image)

1675 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1676

1677 # Check that the neighbors aren't over-subtracted.

1678 for neighbor in [-1, 1]:

1679 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1680 neighbor_image = result.exposure[bbox_neighbor].image.array

1681

1682 neighbor_median = np.median(neighbor_image)

1683 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1684

1685 def test_isrBadPtcGain(self):

1686 """Test processing when an amp has a bad (nan) PTC gain.

1687 """

1688 # We use a flat frame for this test for convenience.

1689 mock_config = self.get_mock_config_no_signal()

1690 mock_config.doAddDark = True

1691 mock_config.doAddFlat = True

1692 # The doAddSky option adds the equivalent of flat-field flux.

1693 mock_config.doAddSky = True

1694

1695 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1696 input_exp = mock.run()

1697

1698 isr_config = self.get_isr_config_electronic_corrections()

1699 isr_config.doBias = True

1700 isr_config.doDark = True

1701 isr_config.doFlat = False

1702 isr_config.doDefect = True

1703

1704 # Set a bad amplifier to a nan gain.

1705 bad_amp = self.detector[0].getName()

1706

1707 ptc = copy.copy(self.ptc)

1708 ptc.gain[bad_amp] = np.nan

1709

1710 # We also want non-zero (but very small) crosstalk values

1711 # to ensure that these don't propagate nans.

1712 crosstalk = copy.copy(self.crosstalk)

1713 for i in range(len(self.detector)):

1714 for j in range(len(self.detector)):

1715 if i == j:

1716 continue

1717 if crosstalk.coeffs[i, j] == 0:

1718 crosstalk.coeffs[i, j] = 1e-10

1719

1720 isr_task = IsrTaskLSST(config=isr_config)

1721 with self.assertLogs(level=logging.WARNING) as cm:

1722 result = isr_task.run(

1723 input_exp.clone(),

1724 bias=self.bias,

1725 dark=self.dark,

1726 crosstalk=crosstalk,

1727 ptc=ptc,

1728 linearizer=self.linearizer,

1729 defects=self.defects,

1730 deferredChargeCalib=self.cti,

1731 )

1732 self.assertIn(f"Amplifier {bad_amp} is bad (non-finite gain)", cm.output[0])

1733

1734 # Confirm that the bad_amp is marked bad and the other amps are not.

1735 # We have to special case the amp with the defect.

1736 mask = result.exposure.mask

1737

1738 for amp in self.detector:

1739 bbox = amp.getBBox()

1740 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1741

1742 if amp.getName() == bad_amp:

1743 self.assertTrue(np.all(bad_in_amp))

1744 elif amp.getName() == "C:0,2":

1745 # This is the amp with the defect.

1746 self.assertEqual(np.sum(bad_in_amp), 51)

1747 else:

1748 self.assertTrue(np.all(~bad_in_amp))

1749

1750 def test_saturationModes(self):

1751 """Test the different saturation modes."""

1752 # Use a simple bias run for these.

1753 mock_config = self.get_mock_config_no_signal()

1754

1755 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1756 input_exp = mock.run()

1757

1758 isr_config = self.get_isr_config_electronic_corrections()

1759 isr_config.doSaturation = True

1760 isr_config.maskNegativeVariance = False

1761 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1762 amp_config = copy.copy(detector_config.defaultAmpConfig)

1763

1764 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1765 isr_config.defaultSaturationSource = mode

1766

1767 # Reset the PTC.

1768 ptc = copy.copy(self.ptc)

1769 # Reset the detector config.

1770 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1771 if mode == "NONE":

1772 # We must use the config.

1773 sat_level = 1.2 * self.saturation_adu

1774 amp_config_new = copy.copy(amp_config)

1775 amp_config_new.saturation = sat_level

1776 detector_config_new = copy.copy(detector_config)

1777 detector_config_new.defaultAmpConfig = amp_config_new

1778 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1779 elif mode == "CAMERAMODEL":

1780 sat_level = input_exp.getDetector()[0].getSaturation()

1781 elif mode == "PTCTURNOFF":

1782 sat_level = 1.3 * self.saturation_adu

1783 for amp_name in ptc.ampNames:

1784 ptc.ptcTurnoff[amp_name] = sat_level

1785

1786 isr_task = IsrTaskLSST(config=isr_config)

1787 with self.assertNoLogs(level=logging.WARNING):

1788 result = isr_task.run(

1789 input_exp.clone(),

1790 bias=self.bias,

1791 crosstalk=self.crosstalk,

1792 ptc=self.ptc,

1793 linearizer=self.linearizer,

1794 deferredChargeCalib=self.cti,

1795 defects=self.defects,

1796 )

1797

1798 metadata = result.exposure.metadata

1799

1800 for amp in self.detector:

1801 amp_name = amp.getName()

1802 key = f"LSST ISR GAIN {amp_name}"

1803 self.assertIn(key, metadata, msg=mode)

1804 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1805 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1806 self.assertIn(key, metadata, msg=mode)

1807 self.assertEqual(metadata[key], sat_level * gain, msg=mode)

1808

1809 def test_noPTC(self):

1810 """Test if we do not supply a PTC."""

1811 mock_config = self.get_mock_config_no_signal()

1812

1813 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1814 input_exp = mock.run()

1815

1816 isr_config = self.get_isr_config_minimal_corrections()

1817 isr_task = IsrTaskLSST(config=isr_config)

1818

1819 with self.assertRaises(RuntimeError) as cm:

1820 _ = isr_task.run(input_exp.clone())

1821 self.assertIn("doBootstrap==False and useGainsFrom =="

1822 " 'PTC' but no PTC provided.",

1823 cm.exception.args[0])

1824

1825 def test_suspectModes(self):

1826 """Test the different suspect modes."""

1827 # Use a simple bias run for these.

1828 mock_config = self.get_mock_config_no_signal()

1829

1830 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1831 input_exp = mock.run()

1832

1833 isr_config = self.get_isr_config_electronic_corrections()

1834 isr_config.doSaturation = True

1835 isr_config.maskNegativeVariance = False

1836 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1837 amp_config = copy.copy(detector_config.defaultAmpConfig)

1838

1839 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1840 isr_config.defaultSuspectSource = mode

1841

1842 # Reset the PTC.

1843 ptc = copy.copy(self.ptc)

1844 # Reset the detector config.

1845 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1846 if mode == "NONE":

1847 # We must use the config.

1848 suspect_level = 1.2 * self.saturation_adu

1849 amp_config_new = copy.copy(amp_config)

1850 amp_config_new.suspectLevel = suspect_level

1851 detector_config_new = copy.copy(detector_config)

1852 detector_config_new.defaultAmpConfig = amp_config_new

1853 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1854 elif mode == "CAMERAMODEL":

1855 suspect_level = input_exp.getDetector()[0].getSuspectLevel()

1856 elif mode == "PTCTURNOFF":

1857 suspect_level = 1.3 * self.saturation_adu

1858 for amp_name in ptc.ampNames:

1859 ptc.ptcTurnoff[amp_name] = suspect_level

1860

1861 isr_task = IsrTaskLSST(config=isr_config)

1862 with self.assertNoLogs(level=logging.WARNING):

1863 result = isr_task.run(

1864 input_exp.clone(),

1865 bias=self.bias,

1866 crosstalk=self.crosstalk,

1867 ptc=self.ptc,

1868 linearizer=self.linearizer,

1869 deferredChargeCalib=self.cti,

1870 defects=self.defects,

1871 )

1872

1873 metadata = result.exposure.metadata

1874

1875 for amp in self.detector:

1876 amp_name = amp.getName()

1877 key = f"LSST ISR GAIN {amp_name}"

1878 self.assertIn(key, metadata, msg=mode)

1879 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1880 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1881 self.assertIn(key, metadata, msg=mode)

1882 self.assertEqual(metadata[key], suspect_level * gain, msg=mode)

1883

1884 def test_sequencerMismatches(self):

1885 """Test with a pile of sequencer mismatches."""

1886 mock_config = self.get_mock_config_no_signal()

1887 mock_config.doAddDark = True

1888 mock_config.doAddFlat = True

1889 # Set this to False until we have fringe correction.

1890 mock_config.doAddFringe = False

1891 mock_config.doAddSky = True

1892 mock_config.doAddSource = True

1893

1894 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1895 input_exp = mock.run()

1896 input_exp.metadata["SEQFILE"] = "a_sequencer"

1897

1898 isr_config = self.get_isr_config_electronic_corrections()

1899 isr_config.doBias = True

1900 isr_config.doDark = True

1901 isr_config.doFlat = True

1902 isr_config.cameraKeywordsToCompare = ["SEQFILE"]

1903

1904 bias = self.bias.clone()

1905 bias.metadata["SEQFILE"] = "b_sequencer"

1906 dark = self.dark.clone()

1907 dark.metadata["SEQFILE"] = "b_sequencer"

1908 flat = self.flat.clone()

1909 flat.metadata["SEQFILE"] = "b_sequencer"

1910 crosstalk = copy.copy(self.crosstalk)

1911 crosstalk.metadata["SEQFILE"] = "b_sequencer"

1912 defects = copy.copy(self.defects)

1913 defects.metadata["SEQFILE"] = "b_sequencer"

1914 ptc = copy.copy(self.ptc)

1915 ptc.metadata["SEQFILE"] = "b_sequencer"

1916 linearizer = copy.copy(self.linearizer)

1917 linearizer.metadata["SEQFILE"] = "b_sequencer"

1918 cti = copy.copy(self.cti)

1919 cti.metadata["SEQFILE"] = "b_sequencer"

1920

1921 isr_task = IsrTaskLSST(config=isr_config)

1922 with self.assertLogs(level=logging.WARNING):

1923 result = isr_task.run(

1924 input_exp.clone(),

1925 bias=bias,

1926 dark=dark,

1927 flat=flat,

1928 crosstalk=crosstalk,

1929 defects=defects,

1930 ptc=ptc,

1931 linearizer=linearizer,

1932 deferredChargeCalib=cti,

1933 )

1934

1935 for ctype in ["BIAS", "DARK", "FLAT", "CROSSTALK", "DEFECTS", "PTC", "LINEARIZER", "CTI"]:

1936 self.assertTrue(result.exposure.metadata[f"ISR {ctype} SEQUENCER MISMATCH"])

1937

1938 def test_highPtcNoiseAmps(self):

1939 """Test for masking of high noise amps (in PTC)."""

1940 # We use a flat frame for this test for convenience.

1941 mock_config = self.get_mock_config_no_signal()

1942 mock_config.doAddDark = True

1943 mock_config.doAddFlat = True

1944 # The doAddSky option adds the equivalent of flat-field flux.

1945 mock_config.doAddSky = True

1946

1947 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1948 input_exp = mock.run()

1949

1950 isr_config = self.get_isr_config_electronic_corrections()

1951 isr_config.doBias = True

1952 isr_config.doDark = True

1953 isr_config.doFlat = False

1954 isr_config.doDefect = True

1955

1956 # Set a bad amplifier to a high noise.

1957 bad_amp = self.detector[0].getName()

1958

1959 ptc = copy.copy(self.ptc)

1960 ptc.noise[bad_amp] = 50.0

1961

1962 isr_task = IsrTaskLSST(config=isr_config)

1963

1964 # With the PTC this should not warn.

1965 with self.assertNoLogs(level=logging.WARNING):

1966 result = isr_task.run(

1967 input_exp.clone(),

1968 bias=self.bias,

1969 dark=self.dark,

1970 crosstalk=self.crosstalk,

1971 ptc=ptc,

1972 linearizer=self.linearizer,

1973 defects=self.defects,

1974 deferredChargeCalib=self.cti,

1975 )

1976

1977 # Confirm that the bad_amp is marked bad and the other amps are not.

1978 # We have to special case the amp with the defect.

1979 mask = result.exposure.mask

1980

1981 for amp in self.detector:

1982 bbox = amp.getBBox()

1983 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1984

1985 if amp.getName() == bad_amp:

1986 self.assertTrue(np.all(bad_in_amp))

1987 elif amp.getName() == "C:0,2":

1988 # This is the amp with the defect.

1989 self.assertEqual(np.sum(bad_in_amp), 51)

1990 else:

1991 self.assertTrue(np.all(~bad_in_amp))

1992

1993 def test_changedOverscanAmps(self):

1994 """Tests for masking of amps where the overscan level changed."""

1995

1996 # We use a flat frame for this test for convenience.

1997 mock_config = self.get_mock_config_no_signal()

1998 mock_config.doAddDark = True

1999 mock_config.doAddFlat = True

2000 # The doAddSky option adds the equivalent of flat-field flux.

2001 mock_config.doAddSky = True

2002

2003 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2004 input_exp = mock.run()

2005

2006 # Offset one amp with a constant value.

2007 bad_amp = "C:0,0"

2008

2009 input_exp2 = input_exp.clone()

2010 input_exp2.image[self.detector[bad_amp].getRawBBox()].array[:, :] -= 2000.0

2011

2012 isr_config = self.get_isr_config_electronic_corrections()

2013 isr_config.doBias = True

2014 isr_config.doDark = True

2015 isr_config.doFlat = False

2016 isr_config.doDefect = True

2017 isr_config.doInterpolate = False

2018 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

2019

2020 isr_task = IsrTaskLSST(config=isr_config)

2021 with self.assertLogs(level=logging.WARNING) as cm:

2022 _ = isr_task.run(

2023 input_exp2,

2024 bias=self.bias,

2025 dark=self.dark,

2026 crosstalk=self.crosstalk,

2027 ptc=self.ptc,

2028 linearizer=self.linearizer,

2029 defects=self.defects,

2030 deferredChargeCalib=self.cti,

2031 )

2032 self.assertEqual(len(cm.output), 1)

2033 self.assertIn(f"Amplifier {bad_amp} has an overscan level", cm.output[0])

2034

2035 # Offset all amps to see that it is now unprocessable.

2036

2037 input_exp2 = input_exp.clone()

2038 input_exp2.image.array[:, :] -= 2000.0

2039

2040 with self.assertLogs(level=logging.WARNING):

2041 with self.assertRaises(UnprocessableDataError):

2042 _ = isr_task.run(

2043 input_exp2,

2044 bias=self.bias,

2045 dark=self.dark,

2046 crosstalk=self.crosstalk,

2047 ptc=self.ptc,

2048 linearizer=self.linearizer,

2049 defects=self.defects,

2050 deferredChargeCalib=self.cti,

2051 )

2052

2053 # Remove the values in the PTC and turn off check.

2054 # This should run without warnings.

2055 ptc = copy.copy(self.ptc)

2056 for amp in self.detector:

2057 ptc.overscanMedian[amp.getName()] = np.nan

2058

2059 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2060

2061 isr_task = IsrTaskLSST(config=isr_config)

2062 with self.assertNoLogs(level=logging.WARNING):

2063 _ = isr_task.run(

2064 input_exp.clone(),

2065 bias=self.bias,

2066 dark=self.dark,

2067 crosstalk=self.crosstalk,

2068 ptc=ptc,

2069 linearizer=self.linearizer,

2070 defects=self.defects,

2071 deferredChargeCalib=self.cti,

2072 )

2073

2074 # Turn the check back on; this should have 1 warning.

2075 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

2076

2077 isr_task = IsrTaskLSST(config=isr_config)

2078 with self.assertLogs(level=logging.WARNING) as cm:

2079 _ = isr_task.run(

2080 input_exp.clone(),

2081 bias=self.bias,

2082 dark=self.dark,

2083 crosstalk=self.crosstalk,

2084 ptc=ptc,

2085 linearizer=self.linearizer,

2086 defects=self.defects,

2087 deferredChargeCalib=self.cti,

2088 )

2089 self.assertEqual(len(cm.output), 1)

2090 self.assertIn("No PTC overscan information", cm.output[0])

2091

2092 def test_highOverscanNoiseAmps(self):

2093 """Test for masking of high noise amps (in overscan)."""

2094

2095 # We use a flat frame for this test for convenience.

2096 mock_config = self.get_mock_config_no_signal()

2097 mock_config.doAddDark = True

2098 mock_config.doAddFlat = True

2099 # The doAddSky option adds the equivalent of flat-field flux.

2100 mock_config.doAddSky = True

2101 mock_config.readNoise = 50.0

2102

2103 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2104 input_exp = mock.run()

2105

2106 isr_config = self.get_isr_config_electronic_corrections()

2107 isr_config.doBias = True

2108 isr_config.doDark = True

2109 isr_config.doFlat = False

2110 isr_config.doDefect = True

2111 isr_config.doInterpolate = False

2112 # Let all the amps fail to check the logging.

2113 isr_config.doCheckUnprocessableData = False

2114

2115 isr_task = IsrTaskLSST(config=isr_config)

2116 with self.assertLogs(level=logging.WARNING) as cm:

2117 result = isr_task.run(

2118 input_exp.clone(),

2119 bias=self.bias,

2120 dark=self.dark,

2121 crosstalk=self.crosstalk,

2122 ptc=self.ptc,

2123 linearizer=self.linearizer,

2124 defects=self.defects,

2125 deferredChargeCalib=self.cti,

2126 )

2127 self.assertEqual(len(cm.output), len(self.detector))

2128

2129 # All pixels should be BAD

2130 bad_value = result.exposure.mask.getPlaneBitMask("BAD")

2131 np.testing.assert_array_equal(result.exposure.mask.array & bad_value, bad_value)

2132

2133 # And run again to check the UnprocessableDataError.

2134 isr_config.doCheckUnprocessableData = True

2135 isr_task = IsrTaskLSST(config=isr_config)

2136

2137 with self.assertRaises(UnprocessableDataError):

2138 with self.assertLogs(level=logging.WARNING):

2139 result = isr_task.run(

2140 input_exp.clone(),

2141 bias=self.bias,

2142 dark=self.dark,

2143 crosstalk=self.crosstalk,

2144 ptc=self.ptc,

2145 linearizer=self.linearizer,

2146 defects=self.defects,

2147 deferredChargeCalib=self.cti,

2148 )

2149

2150 def test_bssVoltageChecks(self):

2151 """Test the BSS voltage checks."""

2152 # We use a flat frame for this test for convenience.

2153 mock_config = self.get_mock_config_no_signal()

2154 mock_config.doAddDark = True

2155 mock_config.doAddFlat = True

2156 # The doAddSky option adds the equivalent of flat-field flux.

2157 mock_config.doAddSky = True

2158

2159 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2160 input_exp = mock.run()

2161

2162 isr_config = self.get_isr_config_electronic_corrections()

2163 isr_config.doBias = True

2164 isr_config.doDark = True

2165 isr_config.doFlat = False

2166 isr_config.doDefect = True

2167

2168 # Set the voltage.

2169 input_exp.metadata["BSSVBS"] = 0.25

2170

2171 # Check that processing runs with checks turned off.

2172 isr_config.doCheckUnprocessableData = False

2173

2174 isr_task = IsrTaskLSST(config=isr_config)

2175 with self.assertNoLogs(level=logging.WARNING):

2176 _ = isr_task.run(

2177 input_exp.clone(),

2178 bias=self.bias,

2179 dark=self.dark,

2180 crosstalk=self.crosstalk,

2181 ptc=self.ptc,

2182 linearizer=self.linearizer,

2183 defects=self.defects,

2184 deferredChargeCalib=self.cti,

2185 )

2186

2187 # Check that processing runs with other way of turning checks off.

2188 isr_config.doCheckUnprocessableData = True

2189 isr_config.bssVoltageMinimum = 0.0

2190

2191 isr_task = IsrTaskLSST(config=isr_config)

2192 with self.assertNoLogs(level=logging.WARNING):

2193 _ = isr_task.run(

2194 input_exp.clone(),

2195 bias=self.bias,

2196 dark=self.dark,

2197 crosstalk=self.crosstalk,

2198 ptc=self.ptc,

2199 linearizer=self.linearizer,

2200 defects=self.defects,

2201 deferredChargeCalib=self.cti,

2202 )

2203

2204 # Check that processing runs but warns if header keyword is None.

2205 isr_config.doCheckUnprocessableData = True

2206 isr_config.bssVoltageMinimum = 10.0

2207

2208 input_exp2 = input_exp.clone()

2209 input_exp2.metadata["BSSVBS"] = None

2210

2211 isr_task = IsrTaskLSST(config=isr_config)

2212 with self.assertLogs(level=logging.WARNING) as cm:

2213 _ = isr_task.run(

2214 input_exp2,

2215 bias=self.bias,

2216 dark=self.dark,

2217 crosstalk=self.crosstalk,

2218 ptc=self.ptc,

2219 linearizer=self.linearizer,

2220 defects=self.defects,

2221 deferredChargeCalib=self.cti,

2222 )

2223 self.assertEqual(len(cm.output), 1)

2224 self.assertIn("Back-side bias voltage BSSVBS not found", cm.output[0])

2225

2226 # Check that it raises.

2227 isr_config.doCheckUnprocessableData = True

2228 isr_config.bssVoltageMinimum = 10.0

2229

2230 isr_task = IsrTaskLSST(config=isr_config)

2231 with self.assertRaises(UnprocessableDataError):

2232 _ = isr_task.run(

2233 input_exp.clone(),

2234 bias=self.bias,

2235 dark=self.dark,

2236 crosstalk=self.crosstalk,

2237 ptc=self.ptc,

2238 linearizer=self.linearizer,

2239 defects=self.defects,

2240 deferredChargeCalib=self.cti,

2241 )

2242

2243 def test_overrideMaskBadAmp(self):

2244 """Test overriding config to mask an amp."""

2245 # We use a flat frame for this test for convenience.

2246 mock_config = self.get_mock_config_no_signal()

2247 mock_config.doAddDark = True

2248 mock_config.doAddFlat = True

2249 # The doAddSky option adds the equivalent of flat-field flux.

2250 mock_config.doAddSky = True

2251

2252 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2253 input_exp = mock.run()

2254

2255 isr_config = self.get_isr_config_electronic_corrections()

2256 isr_config.doBias = True

2257 isr_config.doDark = True

2258 isr_config.doFlat = False

2259 isr_config.doDefect = True

2260

2261 bad_amps = ["C:0,0", "C:0,1"]

2262

2263 detector_name = self.detector.getName()

2264 isr_config.badAmps = [f"{detector_name}_{bad_amp}" for bad_amp in bad_amps]

2265

2266 isr_task = IsrTaskLSST(config=isr_config)

2267 with self.assertNoLogs(level=logging.WARNING):

2268 result = isr_task.run(

2269 input_exp.clone(),

2270 bias=self.bias,

2271 dark=self.dark,

2272 crosstalk=self.crosstalk,

2273 ptc=self.ptc,

2274 linearizer=self.linearizer,

2275 defects=self.defects,

2276 deferredChargeCalib=self.cti,

2277 )

2278

2279 for amp_name in bad_amps:

2280 mask_array = result.exposure[self.detector[amp_name].getBBox()].mask.array

2281 bad_mask = result.exposure.mask.getPlaneBitMask("BAD")

2282 self.assertTrue(np.all((mask_array & bad_mask) > 0))

2283

2284 # Make sure it didn't obliterate everything.

2285 mask_array = result.exposure.mask.array

2286 self.assertFalse(np.all((mask_array & bad_mask) > 0))

2287

2288 def test_hvBiasChecks(self):

2289 """Test the HVBIAS checks."""

2290 # We use a flat frame for this test for convenience.

2291 mock_config = self.get_mock_config_no_signal()

2292 mock_config.doAddDark = True

2293 mock_config.doAddFlat = True

2294 # The doAddSky option adds the equivalent of flat-field flux.

2295 mock_config.doAddSky = True

2296

2297 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2298 input_exp = mock.run()

2299

2300 isr_config = self.get_isr_config_electronic_corrections()

2301 isr_config.doBias = True

2302 isr_config.doDark = True

2303 isr_config.doFlat = False

2304 isr_config.doDefect = True

2305

2306 # Set the voltage.

2307 input_exp.metadata["HVBIAS"] = "OFF"

2308

2309 # Check that processing runs with checks turned off.

2310 isr_config.doCheckUnprocessableData = False

2311

2312 isr_task = IsrTaskLSST(config=isr_config)

2313 with self.assertNoLogs(level=logging.WARNING):

2314 _ = isr_task.run(

2315 input_exp.clone(),

2316 bias=self.bias,

2317 dark=self.dark,

2318 crosstalk=self.crosstalk,

2319 ptc=self.ptc,

2320 linearizer=self.linearizer,

2321 defects=self.defects,

2322 deferredChargeCalib=self.cti,

2323 )

2324

2325 # Check that processing runs with other way of turning checks off.

2326 isr_config.doCheckUnprocessableData = True

2327 isr_config.bssVoltageMinimum = 0.0

2328

2329 isr_task = IsrTaskLSST(config=isr_config)

2330 with self.assertNoLogs(level=logging.WARNING):

2331 _ = isr_task.run(

2332 input_exp.clone(),

2333 bias=self.bias,

2334 dark=self.dark,

2335 crosstalk=self.crosstalk,

2336 ptc=self.ptc,

2337 linearizer=self.linearizer,

2338 defects=self.defects,

2339 deferredChargeCalib=self.cti,

2340 )

2341

2342 # Check that processing runs but warns if header keyword is None.

2343 isr_config.doCheckUnprocessableData = True

2344 isr_config.bssVoltageMinimum = 10.0

2345

2346 input_exp2 = input_exp.clone()

2347 input_exp2.metadata["HVBIAS"] = None

2348

2349 isr_task = IsrTaskLSST(config=isr_config)

2350 with self.assertLogs(level=logging.WARNING) as cm:

2351 _ = isr_task.run(

2352 input_exp2,

2353 bias=self.bias,

2354 dark=self.dark,

2355 crosstalk=self.crosstalk,

2356 ptc=self.ptc,

2357 linearizer=self.linearizer,

2358 defects=self.defects,

2359 deferredChargeCalib=self.cti,

2360 )

2361 self.assertEqual(len(cm.output), 1)

2362 self.assertIn("HV bias on HVBIAS not found in metadata", cm.output[0])

2363

2364 # Check that it raises.

2365 isr_config.doCheckUnprocessableData = True

2366 isr_config.bssVoltageMinimum = 10.0

2367

2368 isr_task = IsrTaskLSST(config=isr_config)

2369 with self.assertRaises(UnprocessableDataError):

2370 _ = isr_task.run(

2371 input_exp.clone(),

2372 bias=self.bias,

2373 dark=self.dark,

2374 crosstalk=self.crosstalk,

2375 ptc=self.ptc,

2376 linearizer=self.linearizer,

2377 defects=self.defects,

2378 deferredChargeCalib=self.cti,

2379 )

2380

2381 def get_mock_config_no_signal(self):

2382 """Get an IsrMockLSSTConfig with all signal set to False.

2383

2384 This will have all the electronic effects turned on (including

2385 2D bias).

2386 """

2387 mock_config = isrMockLSST.IsrMockLSSTConfig()

2388 mock_config.isTrimmed = False

2389 mock_config.doAddDark = False

2390 mock_config.doAddFlat = False

2391 mock_config.doAddFringe = False

2392 mock_config.doAddSky = False

2393 mock_config.doAddSource = False

2394

2395 mock_config.doAdd2DBias = True

2396 mock_config.doAddBias = True

2397 mock_config.doAddCrosstalk = True

2398 mock_config.doAddDeferredCharge = True

2399 mock_config.doAddBrightDefects = True

2400 mock_config.doAddClockInjectedOffset = True

2401 mock_config.doAddParallelOverscanRamp = True

2402 mock_config.doAddSerialOverscanRamp = True

2403 mock_config.doAddHighSignalNonlinearity = True

2404 mock_config.doApplyGain = True

2405 mock_config.doRoundAdu = True

2406

2407 # We always want to generate the image with these configs.

2408 mock_config.doGenerateImage = True

2409

2410 return mock_config

2411

2412 def get_mock_config_clean(self):

2413 """Get an IsrMockLSSTConfig trimmed with all electronic signatures

2414 turned off.

2415 """

2416 mock_config = isrMockLSST.IsrMockLSSTConfig()

2417 mock_config.doAddBias = False

2418 mock_config.doAdd2DBias = False

2419 mock_config.doAddClockInjectedOffset = False

2420 mock_config.doAddDark = False

2421 mock_config.doAddDarkNoiseOnly = False

2422 mock_config.doAddFlat = False

2423 mock_config.doAddFringe = False

2424 mock_config.doAddSky = False

2425 mock_config.doAddSource = False

2426 mock_config.doRoundAdu = False

2427 mock_config.doAddHighSignalNonlinearity = False

2428 mock_config.doApplyGain = False

2429 mock_config.doAddCrosstalk = False

2430 mock_config.doAddBrightDefects = False

2431 mock_config.doAddParallelOverscanRamp = False

2432 mock_config.doAddSerialOverscanRamp = False

2433

2434 mock_config.isTrimmed = True

2435 mock_config.doGenerateImage = True

2436

2437 return mock_config

2438

2439 def get_isr_config_minimal_corrections(self):

2440 """Get an IsrTaskLSSTConfig with minimal corrections.

2441 """

2442 isr_config = IsrTaskLSSTConfig()

2443 isr_config.bssVoltageMinimum = 0.0

2444 isr_config.ampNoiseThreshold = np.inf

2445 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2446 isr_config.doBias = False

2447 isr_config.doDark = False

2448 isr_config.doDeferredCharge = False

2449 isr_config.doLinearize = False

2450 isr_config.doCorrectGains = False

2451 isr_config.doCrosstalk = False

2452 isr_config.doDefect = False

2453 isr_config.doBrighterFatter = False

2454 isr_config.doFlat = False

2455 isr_config.doSaturation = False

2456 isr_config.doE2VEdgeBleedMask = False

2457 isr_config.doITLEdgeBleedMask = False

2458 isr_config.doSuspect = False

2459 # We override the leading/trailing to skip here because of the limited

2460 # size of the test camera overscan regions.

2461 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2462 defaultAmpConfig.doSerialOverscan = True

2463 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2464 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2465 defaultAmpConfig.doParallelOverscan = True

2466 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2467 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2468 # Our strong overscan slope in the tests requires an override.

2469 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2470

2471 isr_config.doAssembleCcd = True

2472 isr_config.crosstalk.doSubtrahendMasking = True

2473 isr_config.crosstalk.minPixelToMask = 1.0

2474

2475 return isr_config

2476

2477 def get_isr_config_electronic_corrections(self):

2478 """Get an IsrTaskLSSTConfig with electronic corrections.

2479

2480 This tests all the corrections that we support in the mocks/ISR.

2481 """

2482 isr_config = IsrTaskLSSTConfig()

2483 # We add these as appropriate in the tests.

2484 isr_config.doBias = False

2485 isr_config.doDark = False

2486 isr_config.doFlat = False

2487

2488 # These are the electronic effects the tests support (in addition

2489 # to overscan).

2490 isr_config.doCrosstalk = True

2491 isr_config.doDefect = True

2492 isr_config.doLinearize = True

2493 isr_config.doDeferredCharge = True

2494

2495 # This is False because it is only used in a single test case

2496 # as it takes a while to solve

2497 isr_config.doBrighterFatter = False

2498

2499 # These are the electronic effects we do not support in tests yet.

2500 isr_config.doCorrectGains = False

2501

2502 # We override the leading/trailing to skip here because of the limited

2503 # size of the test camera overscan regions.

2504 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2505 defaultAmpConfig.doSerialOverscan = True

2506 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2507 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2508 defaultAmpConfig.doParallelOverscan = True

2509 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2510 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2511 # Our strong overscan slope in the tests requires an override.

2512 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2513

2514 isr_config.doAssembleCcd = True

2515 isr_config.crosstalk.doSubtrahendMasking = True

2516 isr_config.crosstalk.minPixelToMask = 1.0

2517

2518 return isr_config

2519

2520 def get_non_defect_pixels(self, mask_origin):

2521 """Get the non-defect pixels to compare.

2522

2523 Parameters

2524 ----------

2525 mask_origin : `lsst.afw.image.MaskX`

2526 The origin mask (for shape and type).

2527

2528 Returns

2529 -------

2530 pix_x, pix_y : `tuple` [`np.ndarray`]

2531 x and y values of good pixels.

2532 """

2533 mask_temp = mask_origin.clone()

2534 mask_temp[:, :] = 0

2535

2536 for defect in self.defects:

2537 mask_temp[defect.getBBox()] = 1

2538

2539 return np.where(mask_temp.array == 0)

2540

2541 def _check_bad_column_crosstalk_correction(

2542 self,

2543 exp,

2544 nsigma_cut=5.0,

2545 ):

2546 """Test bad column crosstalk correction.

2547

2548 This includes possible provblems from parallel overscan

2549 crosstalk and gain mismatches.

2550

2551 The target amp is self.detector[0], "C:0,0".

2552

2553 Parameters

2554 ----------

2555 exp : `lsst.afw.image.Exposure`

2556 Input exposure.

2557 nsigma_cut : `float`, optional

2558 Number of sigma to check for outliers.

2559 """

2560 amp = self.detector[0]

2561 amp_image = exp[amp.getBBox()].image.array

2562 sigma = median_abs_deviation(amp_image.ravel(), scale="normal")

2563

2564 med = np.median(amp_image.ravel())

2565 self.assertLess(amp_image.max(), med + nsigma_cut*sigma)

2566 self.assertGreater(amp_image.min(), med - nsigma_cut*sigma)

2567

2568

2569class MemoryTester(lsst.utils.tests.MemoryTestCase):

2570 pass

2571

2572

2573def setup_module(module):

2574 lsst.utils.tests.init()

2575

2576

2577if __name__ == "__main__": 2577 ↛ 2578line 2577 didn't jump to line 2578 because the condition on line 2577 was never true

2578 lsst.utils.tests.init()

2579 unittest.main()

Coverage for tests / test_isrTaskLSST.py: 4%

1289 statements