Coverage for tests/test_isrTaskLSST.py: 4%

2# LSST Data Management System

5# This product includes software developed by the

6# LSST Project (http://www.lsst.org/).

8# This program is free software: you can redistribute it and/or modify

9# it under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# This program is distributed in the hope that it will be useful,

14# but WITHOUT ANY WARRANTY; without even the implied warranty of

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

16# GNU General Public License for more details.

17#

18# You should have received a copy of the LSST License Statement and

19# the GNU General Public License along with this program. If not,

20# see <https://www.lsstcorp.org/LegalNotices/>.

21#

23import copy

24import unittest

25import numpy as np

26import logging

27import galsim

28from scipy.stats import median_abs_deviation

30import lsst.afw.geom as afwGeom

31import lsst.geom as geom

32from lsst.pipe.base import UnprocessableDataError

33import lsst.ip.isr.isrMockLSST as isrMockLSST

34import lsst.utils.tests

35from lsst.ip.isr.isrTaskLSST import (IsrTaskLSST, IsrTaskLSSTConfig)

36from lsst.ip.isr.crosstalk import CrosstalkCalib

37from lsst.ip.isr import PhotonTransferCurveDataset

38from lsst.ip.isr.vignette import maskVignettedRegion

39from lsst.ip.isr.gainCorrection import GainCorrection

42class IsrTaskLSSTTestCase(lsst.utils.tests.TestCase):

43 """Test IsrTaskLSST"""

44 def setUp(self):

45 mock = isrMockLSST.IsrMockLSST()

46 self.camera = mock.getCamera()

47 self.detector = self.camera[mock.config.detectorIndex]

48 self.namp = len(self.detector)

50 # Create adu (bootstrap) calibration frames

51 self.bias_adu = isrMockLSST.BiasMockLSST(adu=True).run()

52 self.dark_adu = isrMockLSST.DarkMockLSST(adu=True).run()

53 self.flat_adu = isrMockLSST.FlatMockLSST(adu=True).run()

54 self.flat_adu.metadata["FLATSRC"] = "DOME"

56 # Create calibration frames

57 self.bias = isrMockLSST.BiasMockLSST().run()

58 self.dark = isrMockLSST.DarkMockLSST().run()

59 self.flat = isrMockLSST.FlatMockLSST().run()

60 self.flat.metadata["FLATSRC"] = "DOME"

61 self.bf_kernel = isrMockLSST.BfKernelMockLSST().run()

62 self.electroBfDistortionMatrix = isrMockLSST.ElectrostaticBfMockLSST().run()

63 self.cti = isrMockLSST.DeferredChargeMockLSST().run()

65 # The crosstalk ratios in isrMockLSST are in electrons.

66 self.crosstalk = CrosstalkCalib(nAmp=self.namp)

67 self.crosstalk.hasCrosstalk = True

68 self.crosstalk.coeffs = isrMockLSST.CrosstalkCoeffMockLSST().run()

69 for i, amp in enumerate(self.detector):

70 self.crosstalk.fitGains[i] = mock.config.gainDict[amp.getName()]

71 self.crosstalk.crosstalkRatiosUnits = "electron"

73 self.defects = isrMockLSST.DefectMockLSST().run()

75 amp_names = [x.getName() for x in self.detector.getAmplifiers()]

76 self.ptc = PhotonTransferCurveDataset(amp_names,

77 ptcFitType='DUMMY_PTC',

78 covMatrixSide=1)

80 self.saturation_adu = 100_000.0

82 # PTC records noise units in electron, same as the

83 # configuration parameter.

84 for amp_name in amp_names:

85 self.ptc.gain[amp_name] = mock.config.gainDict.get(amp_name, mock.config.gain)

86 self.ptc.noise[amp_name] = mock.config.readNoise

87 self.ptc.ptcTurnoff[amp_name] = self.saturation_adu

88 pre_level = mock.config.clockInjectedOffsetLevel

89 overscan_level = mock.config.biasLevel + (pre_level / self.ptc.gain[amp_name])

90 self.ptc.overscanMedian[amp_name] = overscan_level

91 # We set this sigma level very large because the

92 # noise characteristics of the mock image with

93 # a large serial gradient and small overscan region

94 # amplifies the noise vs LSSTCam.

95 self.ptc.overscanMedianSigma[amp_name] = 10.0

97 # TODO:

98 # self.cti = isrMockLSST.DeferredChargeMockLSST().run()

100 self.linearizer = isrMockLSST.LinearizerMockLSST().run()

101 # We currently only have high-signal non-linearity.

102 mock_config = self.get_mock_config_no_signal()

103 for amp_name in amp_names:

104 coeffs = self.linearizer.linearityCoeffs[amp_name]

105 centers, values = np.split(coeffs, 2)

106 values[centers < mock_config.highSignalNonlinearityThreshold] = 0.0

107 self.linearizer.linearityCoeffs[amp_name] = np.concatenate((centers, values))

108

109 def _check_applied_keys(self, metadata, isr_config, expected_gain_correction=False):

110 """Check if the APPLIED keys have been set properly.

111

112 Parameters

113 ----------

114 metadata : `lsst.daf.base.PropertyList`

115 isr_config : `lsst.ip.isr.IsrTaskLSSTConfig`

116 expected_gain_correction : `bool`, optional

117 Did we expect gain correction to be applied?

118 """

119 key = "LSST ISR GAINCORRECTION APPLIED"

120 self.assertIn(key, metadata)

121 self.assertEqual(metadata[key], expected_gain_correction)

122

123 key = "LSST ISR CROSSTALK APPLIED"

124 self.assertIn(key, metadata)

125 self.assertEqual(metadata[key], isr_config.doCrosstalk)

126

127 key = "LSST ISR OVERSCANLEVEL CHECKED"

128 self.assertIn(key, metadata)

129 self.assertEqual(metadata[key], np.isfinite(isr_config.serialOverscanMedianShiftSigmaThreshold))

130

131 key = "LSST ISR NOISE CHECKED"

132 self.assertIn(key, metadata)

133 self.assertEqual(metadata[key], np.isfinite(isr_config.ampNoiseThreshold))

134

135 key = "LSST ISR LINEARIZER APPLIED"

136 self.assertIn(key, metadata)

137 self.assertEqual(metadata[key], isr_config.doLinearize)

138

139 key = "LSST ISR CTI APPLIED"

140 self.assertIn(key, metadata)

141 self.assertEqual(metadata[key], isr_config.doDeferredCharge)

142

143 key = "LSST ISR BIAS APPLIED"

144 self.assertIn(key, metadata)

145 self.assertEqual(metadata[key], isr_config.doBias)

146

147 key = "LSST ISR DARK APPLIED"

148 self.assertIn(key, metadata)

149 self.assertEqual(metadata[key], isr_config.doDark)

150

151 key = "LSST ISR BF APPLIED"

152 self.assertIn(key, metadata)

153 self.assertEqual(metadata[key], isr_config.doBrighterFatter)

154

155 if isr_config.doBrighterFatter:

156 key = "LSST ISR BF CORR METHOD"

157 self.assertIn(key, metadata)

158 self.assertEqual(

159 metadata[key],

160 isr_config.brighterFatterCorrectionMethod,

161 )

162

163 key = "LSST ISR FLAT APPLIED"

164 self.assertIn(key, metadata)

165 self.assertEqual(metadata[key], isr_config.doFlat)

166

167 if metadata[key]:

168 key2 = "LSST ISR FLAT SOURCE"

169 self.assertIn(key2, metadata)

170 self.assertEqual(metadata[key2], "DOME")

171

172 key = "LSST ISR DEFECTS APPLIED"

173 self.assertIn(key, metadata)

174 self.assertEqual(metadata[key], isr_config.doDefect)

175

176 def test_isrBootstrapBias(self):

177 """Test processing of a ``bootstrap`` bias frame.

178

179 This will be output with ADU units.

180 """

181 mock_config = self.get_mock_config_no_signal()

182

183 mock = isrMockLSST.IsrMockLSST(config=mock_config)

184 input_exp = mock.run()

185

186 isr_config = self.get_isr_config_minimal_corrections()

187 isr_config.doBootstrap = True

188 isr_config.doApplyGains = False

189 isr_config.doBias = True

190 isr_config.doCrosstalk = True

191

192 # Need to make sure we are not masking the negative variance

193 # pixels when directly comparing calibration images and

194 # calibration-corrected calibrations.

195 isr_config.maskNegativeVariance = False

196

197 isr_task = IsrTaskLSST(config=isr_config)

198 with self.assertLogs(level=logging.WARNING) as cm:

199 result = isr_task.run(

200 input_exp.clone(),

201 bias=self.bias_adu,

202 ptc=self.ptc,

203 crosstalk=self.crosstalk,

204 )

205 self.assertIn("Ignoring provided PTC", cm.output[0])

206 self._check_applied_keys(result.exposure.metadata, isr_config)

207

208 # Rerun without doing the bias correction.

209 isr_config.doBias = False

210 isr_task2 = IsrTaskLSST(config=isr_config)

211 with self.assertNoLogs(level=logging.WARNING):

212 result2 = isr_task2.run(input_exp.clone(), crosstalk=self.crosstalk)

213

214 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

215

216 self.assertLess(

217 np.mean(result.exposure.image.array[good_pixels]),

218 np.mean(result2.exposure.image.array[good_pixels]),

219 )

220 self.assertLess(

221 np.std(result.exposure.image.array[good_pixels]),

222 np.std(result2.exposure.image.array[good_pixels]),

223 )

224

225 delta = result2.exposure.image.array - result.exposure.image.array

226 self.assertFloatsAlmostEqual(delta[good_pixels], self.bias_adu.image.array[good_pixels], atol=1e-5)

227

228 metadata = result.exposure.metadata

229

230 key = "LSST ISR BOOTSTRAP"

231 self.assertIn(key, metadata)

232 self.assertEqual(metadata[key], True)

233

234 key = "LSST ISR UNITS"

235 self.assertIn(key, metadata)

236 self.assertEqual(metadata[key], "adu")

237

238 key = "LSST ISR READNOISE UNITS"

239 self.assertIn(key, metadata)

240 self.assertEqual(metadata[key], "electron")

241

242 for amp in self.detector:

243 amp_name = amp.getName()

244 key = f"LSST ISR GAIN {amp_name}"

245 self.assertIn(key, metadata)

246 self.assertEqual(metadata[key], 1.0)

247 key = f"LSST ISR PTCTURNOFF {amp_name}"

248 self.assertIn(key, metadata)

249 self.assertEqual(metadata[key], np.inf)

250

251 self._check_bad_column_crosstalk_correction(result.exposure)

252

253 def test_isrBootstrapDark(self):

254 """Test processing of a ``bootstrap`` dark frame.

255

256 This will be output with ADU units.

257 """

258 mock_config = self.get_mock_config_no_signal()

259 mock_config.doAddDark = True

260

261 mock = isrMockLSST.IsrMockLSST(config=mock_config)

262 input_exp = mock.run()

263

264 isr_config = self.get_isr_config_minimal_corrections()

265 isr_config.doBootstrap = True

266 isr_config.doApplyGains = False

267 isr_config.doBias = True

268 isr_config.doDark = True

269 isr_config.maskNegativeVariance = False

270 isr_config.doCrosstalk = True

271

272 isr_task = IsrTaskLSST(config=isr_config)

273 with self.assertLogs(level=logging.WARNING) as cm:

274 result = isr_task.run(

275 input_exp.clone(),

276 bias=self.bias_adu,

277 dark=self.dark_adu,

278 ptc=self.ptc,

279 crosstalk=self.crosstalk,

280 )

281 self.assertIn("Ignoring provided PTC", cm.output[0])

282 self._check_applied_keys(result.exposure.metadata, isr_config)

283

284 # Rerun without doing the dark correction.

285 isr_config.doDark = False

286 isr_task2 = IsrTaskLSST(config=isr_config)

287 with self.assertNoLogs(level=logging.WARNING):

288 result2 = isr_task2.run(input_exp.clone(), bias=self.bias_adu, crosstalk=self.crosstalk)

289

290 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

291

292 self.assertLess(

293 np.mean(result.exposure.image.array[good_pixels]),

294 np.mean(result2.exposure.image.array[good_pixels]),

295 )

296

297 delta = result2.exposure.image.array - result.exposure.image.array

298 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

299 self.assertFloatsAlmostEqual(

300 delta[good_pixels],

301 self.dark_adu.image.array[good_pixels] * exp_time,

302 atol=1e-5,

303 )

304

305 metadata = result.exposure.metadata

306

307 key = "LSST ISR BOOTSTRAP"

308 self.assertIn(key, metadata)

309 self.assertEqual(metadata[key], True)

310

311 key = "LSST ISR UNITS"

312 self.assertIn(key, metadata)

313 self.assertEqual(metadata[key], "adu")

314

315 self._check_bad_column_crosstalk_correction(result.exposure)

316

317 def test_isrBootstrapFlat(self):

318 """Test processing of a ``bootstrap`` flat frame.

319

320 This will be output with ADU units.

321 """

322 mock_config = self.get_mock_config_no_signal()

323 mock_config.doAddDark = True

324 mock_config.doAddFlat = True

325 # The doAddSky option adds the equivalent of flat-field flux.

326 mock_config.doAddSky = True

327

328 mock = isrMockLSST.IsrMockLSST(config=mock_config)

329 input_exp = mock.run()

330

331 isr_config = self.get_isr_config_minimal_corrections()

332 isr_config.doBootstrap = True

333 isr_config.doApplyGains = False

334 isr_config.doBias = True

335 isr_config.doDark = True

336 isr_config.doFlat = True

337 isr_config.maskNegativeVariance = False

338 isr_config.doCrosstalk = True

339

340 isr_task = IsrTaskLSST(config=isr_config)

341 with self.assertLogs(level=logging.WARNING) as cm:

342 result = isr_task.run(

343 input_exp.clone(),

344 bias=self.bias_adu,

345 dark=self.dark_adu,

346 flat=self.flat_adu,

347 ptc=self.ptc,

348 crosstalk=self.crosstalk,

349 )

350 self.assertIn("Ignoring provided PTC", cm.output[0])

351 self._check_applied_keys(result.exposure.metadata, isr_config)

352

353 # Rerun without doing the flat correction.

354 isr_config.doFlat = False

355 isr_task2 = IsrTaskLSST(config=isr_config)

356 with self.assertNoLogs(level=logging.WARNING):

357 result2 = isr_task2.run(

358 input_exp.clone(),

359 bias=self.bias_adu,

360 dark=self.dark_adu,

361 crosstalk=self.crosstalk,

362 )

363

364 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

365

366 # Applying the flat will increase the counts.

367 self.assertGreater(

368 np.mean(result.exposure.image.array[good_pixels]),

369 np.mean(result2.exposure.image.array[good_pixels]),

370 )

371 # And will decrease the sigma.

372 self.assertLess(

373 np.std(result.exposure.image.array[good_pixels]),

374 np.std(result2.exposure.image.array[good_pixels]),

375 )

376

377 ratio = result2.exposure.image.array / result.exposure.image.array

378 self.assertFloatsAlmostEqual(ratio[good_pixels], self.flat_adu.image.array[good_pixels], atol=1e-5)

379

380 # Test the variance plane in the case of adu units.

381 # The expected variance starts with the image array.

382 expected_variance = result.exposure.image.clone()

383 # We have to remove the flat-fielding from the image pixels.

384 expected_variance.array *= self.flat_adu.image.array

385 # And add in the bias variance.

386 expected_variance.array += self.bias_adu.variance.array

387 # And add in the scaled dark variance.

388 scale = result.exposure.visitInfo.darkTime / self.dark_adu.visitInfo.darkTime

389 expected_variance.array += scale**2. * self.dark_adu.variance.array

390 # And add the gain and read noise (in electron) per amp.

391 for amp in self.detector:

392 # We need to use the gain and read noise from the header

393 # because these are bootstraps.

394 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

395 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

396

397 expected_variance[amp.getBBox()].array /= gain

398 # Read noise is always in electron units, but since this is a

399 # bootstrap, the gain is 1.0.

400 expected_variance[amp.getBBox()].array += (read_noise/gain)**2.

401

402 # And apply the full formula for dividing by the flat with variance.

403 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

404 unflat_image_array = result.exposure.image.array * self.flat_adu.image.array

405 expected_variance.array = ((unflat_image_array**2. * self.flat_adu.variance.array

406 + self.flat_adu.image.array**2. * expected_variance.array)

407 / self.flat_adu.image.array**4.)

408

409 self.assertFloatsAlmostEqual(

410 result.exposure.variance.array[good_pixels],

411 expected_variance.array[good_pixels],

412 rtol=1e-6,

413 )

414

415 metadata = result.exposure.metadata

416

417 key = "LSST ISR BOOTSTRAP"

418 self.assertIn(key, metadata)

419 self.assertEqual(metadata[key], True)

420

421 key = "LSST ISR UNITS"

422 self.assertIn(key, metadata)

423 self.assertEqual(metadata[key], "adu")

424

425 self._check_bad_column_crosstalk_correction(result.exposure)

426

427 def test_isrBootstrapAndRegularFlat(self):

428 """Test that bootstrap and "regular" flat processing are equivalent."""

429 # This is a test for DM-52684, for the linearizer units.

430

431 mock_config = self.get_mock_config_no_signal()

432 mock_config.doAddDark = True

433 mock_config.doAddFlat = True

434 # The doAddSky option adds the equivalent of flat-field flux.

435 mock_config.doAddSky = True

436 # We set the sky/flat level to a range where the "high signal"

437 # non-linearity has kicked in.

438 mock_config.skyLevel = 40000.0

439

440 mock = isrMockLSST.IsrMockLSST(config=mock_config)

441 input_exp = mock.run()

442

443 # First config is with linearizer in bootstrap mode.

444

445 isr_config_bootstrap = self.get_isr_config_minimal_corrections()

446 isr_config_bootstrap.doBootstrap = True

447 isr_config_bootstrap.doApplyGains = False

448 isr_config_bootstrap.doLinearize = True

449 isr_config_bootstrap.doBias = False

450 isr_config_bootstrap.doDark = False

451 isr_config_bootstrap.doFlat = False

452 isr_config_bootstrap.maskNegativeVariance = False

453 isr_config_bootstrap.doCrosstalk = True

454

455 isr_task_bootstrap = IsrTaskLSST(config=isr_config_bootstrap)

456 with self.assertNoLogs(level=logging.WARNING):

457 result = isr_task_bootstrap.run(

458 input_exp.clone(),

459 crosstalk=self.crosstalk,

460 linearizer=self.linearizer,

461 )

462

463 exp_bootstrap = result.exposure

464

465 # Apply the gains (after the linearization); this is

466 # similar to processing in PTC building.

467 for amp in self.detector:

468 exp_bootstrap[amp.getBBox()].image.array *= self.ptc.gain[amp.getName()]

469

470 # Run again with non-bootstrap mode.

471 isr_config = self.get_isr_config_minimal_corrections()

472 isr_config.doBootstrap = False

473 isr_config.doApplyGains = True

474 isr_config.doLinearize = True

475 isr_config.doBias = False

476 isr_config.doDark = False

477 isr_config.doFlat = False

478 isr_config.maskNegativeVariance = False

479 isr_config.doCrosstalk = True

480

481 isr_task = IsrTaskLSST(config=isr_config)

482 with self.assertNoLogs(level=logging.WARNING):

483 result = isr_task.run(

484 input_exp.clone(),

485 crosstalk=self.crosstalk,

486 linearizer=self.linearizer,

487 ptc=self.ptc,

488 )

489

490 exp = result.exposure

491

492 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

493

494 delta = exp.image.array - exp_bootstrap.image.array

495

496 self.assertFloatsAlmostEqual(delta[good_pixels], 0.0, atol=1e-2)

497

498 def test_isrBias(self):

499 """Test processing of a bias frame."""

500 mock_config = self.get_mock_config_no_signal()

501

502 mock = isrMockLSST.IsrMockLSST(config=mock_config)

503 input_exp = mock.run()

504

505 isr_config = self.get_isr_config_electronic_corrections()

506 isr_config.doBias = True

507 # We do not do defect correction when processing biases.

508 isr_config.doDefect = False

509 isr_config.maskNegativeVariance = False

510

511 isr_task = IsrTaskLSST(config=isr_config)

512 with self.assertNoLogs(level=logging.WARNING):

513 result = isr_task.run(

514 input_exp.clone(),

515 bias=self.bias,

516 crosstalk=self.crosstalk,

517 ptc=self.ptc,

518 linearizer=self.linearizer,

519 deferredChargeCalib=self.cti,

520 )

521 self._check_applied_keys(result.exposure.metadata, isr_config)

522

523 # Rerun without doing the bias correction.

524 isr_config.doBias = False

525 isr_task2 = IsrTaskLSST(config=isr_config)

526 with self.assertNoLogs(level=logging.WARNING):

527 result2 = isr_task2.run(

528 input_exp.clone(),

529 crosstalk=self.crosstalk,

530 ptc=self.ptc,

531 linearizer=self.linearizer,

532 deferredChargeCalib=self.cti,

533 )

534

535 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

536

537 self.assertLess(

538 np.mean(result.exposure.image.array[good_pixels]),

539 np.mean(result2.exposure.image.array[good_pixels]),

540 )

541

542 self.assertLess(

543 np.std(result.exposure.image.array[good_pixels]),

544 np.std(result2.exposure.image.array[good_pixels]),

545 )

546

547 # Confirm that it is flat with an arbitrary cutoff that depends

548 # on the read noise.

549 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

550

551 delta = result2.exposure.image.array - result.exposure.image.array

552

553 # Note that the bias is made with bias noise + read noise, and

554 # the image contains read noise.

555 self.assertFloatsAlmostEqual(

556 delta[good_pixels],

557 self.bias.image.array[good_pixels],

558 atol=1e-5,

559 )

560

561 self._check_bad_column_crosstalk_correction(result.exposure)

562

563 def test_isrBiasNoParallelOscanCorrection(self):

564 """Test processing of a bias frame with parallel

565 overscan correction turned off."""

566 mock_config = self.get_mock_config_no_signal()

567 mock_config.doAddParallelOverscanRamp = False

568

569 mock = isrMockLSST.IsrMockLSST(config=mock_config)

570 input_exp = mock.run()

571

572 isr_config = self.get_isr_config_electronic_corrections()

573

574 # Turn off the parallel overscan correction

575 amp_oscan_config = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

576 amp_oscan_config.doParallelOverscan = False

577 isr_config.doBias = True

578

579 # We do not do defect correction when processing biases.

580 isr_config.doDefect = False

581 isr_config.maskNegativeVariance = False

582

583 isr_task = IsrTaskLSST(config=isr_config)

584 with self.assertNoLogs(level=logging.WARNING):

585 result = isr_task.run(

586 input_exp.clone(),

587 bias=self.bias,

588 crosstalk=self.crosstalk,

589 ptc=self.ptc,

590 linearizer=self.linearizer,

591 deferredChargeCalib=self.cti,

592 )

593 self._check_applied_keys(result.exposure.metadata, isr_config)

594

595 # Rerun without doing the bias correction.

596 isr_config.doBias = False

597 isr_task2 = IsrTaskLSST(config=isr_config)

598 with self.assertNoLogs(level=logging.WARNING):

599 result2 = isr_task2.run(

600 input_exp.clone(),

601 crosstalk=self.crosstalk,

602 ptc=self.ptc,

603 linearizer=self.linearizer,

604 deferredChargeCalib=self.cti,

605 )

606

607 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

608

609 self.assertLess(

610 np.mean(result.exposure.image.array[good_pixels]),

611 np.mean(result2.exposure.image.array[good_pixels]),

612 )

613

614 self.assertLess(

615 np.std(result.exposure.image.array[good_pixels]),

616 np.std(result2.exposure.image.array[good_pixels]),

617 )

618

619 # Confirm that it is flat with an arbitrary cutoff that depends

620 # on the read noise.

621 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

622

623 delta = result2.exposure.image.array - result.exposure.image.array

624

625 # Note that the bias is made with bias noise + read noise, and

626 # the image contains read noise.

627 self.assertFloatsAlmostEqual(

628 delta[good_pixels],

629 self.bias.image.array[good_pixels],

630 atol=1e-5,

631 )

632

633 self._check_bad_column_crosstalk_correction(result.exposure)

634

635 def test_isrBiasCti(self):

636 """Test over-correction of bias amp edges from prescan."""

637 mock_config = self.get_mock_config_no_signal()

638 mock_config.doAddBrightDefects = False

639

640 mock = isrMockLSST.IsrMockLSST(config=mock_config)

641 input_exp = mock.run()

642

643 isr_config = self.get_isr_config_electronic_corrections()

644 # We do not do defect correction when processing biases.

645 isr_config.doDefect = False

646 isr_config.maskNegativeVariance = False

647

648 isr_task = IsrTaskLSST(config=isr_config)

649 with self.assertNoLogs(level=logging.WARNING):

650 result = isr_task.run(

651 input_exp.clone(),

652 crosstalk=self.crosstalk,

653 ptc=self.ptc,

654 linearizer=self.linearizer,

655 deferredChargeCalib=self.cti,

656 )

657 self._check_applied_keys(result.exposure.metadata, isr_config)

658

659 # Rerun without doing the CTI correction

660 isr_config.doDeferredCharge = False

661

662 isr_task2 = IsrTaskLSST(config=isr_config)

663 with self.assertNoLogs(level=logging.WARNING):

664 result2 = isr_task2.run(

665 input_exp.clone(),

666 crosstalk=self.crosstalk,

667 ptc=self.ptc,

668 linearizer=self.linearizer,

669 )

670

671 # This confirms that things are *close* to equal. Unfortunately,

672 # the unusual camera geometry in the test camera doesn't completely

673 # zero out the prescan pixels, so we need a higher threshold.

674 std_delta = np.std(result2.exposure.image.array - result.exposure.image.array)

675 self.assertLess(std_delta, 0.15)

676

677 def test_isrDark(self):

678 """Test processing of a dark frame."""

679 mock_config = self.get_mock_config_no_signal()

680 mock_config.doAddDark = True

681 # We turn off the bad parallel overscan column because it does

682 # add more noise to that region.

683 mock_config.doAddBadParallelOverscanColumn = False

684

685 mock = isrMockLSST.IsrMockLSST(config=mock_config)

686 input_exp = mock.run()

687

688 isr_config = self.get_isr_config_electronic_corrections()

689 isr_config.doBias = True

690 isr_config.doDark = True

691 # We do not do defect correction when processing darks.

692 isr_config.doDefect = False

693 isr_config.maskNegativeVariance = False

694

695 isr_task = IsrTaskLSST(config=isr_config)

696 with self.assertNoLogs(level=logging.WARNING):

697 result = isr_task.run(

698 input_exp.clone(),

699 bias=self.bias,

700 dark=self.dark,

701 crosstalk=self.crosstalk,

702 ptc=self.ptc,

703 linearizer=self.linearizer,

704 deferredChargeCalib=self.cti,

705 )

706 self._check_applied_keys(result.exposure.metadata, isr_config)

707

708 # Rerun without doing the dark correction.

709 isr_config.doDark = False

710 isr_task2 = IsrTaskLSST(config=isr_config)

711 with self.assertNoLogs(level=logging.WARNING):

712 result2 = isr_task2.run(

713 input_exp.clone(),

714 bias=self.bias,

715 crosstalk=self.crosstalk,

716 ptc=self.ptc,

717 linearizer=self.linearizer,

718 deferredChargeCalib=self.cti,

719 )

720

721 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

722

723 self.assertLess(

724 np.mean(result.exposure.image.array[good_pixels]),

725 np.mean(result2.exposure.image.array[good_pixels]),

726 )

727 # The mock dark has no noise, so these should be equal.

728 self.assertFloatsAlmostEqual(

729 np.std(result.exposure.image.array[good_pixels]),

730 np.std(result2.exposure.image.array[good_pixels]),

731 atol=1e-12,

732 )

733

734 # This is a somewhat arbitrary comparison that includes a fudge

735 # factor for the extra noise from the overscan subtraction.

736 self.assertLess(

737 np.std(result.exposure.image.array[good_pixels]),

738 1.6*np.sqrt(mock_config.darkRate*mock_config.expTime + mock_config.readNoise),

739 )

740

741 delta = result2.exposure.image.array - result.exposure.image.array

742 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

743

744 # Allow <3 pixels to fail this test due to rounding error

745 # if doRoundAdu=True

746 diff = np.abs(delta[good_pixels] - self.dark.image.array[good_pixels] * exp_time)

747 self.assertLess(np.count_nonzero(diff >= 1e-12), 3)

748

749 self._check_bad_column_crosstalk_correction(result.exposure)

750

751 def test_isrFlat(self):

752 """Test processing of a flat frame."""

753 mock_config = self.get_mock_config_no_signal()

754 mock_config.doAddDark = True

755 mock_config.doAddFlat = True

756 # The doAddSky option adds the equivalent of flat-field flux.

757 mock_config.doAddSky = True

758

759 mock = isrMockLSST.IsrMockLSST(config=mock_config)

760 input_exp = mock.run()

761

762 isr_config = self.get_isr_config_electronic_corrections()

763 isr_config.doBias = True

764 isr_config.doDark = True

765 isr_config.doFlat = True

766 # Although we usually do not do defect interpolation when

767 # processing flats, this is a good test of the interpolation.

768 isr_config.doDefect = True

769 isr_config.maskNegativeVariance = False

770

771 isr_task = IsrTaskLSST(config=isr_config)

772 with self.assertNoLogs(level=logging.WARNING):

773 result = isr_task.run(

774 input_exp.clone(),

775 bias=self.bias,

776 dark=self.dark,

777 flat=self.flat,

778 crosstalk=self.crosstalk,

779 defects=self.defects,

780 ptc=self.ptc,

781 linearizer=self.linearizer,

782 deferredChargeCalib=self.cti,

783 )

784 self._check_applied_keys(result.exposure.metadata, isr_config)

785

786 # Rerun without doing the bias correction.

787 isr_config.doFlat = False

788 isr_task2 = IsrTaskLSST(config=isr_config)

789 with self.assertNoLogs(level=logging.WARNING):

790 result2 = isr_task2.run(

791 input_exp.clone(),

792 bias=self.bias,

793 dark=self.dark,

794 crosstalk=self.crosstalk,

795 defects=self.defects,

796 ptc=self.ptc,

797 linearizer=self.linearizer,

798 deferredChargeCalib=self.cti,

799 )

800

801 # With defect correction, we should not need to filter out bad

802 # pixels.

803

804 # Applying the flat will increase the counts.

805 self.assertGreater(

806 np.mean(result.exposure.image.array),

807 np.mean(result2.exposure.image.array),

808 )

809 # And will decrease the sigma.

810 self.assertLess(

811 np.std(result.exposure.image.array),

812 np.std(result2.exposure.image.array),

813 )

814

815 # Check that the resulting image is approximately flat.

816 # In particular that the noise is consistent with sky + margin.

817 self.assertLess(np.std(result.exposure.image.array), np.sqrt(mock_config.skyLevel) + 3.0)

818

819 # Generate a flat without any defects for comparison

820 # (including interpolation)

821 flat_nodefect_config = isrMockLSST.FlatMockLSST.ConfigClass()

822 flat_nodefect_config.doAddBrightDefects = False

823 flat_nodefects = isrMockLSST.FlatMockLSST(config=flat_nodefect_config).run()

824

825 ratio = result2.exposure.image.array / result.exposure.image.array

826 self.assertFloatsAlmostEqual(ratio, flat_nodefects.image.array, atol=1e-4)

827

828 self._check_bad_column_crosstalk_correction(result.exposure)

829

830 def test_isrNoise(self):

831 """Test the recorded noise and gain in the metadata."""

832 mock_config = self.get_mock_config_no_signal()

833 # Remove the overscan scale so that the only variation

834 # in the overscan is from the read noise.

835 mock_config.overscanScale = 0.0

836

837 mock = isrMockLSST.IsrMockLSST(config=mock_config)

838 input_exp = mock.run()

839

840 isr_config = self.get_isr_config_electronic_corrections()

841 isr_config.doBias = True

842 # We do not do defect correction when processing biases.

843 isr_config.doDefect = False

844 isr_config.maskNegativeVariance = False

845

846 isr_task = IsrTaskLSST(config=isr_config)

847 with self.assertNoLogs(level=logging.WARNING):

848 result = isr_task.run(

849 input_exp.clone(),

850 bias=self.bias,

851 crosstalk=self.crosstalk,

852 ptc=self.ptc,

853 deferredChargeCalib=self.cti,

854 linearizer=self.linearizer,

855 )

856 self._check_applied_keys(result.exposure.metadata, isr_config)

857

858 metadata = result.exposure.metadata

859

860 for amp in self.detector:

861 # The overscan noise is always in adu and the readnoise is always

862 # in electron.

863 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

864 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

865 turnoff = result.exposure.metadata[f"LSST ISR PTCTURNOFF {amp.getName()}"]

866

867 # Check that the gain and read noise are consistent with the

868 # values stored in the PTC.

869 self.assertEqual(gain, self.ptc.gain[amp.getName()])

870 self.assertEqual(read_noise, self.ptc.noise[amp.getName()])

871 self.assertEqual(turnoff, self.ptc.ptcTurnoff[amp.getName()] * self.ptc.gain[amp.getName()])

872

873 key = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {amp.getName()}"

874 self.assertIn(key, metadata)

875

876 # Determine if the residual serial overscan stddev is consistent

877 # with the PTC readnoise within 3xstandard error.

878 serial_overscan_area = amp.getRawHorizontalOverscanBBox().area

879 self.assertFloatsAlmostEqual(

880 metadata[key] * gain,

881 read_noise,

882 atol=3*read_noise / np.sqrt(serial_overscan_area),

883 )

884

885 def test_isrBrighterFatterKernel(self):

886 """Test processing of a flat frame."""

887 # Image with brighter-fatter correction

888 mock_config = self.get_mock_config_no_signal()

889 mock_config.isTrimmed = False

890 mock_config.doAddDark = True

891 mock_config.doAddFlat = True

892 mock_config.doAddSky = True

893 mock_config.doAddSource = True

894 mock_config.sourceFlux = [75000.0]

895 mock_config.doAddBrighterFatter = True

896

897 mock = isrMockLSST.IsrMockLSST(config=mock_config)

898 input_exp = mock.run()

899

900 isr_config = self.get_isr_config_electronic_corrections()

901 isr_config.doBias = True

902 isr_config.doDark = True

903 isr_config.doFlat = True

904 isr_config.doBrighterFatter = True

905

906 isr_task = IsrTaskLSST(config=isr_config)

907 with self.assertNoLogs(level=logging.WARNING):

908 result = isr_task.run(

909 input_exp.clone(),

910 bias=self.bias,

911 dark=self.dark,

912 flat=self.flat,

913 deferredChargeCalib=self.cti,

914 crosstalk=self.crosstalk,

915 defects=self.defects,

916 ptc=self.ptc,

917 linearizer=self.linearizer,

918 bfKernel=self.bf_kernel,

919 )

920 self._check_applied_keys(result.exposure.metadata, isr_config)

921

922 mock_config = self.get_mock_config_no_signal()

923 mock_config.isTrimmed = False

924 mock_config.doAddDark = True

925 mock_config.doAddFlat = True

926 mock_config.doAddSky = True

927 mock_config.doAddSource = True

928 mock_config.sourceFlux = [75000.0]

929 mock_config.doAddBrighterFatter = False

930

931 mock = isrMockLSST.IsrMockLSST(config=mock_config)

932 input_truth = mock.run()

933

934 isr_config = self.get_isr_config_electronic_corrections()

935 isr_config.doBias = True

936 isr_config.doDark = True

937 isr_config.doFlat = True

938 isr_config.doBrighterFatter = False

939 isr_config.brighterFatterCorrectionMethod = "COULTON18"

940

941 isr_task = IsrTaskLSST(config=isr_config)

942 with self.assertNoLogs(level=logging.WARNING):

943 result_truth = isr_task.run(

944 input_truth.clone(),

945 bias=self.bias,

946 dark=self.dark,

947 flat=self.flat,

948 deferredChargeCalib=self.cti,

949 crosstalk=self.crosstalk,

950 defects=self.defects,

951 ptc=self.ptc,

952 linearizer=self.linearizer,

953 bfKernel=self.bf_kernel,

954 )

955

956 # Measure the source size in the BF-corrected image.

957 # The injected source is a Gaussian with 3.0px

958 image = galsim.ImageF(result.exposure.image.array)

959 image_truth = galsim.ImageF(result_truth.exposure.image.array)

960 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

961 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

962 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

963 strict=False)

964 measured_sigma = hsm_result.moments_sigma

965 true_sigma = hsm_result_truth.moments_sigma

966 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

967

968 # Check that the variance in an amp far away from the

969 # source is expected. The source is in amp 0; this will

970 # check the variation in neighboring amp 1

971 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

972 n_pixels = test_amp_bbox.getArea()

973 stdev = np.std(result.exposure[test_amp_bbox].image.array)

974 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

975 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

976

977 # Check that the variance in the amp with a defect is

978 # unchanged as a result of applying the BF correction after

979 # interpolating. The defect was added to amplifier 2.

980 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

981 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

982 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

983 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

984 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

985

986 # Check that BF has converged in the expected number of iterations.

987 metadata = result.exposure.metadata

988 key = "LSST ISR BF ITERS"

989 self.assertIn(key, metadata)

990 self.assertEqual(metadata[key], 2)

991

992 def test_isrElectrostaticBrighterFatter(self):

993 """Test processing of a flat frame."""

994 # Image with brighter-fatter correction

995 mock_config = self.get_mock_config_no_signal()

996 mock_config.isTrimmed = False

997 mock_config.doAddDark = True

998 mock_config.doAddFlat = True

999 mock_config.doAddSky = True

1000 mock_config.doAddSource = True

1001 mock_config.sourceFlux = [75000.0]

1002 mock_config.doAddBrighterFatter = True

1003

1004 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1005 input_exp = mock.run()

1006

1007 isr_config = self.get_isr_config_electronic_corrections()

1008 isr_config.doBias = True

1009 isr_config.doDark = True

1010 isr_config.doFlat = True

1011 isr_config.doBrighterFatter = True

1012 isr_config.brighterFatterCorrectionMethod = "ASTIER23"

1013

1014 isr_task = IsrTaskLSST(config=isr_config)

1015 with self.assertNoLogs(level=logging.WARNING):

1016 result = isr_task.run(

1017 input_exp.clone(),

1018 bias=self.bias,

1019 dark=self.dark,

1020 flat=self.flat,

1021 deferredChargeCalib=self.cti,

1022 crosstalk=self.crosstalk,

1023 defects=self.defects,

1024 ptc=self.ptc,

1025 linearizer=self.linearizer,

1026 electroBfDistortionMatrix=self.electroBfDistortionMatrix,

1027 )

1028 self._check_applied_keys(result.exposure.metadata, isr_config)

1029

1030 mock_config = self.get_mock_config_no_signal()

1031 mock_config.isTrimmed = False

1032 mock_config.doAddDark = True

1033 mock_config.doAddFlat = True

1034 mock_config.doAddSky = True

1035 mock_config.doAddSource = True

1036 mock_config.sourceFlux = [75000.0]

1037 mock_config.doAddBrighterFatter = False

1038

1039 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1040 input_truth = mock.run()

1041

1042 isr_config = self.get_isr_config_electronic_corrections()

1043 isr_config.doBias = True

1044 isr_config.doDark = True

1045 isr_config.doFlat = True

1046 isr_config.doBrighterFatter = False

1047

1048 isr_task = IsrTaskLSST(config=isr_config)

1049 with self.assertNoLogs(level=logging.WARNING):

1050 result_truth = isr_task.run(

1051 input_truth.clone(),

1052 bias=self.bias,

1053 dark=self.dark,

1054 flat=self.flat,

1055 deferredChargeCalib=self.cti,

1056 crosstalk=self.crosstalk,

1057 defects=self.defects,

1058 ptc=self.ptc,

1059 linearizer=self.linearizer,

1060 electroBfDistortionMatrix=self.electroBfDistortionMatrix,

1061 )

1062

1063 # Measure the source size in the BF-corrected image.

1064 # The injected source is a Gaussian with 3.0px

1065 image = galsim.ImageF(result.exposure.image.array)

1066 image_truth = galsim.ImageF(result_truth.exposure.image.array)

1067 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

1068 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

1069 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

1070 strict=False)

1071 measured_sigma = hsm_result.moments_sigma

1072 true_sigma = hsm_result_truth.moments_sigma

1073 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

1074

1075 # Check that the variance in an amp far away from the

1076 # source is expected. The source is in amp 0; this will

1077 # check the variation in neighboring amp 1

1078 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

1079 n_pixels = test_amp_bbox.getArea()

1080 stdev = np.std(result.exposure[test_amp_bbox].image.array)

1081 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

1082 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

1083

1084 # Check that the variance in the amp with a defect is

1085 # unchanged as a result of applying the BF correction after

1086 # interpolating. The defect was added to amplifier 2.

1087 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

1088 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

1089 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

1090 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

1091 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

1092

1093 def test_isrSkyImage(self):

1094 """Test processing of a sky image."""

1095 mock_config = self.get_mock_config_no_signal()

1096 mock_config.doAddDark = True

1097 mock_config.doAddFlat = True

1098 # Set this to False until we have fringe correction.

1099 mock_config.doAddFringe = False

1100 mock_config.doAddSky = True

1101 mock_config.doAddSource = True

1102

1103 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1104 input_exp = mock.run()

1105

1106 isr_config = self.get_isr_config_electronic_corrections()

1107 isr_config.doCorrectGains = True

1108 isr_config.doBias = True

1109 isr_config.doDark = True

1110 isr_config.doFlat = True

1111

1112 ptc = copy.copy(self.ptc)

1113 ptc.gain[ptc.ampNames[0]] *= 0.95

1114

1115 adjustments = np.ones(len(ptc.ampNames))

1116 adjustments[0] /= 0.95

1117 gainCorrection = GainCorrection(ampNames=ptc.ampNames, gainAdjustments=adjustments)

1118

1119 isr_task = IsrTaskLSST(config=isr_config)

1120 with self.assertNoLogs(level=logging.WARNING):

1121 result = isr_task.run(

1122 input_exp.clone(),

1123 bias=self.bias,

1124 dark=self.dark,

1125 flat=self.flat,

1126 crosstalk=self.crosstalk,

1127 defects=self.defects,

1128 ptc=self.ptc,

1129 gainCorrection=gainCorrection,

1130 linearizer=self.linearizer,

1131 deferredChargeCalib=self.cti,

1132 )

1133 self._check_applied_keys(result.exposure.metadata, isr_config, expected_gain_correction=True)

1134

1135 # Confirm that the output has the defect line as bad.

1136 sat_val = 2**result.exposure.mask.getMaskPlane("BAD")

1137 for defect in self.defects:

1138 np.testing.assert_array_equal(

1139 result.exposure.mask[defect.getBBox()].array & sat_val,

1140 sat_val,

1141 )

1142

1143 clean_mock_config = self.get_mock_config_clean()

1144 # We want the dark noise for more direct comparison.

1145 clean_mock_config.doAddDarkNoiseOnly = True

1146 clean_mock_config.doAddSky = True

1147 clean_mock_config.doAddSource = True

1148

1149 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1150 clean_exp = clean_mock.run()

1151

1152 delta = result.exposure.image.array - clean_exp.image.array

1153

1154 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

1155

1156 # We compare the good pixels in the entirety.

1157 self.assertLess(np.std(delta[good_pixels]), 5.0)

1158 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1159

1160 # Make sure the corrected image is overall consistent with the

1161 # straight image.

1162 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1163

1164 # And overall where the interpolation is a bit worse but

1165 # the statistics are still fine.

1166 self.assertLess(np.std(delta), 5.5)

1167

1168 metadata = result.exposure.metadata

1169

1170 key = "LSST ISR BOOTSTRAP"

1171 self.assertIn(key, metadata)

1172 self.assertEqual(metadata[key], False)

1173

1174 key = "LSST ISR UNITS"

1175 self.assertIn(key, metadata)

1176 self.assertEqual(metadata[key], "electron")

1177

1178 for amp in self.detector:

1179 amp_name = amp.getName()

1180 key = f"LSST ISR GAIN {amp_name}"

1181 self.assertIn(key, metadata)

1182 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1183 key = f"LSST ISR READNOISE {amp_name}"

1184 self.assertIn(key, metadata)

1185 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1186 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1187 self.assertIn(key, metadata)

1188 self.assertEqual(metadata[key], self.saturation_adu * gain)

1189 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1190 self.assertIn(key, metadata)

1191 self.assertEqual(metadata[key], self.saturation_adu * gain)

1192 key = f"LSST ISR PTCTURNOFF {amp_name}"

1193 self.assertIn(key, metadata)

1194 self.assertEqual(metadata[key], self.ptc.ptcTurnoff[amp_name] * gain)

1195

1196 # Test the variance plane in the case of electron units.

1197 # The expected variance starts with the image array.

1198 expected_variance = result.exposure.image.clone()

1199 # We have to remove the flat-fielding from the image pixels.

1200 expected_variance.array *= self.flat.image.array

1201 # And add in the bias variance.

1202 expected_variance.array += self.bias.variance.array

1203 # And add in the scaled dark variance.

1204 scale = result.exposure.visitInfo.darkTime / self.dark.visitInfo.darkTime

1205 expected_variance.array += scale**2. * self.dark.variance.array

1206 # And add the read noise (in electrons) per amp.

1207 for amp in self.detector:

1208 gain = self.ptc.gain[amp.getName()]

1209 read_noise = self.ptc.noise[amp.getName()]

1210

1211 # The image, read noise, and variance plane should all have

1212 # units of electrons, electrons, and electrons^2.

1213 expected_variance[amp.getBBox()].array += read_noise**2.

1214

1215 # And apply the full formula for dividing by the flat with variance.

1216 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

1217 unflat_image_array = result.exposure.image.array * self.flat.image.array

1218 expected_variance.array = ((unflat_image_array**2. * self.flat.variance.array

1219 + self.flat.image.array**2. * expected_variance.array)

1220 / self.flat.image.array**4.)

1221

1222 self.assertFloatsAlmostEqual(

1223 result.exposure.variance.array[good_pixels],

1224 expected_variance.array[good_pixels],

1225 rtol=1e-6,

1226 )

1227

1228 def test_isrSkyImageSaturated(self):

1229 """Test processing of a sky image.

1230

1231 This variation uses saturated pixels instead of defects.

1232

1233 This additionally tests the gain config override.

1234 """

1235 mock_config = self.get_mock_config_no_signal()

1236 mock_config.doAddDark = True

1237 mock_config.doAddFlat = True

1238 # Set this to False until we have fringe correction.

1239 mock_config.doAddFringe = False

1240 mock_config.doAddSky = True

1241 mock_config.doAddSource = True

1242 mock_config.brightDefectLevel = 170_000.0 # Above saturation.

1243

1244 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1245 input_exp = mock.run()

1246

1247 isr_config = self.get_isr_config_electronic_corrections()

1248 isr_config.doBias = True

1249 isr_config.doDark = True

1250 isr_config.doFlat = True

1251 # We turn off defect masking to test the saturation code.

1252 # However, the same pixels below should be masked/interpolated.

1253 isr_config.doDefect = False

1254

1255 # Use a config override saturation value, confirm it is picked up.

1256 saturation_level = self.saturation_adu * 1.05

1257

1258 # This code will set the gain of one amp to the same as the ptc

1259 # value, and we will check that it is logged and used but the

1260 # results should be the same.

1261 detectorConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector)

1262 detectorConfig.defaultAmpConfig.saturation = saturation_level

1263 overscanAmpConfig = copy.copy(detectorConfig.defaultAmpConfig)

1264 overscanAmpConfig.gain = self.ptc.gain[self.detector[1].getName()]

1265 detectorConfig.ampRules[self.detector[1].getName()] = overscanAmpConfig

1266

1267 isr_task = IsrTaskLSST(config=isr_config)

1268 with self.assertLogs(level=logging.WARNING) as cm:

1269 result = isr_task.run(

1270 input_exp.clone(),

1271 bias=self.bias,

1272 dark=self.dark,

1273 flat=self.flat,

1274 deferredChargeCalib=self.cti,

1275 crosstalk=self.crosstalk,

1276 defects=self.defects,

1277 ptc=self.ptc,

1278 linearizer=self.linearizer,

1279 )

1280 self.assertIn("Overriding gain", cm.output[0])

1281 self._check_applied_keys(result.exposure.metadata, isr_config)

1282

1283 # Confirm that the output has the defect line as saturated.

1284 sat_val = 2**result.exposure.mask.getMaskPlane("SAT")

1285 for defect in self.defects:

1286 np.testing.assert_array_equal(

1287 result.exposure.mask[defect.getBBox()].array & sat_val,

1288 sat_val,

1289 )

1290

1291 clean_mock_config = self.get_mock_config_clean()

1292 # We want the dark noise for more direct comparison.

1293 clean_mock_config.doAddDarkNoiseOnly = True

1294 clean_mock_config.doAddSky = True

1295 clean_mock_config.doAddSource = True

1296

1297 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1298 clean_exp = clean_mock.run()

1299

1300 delta = result.exposure.image.array - clean_exp.image.array

1301

1302 bad_val = 2**result.exposure.mask.getMaskPlane("BAD")

1303 good_pixels = np.where((result.exposure.mask.array & (sat_val | bad_val)) == 0)

1304

1305 # We compare the good pixels in the entirety.

1306 self.assertLess(np.std(delta[good_pixels]), 5.0)

1307 # This is sensitive to parallel overscan masking.

1308 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1309

1310 # Make sure the corrected image is overall consistent with the

1311 # straight image.

1312 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1313

1314 # And overall where the interpolation is a bit worse but

1315 # the statistics are still fine. Note that this is worse than

1316 # the defect case because of the widening of the saturation

1317 # trail.

1318 self.assertLess(np.std(delta), 7.0)

1319

1320 metadata = result.exposure.metadata

1321

1322 for amp in self.detector:

1323 amp_name = amp.getName()

1324 key = f"LSST ISR GAIN {amp_name}"

1325 self.assertIn(key, metadata)

1326 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1327 key = f"LSST ISR READNOISE {amp_name}"

1328 self.assertIn(key, metadata)

1329 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1330 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1331 self.assertIn(key, metadata)

1332 self.assertEqual(metadata[key], saturation_level * gain)

1333 key = f"LSST ISR PTCTURNOFF {amp_name}"

1334 self.assertIn(key, metadata)

1335 self.assertEqual(metadata[key], self.ptc.ptcTurnoff[amp_name] * gain)

1336

1337 def test_isrFlatVignette(self):

1338 """Test ISR when the flat has a validPolygon and vignetted region."""

1339

1340 # We use a flat frame for this test for convenience.

1341 mock_config = self.get_mock_config_no_signal()

1342 mock_config.doAddDark = True

1343 mock_config.doAddFlat = True

1344 # The doAddSky option adds the equivalent of flat-field flux.

1345 mock_config.doAddSky = True

1346

1347 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1348 input_exp = mock.run()

1349

1350 isr_config = self.get_isr_config_electronic_corrections()

1351 isr_config.doBias = True

1352 isr_config.doDark = True

1353 isr_config.doFlat = True

1354 isr_config.doDefect = True

1355

1356 flat = self.flat.clone()

1357 bbox = geom.Box2D(

1358 corner=geom.Point2D(0, 0),

1359 dimensions=geom.Extent2D(50, 50),

1360 )

1361 polygon = afwGeom.Polygon(bbox)

1362 flat.info.setValidPolygon(polygon)

1363 maskVignettedRegion(flat, polygon, vignetteValue=0.0)

1364

1365 isr_task = IsrTaskLSST(config=isr_config)

1366 with self.assertNoLogs(level=logging.WARNING):

1367 result = isr_task.run(

1368 input_exp.clone(),

1369 bias=self.bias,

1370 dark=self.dark,

1371 flat=flat,

1372 crosstalk=self.crosstalk,

1373 ptc=self.ptc,

1374 linearizer=self.linearizer,

1375 defects=self.defects,

1376 deferredChargeCalib=self.cti,

1377 )

1378

1379 self.assertEqual(result.exposure.info.getValidPolygon(), polygon)

1380

1381 noDataFlat = (flat.mask.array & flat.mask.getPlaneBitMask("NO_DATA")) > 0

1382 noDataExp = (result.exposure.mask.array & result.exposure.mask.getPlaneBitMask("NO_DATA")) > 0

1383 np.testing.assert_array_equal(noDataExp, noDataFlat)

1384 np.testing.assert_array_equal(result.exposure.image.array[noDataExp], 0.0)

1385 np.testing.assert_array_equal(result.exposure.variance.array[noDataExp], 0.0)

1386 self.assertFalse(np.any(~np.isfinite(result.exposure.image.array)))

1387 self.assertFalse(np.any(~np.isfinite(result.exposure.variance.array)))

1388

1389 def test_isrFloodedSaturatedE2V(self):

1390 """Test ISR when the amps are completely saturated.

1391

1392 This version tests what happens when the parallel overscan

1393 region is flooded like E2V detectors, where the saturation

1394 spreads evenly, but at a greater level than the saturation

1395 value.

1396 """

1397 # We are simulating a flat field.

1398 # Note that these aren't very important because we are replacing

1399 # the flux, but we may as well.

1400 mock_config = self.get_mock_config_no_signal()

1401 mock_config.doAddDark = True

1402 mock_config.doAddFlat = True

1403 # The doAddSky option adds the equivalent of flat-field flux.

1404 mock_config.doAddSky = True

1405

1406 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1407 input_exp = mock.run()

1408

1409 isr_config = self.get_isr_config_minimal_corrections()

1410 isr_config.doBootstrap = True

1411 isr_config.doApplyGains = False

1412 isr_config.doBias = True

1413 isr_config.doDark = True

1414 isr_config.doFlat = False

1415 # Tun off saturation masking to simulate a PTC flat.

1416 isr_config.doSaturation = False

1417 isr_config.doE2VEdgeBleedMask = False

1418 isr_config.doITLEdgeBleedMask = False

1419

1420 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1421 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1422

1423 detector = input_exp.getDetector()

1424 for i, amp in enumerate(detector):

1425 # For half of the amps we are testing what happens when the

1426 # parallel overscan region is above the configured saturation

1427 # level; for the other half we are testing the other branch

1428 # when it saturates below this level (which is a priori

1429 # unknown).

1430 if i < len(detector) // 2:

1431 data_level = (parallel_overscan_saturation * 1.05

1432 + mock_config.biasLevel

1433 + mock_config.clockInjectedOffsetLevel)

1434 parallel_overscan_level = (parallel_overscan_saturation * 1.1

1435 + mock_config.biasLevel

1436 + mock_config.clockInjectedOffsetLevel)

1437 else:

1438 data_level = (parallel_overscan_saturation * 0.7

1439 + mock_config.biasLevel

1440 + mock_config.clockInjectedOffsetLevel)

1441 parallel_overscan_level = (parallel_overscan_saturation * 0.75

1442 + mock_config.biasLevel

1443 + mock_config.clockInjectedOffsetLevel)

1444

1445 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1446 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1447

1448 isr_task = IsrTaskLSST(config=isr_config)

1449 with self.assertLogs(level=logging.WARNING) as cm:

1450 result = isr_task.run(

1451 input_exp.clone(),

1452 bias=self.bias_adu,

1453 dark=self.dark_adu,

1454 )

1455 self.assertEqual(len(cm.records), len(detector))

1456

1457 n_all = 0

1458 n_level = 0

1459 for record in cm.records:

1460 if "All overscan pixels masked" in record.message:

1461 n_all += 1

1462 if "The level in the overscan region" in record.message:

1463 n_level += 1

1464

1465 self.assertEqual(n_all, len(detector) // 2)

1466 self.assertEqual(n_level, len(detector) // 2)

1467

1468 # And confirm that the post-ISR levels are high for each amp.

1469 for amp in detector:

1470 med = np.median(result.exposure[amp.getBBox()].image.array)

1471 self.assertGreater(med, parallel_overscan_saturation*0.8)

1472

1473 def test_isrFloodedSaturatedITL(self):

1474 """Test ISR when the amps are completely saturated.

1475

1476 This version tests what happens when the parallel overscan

1477 region is flooded like ITL detectors, where the saturation

1478 is at a lower level than the imaging region, and also

1479 spreads partly into the serial/parallel region.

1480 """

1481 # We are simulating a flat field.

1482 # Note that these aren't very important because we are replacing

1483 # the flux, but we may as well.

1484 mock_config = self.get_mock_config_no_signal()

1485 mock_config.doAddDark = True

1486 mock_config.doAddFlat = True

1487 # The doAddSky option adds the equivalent of flat-field flux.

1488 mock_config.doAddSky = True

1489

1490 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1491 input_exp = mock.run()

1492

1493 isr_config = self.get_isr_config_minimal_corrections()

1494 isr_config.doBootstrap = True

1495 isr_config.doApplyGains = False

1496 isr_config.doBias = True

1497 isr_config.doDark = True

1498 isr_config.doFlat = False

1499 # Tun off saturation masking to simulate a PTC flat.

1500 isr_config.doSaturation = False

1501 isr_config.doE2VEdgeBleedMask = False

1502 isr_config.doITLEdgeBleedMask = False

1503

1504 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1505 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1506

1507 detector = input_exp.getDetector()

1508 for i, amp in enumerate(detector):

1509 # For half of the amps we are testing what happens when the

1510 # parallel overscan region is above the configured saturation

1511 # level; for the other half we are testing the other branch

1512 # when it saturates below this level (which is a priori

1513 # unknown).

1514 if i < len(detector) // 2:

1515 data_level = (parallel_overscan_saturation * 1.1

1516 + mock_config.biasLevel

1517 + mock_config.clockInjectedOffsetLevel)

1518 parallel_overscan_level = (parallel_overscan_saturation * 1.05

1519 + mock_config.biasLevel

1520 + mock_config.clockInjectedOffsetLevel)

1521 else:

1522 data_level = (parallel_overscan_saturation * 0.75

1523 + mock_config.biasLevel

1524 + mock_config.clockInjectedOffsetLevel)

1525 parallel_overscan_level = (parallel_overscan_saturation * 0.7

1526 + mock_config.biasLevel

1527 + mock_config.clockInjectedOffsetLevel)

1528

1529 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1530 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1531 # The serial/parallel region for the test camera looks like this:

1532 serial_overscan_bbox = amp.getRawSerialOverscanBBox()

1533 parallel_overscan_bbox = amp.getRawParallelOverscanBBox()

1534

1535 overscan_corner_bbox = geom.Box2I(

1536 geom.Point2I(

1537 serial_overscan_bbox.getMinX(),

1538 parallel_overscan_bbox.getMinY(),

1539 ),

1540 geom.Extent2I(

1541 serial_overscan_bbox.getWidth(),

1542 parallel_overscan_bbox.getHeight(),

1543 ),

1544 )

1545 input_exp[overscan_corner_bbox].image.array[-2:, :] = parallel_overscan_level

1546

1547 isr_task = IsrTaskLSST(config=isr_config)

1548 with self.assertLogs(level=logging.WARNING) as cm:

1549 result = isr_task.run(

1550 input_exp.clone(),

1551 bias=self.bias_adu,

1552 dark=self.dark_adu,

1553 )

1554 self.assertEqual(len(cm.records), len(detector))

1555

1556 n_all = 0

1557 n_level = 0

1558 for record in cm.records:

1559 if "All overscan pixels masked" in record.message:

1560 n_all += 1

1561 if "The level in the overscan region" in record.message:

1562 n_level += 1

1563

1564 self.assertEqual(n_all, len(detector) // 2)

1565 self.assertEqual(n_level, len(detector) // 2)

1566

1567 # And confirm that the post-ISR levels are high for each amp.

1568 for amp in detector:

1569 med = np.median(result.exposure[amp.getBBox()].image.array)

1570 self.assertGreater(med, parallel_overscan_saturation*0.8)

1571

1572 def test_isrBadParallelOverscanColumnsBootstrap(self):

1573 """Test processing a bias when we have a bad parallel overscan column.

1574

1575 This tests in bootstrap mode.

1576 """

1577 # We base this on the bootstrap bias, and make sure

1578 # that the bad column remains.

1579 mock_config = self.get_mock_config_no_signal()

1580 isr_config = self.get_isr_config_minimal_corrections()

1581 isr_config.doSaturation = False

1582 isr_config.doE2VEdgeBleedMask = False

1583 isr_config.doITLEdgeBleedMask = False

1584 isr_config.doBootstrap = True

1585 isr_config.doApplyGains = False

1586

1587 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1588 overscan_sat_level_adu = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1589 # The defect is in amp 2.

1590 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1591 overscan_sat_level = amp_gain * overscan_sat_level_adu

1592 # The expected defect level is in adu for the bootstrap bias.

1593 expected_defect_level = mock_config.brightDefectLevel / amp_gain

1594

1595 # The levels are set in electron units.

1596 # We test 3 levels:

1597 # * 10.0, a very low outlier, to test median smoothing detection

1598 # code. This value is given by gain*threshold + cushion.

1599 # * 575.0, a lowish but outlier level, given by gain*threshold +

1600 # 100.0 (average of the parallel overscan offset) + 10.0

1601 # (an additional cushion).

1602 # * 1.05*saturation.

1603 # Note that the default parallel overscan saturation level for

1604 # bootstrap (pre-saturation-measure) analysis is very low, in

1605 # order to capture all types of amps, even with low saturation.

1606 # Therefore, we only need to test above this saturation level.

1607 # (c.f. test_isrBadParallelOverscanColumns).

1608 levels = np.array([10.0, 575.0, 1.05*overscan_sat_level])

1609

1610 for level in levels:

1611 mock_config.badParallelOverscanColumnLevel = level

1612 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1613 input_exp = mock.run()

1614

1615 isr_task = IsrTaskLSST(config=isr_config)

1616 with self.assertNoLogs(level=logging.WARNING):

1617 result = isr_task.run(input_exp.clone())

1618

1619 for defect in self.defects:

1620 bbox = defect.getBBox()

1621 defect_image = result.exposure[bbox].image.array

1622

1623 # Check that the defect is the correct level

1624 # (not subtracted away).

1625 defect_median = np.median(defect_image)

1626 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1627

1628 # Check that the neighbors aren't over-subtracted.

1629 for neighbor in [-1, 1]:

1630 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1631 neighbor_image = result.exposure[bbox_neighbor].image.array

1632

1633 neighbor_median = np.median(neighbor_image)

1634 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1635

1636 def test_isrBadParallelOverscanColumns(self):

1637 """Test processing a bias when we have a bad parallel overscan column.

1638

1639 This test uses regular non-bootstrap processing.

1640 """

1641 mock_config = self.get_mock_config_no_signal()

1642 isr_config = self.get_isr_config_electronic_corrections()

1643 # We do not do defect correction when processing biases.

1644 isr_config.doDefect = False

1645

1646 # The defect is in amp 2.

1647 sat_level_adu = self.ptc.ptcTurnoff[self.detector[2].getName()]

1648 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1649 sat_level = amp_gain * sat_level_adu

1650 # The expected defect level is in electron for the full bias.

1651 expected_defect_level = mock_config.brightDefectLevel

1652

1653 # The levels are set in electron units.

1654 # We test 4 levels:

1655 # * 10.0, a very low outlier, to test median smoothing detection

1656 # code. This value is given by gain*threshold + cushion.

1657 # * 575.0, a lowish but outlier level, given by gain*threshold +

1658 # 100.0 (average of the parallel overscan offset) + 10.0

1659 # (an additional cushion).

1660 # * 0.9*saturation, following ITL-style parallel overscan bleeds.

1661 # * 1.05*saturation, following E2V-style parallel overscan bleeds.

1662 levels = np.array([10.0, 575.0, 0.9*sat_level, 1.1*sat_level])

1663

1664 for level in levels:

1665 mock_config.badParallelOverscanColumnLevel = level

1666 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1667 input_exp = mock.run()

1668

1669 isr_task = IsrTaskLSST(config=isr_config)

1670 with self.assertNoLogs(level=logging.WARNING):

1671 result = isr_task.run(

1672 input_exp.clone(),

1673 crosstalk=self.crosstalk,

1674 ptc=self.ptc,

1675 linearizer=self.linearizer,

1676 deferredChargeCalib=self.cti,

1677 )

1678

1679 for defect in self.defects:

1680 bbox = defect.getBBox()

1681 defect_image = result.exposure[bbox].image.array

1682

1683 # Check that the defect is the correct level

1684 # (not subtracted away).

1685 defect_median = np.median(defect_image)

1686 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1687

1688 # Check that the neighbors aren't over-subtracted.

1689 for neighbor in [-1, 1]:

1690 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1691 neighbor_image = result.exposure[bbox_neighbor].image.array

1692

1693 neighbor_median = np.median(neighbor_image)

1694 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1695

1696 def test_isrBadPtcGain(self):

1697 """Test processing when an amp has a bad (nan) PTC gain.

1698 """

1699 # We use a flat frame for this test for convenience.

1700 mock_config = self.get_mock_config_no_signal()

1701 mock_config.doAddDark = True

1702 mock_config.doAddFlat = True

1703 # The doAddSky option adds the equivalent of flat-field flux.

1704 mock_config.doAddSky = True

1705

1706 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1707 input_exp = mock.run()

1708

1709 isr_config = self.get_isr_config_electronic_corrections()

1710 isr_config.doBias = True

1711 isr_config.doDark = True

1712 isr_config.doFlat = False

1713 isr_config.doDefect = True

1714

1715 # Set a bad amplifier to a nan gain.

1716 bad_amp = self.detector[0].getName()

1717

1718 ptc = copy.copy(self.ptc)

1719 ptc.gain[bad_amp] = np.nan

1720

1721 # We also want non-zero (but very small) crosstalk values

1722 # to ensure that these don't propagate nans.

1723 crosstalk = copy.copy(self.crosstalk)

1724 for i in range(len(self.detector)):

1725 for j in range(len(self.detector)):

1726 if i == j:

1727 continue

1728 if crosstalk.coeffs[i, j] == 0:

1729 crosstalk.coeffs[i, j] = 1e-10

1730

1731 isr_task = IsrTaskLSST(config=isr_config)

1732 with self.assertLogs(level=logging.WARNING) as cm:

1733 result = isr_task.run(

1734 input_exp.clone(),

1735 bias=self.bias,

1736 dark=self.dark,

1737 crosstalk=crosstalk,

1738 ptc=ptc,

1739 linearizer=self.linearizer,

1740 defects=self.defects,

1741 deferredChargeCalib=self.cti,

1742 )

1743 self.assertIn(f"Amplifier {bad_amp} is bad (non-finite gain)", cm.output[0])

1744

1745 # Confirm that the bad_amp is marked bad and the other amps are not.

1746 # We have to special case the amp with the defect.

1747 mask = result.exposure.mask

1748

1749 for amp in self.detector:

1750 bbox = amp.getBBox()

1751 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1752

1753 if amp.getName() == bad_amp:

1754 self.assertTrue(np.all(bad_in_amp))

1755 elif amp.getName() == "C:0,2":

1756 # This is the amp with the defect.

1757 self.assertEqual(np.sum(bad_in_amp), 51)

1758 else:

1759 self.assertTrue(np.all(~bad_in_amp))

1760

1761 def test_saturationModes(self):

1762 """Test the different saturation modes."""

1763 # Use a simple bias run for these.

1764 mock_config = self.get_mock_config_no_signal()

1765

1766 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1767 input_exp = mock.run()

1768

1769 isr_config = self.get_isr_config_electronic_corrections()

1770 isr_config.doSaturation = True

1771 isr_config.maskNegativeVariance = False

1772 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1773 amp_config = copy.copy(detector_config.defaultAmpConfig)

1774

1775 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1776 isr_config.defaultSaturationSource = mode

1777

1778 # Reset the PTC.

1779 ptc = copy.copy(self.ptc)

1780 # Reset the detector config.

1781 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1782 if mode == "NONE":

1783 # We must use the config.

1784 sat_level = 1.2 * self.saturation_adu

1785 amp_config_new = copy.copy(amp_config)

1786 amp_config_new.saturation = sat_level

1787 detector_config_new = copy.copy(detector_config)

1788 detector_config_new.defaultAmpConfig = amp_config_new

1789 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1790 elif mode == "CAMERAMODEL":

1791 sat_level = input_exp.getDetector()[0].getSaturation()

1792 elif mode == "PTCTURNOFF":

1793 sat_level = 1.3 * self.saturation_adu

1794 for amp_name in ptc.ampNames:

1795 ptc.ptcTurnoff[amp_name] = sat_level

1796

1797 isr_task = IsrTaskLSST(config=isr_config)

1798 with self.assertNoLogs(level=logging.WARNING):

1799 result = isr_task.run(

1800 input_exp.clone(),

1801 bias=self.bias,

1802 crosstalk=self.crosstalk,

1803 ptc=self.ptc,

1804 linearizer=self.linearizer,

1805 deferredChargeCalib=self.cti,

1806 defects=self.defects,

1807 )

1808

1809 metadata = result.exposure.metadata

1810

1811 for amp in self.detector:

1812 amp_name = amp.getName()

1813 key = f"LSST ISR GAIN {amp_name}"

1814 self.assertIn(key, metadata, msg=mode)

1815 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1816 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1817 self.assertIn(key, metadata, msg=mode)

1818 self.assertEqual(metadata[key], sat_level * gain, msg=mode)

1819

1820 def test_noPTC(self):

1821 """Test if we do not supply a PTC."""

1822 mock_config = self.get_mock_config_no_signal()

1823

1824 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1825 input_exp = mock.run()

1826

1827 isr_config = self.get_isr_config_minimal_corrections()

1828 isr_task = IsrTaskLSST(config=isr_config)

1829

1830 with self.assertRaises(RuntimeError) as cm:

1831 _ = isr_task.run(input_exp.clone())

1832 self.assertIn("doBootstrap==False and useGainsFrom =="

1833 " 'PTC' but no PTC provided.",

1834 cm.exception.args[0])

1835

1836 def test_suspectModes(self):

1837 """Test the different suspect modes."""

1838 # Use a simple bias run for these.

1839 mock_config = self.get_mock_config_no_signal()

1840

1841 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1842 input_exp = mock.run()

1843

1844 isr_config = self.get_isr_config_electronic_corrections()

1845 isr_config.doSaturation = True

1846 isr_config.maskNegativeVariance = False

1847 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1848 amp_config = copy.copy(detector_config.defaultAmpConfig)

1849

1850 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1851 isr_config.defaultSuspectSource = mode

1852

1853 # Reset the PTC.

1854 ptc = copy.copy(self.ptc)

1855 # Reset the detector config.

1856 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1857 if mode == "NONE":

1858 # We must use the config.

1859 suspect_level = 1.2 * self.saturation_adu

1860 amp_config_new = copy.copy(amp_config)

1861 amp_config_new.suspectLevel = suspect_level

1862 detector_config_new = copy.copy(detector_config)

1863 detector_config_new.defaultAmpConfig = amp_config_new

1864 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1865 elif mode == "CAMERAMODEL":

1866 suspect_level = input_exp.getDetector()[0].getSuspectLevel()

1867 elif mode == "PTCTURNOFF":

1868 suspect_level = 1.3 * self.saturation_adu

1869 for amp_name in ptc.ampNames:

1870 ptc.ptcTurnoff[amp_name] = suspect_level

1871

1872 isr_task = IsrTaskLSST(config=isr_config)

1873 with self.assertNoLogs(level=logging.WARNING):

1874 result = isr_task.run(

1875 input_exp.clone(),

1876 bias=self.bias,

1877 crosstalk=self.crosstalk,

1878 ptc=self.ptc,

1879 linearizer=self.linearizer,

1880 deferredChargeCalib=self.cti,

1881 defects=self.defects,

1882 )

1883

1884 metadata = result.exposure.metadata

1885

1886 for amp in self.detector:

1887 amp_name = amp.getName()

1888 key = f"LSST ISR GAIN {amp_name}"

1889 self.assertIn(key, metadata, msg=mode)

1890 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1891 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1892 self.assertIn(key, metadata, msg=mode)

1893 self.assertEqual(metadata[key], suspect_level * gain, msg=mode)

1894

1895 def test_sequencerMismatches(self):

1896 """Test with a pile of sequencer mismatches."""

1897 mock_config = self.get_mock_config_no_signal()

1898 mock_config.doAddDark = True

1899 mock_config.doAddFlat = True

1900 # Set this to False until we have fringe correction.

1901 mock_config.doAddFringe = False

1902 mock_config.doAddSky = True

1903 mock_config.doAddSource = True

1904

1905 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1906 input_exp = mock.run()

1907 input_exp.metadata["SEQFILE"] = "a_sequencer"

1908

1909 isr_config = self.get_isr_config_electronic_corrections()

1910 isr_config.doBias = True

1911 isr_config.doDark = True

1912 isr_config.doFlat = True

1913 isr_config.cameraKeywordsToCompare = ["SEQFILE"]

1914

1915 bias = self.bias.clone()

1916 bias.metadata["SEQFILE"] = "b_sequencer"

1917 dark = self.dark.clone()

1918 dark.metadata["SEQFILE"] = "b_sequencer"

1919 flat = self.flat.clone()

1920 flat.metadata["SEQFILE"] = "b_sequencer"

1921 crosstalk = copy.copy(self.crosstalk)

1922 crosstalk.metadata["SEQFILE"] = "b_sequencer"

1923 defects = copy.copy(self.defects)

1924 defects.metadata["SEQFILE"] = "b_sequencer"

1925 ptc = copy.copy(self.ptc)

1926 ptc.metadata["SEQFILE"] = "b_sequencer"

1927 linearizer = copy.copy(self.linearizer)

1928 linearizer.metadata["SEQFILE"] = "b_sequencer"

1929 cti = copy.copy(self.cti)

1930 cti.metadata["SEQFILE"] = "b_sequencer"

1931

1932 isr_task = IsrTaskLSST(config=isr_config)

1933 with self.assertLogs(level=logging.WARNING):

1934 result = isr_task.run(

1935 input_exp.clone(),

1936 bias=bias,

1937 dark=dark,

1938 flat=flat,

1939 crosstalk=crosstalk,

1940 defects=defects,

1941 ptc=ptc,

1942 linearizer=linearizer,

1943 deferredChargeCalib=cti,

1944 )

1945

1946 for ctype in ["BIAS", "DARK", "FLAT", "CROSSTALK", "DEFECTS", "PTC", "LINEARIZER", "CTI"]:

1947 self.assertTrue(result.exposure.metadata[f"ISR {ctype} SEQUENCER MISMATCH"])

1948

1949 def test_highPtcNoiseAmps(self):

1950 """Test for masking of high noise amps (in PTC)."""

1951 # We use a flat frame for this test for convenience.

1952 mock_config = self.get_mock_config_no_signal()

1953 mock_config.doAddDark = True

1954 mock_config.doAddFlat = True

1955 # The doAddSky option adds the equivalent of flat-field flux.

1956 mock_config.doAddSky = True

1957

1958 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1959 input_exp = mock.run()

1960

1961 isr_config = self.get_isr_config_electronic_corrections()

1962 isr_config.doBias = True

1963 isr_config.doDark = True

1964 isr_config.doFlat = False

1965 isr_config.doDefect = True

1966

1967 # Set a bad amplifier to a high noise.

1968 bad_amp = self.detector[0].getName()

1969

1970 ptc = copy.copy(self.ptc)

1971 ptc.noise[bad_amp] = 50.0

1972

1973 isr_task = IsrTaskLSST(config=isr_config)

1974

1975 # With the PTC this should not warn.

1976 with self.assertNoLogs(level=logging.WARNING):

1977 result = isr_task.run(

1978 input_exp.clone(),

1979 bias=self.bias,

1980 dark=self.dark,

1981 crosstalk=self.crosstalk,

1982 ptc=ptc,

1983 linearizer=self.linearizer,

1984 defects=self.defects,

1985 deferredChargeCalib=self.cti,

1986 )

1987

1988 # Confirm that the bad_amp is marked bad and the other amps are not.

1989 # We have to special case the amp with the defect.

1990 mask = result.exposure.mask

1991

1992 for amp in self.detector:

1993 bbox = amp.getBBox()

1994 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1995

1996 if amp.getName() == bad_amp:

1997 self.assertTrue(np.all(bad_in_amp))

1998 elif amp.getName() == "C:0,2":

1999 # This is the amp with the defect.

2000 self.assertEqual(np.sum(bad_in_amp), 51)

2001 else:

2002 self.assertTrue(np.all(~bad_in_amp))

2003

2004 def test_changedOverscanAmps(self):

2005 """Tests for masking of amps where the overscan level changed."""

2006

2007 # We use a flat frame for this test for convenience.

2008 mock_config = self.get_mock_config_no_signal()

2009 mock_config.doAddDark = True

2010 mock_config.doAddFlat = True

2011 # The doAddSky option adds the equivalent of flat-field flux.

2012 mock_config.doAddSky = True

2013

2014 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2015 input_exp = mock.run()

2016

2017 # Offset one amp with a constant value.

2018 bad_amp = "C:0,0"

2019

2020 input_exp2 = input_exp.clone()

2021 input_exp2.image[self.detector[bad_amp].getRawBBox()].array[:, :] -= 2000.0

2022

2023 isr_config = self.get_isr_config_electronic_corrections()

2024 isr_config.doBias = True

2025 isr_config.doDark = True

2026 isr_config.doFlat = False

2027 isr_config.doDefect = True

2028 isr_config.doInterpolate = False

2029 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

2030

2031 isr_task = IsrTaskLSST(config=isr_config)

2032 with self.assertLogs(level=logging.WARNING) as cm:

2033 _ = isr_task.run(

2034 input_exp2,

2035 bias=self.bias,

2036 dark=self.dark,

2037 crosstalk=self.crosstalk,

2038 ptc=self.ptc,

2039 linearizer=self.linearizer,

2040 defects=self.defects,

2041 deferredChargeCalib=self.cti,

2042 )

2043 self.assertEqual(len(cm.output), 1)

2044 self.assertIn(f"Amplifier {bad_amp} has an overscan level", cm.output[0])

2045

2046 # Offset all amps to see that it is now unprocessable.

2047

2048 input_exp2 = input_exp.clone()

2049 input_exp2.image.array[:, :] -= 2000.0

2050

2051 with self.assertLogs(level=logging.WARNING):

2052 with self.assertRaises(UnprocessableDataError):

2053 _ = isr_task.run(

2054 input_exp2,

2055 bias=self.bias,

2056 dark=self.dark,

2057 crosstalk=self.crosstalk,

2058 ptc=self.ptc,

2059 linearizer=self.linearizer,

2060 defects=self.defects,

2061 deferredChargeCalib=self.cti,

2062 )

2063

2064 # Remove the values in the PTC and turn off check.

2065 # This should run without warnings.

2066 ptc = copy.copy(self.ptc)

2067 for amp in self.detector:

2068 ptc.overscanMedian[amp.getName()] = np.nan

2069

2070 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2071

2072 isr_task = IsrTaskLSST(config=isr_config)

2073 with self.assertNoLogs(level=logging.WARNING):

2074 _ = isr_task.run(

2075 input_exp.clone(),

2076 bias=self.bias,

2077 dark=self.dark,

2078 crosstalk=self.crosstalk,

2079 ptc=ptc,

2080 linearizer=self.linearizer,

2081 defects=self.defects,

2082 deferredChargeCalib=self.cti,

2083 )

2084

2085 # Turn the check back on; this should have 1 warning.

2086 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

2087

2088 isr_task = IsrTaskLSST(config=isr_config)

2089 with self.assertLogs(level=logging.WARNING) as cm:

2090 _ = isr_task.run(

2091 input_exp.clone(),

2092 bias=self.bias,

2093 dark=self.dark,

2094 crosstalk=self.crosstalk,

2095 ptc=ptc,

2096 linearizer=self.linearizer,

2097 defects=self.defects,

2098 deferredChargeCalib=self.cti,

2099 )

2100 self.assertEqual(len(cm.output), 1)

2101 self.assertIn("No PTC overscan information", cm.output[0])

2102

2103 def test_highOverscanNoiseAmps(self):

2104 """Test for masking of high noise amps (in overscan)."""

2105

2106 # We use a flat frame for this test for convenience.

2107 mock_config = self.get_mock_config_no_signal()

2108 mock_config.doAddDark = True

2109 mock_config.doAddFlat = True

2110 # The doAddSky option adds the equivalent of flat-field flux.

2111 mock_config.doAddSky = True

2112 mock_config.readNoise = 50.0

2113

2114 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2115 input_exp = mock.run()

2116

2117 isr_config = self.get_isr_config_electronic_corrections()

2118 isr_config.doBias = True

2119 isr_config.doDark = True

2120 isr_config.doFlat = False

2121 isr_config.doDefect = True

2122 isr_config.doInterpolate = False

2123 # Let all the amps fail to check the logging.

2124 isr_config.doCheckUnprocessableData = False

2125

2126 isr_task = IsrTaskLSST(config=isr_config)

2127 with self.assertLogs(level=logging.WARNING) as cm:

2128 result = isr_task.run(

2129 input_exp.clone(),

2130 bias=self.bias,

2131 dark=self.dark,

2132 crosstalk=self.crosstalk,

2133 ptc=self.ptc,

2134 linearizer=self.linearizer,

2135 defects=self.defects,

2136 deferredChargeCalib=self.cti,

2137 )

2138 self.assertEqual(len(cm.output), len(self.detector))

2139

2140 # All pixels should be BAD

2141 bad_value = result.exposure.mask.getPlaneBitMask("BAD")

2142 np.testing.assert_array_equal(result.exposure.mask.array & bad_value, bad_value)

2143

2144 # And run again to check the UnprocessableDataError.

2145 isr_config.doCheckUnprocessableData = True

2146 isr_task = IsrTaskLSST(config=isr_config)

2147

2148 with self.assertRaises(UnprocessableDataError):

2149 with self.assertLogs(level=logging.WARNING):

2150 result = isr_task.run(

2151 input_exp.clone(),

2152 bias=self.bias,

2153 dark=self.dark,

2154 crosstalk=self.crosstalk,

2155 ptc=self.ptc,

2156 linearizer=self.linearizer,

2157 defects=self.defects,

2158 deferredChargeCalib=self.cti,

2159 )

2160

2161 def test_bssVoltageChecks(self):

2162 """Test the BSS voltage checks."""

2163 # We use a flat frame for this test for convenience.

2164 mock_config = self.get_mock_config_no_signal()

2165 mock_config.doAddDark = True

2166 mock_config.doAddFlat = True

2167 # The doAddSky option adds the equivalent of flat-field flux.

2168 mock_config.doAddSky = True

2169

2170 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2171 input_exp = mock.run()

2172

2173 isr_config = self.get_isr_config_electronic_corrections()

2174 isr_config.doBias = True

2175 isr_config.doDark = True

2176 isr_config.doFlat = False

2177 isr_config.doDefect = True

2178

2179 # Set the voltage.

2180 input_exp.metadata["BSSVBS"] = 0.25

2181

2182 # Check that processing runs with checks turned off.

2183 isr_config.doCheckUnprocessableData = False

2184

2185 isr_task = IsrTaskLSST(config=isr_config)

2186 with self.assertNoLogs(level=logging.WARNING):

2187 _ = isr_task.run(

2188 input_exp.clone(),

2189 bias=self.bias,

2190 dark=self.dark,

2191 crosstalk=self.crosstalk,

2192 ptc=self.ptc,

2193 linearizer=self.linearizer,

2194 defects=self.defects,

2195 deferredChargeCalib=self.cti,

2196 )

2197

2198 # Check that processing runs with other way of turning checks off.

2199 isr_config.doCheckUnprocessableData = True

2200 isr_config.bssVoltageMinimum = 0.0

2201

2202 isr_task = IsrTaskLSST(config=isr_config)

2203 with self.assertNoLogs(level=logging.WARNING):

2204 _ = isr_task.run(

2205 input_exp.clone(),

2206 bias=self.bias,

2207 dark=self.dark,

2208 crosstalk=self.crosstalk,

2209 ptc=self.ptc,

2210 linearizer=self.linearizer,

2211 defects=self.defects,

2212 deferredChargeCalib=self.cti,

2213 )

2214

2215 # Check that processing runs but warns if header keyword is None.

2216 isr_config.doCheckUnprocessableData = True

2217 isr_config.bssVoltageMinimum = 10.0

2218

2219 input_exp2 = input_exp.clone()

2220 input_exp2.metadata["BSSVBS"] = None

2221

2222 isr_task = IsrTaskLSST(config=isr_config)

2223 with self.assertLogs(level=logging.WARNING) as cm:

2224 _ = isr_task.run(

2225 input_exp2,

2226 bias=self.bias,

2227 dark=self.dark,

2228 crosstalk=self.crosstalk,

2229 ptc=self.ptc,

2230 linearizer=self.linearizer,

2231 defects=self.defects,

2232 deferredChargeCalib=self.cti,

2233 )

2234 self.assertEqual(len(cm.output), 1)

2235 self.assertIn("Back-side bias voltage BSSVBS not found", cm.output[0])

2236

2237 # Check that it raises.

2238 isr_config.doCheckUnprocessableData = True

2239 isr_config.bssVoltageMinimum = 10.0

2240

2241 isr_task = IsrTaskLSST(config=isr_config)

2242 with self.assertRaises(UnprocessableDataError):

2243 _ = isr_task.run(

2244 input_exp.clone(),

2245 bias=self.bias,

2246 dark=self.dark,

2247 crosstalk=self.crosstalk,

2248 ptc=self.ptc,

2249 linearizer=self.linearizer,

2250 defects=self.defects,

2251 deferredChargeCalib=self.cti,

2252 )

2253

2254 def test_overrideMaskBadAmp(self):

2255 """Test overriding config to mask an amp."""

2256 # We use a flat frame for this test for convenience.

2257 mock_config = self.get_mock_config_no_signal()

2258 mock_config.doAddDark = True

2259 mock_config.doAddFlat = True

2260 # The doAddSky option adds the equivalent of flat-field flux.

2261 mock_config.doAddSky = True

2262

2263 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2264 input_exp = mock.run()

2265

2266 isr_config = self.get_isr_config_electronic_corrections()

2267 isr_config.doBias = True

2268 isr_config.doDark = True

2269 isr_config.doFlat = False

2270 isr_config.doDefect = True

2271

2272 bad_amps = ["C:0,0", "C:0,1"]

2273

2274 detector_name = self.detector.getName()

2275 isr_config.badAmps = [f"{detector_name}_{bad_amp}" for bad_amp in bad_amps]

2276

2277 isr_task = IsrTaskLSST(config=isr_config)

2278 with self.assertNoLogs(level=logging.WARNING):

2279 result = isr_task.run(

2280 input_exp.clone(),

2281 bias=self.bias,

2282 dark=self.dark,

2283 crosstalk=self.crosstalk,

2284 ptc=self.ptc,

2285 linearizer=self.linearizer,

2286 defects=self.defects,

2287 deferredChargeCalib=self.cti,

2288 )

2289

2290 for amp_name in bad_amps:

2291 mask_array = result.exposure[self.detector[amp_name].getBBox()].mask.array

2292 bad_mask = result.exposure.mask.getPlaneBitMask("BAD")

2293 self.assertTrue(np.all((mask_array & bad_mask) > 0))

2294

2295 # Make sure it didn't obliterate everything.

2296 mask_array = result.exposure.mask.array

2297 self.assertFalse(np.all((mask_array & bad_mask) > 0))

2298

2299 def test_hvBiasChecks(self):

2300 """Test the HVBIAS checks."""

2301 # We use a flat frame for this test for convenience.

2302 mock_config = self.get_mock_config_no_signal()

2303 mock_config.doAddDark = True

2304 mock_config.doAddFlat = True

2305 # The doAddSky option adds the equivalent of flat-field flux.

2306 mock_config.doAddSky = True

2307

2308 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2309 input_exp = mock.run()

2310

2311 isr_config = self.get_isr_config_electronic_corrections()

2312 isr_config.doBias = True

2313 isr_config.doDark = True

2314 isr_config.doFlat = False

2315 isr_config.doDefect = True

2316

2317 # Set the voltage.

2318 input_exp.metadata["HVBIAS"] = "OFF"

2319

2320 # Check that processing runs with checks turned off.

2321 isr_config.doCheckUnprocessableData = False

2322

2323 isr_task = IsrTaskLSST(config=isr_config)

2324 with self.assertNoLogs(level=logging.WARNING):

2325 _ = isr_task.run(

2326 input_exp.clone(),

2327 bias=self.bias,

2328 dark=self.dark,

2329 crosstalk=self.crosstalk,

2330 ptc=self.ptc,

2331 linearizer=self.linearizer,

2332 defects=self.defects,

2333 deferredChargeCalib=self.cti,

2334 )

2335

2336 # Check that processing runs with other way of turning checks off.

2337 isr_config.doCheckUnprocessableData = True

2338 isr_config.bssVoltageMinimum = 0.0

2339

2340 isr_task = IsrTaskLSST(config=isr_config)

2341 with self.assertNoLogs(level=logging.WARNING):

2342 _ = isr_task.run(

2343 input_exp.clone(),

2344 bias=self.bias,

2345 dark=self.dark,

2346 crosstalk=self.crosstalk,

2347 ptc=self.ptc,

2348 linearizer=self.linearizer,

2349 defects=self.defects,

2350 deferredChargeCalib=self.cti,

2351 )

2352

2353 # Check that processing runs but warns if header keyword is None.

2354 isr_config.doCheckUnprocessableData = True

2355 isr_config.bssVoltageMinimum = 10.0

2356

2357 input_exp2 = input_exp.clone()

2358 input_exp2.metadata["HVBIAS"] = None

2359

2360 isr_task = IsrTaskLSST(config=isr_config)

2361 with self.assertLogs(level=logging.WARNING) as cm:

2362 _ = isr_task.run(

2363 input_exp2,

2364 bias=self.bias,

2365 dark=self.dark,

2366 crosstalk=self.crosstalk,

2367 ptc=self.ptc,

2368 linearizer=self.linearizer,

2369 defects=self.defects,

2370 deferredChargeCalib=self.cti,

2371 )

2372 self.assertEqual(len(cm.output), 1)

2373 self.assertIn("HV bias on HVBIAS not found in metadata", cm.output[0])

2374

2375 # Check that it raises.

2376 isr_config.doCheckUnprocessableData = True

2377 isr_config.bssVoltageMinimum = 10.0

2378

2379 isr_task = IsrTaskLSST(config=isr_config)

2380 with self.assertRaises(UnprocessableDataError):

2381 _ = isr_task.run(

2382 input_exp.clone(),

2383 bias=self.bias,

2384 dark=self.dark,

2385 crosstalk=self.crosstalk,

2386 ptc=self.ptc,

2387 linearizer=self.linearizer,

2388 defects=self.defects,

2389 deferredChargeCalib=self.cti,

2390 )

2391

2392 def get_mock_config_no_signal(self):

2393 """Get an IsrMockLSSTConfig with all signal set to False.

2394

2395 This will have all the electronic effects turned on (including

2396 2D bias).

2397 """

2398 mock_config = isrMockLSST.IsrMockLSSTConfig()

2399 mock_config.isTrimmed = False

2400 mock_config.doAddDark = False

2401 mock_config.doAddFlat = False

2402 mock_config.doAddFringe = False

2403 mock_config.doAddSky = False

2404 mock_config.doAddSource = False

2405

2406 mock_config.doAdd2DBias = True

2407 mock_config.doAddBias = True

2408 mock_config.doAddCrosstalk = True

2409 mock_config.doAddDeferredCharge = True

2410 mock_config.doAddBrightDefects = True

2411 mock_config.doAddClockInjectedOffset = True

2412 mock_config.doAddParallelOverscanRamp = True

2413 mock_config.doAddSerialOverscanRamp = True

2414 mock_config.doAddHighSignalNonlinearity = True

2415 mock_config.doApplyGain = True

2416 mock_config.doRoundAdu = True

2417

2418 # We always want to generate the image with these configs.

2419 mock_config.doGenerateImage = True

2420

2421 return mock_config

2422

2423 def get_mock_config_clean(self):

2424 """Get an IsrMockLSSTConfig trimmed with all electronic signatures

2425 turned off.

2426 """

2427 mock_config = isrMockLSST.IsrMockLSSTConfig()

2428 mock_config.doAddBias = False

2429 mock_config.doAdd2DBias = False

2430 mock_config.doAddClockInjectedOffset = False

2431 mock_config.doAddDark = False

2432 mock_config.doAddDarkNoiseOnly = False

2433 mock_config.doAddFlat = False

2434 mock_config.doAddFringe = False

2435 mock_config.doAddSky = False

2436 mock_config.doAddSource = False

2437 mock_config.doRoundAdu = False

2438 mock_config.doAddHighSignalNonlinearity = False

2439 mock_config.doApplyGain = False

2440 mock_config.doAddCrosstalk = False

2441 mock_config.doAddBrightDefects = False

2442 mock_config.doAddParallelOverscanRamp = False

2443 mock_config.doAddSerialOverscanRamp = False

2444

2445 mock_config.isTrimmed = True

2446 mock_config.doGenerateImage = True

2447

2448 return mock_config

2449

2450 def get_isr_config_minimal_corrections(self):

2451 """Get an IsrTaskLSSTConfig with minimal corrections.

2452 """

2453 isr_config = IsrTaskLSSTConfig()

2454 isr_config.bssVoltageMinimum = 0.0

2455 isr_config.ampNoiseThreshold = np.inf

2456 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2457 isr_config.doBias = False

2458 isr_config.doDark = False

2459 isr_config.doDeferredCharge = False

2460 isr_config.doLinearize = False

2461 isr_config.doCorrectGains = False

2462 isr_config.doCrosstalk = False

2463 isr_config.doDefect = False

2464 isr_config.doBrighterFatter = False

2465 isr_config.doFlat = False

2466 isr_config.doSaturation = False

2467 isr_config.doE2VEdgeBleedMask = False

2468 isr_config.doITLEdgeBleedMask = False

2469 isr_config.doSuspect = False

2470 # We override the leading/trailing to skip here because of the limited

2471 # size of the test camera overscan regions.

2472 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2473 defaultAmpConfig.doSerialOverscan = True

2474 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2475 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2476 defaultAmpConfig.doParallelOverscan = True

2477 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2478 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2479 # Our strong overscan slope in the tests requires an override.

2480 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2481

2482 isr_config.doAssembleCcd = True

2483 isr_config.crosstalk.doSubtrahendMasking = True

2484 isr_config.crosstalk.minPixelToMask = 1.0

2485

2486 return isr_config

2487

2488 def get_isr_config_electronic_corrections(self):

2489 """Get an IsrTaskLSSTConfig with electronic corrections.

2490

2491 This tests all the corrections that we support in the mocks/ISR.

2492 """

2493 isr_config = IsrTaskLSSTConfig()

2494 # We add these as appropriate in the tests.

2495 isr_config.doBias = False

2496 isr_config.doDark = False

2497 isr_config.doFlat = False

2498

2499 # These are the electronic effects the tests support (in addition

2500 # to overscan).

2501 isr_config.doCrosstalk = True

2502 isr_config.doDefect = True

2503 isr_config.doLinearize = True

2504 isr_config.doDeferredCharge = True

2505

2506 # This is False because it is only used in a single test case

2507 # as it takes a while to solve

2508 isr_config.doBrighterFatter = False

2509

2510 # These are the electronic effects we do not support in tests yet.

2511 isr_config.doCorrectGains = False

2512

2513 # We override the leading/trailing to skip here because of the limited

2514 # size of the test camera overscan regions.

2515 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2516 defaultAmpConfig.doSerialOverscan = True

2517 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2518 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2519 defaultAmpConfig.doParallelOverscan = True

2520 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2521 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2522 # Our strong overscan slope in the tests requires an override.

2523 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2524

2525 isr_config.doAssembleCcd = True

2526 isr_config.crosstalk.doSubtrahendMasking = True

2527 isr_config.crosstalk.minPixelToMask = 1.0

2528

2529 return isr_config

2530

2531 def get_non_defect_pixels(self, mask_origin):

2532 """Get the non-defect pixels to compare.

2533

2534 Parameters

2535 ----------

2536 mask_origin : `lsst.afw.image.MaskX`

2537 The origin mask (for shape and type).

2538

2539 Returns

2540 -------

2541 pix_x, pix_y : `tuple` [`np.ndarray`]

2542 x and y values of good pixels.

2543 """

2544 mask_temp = mask_origin.clone()

2545 mask_temp[:, :] = 0

2546

2547 for defect in self.defects:

2548 mask_temp[defect.getBBox()] = 1

2549

2550 return np.where(mask_temp.array == 0)

2551

2552 def _check_bad_column_crosstalk_correction(

2553 self,

2554 exp,

2555 nsigma_cut=5.0,

2556 ):

2557 """Test bad column crosstalk correction.

2558

2559 This includes possible provblems from parallel overscan

2560 crosstalk and gain mismatches.

2561

2562 The target amp is self.detector[0], "C:0,0".

2563

2564 Parameters

2565 ----------

2566 exp : `lsst.afw.image.Exposure`

2567 Input exposure.

2568 nsigma_cut : `float`, optional

2569 Number of sigma to check for outliers.

2570 """

2571 amp = self.detector[0]

2572 amp_image = exp[amp.getBBox()].image.array

2573 sigma = median_abs_deviation(amp_image.ravel(), scale="normal")

2574

2575 med = np.median(amp_image.ravel())

2576 self.assertLess(amp_image.max(), med + nsigma_cut*sigma)

2577 self.assertGreater(amp_image.min(), med - nsigma_cut*sigma)

2578

2579

2580class MemoryTester(lsst.utils.tests.MemoryTestCase):

2581 pass

2582

2583

2584def setup_module(module):

2585 lsst.utils.tests.init()

2586

2587

2588if __name__ == "__main__": 2588 ↛ 2589line 2588 didn't jump to line 2589 because the condition on line 2588 was never true

2589 lsst.utils.tests.init()

2590 unittest.main()

Coverage for tests / test_isrTaskLSST.py: 4%

1300 statements