Coverage for tests/test_isrTaskLSST.py: 4%

2# LSST Data Management System

5# This product includes software developed by the

6# LSST Project (http://www.lsst.org/).

8# This program is free software: you can redistribute it and/or modify

9# it under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# This program is distributed in the hope that it will be useful,

14# but WITHOUT ANY WARRANTY; without even the implied warranty of

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

16# GNU General Public License for more details.

17#

18# You should have received a copy of the LSST License Statement and

19# the GNU General Public License along with this program. If not,

20# see <https://www.lsstcorp.org/LegalNotices/>.

21#

23import copy

24import unittest

25import numpy as np

26import logging

27import galsim

28from scipy.stats import median_abs_deviation

30import lsst.afw.geom as afwGeom

31import lsst.geom as geom

32from lsst.pipe.base import UnprocessableDataError

33import lsst.ip.isr.isrMockLSST as isrMockLSST

34import lsst.utils.tests

35from lsst.ip.isr.isrTaskLSST import (IsrTaskLSST, IsrTaskLSSTConfig)

36from lsst.ip.isr.crosstalk import CrosstalkCalib

37from lsst.ip.isr import PhotonTransferCurveDataset

38from lsst.ip.isr.vignette import maskVignettedRegion

39from lsst.ip.isr.gainCorrection import GainCorrection

42class IsrTaskLSSTTestCase(lsst.utils.tests.TestCase):

43 """Test IsrTaskLSST"""

44 def setUp(self):

45 mock = isrMockLSST.IsrMockLSST()

46 self.camera = mock.getCamera()

47 self.detector = self.camera[mock.config.detectorIndex]

48 self.namp = len(self.detector)

50 # Create adu (bootstrap) calibration frames

51 self.bias_adu = isrMockLSST.BiasMockLSST(adu=True).run()

52 self.dark_adu = isrMockLSST.DarkMockLSST(adu=True).run()

53 self.flat_adu = isrMockLSST.FlatMockLSST(adu=True).run()

54 self.flat_adu.metadata["FLATSRC"] = "DOME"

56 # Create calibration frames

57 self.bias = isrMockLSST.BiasMockLSST().run()

58 self.dark = isrMockLSST.DarkMockLSST().run()

59 self.flat = isrMockLSST.FlatMockLSST().run()

60 self.flat.metadata["FLATSRC"] = "DOME"

61 self.bf_kernel = isrMockLSST.BfKernelMockLSST().run()

62 self.cti = isrMockLSST.DeferredChargeMockLSST().run()

64 # The crosstalk ratios in isrMockLSST are in electrons.

65 self.crosstalk = CrosstalkCalib(nAmp=self.namp)

66 self.crosstalk.hasCrosstalk = True

67 self.crosstalk.coeffs = isrMockLSST.CrosstalkCoeffMockLSST().run()

68 for i, amp in enumerate(self.detector):

69 self.crosstalk.fitGains[i] = mock.config.gainDict[amp.getName()]

70 self.crosstalk.crosstalkRatiosUnits = "electron"

72 self.defects = isrMockLSST.DefectMockLSST().run()

74 amp_names = [x.getName() for x in self.detector.getAmplifiers()]

75 self.ptc = PhotonTransferCurveDataset(amp_names,

76 ptcFitType='DUMMY_PTC',

77 covMatrixSide=1)

79 self.saturation_adu = 100_000.0

81 # PTC records noise units in electron, same as the

82 # configuration parameter.

83 for amp_name in amp_names:

84 self.ptc.gain[amp_name] = mock.config.gainDict.get(amp_name, mock.config.gain)

85 self.ptc.noise[amp_name] = mock.config.readNoise

86 self.ptc.ptcTurnoff[amp_name] = self.saturation_adu

87 pre_level = mock.config.clockInjectedOffsetLevel

88 overscan_level = mock.config.biasLevel + (pre_level / self.ptc.gain[amp_name])

89 self.ptc.overscanMedian[amp_name] = overscan_level

90 # We set this sigma level very large because the

91 # noise characteristics of the mock image with

92 # a large serial gradient and small overscan region

93 # amplifies the noise vs LSSTCam.

94 self.ptc.overscanMedianSigma[amp_name] = 10.0

96 # TODO:

97 # self.cti = isrMockLSST.DeferredChargeMockLSST().run()

99 self.linearizer = isrMockLSST.LinearizerMockLSST().run()

100 # We currently only have high-signal non-linearity.

101 mock_config = self.get_mock_config_no_signal()

102 for amp_name in amp_names:

103 coeffs = self.linearizer.linearityCoeffs[amp_name]

104 centers, values = np.split(coeffs, 2)

105 values[centers < mock_config.highSignalNonlinearityThreshold] = 0.0

106 self.linearizer.linearityCoeffs[amp_name] = np.concatenate((centers, values))

107

108 def _check_applied_keys(self, metadata, isr_config, expected_gain_correction=False):

109 """Check if the APPLIED keys have been set properly.

110

111 Parameters

112 ----------

113 metadata : `lsst.daf.base.PropertyList`

114 isr_config : `lsst.ip.isr.IsrTaskLSSTConfig`

115 expected_gain_correction : `bool`, optional

116 Did we expect gain correction to be applied?

117 """

118 key = "LSST ISR GAINCORRECTION APPLIED"

119 self.assertIn(key, metadata)

120 self.assertEqual(metadata[key], expected_gain_correction)

121

122 key = "LSST ISR CROSSTALK APPLIED"

123 self.assertIn(key, metadata)

124 self.assertEqual(metadata[key], isr_config.doCrosstalk)

125

126 key = "LSST ISR OVERSCANLEVEL CHECKED"

127 self.assertIn(key, metadata)

128 self.assertEqual(metadata[key], np.isfinite(isr_config.serialOverscanMedianShiftSigmaThreshold))

129

130 key = "LSST ISR NOISE CHECKED"

131 self.assertIn(key, metadata)

132 self.assertEqual(metadata[key], np.isfinite(isr_config.ampNoiseThreshold))

133

134 key = "LSST ISR LINEARIZER APPLIED"

135 self.assertIn(key, metadata)

136 self.assertEqual(metadata[key], isr_config.doLinearize)

137

138 key = "LSST ISR CTI APPLIED"

139 self.assertIn(key, metadata)

140 self.assertEqual(metadata[key], isr_config.doDeferredCharge)

141

142 key = "LSST ISR BIAS APPLIED"

143 self.assertIn(key, metadata)

144 self.assertEqual(metadata[key], isr_config.doBias)

145

146 key = "LSST ISR DARK APPLIED"

147 self.assertIn(key, metadata)

148 self.assertEqual(metadata[key], isr_config.doDark)

149

150 key = "LSST ISR BF APPLIED"

151 self.assertIn(key, metadata)

152 self.assertEqual(metadata[key], isr_config.doBrighterFatter)

153

154 key = "LSST ISR FLAT APPLIED"

155 self.assertIn(key, metadata)

156 self.assertEqual(metadata[key], isr_config.doFlat)

157

158 if metadata[key]:

159 key2 = "LSST ISR FLAT SOURCE"

160 self.assertIn(key2, metadata)

161 self.assertEqual(metadata[key2], "DOME")

162

163 key = "LSST ISR DEFECTS APPLIED"

164 self.assertIn(key, metadata)

165 self.assertEqual(metadata[key], isr_config.doDefect)

166

167 def test_isrBootstrapBias(self):

168 """Test processing of a ``bootstrap`` bias frame.

169

170 This will be output with ADU units.

171 """

172 mock_config = self.get_mock_config_no_signal()

173

174 mock = isrMockLSST.IsrMockLSST(config=mock_config)

175 input_exp = mock.run()

176

177 isr_config = self.get_isr_config_minimal_corrections()

178 isr_config.doBootstrap = True

179 isr_config.doApplyGains = False

180 isr_config.doBias = True

181 isr_config.doCrosstalk = True

182

183 # Need to make sure we are not masking the negative variance

184 # pixels when directly comparing calibration images and

185 # calibration-corrected calibrations.

186 isr_config.maskNegativeVariance = False

187

188 isr_task = IsrTaskLSST(config=isr_config)

189 with self.assertLogs(level=logging.WARNING) as cm:

190 result = isr_task.run(

191 input_exp.clone(),

192 bias=self.bias_adu,

193 ptc=self.ptc,

194 crosstalk=self.crosstalk,

195 )

196 self.assertIn("Ignoring provided PTC", cm.output[0])

197 self._check_applied_keys(result.exposure.metadata, isr_config)

198

199 # Rerun without doing the bias correction.

200 isr_config.doBias = False

201 isr_task2 = IsrTaskLSST(config=isr_config)

202 with self.assertNoLogs(level=logging.WARNING):

203 result2 = isr_task2.run(input_exp.clone(), crosstalk=self.crosstalk)

204

205 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

206

207 self.assertLess(

208 np.mean(result.exposure.image.array[good_pixels]),

209 np.mean(result2.exposure.image.array[good_pixels]),

210 )

211 self.assertLess(

212 np.std(result.exposure.image.array[good_pixels]),

213 np.std(result2.exposure.image.array[good_pixels]),

214 )

215

216 delta = result2.exposure.image.array - result.exposure.image.array

217 self.assertFloatsAlmostEqual(delta[good_pixels], self.bias_adu.image.array[good_pixels], atol=1e-5)

218

219 metadata = result.exposure.metadata

220

221 key = "LSST ISR BOOTSTRAP"

222 self.assertIn(key, metadata)

223 self.assertEqual(metadata[key], True)

224

225 key = "LSST ISR UNITS"

226 self.assertIn(key, metadata)

227 self.assertEqual(metadata[key], "adu")

228

229 key = "LSST ISR READNOISE UNITS"

230 self.assertIn(key, metadata)

231 self.assertEqual(metadata[key], "electron")

232

233 for amp in self.detector:

234 amp_name = amp.getName()

235 key = f"LSST ISR GAIN {amp_name}"

236 self.assertIn(key, metadata)

237 self.assertEqual(metadata[key], 1.0)

238

239 self._check_bad_column_crosstalk_correction(result.exposure)

240

241 def test_isrBootstrapDark(self):

242 """Test processing of a ``bootstrap`` dark frame.

243

244 This will be output with ADU units.

245 """

246 mock_config = self.get_mock_config_no_signal()

247 mock_config.doAddDark = True

248

249 mock = isrMockLSST.IsrMockLSST(config=mock_config)

250 input_exp = mock.run()

251

252 isr_config = self.get_isr_config_minimal_corrections()

253 isr_config.doBootstrap = True

254 isr_config.doApplyGains = False

255 isr_config.doBias = True

256 isr_config.doDark = True

257 isr_config.maskNegativeVariance = False

258 isr_config.doCrosstalk = True

259

260 isr_task = IsrTaskLSST(config=isr_config)

261 with self.assertLogs(level=logging.WARNING) as cm:

262 result = isr_task.run(

263 input_exp.clone(),

264 bias=self.bias_adu,

265 dark=self.dark_adu,

266 ptc=self.ptc,

267 crosstalk=self.crosstalk,

268 )

269 self.assertIn("Ignoring provided PTC", cm.output[0])

270 self._check_applied_keys(result.exposure.metadata, isr_config)

271

272 # Rerun without doing the dark correction.

273 isr_config.doDark = False

274 isr_task2 = IsrTaskLSST(config=isr_config)

275 with self.assertNoLogs(level=logging.WARNING):

276 result2 = isr_task2.run(input_exp.clone(), bias=self.bias_adu, crosstalk=self.crosstalk)

277

278 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

279

280 self.assertLess(

281 np.mean(result.exposure.image.array[good_pixels]),

282 np.mean(result2.exposure.image.array[good_pixels]),

283 )

284

285 delta = result2.exposure.image.array - result.exposure.image.array

286 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

287 self.assertFloatsAlmostEqual(

288 delta[good_pixels],

289 self.dark_adu.image.array[good_pixels] * exp_time,

290 atol=1e-5,

291 )

292

293 metadata = result.exposure.metadata

294

295 key = "LSST ISR BOOTSTRAP"

296 self.assertIn(key, metadata)

297 self.assertEqual(metadata[key], True)

298

299 key = "LSST ISR UNITS"

300 self.assertIn(key, metadata)

301 self.assertEqual(metadata[key], "adu")

302

303 self._check_bad_column_crosstalk_correction(result.exposure)

304

305 def test_isrBootstrapFlat(self):

306 """Test processing of a ``bootstrap`` flat frame.

307

308 This will be output with ADU units.

309 """

310 mock_config = self.get_mock_config_no_signal()

311 mock_config.doAddDark = True

312 mock_config.doAddFlat = True

313 # The doAddSky option adds the equivalent of flat-field flux.

314 mock_config.doAddSky = True

315

316 mock = isrMockLSST.IsrMockLSST(config=mock_config)

317 input_exp = mock.run()

318

319 isr_config = self.get_isr_config_minimal_corrections()

320 isr_config.doBootstrap = True

321 isr_config.doApplyGains = False

322 isr_config.doBias = True

323 isr_config.doDark = True

324 isr_config.doFlat = True

325 isr_config.maskNegativeVariance = False

326 isr_config.doCrosstalk = True

327

328 isr_task = IsrTaskLSST(config=isr_config)

329 with self.assertLogs(level=logging.WARNING) as cm:

330 result = isr_task.run(

331 input_exp.clone(),

332 bias=self.bias_adu,

333 dark=self.dark_adu,

334 flat=self.flat_adu,

335 ptc=self.ptc,

336 crosstalk=self.crosstalk,

337 )

338 self.assertIn("Ignoring provided PTC", cm.output[0])

339 self._check_applied_keys(result.exposure.metadata, isr_config)

340

341 # Rerun without doing the flat correction.

342 isr_config.doFlat = False

343 isr_task2 = IsrTaskLSST(config=isr_config)

344 with self.assertNoLogs(level=logging.WARNING):

345 result2 = isr_task2.run(

346 input_exp.clone(),

347 bias=self.bias_adu,

348 dark=self.dark_adu,

349 crosstalk=self.crosstalk,

350 )

351

352 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

353

354 # Applying the flat will increase the counts.

355 self.assertGreater(

356 np.mean(result.exposure.image.array[good_pixels]),

357 np.mean(result2.exposure.image.array[good_pixels]),

358 )

359 # And will decrease the sigma.

360 self.assertLess(

361 np.std(result.exposure.image.array[good_pixels]),

362 np.std(result2.exposure.image.array[good_pixels]),

363 )

364

365 ratio = result2.exposure.image.array / result.exposure.image.array

366 self.assertFloatsAlmostEqual(ratio[good_pixels], self.flat_adu.image.array[good_pixels], atol=1e-5)

367

368 # Test the variance plane in the case of adu units.

369 # The expected variance starts with the image array.

370 expected_variance = result.exposure.image.clone()

371 # We have to remove the flat-fielding from the image pixels.

372 expected_variance.array *= self.flat_adu.image.array

373 # And add in the bias variance.

374 expected_variance.array += self.bias_adu.variance.array

375 # And add in the scaled dark variance.

376 scale = result.exposure.visitInfo.darkTime / self.dark_adu.visitInfo.darkTime

377 expected_variance.array += scale**2. * self.dark_adu.variance.array

378 # And add the gain and read noise (in electron) per amp.

379 for amp in self.detector:

380 # We need to use the gain and read noise from the header

381 # because these are bootstraps.

382 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

383 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

384

385 expected_variance[amp.getBBox()].array /= gain

386 # Read noise is always in electron units, but since this is a

387 # bootstrap, the gain is 1.0.

388 expected_variance[amp.getBBox()].array += (read_noise/gain)**2.

389

390 # And apply the full formula for dividing by the flat with variance.

391 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

392 unflat_image_array = result.exposure.image.array * self.flat_adu.image.array

393 expected_variance.array = ((unflat_image_array**2. * self.flat_adu.variance.array

394 + self.flat_adu.image.array**2. * expected_variance.array)

395 / self.flat_adu.image.array**4.)

396

397 self.assertFloatsAlmostEqual(

398 result.exposure.variance.array[good_pixels],

399 expected_variance.array[good_pixels],

400 rtol=1e-6,

401 )

402

403 metadata = result.exposure.metadata

404

405 key = "LSST ISR BOOTSTRAP"

406 self.assertIn(key, metadata)

407 self.assertEqual(metadata[key], True)

408

409 key = "LSST ISR UNITS"

410 self.assertIn(key, metadata)

411 self.assertEqual(metadata[key], "adu")

412

413 self._check_bad_column_crosstalk_correction(result.exposure)

414

415 def test_isrBootstrapAndRegularFlat(self):

416 """Test that bootstrap and "regular" flat processing are equivalent."""

417 # This is a test for DM-52684, for the linearizer units.

418

419 mock_config = self.get_mock_config_no_signal()

420 mock_config.doAddDark = True

421 mock_config.doAddFlat = True

422 # The doAddSky option adds the equivalent of flat-field flux.

423 mock_config.doAddSky = True

424 # We set the sky/flat level to a range where the "high signal"

425 # non-linearity has kicked in.

426 mock_config.skyLevel = 40000.0

427

428 mock = isrMockLSST.IsrMockLSST(config=mock_config)

429 input_exp = mock.run()

430

431 # First config is with linearizer in bootstrap mode.

432

433 isr_config_bootstrap = self.get_isr_config_minimal_corrections()

434 isr_config_bootstrap.doBootstrap = True

435 isr_config_bootstrap.doApplyGains = False

436 isr_config_bootstrap.doLinearize = True

437 isr_config_bootstrap.doBias = False

438 isr_config_bootstrap.doDark = False

439 isr_config_bootstrap.doFlat = False

440 isr_config_bootstrap.maskNegativeVariance = False

441 isr_config_bootstrap.doCrosstalk = True

442

443 isr_task_bootstrap = IsrTaskLSST(config=isr_config_bootstrap)

444 with self.assertNoLogs(level=logging.WARNING):

445 result = isr_task_bootstrap.run(

446 input_exp.clone(),

447 crosstalk=self.crosstalk,

448 linearizer=self.linearizer,

449 )

450

451 exp_bootstrap = result.exposure

452

453 # Apply the gains (after the linearization); this is

454 # similar to processing in PTC building.

455 for amp in self.detector:

456 exp_bootstrap[amp.getBBox()].image.array *= self.ptc.gain[amp.getName()]

457

458 # Run again with non-bootstrap mode.

459 isr_config = self.get_isr_config_minimal_corrections()

460 isr_config.doBootstrap = False

461 isr_config.doApplyGains = True

462 isr_config.doLinearize = True

463 isr_config.doBias = False

464 isr_config.doDark = False

465 isr_config.doFlat = False

466 isr_config.maskNegativeVariance = False

467 isr_config.doCrosstalk = True

468

469 isr_task = IsrTaskLSST(config=isr_config)

470 with self.assertNoLogs(level=logging.WARNING):

471 result = isr_task.run(

472 input_exp.clone(),

473 crosstalk=self.crosstalk,

474 linearizer=self.linearizer,

475 ptc=self.ptc,

476 )

477

478 exp = result.exposure

479

480 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

481

482 delta = exp.image.array - exp_bootstrap.image.array

483

484 self.assertFloatsAlmostEqual(delta[good_pixels], 0.0, atol=1e-2)

485

486 def test_isrBias(self):

487 """Test processing of a bias frame."""

488 mock_config = self.get_mock_config_no_signal()

489

490 mock = isrMockLSST.IsrMockLSST(config=mock_config)

491 input_exp = mock.run()

492

493 isr_config = self.get_isr_config_electronic_corrections()

494 isr_config.doBias = True

495 # We do not do defect correction when processing biases.

496 isr_config.doDefect = False

497 isr_config.maskNegativeVariance = False

498

499 isr_task = IsrTaskLSST(config=isr_config)

500 with self.assertNoLogs(level=logging.WARNING):

501 result = isr_task.run(

502 input_exp.clone(),

503 bias=self.bias,

504 crosstalk=self.crosstalk,

505 ptc=self.ptc,

506 linearizer=self.linearizer,

507 deferredChargeCalib=self.cti,

508 )

509 self._check_applied_keys(result.exposure.metadata, isr_config)

510

511 # Rerun without doing the bias correction.

512 isr_config.doBias = False

513 isr_task2 = IsrTaskLSST(config=isr_config)

514 with self.assertNoLogs(level=logging.WARNING):

515 result2 = isr_task2.run(

516 input_exp.clone(),

517 crosstalk=self.crosstalk,

518 ptc=self.ptc,

519 linearizer=self.linearizer,

520 deferredChargeCalib=self.cti,

521 )

522

523 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

524

525 self.assertLess(

526 np.mean(result.exposure.image.array[good_pixels]),

527 np.mean(result2.exposure.image.array[good_pixels]),

528 )

529

530 self.assertLess(

531 np.std(result.exposure.image.array[good_pixels]),

532 np.std(result2.exposure.image.array[good_pixels]),

533 )

534

535 # Confirm that it is flat with an arbitrary cutoff that depends

536 # on the read noise.

537 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

538

539 delta = result2.exposure.image.array - result.exposure.image.array

540

541 # Note that the bias is made with bias noise + read noise, and

542 # the image contains read noise.

543 self.assertFloatsAlmostEqual(

544 delta[good_pixels],

545 self.bias.image.array[good_pixels],

546 atol=1e-5,

547 )

548

549 self._check_bad_column_crosstalk_correction(result.exposure)

550

551 def test_isrBiasNoParallelOscanCorrection(self):

552 """Test processing of a bias frame with parallel

553 overscan correction turned off."""

554 mock_config = self.get_mock_config_no_signal()

555 mock_config.doAddParallelOverscanRamp = False

556

557 mock = isrMockLSST.IsrMockLSST(config=mock_config)

558 input_exp = mock.run()

559

560 isr_config = self.get_isr_config_electronic_corrections()

561

562 # Turn off the parallel overscan correction

563 amp_oscan_config = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

564 amp_oscan_config.doParallelOverscan = False

565 isr_config.doBias = True

566

567 # We do not do defect correction when processing biases.

568 isr_config.doDefect = False

569 isr_config.maskNegativeVariance = False

570

571 isr_task = IsrTaskLSST(config=isr_config)

572 with self.assertNoLogs(level=logging.WARNING):

573 result = isr_task.run(

574 input_exp.clone(),

575 bias=self.bias,

576 crosstalk=self.crosstalk,

577 ptc=self.ptc,

578 linearizer=self.linearizer,

579 deferredChargeCalib=self.cti,

580 )

581 self._check_applied_keys(result.exposure.metadata, isr_config)

582

583 # Rerun without doing the bias correction.

584 isr_config.doBias = False

585 isr_task2 = IsrTaskLSST(config=isr_config)

586 with self.assertNoLogs(level=logging.WARNING):

587 result2 = isr_task2.run(

588 input_exp.clone(),

589 crosstalk=self.crosstalk,

590 ptc=self.ptc,

591 linearizer=self.linearizer,

592 deferredChargeCalib=self.cti,

593 )

594

595 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

596

597 self.assertLess(

598 np.mean(result.exposure.image.array[good_pixels]),

599 np.mean(result2.exposure.image.array[good_pixels]),

600 )

601

602 self.assertLess(

603 np.std(result.exposure.image.array[good_pixels]),

604 np.std(result2.exposure.image.array[good_pixels]),

605 )

606

607 # Confirm that it is flat with an arbitrary cutoff that depends

608 # on the read noise.

609 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

610

611 delta = result2.exposure.image.array - result.exposure.image.array

612

613 # Note that the bias is made with bias noise + read noise, and

614 # the image contains read noise.

615 self.assertFloatsAlmostEqual(

616 delta[good_pixels],

617 self.bias.image.array[good_pixels],

618 atol=1e-5,

619 )

620

621 self._check_bad_column_crosstalk_correction(result.exposure)

622

623 def test_isrBiasCti(self):

624 """Test over-correction of bias amp edges from prescan."""

625 mock_config = self.get_mock_config_no_signal()

626 mock_config.doAddBrightDefects = False

627

628 mock = isrMockLSST.IsrMockLSST(config=mock_config)

629 input_exp = mock.run()

630

631 isr_config = self.get_isr_config_electronic_corrections()

632 # We do not do defect correction when processing biases.

633 isr_config.doDefect = False

634 isr_config.maskNegativeVariance = False

635

636 isr_task = IsrTaskLSST(config=isr_config)

637 with self.assertNoLogs(level=logging.WARNING):

638 result = isr_task.run(

639 input_exp.clone(),

640 crosstalk=self.crosstalk,

641 ptc=self.ptc,

642 linearizer=self.linearizer,

643 deferredChargeCalib=self.cti,

644 )

645 self._check_applied_keys(result.exposure.metadata, isr_config)

646

647 # Rerun without doing the CTI correction

648 isr_config.doDeferredCharge = False

649

650 isr_task2 = IsrTaskLSST(config=isr_config)

651 with self.assertNoLogs(level=logging.WARNING):

652 result2 = isr_task2.run(

653 input_exp.clone(),

654 crosstalk=self.crosstalk,

655 ptc=self.ptc,

656 linearizer=self.linearizer,

657 )

658

659 # This confirms that things are *close* to equal. Unfortunately,

660 # the unusual camera geometry in the test camera doesn't completely

661 # zero out the prescan pixels, so we need a higher threshold.

662 std_delta = np.std(result2.exposure.image.array - result.exposure.image.array)

663 self.assertLess(std_delta, 0.15)

664

665 def test_isrDark(self):

666 """Test processing of a dark frame."""

667 mock_config = self.get_mock_config_no_signal()

668 mock_config.doAddDark = True

669 # We turn off the bad parallel overscan column because it does

670 # add more noise to that region.

671 mock_config.doAddBadParallelOverscanColumn = False

672

673 mock = isrMockLSST.IsrMockLSST(config=mock_config)

674 input_exp = mock.run()

675

676 isr_config = self.get_isr_config_electronic_corrections()

677 isr_config.doBias = True

678 isr_config.doDark = True

679 # We do not do defect correction when processing darks.

680 isr_config.doDefect = False

681 isr_config.maskNegativeVariance = False

682

683 isr_task = IsrTaskLSST(config=isr_config)

684 with self.assertNoLogs(level=logging.WARNING):

685 result = isr_task.run(

686 input_exp.clone(),

687 bias=self.bias,

688 dark=self.dark,

689 crosstalk=self.crosstalk,

690 ptc=self.ptc,

691 linearizer=self.linearizer,

692 deferredChargeCalib=self.cti,

693 )

694 self._check_applied_keys(result.exposure.metadata, isr_config)

695

696 # Rerun without doing the dark correction.

697 isr_config.doDark = False

698 isr_task2 = IsrTaskLSST(config=isr_config)

699 with self.assertNoLogs(level=logging.WARNING):

700 result2 = isr_task2.run(

701 input_exp.clone(),

702 bias=self.bias,

703 crosstalk=self.crosstalk,

704 ptc=self.ptc,

705 linearizer=self.linearizer,

706 deferredChargeCalib=self.cti,

707 )

708

709 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

710

711 self.assertLess(

712 np.mean(result.exposure.image.array[good_pixels]),

713 np.mean(result2.exposure.image.array[good_pixels]),

714 )

715 # The mock dark has no noise, so these should be equal.

716 self.assertFloatsAlmostEqual(

717 np.std(result.exposure.image.array[good_pixels]),

718 np.std(result2.exposure.image.array[good_pixels]),

719 atol=1e-12,

720 )

721

722 # This is a somewhat arbitrary comparison that includes a fudge

723 # factor for the extra noise from the overscan subtraction.

724 self.assertLess(

725 np.std(result.exposure.image.array[good_pixels]),

726 1.6*np.sqrt(mock_config.darkRate*mock_config.expTime + mock_config.readNoise),

727 )

728

729 delta = result2.exposure.image.array - result.exposure.image.array

730 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

731

732 # Allow <3 pixels to fail this test due to rounding error

733 # if doRoundAdu=True

734 diff = np.abs(delta[good_pixels] - self.dark.image.array[good_pixels] * exp_time)

735 self.assertLess(np.count_nonzero(diff >= 1e-12), 3)

736

737 self._check_bad_column_crosstalk_correction(result.exposure)

738

739 def test_isrFlat(self):

740 """Test processing of a flat frame."""

741 mock_config = self.get_mock_config_no_signal()

742 mock_config.doAddDark = True

743 mock_config.doAddFlat = True

744 # The doAddSky option adds the equivalent of flat-field flux.

745 mock_config.doAddSky = True

746

747 mock = isrMockLSST.IsrMockLSST(config=mock_config)

748 input_exp = mock.run()

749

750 isr_config = self.get_isr_config_electronic_corrections()

751 isr_config.doBias = True

752 isr_config.doDark = True

753 isr_config.doFlat = True

754 # Although we usually do not do defect interpolation when

755 # processing flats, this is a good test of the interpolation.

756 isr_config.doDefect = True

757 isr_config.maskNegativeVariance = False

758

759 isr_task = IsrTaskLSST(config=isr_config)

760 with self.assertNoLogs(level=logging.WARNING):

761 result = isr_task.run(

762 input_exp.clone(),

763 bias=self.bias,

764 dark=self.dark,

765 flat=self.flat,

766 crosstalk=self.crosstalk,

767 defects=self.defects,

768 ptc=self.ptc,

769 linearizer=self.linearizer,

770 deferredChargeCalib=self.cti,

771 )

772 self._check_applied_keys(result.exposure.metadata, isr_config)

773

774 # Rerun without doing the bias correction.

775 isr_config.doFlat = False

776 isr_task2 = IsrTaskLSST(config=isr_config)

777 with self.assertNoLogs(level=logging.WARNING):

778 result2 = isr_task2.run(

779 input_exp.clone(),

780 bias=self.bias,

781 dark=self.dark,

782 crosstalk=self.crosstalk,

783 defects=self.defects,

784 ptc=self.ptc,

785 linearizer=self.linearizer,

786 deferredChargeCalib=self.cti,

787 )

788

789 # With defect correction, we should not need to filter out bad

790 # pixels.

791

792 # Applying the flat will increase the counts.

793 self.assertGreater(

794 np.mean(result.exposure.image.array),

795 np.mean(result2.exposure.image.array),

796 )

797 # And will decrease the sigma.

798 self.assertLess(

799 np.std(result.exposure.image.array),

800 np.std(result2.exposure.image.array),

801 )

802

803 # Check that the resulting image is approximately flat.

804 # In particular that the noise is consistent with sky + margin.

805 self.assertLess(np.std(result.exposure.image.array), np.sqrt(mock_config.skyLevel) + 3.0)

806

807 # Generate a flat without any defects for comparison

808 # (including interpolation)

809 flat_nodefect_config = isrMockLSST.FlatMockLSST.ConfigClass()

810 flat_nodefect_config.doAddBrightDefects = False

811 flat_nodefects = isrMockLSST.FlatMockLSST(config=flat_nodefect_config).run()

812

813 ratio = result2.exposure.image.array / result.exposure.image.array

814 self.assertFloatsAlmostEqual(ratio, flat_nodefects.image.array, atol=1e-4)

815

816 self._check_bad_column_crosstalk_correction(result.exposure)

817

818 def test_isrNoise(self):

819 """Test the recorded noise and gain in the metadata."""

820 mock_config = self.get_mock_config_no_signal()

821 # Remove the overscan scale so that the only variation

822 # in the overscan is from the read noise.

823 mock_config.overscanScale = 0.0

824

825 mock = isrMockLSST.IsrMockLSST(config=mock_config)

826 input_exp = mock.run()

827

828 isr_config = self.get_isr_config_electronic_corrections()

829 isr_config.doBias = True

830 # We do not do defect correction when processing biases.

831 isr_config.doDefect = False

832 isr_config.maskNegativeVariance = False

833

834 isr_task = IsrTaskLSST(config=isr_config)

835 with self.assertNoLogs(level=logging.WARNING):

836 result = isr_task.run(

837 input_exp.clone(),

838 bias=self.bias,

839 crosstalk=self.crosstalk,

840 ptc=self.ptc,

841 deferredChargeCalib=self.cti,

842 linearizer=self.linearizer,

843 )

844 self._check_applied_keys(result.exposure.metadata, isr_config)

845

846 metadata = result.exposure.metadata

847

848 for amp in self.detector:

849 # The overscan noise is always in adu and the readnoise is always

850 # in electron.

851 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

852 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

853

854 # Check that the gain and read noise are consistent with the

855 # values stored in the PTC.

856 self.assertEqual(gain, self.ptc.gain[amp.getName()])

857 self.assertEqual(read_noise, self.ptc.noise[amp.getName()])

858

859 key = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {amp.getName()}"

860 self.assertIn(key, metadata)

861

862 # Determine if the residual serial overscan stddev is consistent

863 # with the PTC readnoise within 3xstandard error.

864 serial_overscan_area = amp.getRawHorizontalOverscanBBox().area

865 self.assertFloatsAlmostEqual(

866 metadata[key] * gain,

867 read_noise,

868 atol=3*read_noise / np.sqrt(serial_overscan_area),

869 )

870

871 def test_isrBrighterFatter(self):

872 """Test processing of a flat frame."""

873 # Image with brighter-fatter correction

874 mock_config = self.get_mock_config_no_signal()

875 mock_config.isTrimmed = False

876 mock_config.doAddDark = True

877 mock_config.doAddFlat = True

878 mock_config.doAddSky = True

879 mock_config.doAddSource = True

880 mock_config.sourceFlux = [75000.0]

881 mock_config.doAddBrighterFatter = True

882

883 mock = isrMockLSST.IsrMockLSST(config=mock_config)

884 input_exp = mock.run()

885

886 isr_config = self.get_isr_config_electronic_corrections()

887 isr_config.doBias = True

888 isr_config.doDark = True

889 isr_config.doFlat = True

890 isr_config.doBrighterFatter = True

891

892 isr_task = IsrTaskLSST(config=isr_config)

893 with self.assertNoLogs(level=logging.WARNING):

894 result = isr_task.run(

895 input_exp.clone(),

896 bias=self.bias,

897 dark=self.dark,

898 flat=self.flat,

899 deferredChargeCalib=self.cti,

900 crosstalk=self.crosstalk,

901 defects=self.defects,

902 ptc=self.ptc,

903 linearizer=self.linearizer,

904 bfKernel=self.bf_kernel,

905 )

906 self._check_applied_keys(result.exposure.metadata, isr_config)

907

908 mock_config = self.get_mock_config_no_signal()

909 mock_config.isTrimmed = False

910 mock_config.doAddDark = True

911 mock_config.doAddFlat = True

912 mock_config.doAddSky = True

913 mock_config.doAddSource = True

914 mock_config.sourceFlux = [75000.0]

915 mock_config.doAddBrighterFatter = False

916

917 mock = isrMockLSST.IsrMockLSST(config=mock_config)

918 input_truth = mock.run()

919

920 isr_config = self.get_isr_config_electronic_corrections()

921 isr_config.doBias = True

922 isr_config.doDark = True

923 isr_config.doFlat = True

924 isr_config.doBrighterFatter = False

925

926 isr_task = IsrTaskLSST(config=isr_config)

927 with self.assertNoLogs(level=logging.WARNING):

928 result_truth = isr_task.run(

929 input_truth.clone(),

930 bias=self.bias,

931 dark=self.dark,

932 flat=self.flat,

933 deferredChargeCalib=self.cti,

934 crosstalk=self.crosstalk,

935 defects=self.defects,

936 ptc=self.ptc,

937 linearizer=self.linearizer,

938 bfKernel=self.bf_kernel,

939 )

940

941 # Measure the source size in the BF-corrected image.

942 # The injected source is a Gaussian with 3.0px

943 image = galsim.ImageF(result.exposure.image.array)

944 image_truth = galsim.ImageF(result_truth.exposure.image.array)

945 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

946 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

947 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

948 strict=False)

949 measured_sigma = hsm_result.moments_sigma

950 true_sigma = hsm_result_truth.moments_sigma

951 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

952

953 # Check that the variance in an amp far away from the

954 # source is expected. The source is in amp 0; this will

955 # check the variation in neighboring amp 1

956 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

957 n_pixels = test_amp_bbox.getArea()

958 stdev = np.std(result.exposure[test_amp_bbox].image.array)

959 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

960 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

961

962 # Check that the variance in the amp with a defect is

963 # unchanged as a result of applying the BF correction after

964 # interpolating. The defect was added to amplifier 2.

965 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

966 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

967 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

968 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

969 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

970

971 # Check that BF has converged in the expected number of iterations.

972 metadata = result.exposure.metadata

973 key = "LSST ISR BF ITERS"

974 self.assertIn(key, metadata)

975 self.assertEqual(metadata[key], 2)

976

977 def test_isrSkyImage(self):

978 """Test processing of a sky image."""

979 mock_config = self.get_mock_config_no_signal()

980 mock_config.doAddDark = True

981 mock_config.doAddFlat = True

982 # Set this to False until we have fringe correction.

983 mock_config.doAddFringe = False

984 mock_config.doAddSky = True

985 mock_config.doAddSource = True

986

987 mock = isrMockLSST.IsrMockLSST(config=mock_config)

988 input_exp = mock.run()

989

990 isr_config = self.get_isr_config_electronic_corrections()

991 isr_config.doCorrectGains = True

992 isr_config.doBias = True

993 isr_config.doDark = True

994 isr_config.doFlat = True

995

996 ptc = copy.copy(self.ptc)

997 ptc.gain[ptc.ampNames[0]] *= 0.95

998

999 adjustments = np.ones(len(ptc.ampNames))

1000 adjustments[0] /= 0.95

1001 gainCorrection = GainCorrection(ampNames=ptc.ampNames, gainAdjustments=adjustments)

1002

1003 isr_task = IsrTaskLSST(config=isr_config)

1004 with self.assertNoLogs(level=logging.WARNING):

1005 result = isr_task.run(

1006 input_exp.clone(),

1007 bias=self.bias,

1008 dark=self.dark,

1009 flat=self.flat,

1010 crosstalk=self.crosstalk,

1011 defects=self.defects,

1012 ptc=self.ptc,

1013 gainCorrection=gainCorrection,

1014 linearizer=self.linearizer,

1015 deferredChargeCalib=self.cti,

1016 )

1017 self._check_applied_keys(result.exposure.metadata, isr_config, expected_gain_correction=True)

1018

1019 # Confirm that the output has the defect line as bad.

1020 sat_val = 2**result.exposure.mask.getMaskPlane("BAD")

1021 for defect in self.defects:

1022 np.testing.assert_array_equal(

1023 result.exposure.mask[defect.getBBox()].array & sat_val,

1024 sat_val,

1025 )

1026

1027 clean_mock_config = self.get_mock_config_clean()

1028 # We want the dark noise for more direct comparison.

1029 clean_mock_config.doAddDarkNoiseOnly = True

1030 clean_mock_config.doAddSky = True

1031 clean_mock_config.doAddSource = True

1032

1033 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1034 clean_exp = clean_mock.run()

1035

1036 delta = result.exposure.image.array - clean_exp.image.array

1037

1038 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

1039

1040 # We compare the good pixels in the entirety.

1041 self.assertLess(np.std(delta[good_pixels]), 5.0)

1042 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1043

1044 # Make sure the corrected image is overall consistent with the

1045 # straight image.

1046 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1047

1048 # And overall where the interpolation is a bit worse but

1049 # the statistics are still fine.

1050 self.assertLess(np.std(delta), 5.5)

1051

1052 metadata = result.exposure.metadata

1053

1054 key = "LSST ISR BOOTSTRAP"

1055 self.assertIn(key, metadata)

1056 self.assertEqual(metadata[key], False)

1057

1058 key = "LSST ISR UNITS"

1059 self.assertIn(key, metadata)

1060 self.assertEqual(metadata[key], "electron")

1061

1062 for amp in self.detector:

1063 amp_name = amp.getName()

1064 key = f"LSST ISR GAIN {amp_name}"

1065 self.assertIn(key, metadata)

1066 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1067 key = f"LSST ISR READNOISE {amp_name}"

1068 self.assertIn(key, metadata)

1069 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1070 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1071 self.assertIn(key, metadata)

1072 self.assertEqual(metadata[key], self.saturation_adu * gain)

1073 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1074 self.assertIn(key, metadata)

1075 self.assertEqual(metadata[key], self.saturation_adu * gain)

1076

1077 # Test the variance plane in the case of electron units.

1078 # The expected variance starts with the image array.

1079 expected_variance = result.exposure.image.clone()

1080 # We have to remove the flat-fielding from the image pixels.

1081 expected_variance.array *= self.flat.image.array

1082 # And add in the bias variance.

1083 expected_variance.array += self.bias.variance.array

1084 # And add in the scaled dark variance.

1085 scale = result.exposure.visitInfo.darkTime / self.dark.visitInfo.darkTime

1086 expected_variance.array += scale**2. * self.dark.variance.array

1087 # And add the read noise (in electrons) per amp.

1088 for amp in self.detector:

1089 gain = self.ptc.gain[amp.getName()]

1090 read_noise = self.ptc.noise[amp.getName()]

1091

1092 # The image, read noise, and variance plane should all have

1093 # units of electrons, electrons, and electrons^2.

1094 expected_variance[amp.getBBox()].array += read_noise**2.

1095

1096 # And apply the full formula for dividing by the flat with variance.

1097 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

1098 unflat_image_array = result.exposure.image.array * self.flat.image.array

1099 expected_variance.array = ((unflat_image_array**2. * self.flat.variance.array

1100 + self.flat.image.array**2. * expected_variance.array)

1101 / self.flat.image.array**4.)

1102

1103 self.assertFloatsAlmostEqual(

1104 result.exposure.variance.array[good_pixels],

1105 expected_variance.array[good_pixels],

1106 rtol=1e-6,

1107 )

1108

1109 def test_isrSkyImageSaturated(self):

1110 """Test processing of a sky image.

1111

1112 This variation uses saturated pixels instead of defects.

1113

1114 This additionally tests the gain config override.

1115 """

1116 mock_config = self.get_mock_config_no_signal()

1117 mock_config.doAddDark = True

1118 mock_config.doAddFlat = True

1119 # Set this to False until we have fringe correction.

1120 mock_config.doAddFringe = False

1121 mock_config.doAddSky = True

1122 mock_config.doAddSource = True

1123 mock_config.brightDefectLevel = 170_000.0 # Above saturation.

1124

1125 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1126 input_exp = mock.run()

1127

1128 isr_config = self.get_isr_config_electronic_corrections()

1129 isr_config.doBias = True

1130 isr_config.doDark = True

1131 isr_config.doFlat = True

1132 # We turn off defect masking to test the saturation code.

1133 # However, the same pixels below should be masked/interpolated.

1134 isr_config.doDefect = False

1135

1136 # Use a config override saturation value, confirm it is picked up.

1137 saturation_level = self.saturation_adu * 1.05

1138

1139 # This code will set the gain of one amp to the same as the ptc

1140 # value, and we will check that it is logged and used but the

1141 # results should be the same.

1142 detectorConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector)

1143 detectorConfig.defaultAmpConfig.saturation = saturation_level

1144 overscanAmpConfig = copy.copy(detectorConfig.defaultAmpConfig)

1145 overscanAmpConfig.gain = self.ptc.gain[self.detector[1].getName()]

1146 detectorConfig.ampRules[self.detector[1].getName()] = overscanAmpConfig

1147

1148 isr_task = IsrTaskLSST(config=isr_config)

1149 with self.assertLogs(level=logging.WARNING) as cm:

1150 result = isr_task.run(

1151 input_exp.clone(),

1152 bias=self.bias,

1153 dark=self.dark,

1154 flat=self.flat,

1155 deferredChargeCalib=self.cti,

1156 crosstalk=self.crosstalk,

1157 defects=self.defects,

1158 ptc=self.ptc,

1159 linearizer=self.linearizer,

1160 )

1161 self.assertIn("Overriding gain", cm.output[0])

1162 self._check_applied_keys(result.exposure.metadata, isr_config)

1163

1164 # Confirm that the output has the defect line as saturated.

1165 sat_val = 2**result.exposure.mask.getMaskPlane("SAT")

1166 for defect in self.defects:

1167 np.testing.assert_array_equal(

1168 result.exposure.mask[defect.getBBox()].array & sat_val,

1169 sat_val,

1170 )

1171

1172 clean_mock_config = self.get_mock_config_clean()

1173 # We want the dark noise for more direct comparison.

1174 clean_mock_config.doAddDarkNoiseOnly = True

1175 clean_mock_config.doAddSky = True

1176 clean_mock_config.doAddSource = True

1177

1178 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1179 clean_exp = clean_mock.run()

1180

1181 delta = result.exposure.image.array - clean_exp.image.array

1182

1183 bad_val = 2**result.exposure.mask.getMaskPlane("BAD")

1184 good_pixels = np.where((result.exposure.mask.array & (sat_val | bad_val)) == 0)

1185

1186 # We compare the good pixels in the entirety.

1187 self.assertLess(np.std(delta[good_pixels]), 5.0)

1188 # This is sensitive to parallel overscan masking.

1189 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1190

1191 # Make sure the corrected image is overall consistent with the

1192 # straight image.

1193 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1194

1195 # And overall where the interpolation is a bit worse but

1196 # the statistics are still fine. Note that this is worse than

1197 # the defect case because of the widening of the saturation

1198 # trail.

1199 self.assertLess(np.std(delta), 7.0)

1200

1201 metadata = result.exposure.metadata

1202

1203 for amp in self.detector:

1204 amp_name = amp.getName()

1205 key = f"LSST ISR GAIN {amp_name}"

1206 self.assertIn(key, metadata)

1207 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1208 key = f"LSST ISR READNOISE {amp_name}"

1209 self.assertIn(key, metadata)

1210 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1211 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1212 self.assertIn(key, metadata)

1213 self.assertEqual(metadata[key], saturation_level * gain)

1214

1215 def test_isrFlatVignette(self):

1216 """Test ISR when the flat has a validPolygon and vignetted region."""

1217

1218 # We use a flat frame for this test for convenience.

1219 mock_config = self.get_mock_config_no_signal()

1220 mock_config.doAddDark = True

1221 mock_config.doAddFlat = True

1222 # The doAddSky option adds the equivalent of flat-field flux.

1223 mock_config.doAddSky = True

1224

1225 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1226 input_exp = mock.run()

1227

1228 isr_config = self.get_isr_config_electronic_corrections()

1229 isr_config.doBias = True

1230 isr_config.doDark = True

1231 isr_config.doFlat = True

1232 isr_config.doDefect = True

1233

1234 flat = self.flat.clone()

1235 bbox = geom.Box2D(

1236 corner=geom.Point2D(0, 0),

1237 dimensions=geom.Extent2D(50, 50),

1238 )

1239 polygon = afwGeom.Polygon(bbox)

1240 flat.info.setValidPolygon(polygon)

1241 maskVignettedRegion(flat, polygon, vignetteValue=0.0)

1242

1243 isr_task = IsrTaskLSST(config=isr_config)

1244 with self.assertNoLogs(level=logging.WARNING):

1245 result = isr_task.run(

1246 input_exp.clone(),

1247 bias=self.bias,

1248 dark=self.dark,

1249 flat=flat,

1250 crosstalk=self.crosstalk,

1251 ptc=self.ptc,

1252 linearizer=self.linearizer,

1253 defects=self.defects,

1254 deferredChargeCalib=self.cti,

1255 )

1256

1257 self.assertEqual(result.exposure.info.getValidPolygon(), polygon)

1258

1259 noDataFlat = (flat.mask.array & flat.mask.getPlaneBitMask("NO_DATA")) > 0

1260 noDataExp = (result.exposure.mask.array & result.exposure.mask.getPlaneBitMask("NO_DATA")) > 0

1261 np.testing.assert_array_equal(noDataExp, noDataFlat)

1262 np.testing.assert_array_equal(result.exposure.image.array[noDataExp], 0.0)

1263 np.testing.assert_array_equal(result.exposure.variance.array[noDataExp], 0.0)

1264 self.assertFalse(np.any(~np.isfinite(result.exposure.image.array)))

1265 self.assertFalse(np.any(~np.isfinite(result.exposure.variance.array)))

1266

1267 def test_isrFloodedSaturatedE2V(self):

1268 """Test ISR when the amps are completely saturated.

1269

1270 This version tests what happens when the parallel overscan

1271 region is flooded like E2V detectors, where the saturation

1272 spreads evenly, but at a greater level than the saturation

1273 value.

1274 """

1275 # We are simulating a flat field.

1276 # Note that these aren't very important because we are replacing

1277 # the flux, but we may as well.

1278 mock_config = self.get_mock_config_no_signal()

1279 mock_config.doAddDark = True

1280 mock_config.doAddFlat = True

1281 # The doAddSky option adds the equivalent of flat-field flux.

1282 mock_config.doAddSky = True

1283

1284 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1285 input_exp = mock.run()

1286

1287 isr_config = self.get_isr_config_minimal_corrections()

1288 isr_config.doBootstrap = True

1289 isr_config.doApplyGains = False

1290 isr_config.doBias = True

1291 isr_config.doDark = True

1292 isr_config.doFlat = False

1293 # Tun off saturation masking to simulate a PTC flat.

1294 isr_config.doSaturation = False

1295 isr_config.doE2VEdgeBleedMask = False

1296 isr_config.doITLEdgeBleedMask = False

1297

1298 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1299 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1300

1301 detector = input_exp.getDetector()

1302 for i, amp in enumerate(detector):

1303 # For half of the amps we are testing what happens when the

1304 # parallel overscan region is above the configured saturation

1305 # level; for the other half we are testing the other branch

1306 # when it saturates below this level (which is a priori

1307 # unknown).

1308 if i < len(detector) // 2:

1309 data_level = (parallel_overscan_saturation * 1.05

1310 + mock_config.biasLevel

1311 + mock_config.clockInjectedOffsetLevel)

1312 parallel_overscan_level = (parallel_overscan_saturation * 1.1

1313 + mock_config.biasLevel

1314 + mock_config.clockInjectedOffsetLevel)

1315 else:

1316 data_level = (parallel_overscan_saturation * 0.7

1317 + mock_config.biasLevel

1318 + mock_config.clockInjectedOffsetLevel)

1319 parallel_overscan_level = (parallel_overscan_saturation * 0.75

1320 + mock_config.biasLevel

1321 + mock_config.clockInjectedOffsetLevel)

1322

1323 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1324 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1325

1326 isr_task = IsrTaskLSST(config=isr_config)

1327 with self.assertLogs(level=logging.WARNING) as cm:

1328 result = isr_task.run(

1329 input_exp.clone(),

1330 bias=self.bias_adu,

1331 dark=self.dark_adu,

1332 )

1333 self.assertEqual(len(cm.records), len(detector))

1334

1335 n_all = 0

1336 n_level = 0

1337 for record in cm.records:

1338 if "All overscan pixels masked" in record.message:

1339 n_all += 1

1340 if "The level in the overscan region" in record.message:

1341 n_level += 1

1342

1343 self.assertEqual(n_all, len(detector) // 2)

1344 self.assertEqual(n_level, len(detector) // 2)

1345

1346 # And confirm that the post-ISR levels are high for each amp.

1347 for amp in detector:

1348 med = np.median(result.exposure[amp.getBBox()].image.array)

1349 self.assertGreater(med, parallel_overscan_saturation*0.8)

1350

1351 def test_isrFloodedSaturatedITL(self):

1352 """Test ISR when the amps are completely saturated.

1353

1354 This version tests what happens when the parallel overscan

1355 region is flooded like ITL detectors, where the saturation

1356 is at a lower level than the imaging region, and also

1357 spreads partly into the serial/parallel region.

1358 """

1359 # We are simulating a flat field.

1360 # Note that these aren't very important because we are replacing

1361 # the flux, but we may as well.

1362 mock_config = self.get_mock_config_no_signal()

1363 mock_config.doAddDark = True

1364 mock_config.doAddFlat = True

1365 # The doAddSky option adds the equivalent of flat-field flux.

1366 mock_config.doAddSky = True

1367

1368 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1369 input_exp = mock.run()

1370

1371 isr_config = self.get_isr_config_minimal_corrections()

1372 isr_config.doBootstrap = True

1373 isr_config.doApplyGains = False

1374 isr_config.doBias = True

1375 isr_config.doDark = True

1376 isr_config.doFlat = False

1377 # Tun off saturation masking to simulate a PTC flat.

1378 isr_config.doSaturation = False

1379 isr_config.doE2VEdgeBleedMask = False

1380 isr_config.doITLEdgeBleedMask = False

1381

1382 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1383 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1384

1385 detector = input_exp.getDetector()

1386 for i, amp in enumerate(detector):

1387 # For half of the amps we are testing what happens when the

1388 # parallel overscan region is above the configured saturation

1389 # level; for the other half we are testing the other branch

1390 # when it saturates below this level (which is a priori

1391 # unknown).

1392 if i < len(detector) // 2:

1393 data_level = (parallel_overscan_saturation * 1.1

1394 + mock_config.biasLevel

1395 + mock_config.clockInjectedOffsetLevel)

1396 parallel_overscan_level = (parallel_overscan_saturation * 1.05

1397 + mock_config.biasLevel

1398 + mock_config.clockInjectedOffsetLevel)

1399 else:

1400 data_level = (parallel_overscan_saturation * 0.75

1401 + mock_config.biasLevel

1402 + mock_config.clockInjectedOffsetLevel)

1403 parallel_overscan_level = (parallel_overscan_saturation * 0.7

1404 + mock_config.biasLevel

1405 + mock_config.clockInjectedOffsetLevel)

1406

1407 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1408 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1409 # The serial/parallel region for the test camera looks like this:

1410 serial_overscan_bbox = amp.getRawSerialOverscanBBox()

1411 parallel_overscan_bbox = amp.getRawParallelOverscanBBox()

1412

1413 overscan_corner_bbox = geom.Box2I(

1414 geom.Point2I(

1415 serial_overscan_bbox.getMinX(),

1416 parallel_overscan_bbox.getMinY(),

1417 ),

1418 geom.Extent2I(

1419 serial_overscan_bbox.getWidth(),

1420 parallel_overscan_bbox.getHeight(),

1421 ),

1422 )

1423 input_exp[overscan_corner_bbox].image.array[-2:, :] = parallel_overscan_level

1424

1425 isr_task = IsrTaskLSST(config=isr_config)

1426 with self.assertLogs(level=logging.WARNING) as cm:

1427 result = isr_task.run(

1428 input_exp.clone(),

1429 bias=self.bias_adu,

1430 dark=self.dark_adu,

1431 )

1432 self.assertEqual(len(cm.records), len(detector))

1433

1434 n_all = 0

1435 n_level = 0

1436 for record in cm.records:

1437 if "All overscan pixels masked" in record.message:

1438 n_all += 1

1439 if "The level in the overscan region" in record.message:

1440 n_level += 1

1441

1442 self.assertEqual(n_all, len(detector) // 2)

1443 self.assertEqual(n_level, len(detector) // 2)

1444

1445 # And confirm that the post-ISR levels are high for each amp.

1446 for amp in detector:

1447 med = np.median(result.exposure[amp.getBBox()].image.array)

1448 self.assertGreater(med, parallel_overscan_saturation*0.8)

1449

1450 def test_isrBadParallelOverscanColumnsBootstrap(self):

1451 """Test processing a bias when we have a bad parallel overscan column.

1452

1453 This tests in bootstrap mode.

1454 """

1455 # We base this on the bootstrap bias, and make sure

1456 # that the bad column remains.

1457 mock_config = self.get_mock_config_no_signal()

1458 isr_config = self.get_isr_config_minimal_corrections()

1459 isr_config.doSaturation = False

1460 isr_config.doE2VEdgeBleedMask = False

1461 isr_config.doITLEdgeBleedMask = False

1462 isr_config.doBootstrap = True

1463 isr_config.doApplyGains = False

1464

1465 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1466 overscan_sat_level_adu = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1467 # The defect is in amp 2.

1468 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1469 overscan_sat_level = amp_gain * overscan_sat_level_adu

1470 # The expected defect level is in adu for the bootstrap bias.

1471 expected_defect_level = mock_config.brightDefectLevel / amp_gain

1472

1473 # The levels are set in electron units.

1474 # We test 3 levels:

1475 # * 10.0, a very low outlier, to test median smoothing detection

1476 # code. This value is given by gain*threshold + cushion.

1477 # * 575.0, a lowish but outlier level, given by gain*threshold +

1478 # 100.0 (average of the parallel overscan offset) + 10.0

1479 # (an additional cushion).

1480 # * 1.05*saturation.

1481 # Note that the default parallel overscan saturation level for

1482 # bootstrap (pre-saturation-measure) analysis is very low, in

1483 # order to capture all types of amps, even with low saturation.

1484 # Therefore, we only need to test above this saturation level.

1485 # (c.f. test_isrBadParallelOverscanColumns).

1486 levels = np.array([10.0, 575.0, 1.05*overscan_sat_level])

1487

1488 for level in levels:

1489 mock_config.badParallelOverscanColumnLevel = level

1490 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1491 input_exp = mock.run()

1492

1493 isr_task = IsrTaskLSST(config=isr_config)

1494 with self.assertNoLogs(level=logging.WARNING):

1495 result = isr_task.run(input_exp.clone())

1496

1497 for defect in self.defects:

1498 bbox = defect.getBBox()

1499 defect_image = result.exposure[bbox].image.array

1500

1501 # Check that the defect is the correct level

1502 # (not subtracted away).

1503 defect_median = np.median(defect_image)

1504 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1505

1506 # Check that the neighbors aren't over-subtracted.

1507 for neighbor in [-1, 1]:

1508 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1509 neighbor_image = result.exposure[bbox_neighbor].image.array

1510

1511 neighbor_median = np.median(neighbor_image)

1512 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1513

1514 def test_isrBadParallelOverscanColumns(self):

1515 """Test processing a bias when we have a bad parallel overscan column.

1516

1517 This test uses regular non-bootstrap processing.

1518 """

1519 mock_config = self.get_mock_config_no_signal()

1520 isr_config = self.get_isr_config_electronic_corrections()

1521 # We do not do defect correction when processing biases.

1522 isr_config.doDefect = False

1523

1524 # The defect is in amp 2.

1525 sat_level_adu = self.ptc.ptcTurnoff[self.detector[2].getName()]

1526 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1527 sat_level = amp_gain * sat_level_adu

1528 # The expected defect level is in electron for the full bias.

1529 expected_defect_level = mock_config.brightDefectLevel

1530

1531 # The levels are set in electron units.

1532 # We test 4 levels:

1533 # * 10.0, a very low outlier, to test median smoothing detection

1534 # code. This value is given by gain*threshold + cushion.

1535 # * 575.0, a lowish but outlier level, given by gain*threshold +

1536 # 100.0 (average of the parallel overscan offset) + 10.0

1537 # (an additional cushion).

1538 # * 0.9*saturation, following ITL-style parallel overscan bleeds.

1539 # * 1.05*saturation, following E2V-style parallel overscan bleeds.

1540 levels = np.array([10.0, 575.0, 0.9*sat_level, 1.1*sat_level])

1541

1542 for level in levels:

1543 mock_config.badParallelOverscanColumnLevel = level

1544 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1545 input_exp = mock.run()

1546

1547 isr_task = IsrTaskLSST(config=isr_config)

1548 with self.assertNoLogs(level=logging.WARNING):

1549 result = isr_task.run(

1550 input_exp.clone(),

1551 crosstalk=self.crosstalk,

1552 ptc=self.ptc,

1553 linearizer=self.linearizer,

1554 deferredChargeCalib=self.cti,

1555 )

1556

1557 for defect in self.defects:

1558 bbox = defect.getBBox()

1559 defect_image = result.exposure[bbox].image.array

1560

1561 # Check that the defect is the correct level

1562 # (not subtracted away).

1563 defect_median = np.median(defect_image)

1564 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1565

1566 # Check that the neighbors aren't over-subtracted.

1567 for neighbor in [-1, 1]:

1568 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1569 neighbor_image = result.exposure[bbox_neighbor].image.array

1570

1571 neighbor_median = np.median(neighbor_image)

1572 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1573

1574 def test_isrBadPtcGain(self):

1575 """Test processing when an amp has a bad (nan) PTC gain.

1576 """

1577 # We use a flat frame for this test for convenience.

1578 mock_config = self.get_mock_config_no_signal()

1579 mock_config.doAddDark = True

1580 mock_config.doAddFlat = True

1581 # The doAddSky option adds the equivalent of flat-field flux.

1582 mock_config.doAddSky = True

1583

1584 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1585 input_exp = mock.run()

1586

1587 isr_config = self.get_isr_config_electronic_corrections()

1588 isr_config.doBias = True

1589 isr_config.doDark = True

1590 isr_config.doFlat = False

1591 isr_config.doDefect = True

1592

1593 # Set a bad amplifier to a nan gain.

1594 bad_amp = self.detector[0].getName()

1595

1596 ptc = copy.copy(self.ptc)

1597 ptc.gain[bad_amp] = np.nan

1598

1599 # We also want non-zero (but very small) crosstalk values

1600 # to ensure that these don't propagate nans.

1601 crosstalk = copy.copy(self.crosstalk)

1602 for i in range(len(self.detector)):

1603 for j in range(len(self.detector)):

1604 if i == j:

1605 continue

1606 if crosstalk.coeffs[i, j] == 0:

1607 crosstalk.coeffs[i, j] = 1e-10

1608

1609 isr_task = IsrTaskLSST(config=isr_config)

1610 with self.assertLogs(level=logging.WARNING) as cm:

1611 result = isr_task.run(

1612 input_exp.clone(),

1613 bias=self.bias,

1614 dark=self.dark,

1615 crosstalk=crosstalk,

1616 ptc=ptc,

1617 linearizer=self.linearizer,

1618 defects=self.defects,

1619 deferredChargeCalib=self.cti,

1620 )

1621 self.assertIn(f"Amplifier {bad_amp} is bad (non-finite gain)", cm.output[0])

1622

1623 # Confirm that the bad_amp is marked bad and the other amps are not.

1624 # We have to special case the amp with the defect.

1625 mask = result.exposure.mask

1626

1627 for amp in self.detector:

1628 bbox = amp.getBBox()

1629 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1630

1631 if amp.getName() == bad_amp:

1632 self.assertTrue(np.all(bad_in_amp))

1633 elif amp.getName() == "C:0,2":

1634 # This is the amp with the defect.

1635 self.assertEqual(np.sum(bad_in_amp), 51)

1636 else:

1637 self.assertTrue(np.all(~bad_in_amp))

1638

1639 def test_saturationModes(self):

1640 """Test the different saturation modes."""

1641 # Use a simple bias run for these.

1642 mock_config = self.get_mock_config_no_signal()

1643

1644 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1645 input_exp = mock.run()

1646

1647 isr_config = self.get_isr_config_electronic_corrections()

1648 isr_config.doSaturation = True

1649 isr_config.maskNegativeVariance = False

1650 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1651 amp_config = copy.copy(detector_config.defaultAmpConfig)

1652

1653 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1654 isr_config.defaultSaturationSource = mode

1655

1656 # Reset the PTC.

1657 ptc = copy.copy(self.ptc)

1658 # Reset the detector config.

1659 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1660 if mode == "NONE":

1661 # We must use the config.

1662 sat_level = 1.2 * self.saturation_adu

1663 amp_config_new = copy.copy(amp_config)

1664 amp_config_new.saturation = sat_level

1665 detector_config_new = copy.copy(detector_config)

1666 detector_config_new.defaultAmpConfig = amp_config_new

1667 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1668 elif mode == "CAMERAMODEL":

1669 sat_level = input_exp.getDetector()[0].getSaturation()

1670 elif mode == "PTCTURNOFF":

1671 sat_level = 1.3 * self.saturation_adu

1672 for amp_name in ptc.ampNames:

1673 ptc.ptcTurnoff[amp_name] = sat_level

1674

1675 isr_task = IsrTaskLSST(config=isr_config)

1676 with self.assertNoLogs(level=logging.WARNING):

1677 result = isr_task.run(

1678 input_exp.clone(),

1679 bias=self.bias,

1680 crosstalk=self.crosstalk,

1681 ptc=self.ptc,

1682 linearizer=self.linearizer,

1683 deferredChargeCalib=self.cti,

1684 defects=self.defects,

1685 )

1686

1687 metadata = result.exposure.metadata

1688

1689 for amp in self.detector:

1690 amp_name = amp.getName()

1691 key = f"LSST ISR GAIN {amp_name}"

1692 self.assertIn(key, metadata, msg=mode)

1693 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1694 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1695 self.assertIn(key, metadata, msg=mode)

1696 self.assertEqual(metadata[key], sat_level * gain, msg=mode)

1697

1698 def test_noPTC(self):

1699 """Test if we do not supply a PTC."""

1700 mock_config = self.get_mock_config_no_signal()

1701

1702 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1703 input_exp = mock.run()

1704

1705 isr_config = self.get_isr_config_minimal_corrections()

1706 isr_task = IsrTaskLSST(config=isr_config)

1707

1708 with self.assertRaises(RuntimeError) as cm:

1709 _ = isr_task.run(input_exp.clone())

1710 self.assertIn("doBootstrap==False and useGainsFrom =="

1711 " 'PTC' but no PTC provided.",

1712 cm.exception.args[0])

1713

1714 def test_suspectModes(self):

1715 """Test the different suspect modes."""

1716 # Use a simple bias run for these.

1717 mock_config = self.get_mock_config_no_signal()

1718

1719 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1720 input_exp = mock.run()

1721

1722 isr_config = self.get_isr_config_electronic_corrections()

1723 isr_config.doSaturation = True

1724 isr_config.maskNegativeVariance = False

1725 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1726 amp_config = copy.copy(detector_config.defaultAmpConfig)

1727

1728 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1729 isr_config.defaultSuspectSource = mode

1730

1731 # Reset the PTC.

1732 ptc = copy.copy(self.ptc)

1733 # Reset the detector config.

1734 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1735 if mode == "NONE":

1736 # We must use the config.

1737 suspect_level = 1.2 * self.saturation_adu

1738 amp_config_new = copy.copy(amp_config)

1739 amp_config_new.suspectLevel = suspect_level

1740 detector_config_new = copy.copy(detector_config)

1741 detector_config_new.defaultAmpConfig = amp_config_new

1742 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1743 elif mode == "CAMERAMODEL":

1744 suspect_level = input_exp.getDetector()[0].getSuspectLevel()

1745 elif mode == "PTCTURNOFF":

1746 suspect_level = 1.3 * self.saturation_adu

1747 for amp_name in ptc.ampNames:

1748 ptc.ptcTurnoff[amp_name] = suspect_level

1749

1750 isr_task = IsrTaskLSST(config=isr_config)

1751 with self.assertNoLogs(level=logging.WARNING):

1752 result = isr_task.run(

1753 input_exp.clone(),

1754 bias=self.bias,

1755 crosstalk=self.crosstalk,

1756 ptc=self.ptc,

1757 linearizer=self.linearizer,

1758 deferredChargeCalib=self.cti,

1759 defects=self.defects,

1760 )

1761

1762 metadata = result.exposure.metadata

1763

1764 for amp in self.detector:

1765 amp_name = amp.getName()

1766 key = f"LSST ISR GAIN {amp_name}"

1767 self.assertIn(key, metadata, msg=mode)

1768 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1769 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1770 self.assertIn(key, metadata, msg=mode)

1771 self.assertEqual(metadata[key], suspect_level * gain, msg=mode)

1772

1773 def test_sequencerMismatches(self):

1774 """Test with a pile of sequencer mismatches."""

1775 mock_config = self.get_mock_config_no_signal()

1776 mock_config.doAddDark = True

1777 mock_config.doAddFlat = True

1778 # Set this to False until we have fringe correction.

1779 mock_config.doAddFringe = False

1780 mock_config.doAddSky = True

1781 mock_config.doAddSource = True

1782

1783 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1784 input_exp = mock.run()

1785 input_exp.metadata["SEQFILE"] = "a_sequencer"

1786

1787 isr_config = self.get_isr_config_electronic_corrections()

1788 isr_config.doBias = True

1789 isr_config.doDark = True

1790 isr_config.doFlat = True

1791 isr_config.cameraKeywordsToCompare = ["SEQFILE"]

1792

1793 bias = self.bias.clone()

1794 bias.metadata["SEQFILE"] = "b_sequencer"

1795 dark = self.dark.clone()

1796 dark.metadata["SEQFILE"] = "b_sequencer"

1797 flat = self.flat.clone()

1798 flat.metadata["SEQFILE"] = "b_sequencer"

1799 crosstalk = copy.copy(self.crosstalk)

1800 crosstalk.metadata["SEQFILE"] = "b_sequencer"

1801 defects = copy.copy(self.defects)

1802 defects.metadata["SEQFILE"] = "b_sequencer"

1803 ptc = copy.copy(self.ptc)

1804 ptc.metadata["SEQFILE"] = "b_sequencer"

1805 linearizer = copy.copy(self.linearizer)

1806 linearizer.metadata["SEQFILE"] = "b_sequencer"

1807 cti = copy.copy(self.cti)

1808 cti.metadata["SEQFILE"] = "b_sequencer"

1809

1810 isr_task = IsrTaskLSST(config=isr_config)

1811 with self.assertLogs(level=logging.WARNING):

1812 result = isr_task.run(

1813 input_exp.clone(),

1814 bias=bias,

1815 dark=dark,

1816 flat=flat,

1817 crosstalk=crosstalk,

1818 defects=defects,

1819 ptc=ptc,

1820 linearizer=linearizer,

1821 deferredChargeCalib=cti,

1822 )

1823

1824 for ctype in ["BIAS", "DARK", "FLAT", "CROSSTALK", "DEFECTS", "PTC", "LINEARIZER", "CTI"]:

1825 self.assertTrue(result.exposure.metadata[f"ISR {ctype} SEQUENCER MISMATCH"])

1826

1827 def test_highPtcNoiseAmps(self):

1828 """Test for masking of high noise amps (in PTC)."""

1829 # We use a flat frame for this test for convenience.

1830 mock_config = self.get_mock_config_no_signal()

1831 mock_config.doAddDark = True

1832 mock_config.doAddFlat = True

1833 # The doAddSky option adds the equivalent of flat-field flux.

1834 mock_config.doAddSky = True

1835

1836 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1837 input_exp = mock.run()

1838

1839 isr_config = self.get_isr_config_electronic_corrections()

1840 isr_config.doBias = True

1841 isr_config.doDark = True

1842 isr_config.doFlat = False

1843 isr_config.doDefect = True

1844

1845 # Set a bad amplifier to a high noise.

1846 bad_amp = self.detector[0].getName()

1847

1848 ptc = copy.copy(self.ptc)

1849 ptc.noise[bad_amp] = 50.0

1850

1851 isr_task = IsrTaskLSST(config=isr_config)

1852

1853 # With the PTC this should not warn.

1854 with self.assertNoLogs(level=logging.WARNING):

1855 result = isr_task.run(

1856 input_exp.clone(),

1857 bias=self.bias,

1858 dark=self.dark,

1859 crosstalk=self.crosstalk,

1860 ptc=ptc,

1861 linearizer=self.linearizer,

1862 defects=self.defects,

1863 deferredChargeCalib=self.cti,

1864 )

1865

1866 # Confirm that the bad_amp is marked bad and the other amps are not.

1867 # We have to special case the amp with the defect.

1868 mask = result.exposure.mask

1869

1870 for amp in self.detector:

1871 bbox = amp.getBBox()

1872 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1873

1874 if amp.getName() == bad_amp:

1875 self.assertTrue(np.all(bad_in_amp))

1876 elif amp.getName() == "C:0,2":

1877 # This is the amp with the defect.

1878 self.assertEqual(np.sum(bad_in_amp), 51)

1879 else:

1880 self.assertTrue(np.all(~bad_in_amp))

1881

1882 def test_changedOverscanAmps(self):

1883 """Tests for masking of amps where the overscan level changed."""

1884

1885 # We use a flat frame for this test for convenience.

1886 mock_config = self.get_mock_config_no_signal()

1887 mock_config.doAddDark = True

1888 mock_config.doAddFlat = True

1889 # The doAddSky option adds the equivalent of flat-field flux.

1890 mock_config.doAddSky = True

1891

1892 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1893 input_exp = mock.run()

1894

1895 # Offset one amp with a constant value.

1896 bad_amp = "C:0,0"

1897

1898 input_exp2 = input_exp.clone()

1899 input_exp2.image[self.detector[bad_amp].getRawBBox()].array[:, :] -= 2000.0

1900

1901 isr_config = self.get_isr_config_electronic_corrections()

1902 isr_config.doBias = True

1903 isr_config.doDark = True

1904 isr_config.doFlat = False

1905 isr_config.doDefect = True

1906 isr_config.doInterpolate = False

1907 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

1908

1909 isr_task = IsrTaskLSST(config=isr_config)

1910 with self.assertLogs(level=logging.WARNING) as cm:

1911 _ = isr_task.run(

1912 input_exp2,

1913 bias=self.bias,

1914 dark=self.dark,

1915 crosstalk=self.crosstalk,

1916 ptc=self.ptc,

1917 linearizer=self.linearizer,

1918 defects=self.defects,

1919 deferredChargeCalib=self.cti,

1920 )

1921 self.assertEqual(len(cm.output), 1)

1922 self.assertIn(f"Amplifier {bad_amp} has an overscan level", cm.output[0])

1923

1924 # Offset all amps to see that it is now unprocessable.

1925

1926 input_exp2 = input_exp.clone()

1927 input_exp2.image.array[:, :] -= 2000.0

1928

1929 with self.assertLogs(level=logging.WARNING):

1930 with self.assertRaises(UnprocessableDataError):

1931 _ = isr_task.run(

1932 input_exp2,

1933 bias=self.bias,

1934 dark=self.dark,

1935 crosstalk=self.crosstalk,

1936 ptc=self.ptc,

1937 linearizer=self.linearizer,

1938 defects=self.defects,

1939 deferredChargeCalib=self.cti,

1940 )

1941

1942 # Remove the values in the PTC and turn off check.

1943 # This should run without warnings.

1944 ptc = copy.copy(self.ptc)

1945 for amp in self.detector:

1946 ptc.overscanMedian[amp.getName()] = np.nan

1947

1948 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

1949

1950 isr_task = IsrTaskLSST(config=isr_config)

1951 with self.assertNoLogs(level=logging.WARNING):

1952 _ = isr_task.run(

1953 input_exp.clone(),

1954 bias=self.bias,

1955 dark=self.dark,

1956 crosstalk=self.crosstalk,

1957 ptc=ptc,

1958 linearizer=self.linearizer,

1959 defects=self.defects,

1960 deferredChargeCalib=self.cti,

1961 )

1962

1963 # Turn the check back on; this should have 1 warning.

1964 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

1965

1966 isr_task = IsrTaskLSST(config=isr_config)

1967 with self.assertLogs(level=logging.WARNING) as cm:

1968 _ = isr_task.run(

1969 input_exp.clone(),

1970 bias=self.bias,

1971 dark=self.dark,

1972 crosstalk=self.crosstalk,

1973 ptc=ptc,

1974 linearizer=self.linearizer,

1975 defects=self.defects,

1976 deferredChargeCalib=self.cti,

1977 )

1978 self.assertEqual(len(cm.output), 1)

1979 self.assertIn("No PTC overscan information", cm.output[0])

1980

1981 def test_highOverscanNoiseAmps(self):

1982 """Test for masking of high noise amps (in overscan)."""

1983

1984 # We use a flat frame for this test for convenience.

1985 mock_config = self.get_mock_config_no_signal()

1986 mock_config.doAddDark = True

1987 mock_config.doAddFlat = True

1988 # The doAddSky option adds the equivalent of flat-field flux.

1989 mock_config.doAddSky = True

1990 mock_config.readNoise = 50.0

1991

1992 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1993 input_exp = mock.run()

1994

1995 isr_config = self.get_isr_config_electronic_corrections()

1996 isr_config.doBias = True

1997 isr_config.doDark = True

1998 isr_config.doFlat = False

1999 isr_config.doDefect = True

2000 isr_config.doInterpolate = False

2001 # Let all the amps fail to check the logging.

2002 isr_config.doCheckUnprocessableData = False

2003

2004 isr_task = IsrTaskLSST(config=isr_config)

2005 with self.assertLogs(level=logging.WARNING) as cm:

2006 result = isr_task.run(

2007 input_exp.clone(),

2008 bias=self.bias,

2009 dark=self.dark,

2010 crosstalk=self.crosstalk,

2011 ptc=self.ptc,

2012 linearizer=self.linearizer,

2013 defects=self.defects,

2014 deferredChargeCalib=self.cti,

2015 )

2016 self.assertEqual(len(cm.output), len(self.detector))

2017

2018 # All pixels should be BAD

2019 bad_value = result.exposure.mask.getPlaneBitMask("BAD")

2020 np.testing.assert_array_equal(result.exposure.mask.array & bad_value, bad_value)

2021

2022 # And run again to check the UnprocessableDataError.

2023 isr_config.doCheckUnprocessableData = True

2024 isr_task = IsrTaskLSST(config=isr_config)

2025

2026 with self.assertRaises(UnprocessableDataError):

2027 with self.assertLogs(level=logging.WARNING):

2028 result = isr_task.run(

2029 input_exp.clone(),

2030 bias=self.bias,

2031 dark=self.dark,

2032 crosstalk=self.crosstalk,

2033 ptc=self.ptc,

2034 linearizer=self.linearizer,

2035 defects=self.defects,

2036 deferredChargeCalib=self.cti,

2037 )

2038

2039 def test_bssVoltageChecks(self):

2040 """Test the BSS voltage checks."""

2041 # We use a flat frame for this test for convenience.

2042 mock_config = self.get_mock_config_no_signal()

2043 mock_config.doAddDark = True

2044 mock_config.doAddFlat = True

2045 # The doAddSky option adds the equivalent of flat-field flux.

2046 mock_config.doAddSky = True

2047

2048 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2049 input_exp = mock.run()

2050

2051 isr_config = self.get_isr_config_electronic_corrections()

2052 isr_config.doBias = True

2053 isr_config.doDark = True

2054 isr_config.doFlat = False

2055 isr_config.doDefect = True

2056

2057 # Set the voltage.

2058 input_exp.metadata["BSSVBS"] = 0.25

2059

2060 # Check that processing runs with checks turned off.

2061 isr_config.doCheckUnprocessableData = False

2062

2063 isr_task = IsrTaskLSST(config=isr_config)

2064 with self.assertNoLogs(level=logging.WARNING):

2065 _ = isr_task.run(

2066 input_exp.clone(),

2067 bias=self.bias,

2068 dark=self.dark,

2069 crosstalk=self.crosstalk,

2070 ptc=self.ptc,

2071 linearizer=self.linearizer,

2072 defects=self.defects,

2073 deferredChargeCalib=self.cti,

2074 )

2075

2076 # Check that processing runs with other way of turning checks off.

2077 isr_config.doCheckUnprocessableData = True

2078 isr_config.bssVoltageMinimum = 0.0

2079

2080 isr_task = IsrTaskLSST(config=isr_config)

2081 with self.assertNoLogs(level=logging.WARNING):

2082 _ = isr_task.run(

2083 input_exp.clone(),

2084 bias=self.bias,

2085 dark=self.dark,

2086 crosstalk=self.crosstalk,

2087 ptc=self.ptc,

2088 linearizer=self.linearizer,

2089 defects=self.defects,

2090 deferredChargeCalib=self.cti,

2091 )

2092

2093 # Check that processing runs but warns if header keyword is None.

2094 isr_config.doCheckUnprocessableData = True

2095 isr_config.bssVoltageMinimum = 10.0

2096

2097 input_exp2 = input_exp.clone()

2098 input_exp2.metadata["BSSVBS"] = None

2099

2100 isr_task = IsrTaskLSST(config=isr_config)

2101 with self.assertLogs(level=logging.WARNING) as cm:

2102 _ = isr_task.run(

2103 input_exp2,

2104 bias=self.bias,

2105 dark=self.dark,

2106 crosstalk=self.crosstalk,

2107 ptc=self.ptc,

2108 linearizer=self.linearizer,

2109 defects=self.defects,

2110 deferredChargeCalib=self.cti,

2111 )

2112 self.assertEqual(len(cm.output), 1)

2113 self.assertIn("Back-side bias voltage BSSVBS not found", cm.output[0])

2114

2115 # Check that it raises.

2116 isr_config.doCheckUnprocessableData = True

2117 isr_config.bssVoltageMinimum = 10.0

2118

2119 isr_task = IsrTaskLSST(config=isr_config)

2120 with self.assertRaises(UnprocessableDataError):

2121 _ = isr_task.run(

2122 input_exp.clone(),

2123 bias=self.bias,

2124 dark=self.dark,

2125 crosstalk=self.crosstalk,

2126 ptc=self.ptc,

2127 linearizer=self.linearizer,

2128 defects=self.defects,

2129 deferredChargeCalib=self.cti,

2130 )

2131

2132 def test_overrideMaskBadAmp(self):

2133 """Test overriding config to mask an amp."""

2134 # We use a flat frame for this test for convenience.

2135 mock_config = self.get_mock_config_no_signal()

2136 mock_config.doAddDark = True

2137 mock_config.doAddFlat = True

2138 # The doAddSky option adds the equivalent of flat-field flux.

2139 mock_config.doAddSky = True

2140

2141 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2142 input_exp = mock.run()

2143

2144 isr_config = self.get_isr_config_electronic_corrections()

2145 isr_config.doBias = True

2146 isr_config.doDark = True

2147 isr_config.doFlat = False

2148 isr_config.doDefect = True

2149

2150 bad_amps = ["C:0,0", "C:0,1"]

2151

2152 detector_name = self.detector.getName()

2153 isr_config.badAmps = [f"{detector_name}_{bad_amp}" for bad_amp in bad_amps]

2154

2155 isr_task = IsrTaskLSST(config=isr_config)

2156 with self.assertNoLogs(level=logging.WARNING):

2157 result = isr_task.run(

2158 input_exp.clone(),

2159 bias=self.bias,

2160 dark=self.dark,

2161 crosstalk=self.crosstalk,

2162 ptc=self.ptc,

2163 linearizer=self.linearizer,

2164 defects=self.defects,

2165 deferredChargeCalib=self.cti,

2166 )

2167

2168 for amp_name in bad_amps:

2169 mask_array = result.exposure[self.detector[amp_name].getBBox()].mask.array

2170 bad_mask = result.exposure.mask.getPlaneBitMask("BAD")

2171 self.assertTrue(np.all((mask_array & bad_mask) > 0))

2172

2173 # Make sure it didn't obliterate everything.

2174 mask_array = result.exposure.mask.array

2175 self.assertFalse(np.all((mask_array & bad_mask) > 0))

2176

2177 def test_hvBiasChecks(self):

2178 """Test the HVBIAS checks."""

2179 # We use a flat frame for this test for convenience.

2180 mock_config = self.get_mock_config_no_signal()

2181 mock_config.doAddDark = True

2182 mock_config.doAddFlat = True

2183 # The doAddSky option adds the equivalent of flat-field flux.

2184 mock_config.doAddSky = True

2185

2186 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2187 input_exp = mock.run()

2188

2189 isr_config = self.get_isr_config_electronic_corrections()

2190 isr_config.doBias = True

2191 isr_config.doDark = True

2192 isr_config.doFlat = False

2193 isr_config.doDefect = True

2194

2195 # Set the voltage.

2196 input_exp.metadata["HVBIAS"] = "OFF"

2197

2198 # Check that processing runs with checks turned off.

2199 isr_config.doCheckUnprocessableData = False

2200

2201 isr_task = IsrTaskLSST(config=isr_config)

2202 with self.assertNoLogs(level=logging.WARNING):

2203 _ = isr_task.run(

2204 input_exp.clone(),

2205 bias=self.bias,

2206 dark=self.dark,

2207 crosstalk=self.crosstalk,

2208 ptc=self.ptc,

2209 linearizer=self.linearizer,

2210 defects=self.defects,

2211 deferredChargeCalib=self.cti,

2212 )

2213

2214 # Check that processing runs with other way of turning checks off.

2215 isr_config.doCheckUnprocessableData = True

2216 isr_config.bssVoltageMinimum = 0.0

2217

2218 isr_task = IsrTaskLSST(config=isr_config)

2219 with self.assertNoLogs(level=logging.WARNING):

2220 _ = isr_task.run(

2221 input_exp.clone(),

2222 bias=self.bias,

2223 dark=self.dark,

2224 crosstalk=self.crosstalk,

2225 ptc=self.ptc,

2226 linearizer=self.linearizer,

2227 defects=self.defects,

2228 deferredChargeCalib=self.cti,

2229 )

2230

2231 # Check that processing runs but warns if header keyword is None.

2232 isr_config.doCheckUnprocessableData = True

2233 isr_config.bssVoltageMinimum = 10.0

2234

2235 input_exp2 = input_exp.clone()

2236 input_exp2.metadata["HVBIAS"] = None

2237

2238 isr_task = IsrTaskLSST(config=isr_config)

2239 with self.assertLogs(level=logging.WARNING) as cm:

2240 _ = isr_task.run(

2241 input_exp2,

2242 bias=self.bias,

2243 dark=self.dark,

2244 crosstalk=self.crosstalk,

2245 ptc=self.ptc,

2246 linearizer=self.linearizer,

2247 defects=self.defects,

2248 deferredChargeCalib=self.cti,

2249 )

2250 self.assertEqual(len(cm.output), 1)

2251 self.assertIn("HV bias on HVBIAS not found in metadata", cm.output[0])

2252

2253 # Check that it raises.

2254 isr_config.doCheckUnprocessableData = True

2255 isr_config.bssVoltageMinimum = 10.0

2256

2257 isr_task = IsrTaskLSST(config=isr_config)

2258 with self.assertRaises(UnprocessableDataError):

2259 _ = isr_task.run(

2260 input_exp.clone(),

2261 bias=self.bias,

2262 dark=self.dark,

2263 crosstalk=self.crosstalk,

2264 ptc=self.ptc,

2265 linearizer=self.linearizer,

2266 defects=self.defects,

2267 deferredChargeCalib=self.cti,

2268 )

2269

2270 def get_mock_config_no_signal(self):

2271 """Get an IsrMockLSSTConfig with all signal set to False.

2272

2273 This will have all the electronic effects turned on (including

2274 2D bias).

2275 """

2276 mock_config = isrMockLSST.IsrMockLSSTConfig()

2277 mock_config.isTrimmed = False

2278 mock_config.doAddDark = False

2279 mock_config.doAddFlat = False

2280 mock_config.doAddFringe = False

2281 mock_config.doAddSky = False

2282 mock_config.doAddSource = False

2283

2284 mock_config.doAdd2DBias = True

2285 mock_config.doAddBias = True

2286 mock_config.doAddCrosstalk = True

2287 mock_config.doAddDeferredCharge = True

2288 mock_config.doAddBrightDefects = True

2289 mock_config.doAddClockInjectedOffset = True

2290 mock_config.doAddParallelOverscanRamp = True

2291 mock_config.doAddSerialOverscanRamp = True

2292 mock_config.doAddHighSignalNonlinearity = True

2293 mock_config.doApplyGain = True

2294 mock_config.doRoundAdu = True

2295

2296 # We always want to generate the image with these configs.

2297 mock_config.doGenerateImage = True

2298

2299 return mock_config

2300

2301 def get_mock_config_clean(self):

2302 """Get an IsrMockLSSTConfig trimmed with all electronic signatures

2303 turned off.

2304 """

2305 mock_config = isrMockLSST.IsrMockLSSTConfig()

2306 mock_config.doAddBias = False

2307 mock_config.doAdd2DBias = False

2308 mock_config.doAddClockInjectedOffset = False

2309 mock_config.doAddDark = False

2310 mock_config.doAddDarkNoiseOnly = False

2311 mock_config.doAddFlat = False

2312 mock_config.doAddFringe = False

2313 mock_config.doAddSky = False

2314 mock_config.doAddSource = False

2315 mock_config.doRoundAdu = False

2316 mock_config.doAddHighSignalNonlinearity = False

2317 mock_config.doApplyGain = False

2318 mock_config.doAddCrosstalk = False

2319 mock_config.doAddBrightDefects = False

2320 mock_config.doAddParallelOverscanRamp = False

2321 mock_config.doAddSerialOverscanRamp = False

2322

2323 mock_config.isTrimmed = True

2324 mock_config.doGenerateImage = True

2325

2326 return mock_config

2327

2328 def get_isr_config_minimal_corrections(self):

2329 """Get an IsrTaskLSSTConfig with minimal corrections.

2330 """

2331 isr_config = IsrTaskLSSTConfig()

2332 isr_config.bssVoltageMinimum = 0.0

2333 isr_config.ampNoiseThreshold = np.inf

2334 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2335 isr_config.doBias = False

2336 isr_config.doDark = False

2337 isr_config.doDeferredCharge = False

2338 isr_config.doLinearize = False

2339 isr_config.doCorrectGains = False

2340 isr_config.doCrosstalk = False

2341 isr_config.doDefect = False

2342 isr_config.doBrighterFatter = False

2343 isr_config.doFlat = False

2344 isr_config.doSaturation = False

2345 isr_config.doE2VEdgeBleedMask = False

2346 isr_config.doITLEdgeBleedMask = False

2347 isr_config.doSuspect = False

2348 # We override the leading/trailing to skip here because of the limited

2349 # size of the test camera overscan regions.

2350 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2351 defaultAmpConfig.doSerialOverscan = True

2352 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2353 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2354 defaultAmpConfig.doParallelOverscan = True

2355 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2356 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2357 # Our strong overscan slope in the tests requires an override.

2358 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2359

2360 isr_config.doAssembleCcd = True

2361 isr_config.crosstalk.doSubtrahendMasking = True

2362 isr_config.crosstalk.minPixelToMask = 1.0

2363

2364 return isr_config

2365

2366 def get_isr_config_electronic_corrections(self):

2367 """Get an IsrTaskLSSTConfig with electronic corrections.

2368

2369 This tests all the corrections that we support in the mocks/ISR.

2370 """

2371 isr_config = IsrTaskLSSTConfig()

2372 # We add these as appropriate in the tests.

2373 isr_config.doBias = False

2374 isr_config.doDark = False

2375 isr_config.doFlat = False

2376

2377 # These are the electronic effects the tests support (in addition

2378 # to overscan).

2379 isr_config.doCrosstalk = True

2380 isr_config.doDefect = True

2381 isr_config.doLinearize = True

2382 isr_config.doDeferredCharge = True

2383

2384 # This is False because it is only used in a single test case

2385 # as it takes a while to solve

2386 isr_config.doBrighterFatter = False

2387

2388 # These are the electronic effects we do not support in tests yet.

2389 isr_config.doCorrectGains = False

2390

2391 # We override the leading/trailing to skip here because of the limited

2392 # size of the test camera overscan regions.

2393 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2394 defaultAmpConfig.doSerialOverscan = True

2395 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2396 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2397 defaultAmpConfig.doParallelOverscan = True

2398 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2399 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2400 # Our strong overscan slope in the tests requires an override.

2401 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2402

2403 isr_config.doAssembleCcd = True

2404 isr_config.crosstalk.doSubtrahendMasking = True

2405 isr_config.crosstalk.minPixelToMask = 1.0

2406

2407 return isr_config

2408

2409 def get_non_defect_pixels(self, mask_origin):

2410 """Get the non-defect pixels to compare.

2411

2412 Parameters

2413 ----------

2414 mask_origin : `lsst.afw.image.MaskX`

2415 The origin mask (for shape and type).

2416

2417 Returns

2418 -------

2419 pix_x, pix_y : `tuple` [`np.ndarray`]

2420 x and y values of good pixels.

2421 """

2422 mask_temp = mask_origin.clone()

2423 mask_temp[:, :] = 0

2424

2425 for defect in self.defects:

2426 mask_temp[defect.getBBox()] = 1

2427

2428 return np.where(mask_temp.array == 0)

2429

2430 def _check_bad_column_crosstalk_correction(

2431 self,

2432 exp,

2433 nsigma_cut=5.0,

2434 ):

2435 """Test bad column crosstalk correction.

2436

2437 This includes possible provblems from parallel overscan

2438 crosstalk and gain mismatches.

2439

2440 The target amp is self.detector[0], "C:0,0".

2441

2442 Parameters

2443 ----------

2444 exp : `lsst.afw.image.Exposure`

2445 Input exposure.

2446 nsigma_cut : `float`, optional

2447 Number of sigma to check for outliers.

2448 """

2449 amp = self.detector[0]

2450 amp_image = exp[amp.getBBox()].image.array

2451 sigma = median_abs_deviation(amp_image.ravel(), scale="normal")

2452

2453 med = np.median(amp_image.ravel())

2454 self.assertLess(amp_image.max(), med + nsigma_cut*sigma)

2455 self.assertGreater(amp_image.min(), med - nsigma_cut*sigma)

2456

2457

2458class MemoryTester(lsst.utils.tests.MemoryTestCase):

2459 pass

2460

2461

2462def setup_module(module):

2463 lsst.utils.tests.init()

2464

2465

2466if __name__ == "__main__": 2466 ↛ 2467line 2466 didn't jump to line 2467 because the condition on line 2466 was never true

2467 lsst.utils.tests.init()

2468 unittest.main()