Coverage for tests/test_isrTaskLSST.py: 4%

2# LSST Data Management System

5# This product includes software developed by the

6# LSST Project (http://www.lsst.org/).

8# This program is free software: you can redistribute it and/or modify

9# it under the terms of the GNU General Public License as published by

10# the Free Software Foundation, either version 3 of the License, or

11# (at your option) any later version.

12#

13# This program is distributed in the hope that it will be useful,

14# but WITHOUT ANY WARRANTY; without even the implied warranty of

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

16# GNU General Public License for more details.

17#

18# You should have received a copy of the LSST License Statement and

19# the GNU General Public License along with this program. If not,

20# see <https://www.lsstcorp.org/LegalNotices/>.

21#

23import copy

24import unittest

25import numpy as np

26import logging

27import galsim

28from scipy.stats import median_abs_deviation

30import lsst.afw.geom as afwGeom

31import lsst.geom as geom

32from lsst.pipe.base import UnprocessableDataError

33import lsst.ip.isr.isrMockLSST as isrMockLSST

34import lsst.utils.tests

35from lsst.ip.isr.isrTaskLSST import (IsrTaskLSST, IsrTaskLSSTConfig)

36from lsst.ip.isr.crosstalk import CrosstalkCalib

37from lsst.ip.isr import PhotonTransferCurveDataset

38from lsst.ip.isr.vignette import maskVignettedRegion

41class IsrTaskLSSTTestCase(lsst.utils.tests.TestCase):

42 """Test IsrTaskLSST"""

43 def setUp(self):

44 mock = isrMockLSST.IsrMockLSST()

45 self.camera = mock.getCamera()

46 self.detector = self.camera[mock.config.detectorIndex]

47 self.namp = len(self.detector)

49 # Create adu (bootstrap) calibration frames

50 self.bias_adu = isrMockLSST.BiasMockLSST(adu=True).run()

51 self.dark_adu = isrMockLSST.DarkMockLSST(adu=True).run()

52 self.flat_adu = isrMockLSST.FlatMockLSST(adu=True).run()

53 self.flat_adu.metadata["FLATSRC"] = "DOME"

55 # Create calibration frames

56 self.bias = isrMockLSST.BiasMockLSST().run()

57 self.dark = isrMockLSST.DarkMockLSST().run()

58 self.flat = isrMockLSST.FlatMockLSST().run()

59 self.flat.metadata["FLATSRC"] = "DOME"

60 self.bf_kernel = isrMockLSST.BfKernelMockLSST().run()

61 self.cti = isrMockLSST.DeferredChargeMockLSST().run()

63 # The crosstalk ratios in isrMockLSST are in electrons.

64 self.crosstalk = CrosstalkCalib(nAmp=self.namp)

65 self.crosstalk.hasCrosstalk = True

66 self.crosstalk.coeffs = isrMockLSST.CrosstalkCoeffMockLSST().run()

67 for i, amp in enumerate(self.detector):

68 self.crosstalk.fitGains[i] = mock.config.gainDict[amp.getName()]

69 self.crosstalk.crosstalkRatiosUnits = "electron"

71 self.defects = isrMockLSST.DefectMockLSST().run()

73 amp_names = [x.getName() for x in self.detector.getAmplifiers()]

74 self.ptc = PhotonTransferCurveDataset(amp_names,

75 ptcFitType='DUMMY_PTC',

76 covMatrixSide=1)

78 self.saturation_adu = 100_000.0

80 # PTC records noise units in electron, same as the

81 # configuration parameter.

82 for amp_name in amp_names:

83 self.ptc.gain[amp_name] = mock.config.gainDict.get(amp_name, mock.config.gain)

84 self.ptc.noise[amp_name] = mock.config.readNoise

85 self.ptc.ptcTurnoff[amp_name] = self.saturation_adu

86 pre_level = mock.config.clockInjectedOffsetLevel

87 overscan_level = mock.config.biasLevel + (pre_level / self.ptc.gain[amp_name])

88 self.ptc.overscanMedian[amp_name] = overscan_level

89 # We set this sigma level very large because the

90 # noise characteristics of the mock image with

91 # a large serial gradient and small overscan region

92 # amplifies the noise vs LSSTCam.

93 self.ptc.overscanMedianSigma[amp_name] = 10.0

95 # TODO:

96 # self.cti = isrMockLSST.DeferredChargeMockLSST().run()

98 self.linearizer = isrMockLSST.LinearizerMockLSST().run()

99 # We currently only have high-signal non-linearity.

100 mock_config = self.get_mock_config_no_signal()

101 for amp_name in amp_names:

102 coeffs = self.linearizer.linearityCoeffs[amp_name]

103 centers, values = np.split(coeffs, 2)

104 values[centers < mock_config.highSignalNonlinearityThreshold] = 0.0

105 self.linearizer.linearityCoeffs[amp_name] = np.concatenate((centers, values))

106

107 def _check_applied_keys(self, metadata, isr_config):

108 """Check if the APPLIED keys have been set properly.

109

110 Parameters

111 ----------

112 metadata : `lsst.daf.base.PropertyList`

113 isr_config : `lsst.ip.isr.IsrTaskLSSTConfig`

114

115 """

116 key = "LSST ISR CROSSTALK APPLIED"

117 self.assertIn(key, metadata)

118 self.assertEqual(metadata[key], isr_config.doCrosstalk)

119

120 key = "LSST ISR OVERSCANLEVEL CHECKED"

121 self.assertIn(key, metadata)

122 self.assertEqual(metadata[key], np.isfinite(isr_config.serialOverscanMedianShiftSigmaThreshold))

123

124 key = "LSST ISR NOISE CHECKED"

125 self.assertIn(key, metadata)

126 self.assertEqual(metadata[key], np.isfinite(isr_config.ampNoiseThreshold))

127

128 key = "LSST ISR LINEARIZER APPLIED"

129 self.assertIn(key, metadata)

130 self.assertEqual(metadata[key], isr_config.doLinearize)

131

132 key = "LSST ISR CTI APPLIED"

133 self.assertIn(key, metadata)

134 self.assertEqual(metadata[key], isr_config.doDeferredCharge)

135

136 key = "LSST ISR BIAS APPLIED"

137 self.assertIn(key, metadata)

138 self.assertEqual(metadata[key], isr_config.doBias)

139

140 key = "LSST ISR DARK APPLIED"

141 self.assertIn(key, metadata)

142 self.assertEqual(metadata[key], isr_config.doDark)

143

144 key = "LSST ISR BF APPLIED"

145 self.assertIn(key, metadata)

146 self.assertEqual(metadata[key], isr_config.doBrighterFatter)

147

148 key = "LSST ISR FLAT APPLIED"

149 self.assertIn(key, metadata)

150 self.assertEqual(metadata[key], isr_config.doFlat)

151

152 if metadata[key]:

153 key2 = "LSST ISR FLAT SOURCE"

154 self.assertIn(key2, metadata)

155 self.assertEqual(metadata[key2], "DOME")

156

157 key = "LSST ISR DEFECTS APPLIED"

158 self.assertIn(key, metadata)

159 self.assertEqual(metadata[key], isr_config.doDefect)

160

161 def test_isrBootstrapBias(self):

162 """Test processing of a ``bootstrap`` bias frame.

163

164 This will be output with ADU units.

165 """

166 mock_config = self.get_mock_config_no_signal()

167

168 mock = isrMockLSST.IsrMockLSST(config=mock_config)

169 input_exp = mock.run()

170

171 isr_config = self.get_isr_config_minimal_corrections()

172 isr_config.doBootstrap = True

173 isr_config.doApplyGains = False

174 isr_config.doBias = True

175 isr_config.doCrosstalk = True

176

177 # Need to make sure we are not masking the negative variance

178 # pixels when directly comparing calibration images and

179 # calibration-corrected calibrations.

180 isr_config.maskNegativeVariance = False

181

182 isr_task = IsrTaskLSST(config=isr_config)

183 with self.assertLogs(level=logging.WARNING) as cm:

184 result = isr_task.run(

185 input_exp.clone(),

186 bias=self.bias_adu,

187 ptc=self.ptc,

188 crosstalk=self.crosstalk,

189 )

190 self.assertIn("Ignoring provided PTC", cm.output[0])

191 self._check_applied_keys(result.exposure.metadata, isr_config)

192

193 # Rerun without doing the bias correction.

194 isr_config.doBias = False

195 isr_task2 = IsrTaskLSST(config=isr_config)

196 with self.assertNoLogs(level=logging.WARNING):

197 result2 = isr_task2.run(input_exp.clone(), crosstalk=self.crosstalk)

198

199 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

200

201 self.assertLess(

202 np.mean(result.exposure.image.array[good_pixels]),

203 np.mean(result2.exposure.image.array[good_pixels]),

204 )

205 self.assertLess(

206 np.std(result.exposure.image.array[good_pixels]),

207 np.std(result2.exposure.image.array[good_pixels]),

208 )

209

210 delta = result2.exposure.image.array - result.exposure.image.array

211 self.assertFloatsAlmostEqual(delta[good_pixels], self.bias_adu.image.array[good_pixels], atol=1e-5)

212

213 metadata = result.exposure.metadata

214

215 key = "LSST ISR BOOTSTRAP"

216 self.assertIn(key, metadata)

217 self.assertEqual(metadata[key], True)

218

219 key = "LSST ISR UNITS"

220 self.assertIn(key, metadata)

221 self.assertEqual(metadata[key], "adu")

222

223 key = "LSST ISR READNOISE UNITS"

224 self.assertIn(key, metadata)

225 self.assertEqual(metadata[key], "electron")

226

227 for amp in self.detector:

228 amp_name = amp.getName()

229 key = f"LSST ISR GAIN {amp_name}"

230 self.assertIn(key, metadata)

231 self.assertEqual(metadata[key], 1.0)

232

233 self._check_bad_column_crosstalk_correction(result.exposure)

234

235 def test_isrBootstrapDark(self):

236 """Test processing of a ``bootstrap`` dark frame.

237

238 This will be output with ADU units.

239 """

240 mock_config = self.get_mock_config_no_signal()

241 mock_config.doAddDark = True

242

243 mock = isrMockLSST.IsrMockLSST(config=mock_config)

244 input_exp = mock.run()

245

246 isr_config = self.get_isr_config_minimal_corrections()

247 isr_config.doBootstrap = True

248 isr_config.doApplyGains = False

249 isr_config.doBias = True

250 isr_config.doDark = True

251 isr_config.maskNegativeVariance = False

252 isr_config.doCrosstalk = True

253

254 isr_task = IsrTaskLSST(config=isr_config)

255 with self.assertLogs(level=logging.WARNING) as cm:

256 result = isr_task.run(

257 input_exp.clone(),

258 bias=self.bias_adu,

259 dark=self.dark_adu,

260 ptc=self.ptc,

261 crosstalk=self.crosstalk,

262 )

263 self.assertIn("Ignoring provided PTC", cm.output[0])

264 self._check_applied_keys(result.exposure.metadata, isr_config)

265

266 # Rerun without doing the dark correction.

267 isr_config.doDark = False

268 isr_task2 = IsrTaskLSST(config=isr_config)

269 with self.assertNoLogs(level=logging.WARNING):

270 result2 = isr_task2.run(input_exp.clone(), bias=self.bias_adu, crosstalk=self.crosstalk)

271

272 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

273

274 self.assertLess(

275 np.mean(result.exposure.image.array[good_pixels]),

276 np.mean(result2.exposure.image.array[good_pixels]),

277 )

278

279 delta = result2.exposure.image.array - result.exposure.image.array

280 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

281 self.assertFloatsAlmostEqual(

282 delta[good_pixels],

283 self.dark_adu.image.array[good_pixels] * exp_time,

284 atol=1e-5,

285 )

286

287 metadata = result.exposure.metadata

288

289 key = "LSST ISR BOOTSTRAP"

290 self.assertIn(key, metadata)

291 self.assertEqual(metadata[key], True)

292

293 key = "LSST ISR UNITS"

294 self.assertIn(key, metadata)

295 self.assertEqual(metadata[key], "adu")

296

297 self._check_bad_column_crosstalk_correction(result.exposure)

298

299 def test_isrBootstrapFlat(self):

300 """Test processing of a ``bootstrap`` flat frame.

301

302 This will be output with ADU units.

303 """

304 mock_config = self.get_mock_config_no_signal()

305 mock_config.doAddDark = True

306 mock_config.doAddFlat = True

307 # The doAddSky option adds the equivalent of flat-field flux.

308 mock_config.doAddSky = True

309

310 mock = isrMockLSST.IsrMockLSST(config=mock_config)

311 input_exp = mock.run()

312

313 isr_config = self.get_isr_config_minimal_corrections()

314 isr_config.doBootstrap = True

315 isr_config.doApplyGains = False

316 isr_config.doBias = True

317 isr_config.doDark = True

318 isr_config.doFlat = True

319 isr_config.maskNegativeVariance = False

320 isr_config.doCrosstalk = True

321

322 isr_task = IsrTaskLSST(config=isr_config)

323 with self.assertLogs(level=logging.WARNING) as cm:

324 result = isr_task.run(

325 input_exp.clone(),

326 bias=self.bias_adu,

327 dark=self.dark_adu,

328 flat=self.flat_adu,

329 ptc=self.ptc,

330 crosstalk=self.crosstalk,

331 )

332 self.assertIn("Ignoring provided PTC", cm.output[0])

333 self._check_applied_keys(result.exposure.metadata, isr_config)

334

335 # Rerun without doing the flat correction.

336 isr_config.doFlat = False

337 isr_task2 = IsrTaskLSST(config=isr_config)

338 with self.assertNoLogs(level=logging.WARNING):

339 result2 = isr_task2.run(

340 input_exp.clone(),

341 bias=self.bias_adu,

342 dark=self.dark_adu,

343 crosstalk=self.crosstalk,

344 )

345

346 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

347

348 # Applying the flat will increase the counts.

349 self.assertGreater(

350 np.mean(result.exposure.image.array[good_pixels]),

351 np.mean(result2.exposure.image.array[good_pixels]),

352 )

353 # And will decrease the sigma.

354 self.assertLess(

355 np.std(result.exposure.image.array[good_pixels]),

356 np.std(result2.exposure.image.array[good_pixels]),

357 )

358

359 ratio = result2.exposure.image.array / result.exposure.image.array

360 self.assertFloatsAlmostEqual(ratio[good_pixels], self.flat_adu.image.array[good_pixels], atol=1e-5)

361

362 # Test the variance plane in the case of adu units.

363 # The expected variance starts with the image array.

364 expected_variance = result.exposure.image.clone()

365 # We have to remove the flat-fielding from the image pixels.

366 expected_variance.array *= self.flat_adu.image.array

367 # And add in the bias variance.

368 expected_variance.array += self.bias_adu.variance.array

369 # And add in the scaled dark variance.

370 scale = result.exposure.visitInfo.darkTime / self.dark_adu.visitInfo.darkTime

371 expected_variance.array += scale**2. * self.dark_adu.variance.array

372 # And add the gain and read noise (in electron) per amp.

373 for amp in self.detector:

374 # We need to use the gain and read noise from the header

375 # because these are bootstraps.

376 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

377 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

378

379 expected_variance[amp.getBBox()].array /= gain

380 # Read noise is always in electron units, but since this is a

381 # bootstrap, the gain is 1.0.

382 expected_variance[amp.getBBox()].array += (read_noise/gain)**2.

383

384 # And apply the full formula for dividing by the flat with variance.

385 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

386 unflat_image_array = result.exposure.image.array * self.flat_adu.image.array

387 expected_variance.array = ((unflat_image_array**2. * self.flat_adu.variance.array

388 + self.flat_adu.image.array**2. * expected_variance.array)

389 / self.flat_adu.image.array**4.)

390

391 self.assertFloatsAlmostEqual(

392 result.exposure.variance.array[good_pixels],

393 expected_variance.array[good_pixels],

394 rtol=1e-6,

395 )

396

397 metadata = result.exposure.metadata

398

399 key = "LSST ISR BOOTSTRAP"

400 self.assertIn(key, metadata)

401 self.assertEqual(metadata[key], True)

402

403 key = "LSST ISR UNITS"

404 self.assertIn(key, metadata)

405 self.assertEqual(metadata[key], "adu")

406

407 self._check_bad_column_crosstalk_correction(result.exposure)

408

409 def test_isrBootstrapAndRegularFlat(self):

410 """Test that bootstrap and "regular" flat processing are equivalent."""

411 # This is a test for DM-52684, for the linearizer units.

412

413 mock_config = self.get_mock_config_no_signal()

414 mock_config.doAddDark = True

415 mock_config.doAddFlat = True

416 # The doAddSky option adds the equivalent of flat-field flux.

417 mock_config.doAddSky = True

418 # We set the sky/flat level to a range where the "high signal"

419 # non-linearity has kicked in.

420 mock_config.skyLevel = 40000.0

421

422 mock = isrMockLSST.IsrMockLSST(config=mock_config)

423 input_exp = mock.run()

424

425 # First config is with linearizer in bootstrap mode.

426

427 isr_config_bootstrap = self.get_isr_config_minimal_corrections()

428 isr_config_bootstrap.doBootstrap = True

429 isr_config_bootstrap.doApplyGains = False

430 isr_config_bootstrap.doLinearize = True

431 isr_config_bootstrap.doBias = False

432 isr_config_bootstrap.doDark = False

433 isr_config_bootstrap.doFlat = False

434 isr_config_bootstrap.maskNegativeVariance = False

435 isr_config_bootstrap.doCrosstalk = True

436

437 isr_task_bootstrap = IsrTaskLSST(config=isr_config_bootstrap)

438 with self.assertNoLogs(level=logging.WARNING):

439 result = isr_task_bootstrap.run(

440 input_exp.clone(),

441 crosstalk=self.crosstalk,

442 linearizer=self.linearizer,

443 )

444

445 exp_bootstrap = result.exposure

446

447 # Apply the gains (after the linearization); this is

448 # similar to processing in PTC building.

449 for amp in self.detector:

450 exp_bootstrap[amp.getBBox()].image.array *= self.ptc.gain[amp.getName()]

451

452 # Run again with non-bootstrap mode.

453 isr_config = self.get_isr_config_minimal_corrections()

454 isr_config.doBootstrap = False

455 isr_config.doApplyGains = True

456 isr_config.doLinearize = True

457 isr_config.doBias = False

458 isr_config.doDark = False

459 isr_config.doFlat = False

460 isr_config.maskNegativeVariance = False

461 isr_config.doCrosstalk = True

462

463 isr_task = IsrTaskLSST(config=isr_config)

464 with self.assertNoLogs(level=logging.WARNING):

465 result = isr_task.run(

466 input_exp.clone(),

467 crosstalk=self.crosstalk,

468 linearizer=self.linearizer,

469 ptc=self.ptc,

470 )

471

472 exp = result.exposure

473

474 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

475

476 delta = exp.image.array - exp_bootstrap.image.array

477

478 self.assertFloatsAlmostEqual(delta[good_pixels], 0.0, atol=1e-2)

479

480 def test_isrBias(self):

481 """Test processing of a bias frame."""

482 mock_config = self.get_mock_config_no_signal()

483

484 mock = isrMockLSST.IsrMockLSST(config=mock_config)

485 input_exp = mock.run()

486

487 isr_config = self.get_isr_config_electronic_corrections()

488 isr_config.doBias = True

489 # We do not do defect correction when processing biases.

490 isr_config.doDefect = False

491 isr_config.maskNegativeVariance = False

492

493 isr_task = IsrTaskLSST(config=isr_config)

494 with self.assertNoLogs(level=logging.WARNING):

495 result = isr_task.run(

496 input_exp.clone(),

497 bias=self.bias,

498 crosstalk=self.crosstalk,

499 ptc=self.ptc,

500 linearizer=self.linearizer,

501 deferredChargeCalib=self.cti,

502 )

503 self._check_applied_keys(result.exposure.metadata, isr_config)

504

505 # Rerun without doing the bias correction.

506 isr_config.doBias = False

507 isr_task2 = IsrTaskLSST(config=isr_config)

508 with self.assertNoLogs(level=logging.WARNING):

509 result2 = isr_task2.run(

510 input_exp.clone(),

511 crosstalk=self.crosstalk,

512 ptc=self.ptc,

513 linearizer=self.linearizer,

514 deferredChargeCalib=self.cti,

515 )

516

517 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

518

519 self.assertLess(

520 np.mean(result.exposure.image.array[good_pixels]),

521 np.mean(result2.exposure.image.array[good_pixels]),

522 )

523

524 self.assertLess(

525 np.std(result.exposure.image.array[good_pixels]),

526 np.std(result2.exposure.image.array[good_pixels]),

527 )

528

529 # Confirm that it is flat with an arbitrary cutoff that depends

530 # on the read noise.

531 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

532

533 delta = result2.exposure.image.array - result.exposure.image.array

534

535 # Note that the bias is made with bias noise + read noise, and

536 # the image contains read noise.

537 self.assertFloatsAlmostEqual(

538 delta[good_pixels],

539 self.bias.image.array[good_pixels],

540 atol=1e-5,

541 )

542

543 self._check_bad_column_crosstalk_correction(result.exposure)

544

545 def test_isrBiasNoParallelOscanCorrection(self):

546 """Test processing of a bias frame with parallel

547 overscan correction turned off."""

548 mock_config = self.get_mock_config_no_signal()

549 mock_config.doAddParallelOverscanRamp = False

550

551 mock = isrMockLSST.IsrMockLSST(config=mock_config)

552 input_exp = mock.run()

553

554 isr_config = self.get_isr_config_electronic_corrections()

555

556 # Turn off the parallel overscan correction

557 amp_oscan_config = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

558 amp_oscan_config.doParallelOverscan = False

559 isr_config.doBias = True

560

561 # We do not do defect correction when processing biases.

562 isr_config.doDefect = False

563 isr_config.maskNegativeVariance = False

564

565 isr_task = IsrTaskLSST(config=isr_config)

566 with self.assertNoLogs(level=logging.WARNING):

567 result = isr_task.run(

568 input_exp.clone(),

569 bias=self.bias,

570 crosstalk=self.crosstalk,

571 ptc=self.ptc,

572 linearizer=self.linearizer,

573 deferredChargeCalib=self.cti,

574 )

575 self._check_applied_keys(result.exposure.metadata, isr_config)

576

577 # Rerun without doing the bias correction.

578 isr_config.doBias = False

579 isr_task2 = IsrTaskLSST(config=isr_config)

580 with self.assertNoLogs(level=logging.WARNING):

581 result2 = isr_task2.run(

582 input_exp.clone(),

583 crosstalk=self.crosstalk,

584 ptc=self.ptc,

585 linearizer=self.linearizer,

586 deferredChargeCalib=self.cti,

587 )

588

589 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

590

591 self.assertLess(

592 np.mean(result.exposure.image.array[good_pixels]),

593 np.mean(result2.exposure.image.array[good_pixels]),

594 )

595

596 self.assertLess(

597 np.std(result.exposure.image.array[good_pixels]),

598 np.std(result2.exposure.image.array[good_pixels]),

599 )

600

601 # Confirm that it is flat with an arbitrary cutoff that depends

602 # on the read noise.

603 self.assertLess(np.std(result.exposure.image.array[good_pixels]), 2.0*mock_config.readNoise)

604

605 delta = result2.exposure.image.array - result.exposure.image.array

606

607 # Note that the bias is made with bias noise + read noise, and

608 # the image contains read noise.

609 self.assertFloatsAlmostEqual(

610 delta[good_pixels],

611 self.bias.image.array[good_pixels],

612 atol=1e-5,

613 )

614

615 self._check_bad_column_crosstalk_correction(result.exposure)

616

617 def test_isrBiasCti(self):

618 """Test over-correction of bias amp edges from prescan."""

619 mock_config = self.get_mock_config_no_signal()

620 mock_config.doAddBrightDefects = False

621

622 mock = isrMockLSST.IsrMockLSST(config=mock_config)

623 input_exp = mock.run()

624

625 isr_config = self.get_isr_config_electronic_corrections()

626 # We do not do defect correction when processing biases.

627 isr_config.doDefect = False

628 isr_config.maskNegativeVariance = False

629

630 isr_task = IsrTaskLSST(config=isr_config)

631 with self.assertNoLogs(level=logging.WARNING):

632 result = isr_task.run(

633 input_exp.clone(),

634 crosstalk=self.crosstalk,

635 ptc=self.ptc,

636 linearizer=self.linearizer,

637 deferredChargeCalib=self.cti,

638 )

639 self._check_applied_keys(result.exposure.metadata, isr_config)

640

641 # Rerun without doing the CTI correction

642 isr_config.doDeferredCharge = False

643

644 isr_task2 = IsrTaskLSST(config=isr_config)

645 with self.assertNoLogs(level=logging.WARNING):

646 result2 = isr_task2.run(

647 input_exp.clone(),

648 crosstalk=self.crosstalk,

649 ptc=self.ptc,

650 linearizer=self.linearizer,

651 )

652

653 # This confirms that things are *close* to equal. Unfortunately,

654 # the unusual camera geometry in the test camera doesn't completely

655 # zero out the prescan pixels, so we need a higher threshold.

656 std_delta = np.std(result2.exposure.image.array - result.exposure.image.array)

657 self.assertLess(std_delta, 0.15)

658

659 def test_isrDark(self):

660 """Test processing of a dark frame."""

661 mock_config = self.get_mock_config_no_signal()

662 mock_config.doAddDark = True

663 # We turn off the bad parallel overscan column because it does

664 # add more noise to that region.

665 mock_config.doAddBadParallelOverscanColumn = False

666

667 mock = isrMockLSST.IsrMockLSST(config=mock_config)

668 input_exp = mock.run()

669

670 isr_config = self.get_isr_config_electronic_corrections()

671 isr_config.doBias = True

672 isr_config.doDark = True

673 # We do not do defect correction when processing darks.

674 isr_config.doDefect = False

675 isr_config.maskNegativeVariance = False

676

677 isr_task = IsrTaskLSST(config=isr_config)

678 with self.assertNoLogs(level=logging.WARNING):

679 result = isr_task.run(

680 input_exp.clone(),

681 bias=self.bias,

682 dark=self.dark,

683 crosstalk=self.crosstalk,

684 ptc=self.ptc,

685 linearizer=self.linearizer,

686 deferredChargeCalib=self.cti,

687 )

688 self._check_applied_keys(result.exposure.metadata, isr_config)

689

690 # Rerun without doing the dark correction.

691 isr_config.doDark = False

692 isr_task2 = IsrTaskLSST(config=isr_config)

693 with self.assertNoLogs(level=logging.WARNING):

694 result2 = isr_task2.run(

695 input_exp.clone(),

696 bias=self.bias,

697 crosstalk=self.crosstalk,

698 ptc=self.ptc,

699 linearizer=self.linearizer,

700 deferredChargeCalib=self.cti,

701 )

702

703 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

704

705 self.assertLess(

706 np.mean(result.exposure.image.array[good_pixels]),

707 np.mean(result2.exposure.image.array[good_pixels]),

708 )

709 # The mock dark has no noise, so these should be equal.

710 self.assertFloatsAlmostEqual(

711 np.std(result.exposure.image.array[good_pixels]),

712 np.std(result2.exposure.image.array[good_pixels]),

713 atol=1e-12,

714 )

715

716 # This is a somewhat arbitrary comparison that includes a fudge

717 # factor for the extra noise from the overscan subtraction.

718 self.assertLess(

719 np.std(result.exposure.image.array[good_pixels]),

720 1.6*np.sqrt(mock_config.darkRate*mock_config.expTime + mock_config.readNoise),

721 )

722

723 delta = result2.exposure.image.array - result.exposure.image.array

724 exp_time = input_exp.getInfo().getVisitInfo().getExposureTime()

725

726 # Allow <3 pixels to fail this test due to rounding error

727 # if doRoundAdu=True

728 diff = np.abs(delta[good_pixels] - self.dark.image.array[good_pixels] * exp_time)

729 self.assertLess(np.count_nonzero(diff >= 1e-12), 3)

730

731 self._check_bad_column_crosstalk_correction(result.exposure)

732

733 def test_isrFlat(self):

734 """Test processing of a flat frame."""

735 mock_config = self.get_mock_config_no_signal()

736 mock_config.doAddDark = True

737 mock_config.doAddFlat = True

738 # The doAddSky option adds the equivalent of flat-field flux.

739 mock_config.doAddSky = True

740

741 mock = isrMockLSST.IsrMockLSST(config=mock_config)

742 input_exp = mock.run()

743

744 isr_config = self.get_isr_config_electronic_corrections()

745 isr_config.doBias = True

746 isr_config.doDark = True

747 isr_config.doFlat = True

748 # Although we usually do not do defect interpolation when

749 # processing flats, this is a good test of the interpolation.

750 isr_config.doDefect = True

751 isr_config.maskNegativeVariance = False

752

753 isr_task = IsrTaskLSST(config=isr_config)

754 with self.assertNoLogs(level=logging.WARNING):

755 result = isr_task.run(

756 input_exp.clone(),

757 bias=self.bias,

758 dark=self.dark,

759 flat=self.flat,

760 crosstalk=self.crosstalk,

761 defects=self.defects,

762 ptc=self.ptc,

763 linearizer=self.linearizer,

764 deferredChargeCalib=self.cti,

765 )

766 self._check_applied_keys(result.exposure.metadata, isr_config)

767

768 # Rerun without doing the bias correction.

769 isr_config.doFlat = False

770 isr_task2 = IsrTaskLSST(config=isr_config)

771 with self.assertNoLogs(level=logging.WARNING):

772 result2 = isr_task2.run(

773 input_exp.clone(),

774 bias=self.bias,

775 dark=self.dark,

776 crosstalk=self.crosstalk,

777 defects=self.defects,

778 ptc=self.ptc,

779 linearizer=self.linearizer,

780 deferredChargeCalib=self.cti,

781 )

782

783 # With defect correction, we should not need to filter out bad

784 # pixels.

785

786 # Applying the flat will increase the counts.

787 self.assertGreater(

788 np.mean(result.exposure.image.array),

789 np.mean(result2.exposure.image.array),

790 )

791 # And will decrease the sigma.

792 self.assertLess(

793 np.std(result.exposure.image.array),

794 np.std(result2.exposure.image.array),

795 )

796

797 # Check that the resulting image is approximately flat.

798 # In particular that the noise is consistent with sky + margin.

799 self.assertLess(np.std(result.exposure.image.array), np.sqrt(mock_config.skyLevel) + 3.0)

800

801 # Generate a flat without any defects for comparison

802 # (including interpolation)

803 flat_nodefect_config = isrMockLSST.FlatMockLSST.ConfigClass()

804 flat_nodefect_config.doAddBrightDefects = False

805 flat_nodefects = isrMockLSST.FlatMockLSST(config=flat_nodefect_config).run()

806

807 ratio = result2.exposure.image.array / result.exposure.image.array

808 self.assertFloatsAlmostEqual(ratio, flat_nodefects.image.array, atol=1e-4)

809

810 self._check_bad_column_crosstalk_correction(result.exposure)

811

812 def test_isrNoise(self):

813 """Test the recorded noise and gain in the metadata."""

814 mock_config = self.get_mock_config_no_signal()

815 # Remove the overscan scale so that the only variation

816 # in the overscan is from the read noise.

817 mock_config.overscanScale = 0.0

818

819 mock = isrMockLSST.IsrMockLSST(config=mock_config)

820 input_exp = mock.run()

821

822 isr_config = self.get_isr_config_electronic_corrections()

823 isr_config.doBias = True

824 # We do not do defect correction when processing biases.

825 isr_config.doDefect = False

826 isr_config.maskNegativeVariance = False

827

828 isr_task = IsrTaskLSST(config=isr_config)

829 with self.assertNoLogs(level=logging.WARNING):

830 result = isr_task.run(

831 input_exp.clone(),

832 bias=self.bias,

833 crosstalk=self.crosstalk,

834 ptc=self.ptc,

835 deferredChargeCalib=self.cti,

836 linearizer=self.linearizer,

837 )

838 self._check_applied_keys(result.exposure.metadata, isr_config)

839

840 metadata = result.exposure.metadata

841

842 for amp in self.detector:

843 # The overscan noise is always in adu and the readnoise is always

844 # in electron.

845 gain = result.exposure.metadata[f"LSST ISR GAIN {amp.getName()}"]

846 read_noise = result.exposure.metadata[f"LSST ISR READNOISE {amp.getName()}"]

847

848 # Check that the gain and read noise are consistent with the

849 # values stored in the PTC.

850 self.assertEqual(gain, self.ptc.gain[amp.getName()])

851 self.assertEqual(read_noise, self.ptc.noise[amp.getName()])

852

853 key = f"LSST ISR OVERSCAN RESIDUAL SERIAL STDEV {amp.getName()}"

854 self.assertIn(key, metadata)

855

856 # Determine if the residual serial overscan stddev is consistent

857 # with the PTC readnoise within 3xstandard error.

858 serial_overscan_area = amp.getRawHorizontalOverscanBBox().area

859 self.assertFloatsAlmostEqual(

860 metadata[key] * gain,

861 read_noise,

862 atol=3*read_noise / np.sqrt(serial_overscan_area),

863 )

864

865 def test_isrBrighterFatter(self):

866 """Test processing of a flat frame."""

867 # Image with brighter-fatter correction

868 mock_config = self.get_mock_config_no_signal()

869 mock_config.isTrimmed = False

870 mock_config.doAddDark = True

871 mock_config.doAddFlat = True

872 mock_config.doAddSky = True

873 mock_config.doAddSource = True

874 mock_config.sourceFlux = [75000.0]

875 mock_config.doAddBrighterFatter = True

876

877 mock = isrMockLSST.IsrMockLSST(config=mock_config)

878 input_exp = mock.run()

879

880 isr_config = self.get_isr_config_electronic_corrections()

881 isr_config.doBias = True

882 isr_config.doDark = True

883 isr_config.doFlat = True

884 isr_config.doBrighterFatter = True

885

886 isr_task = IsrTaskLSST(config=isr_config)

887 with self.assertNoLogs(level=logging.WARNING):

888 result = isr_task.run(

889 input_exp.clone(),

890 bias=self.bias,

891 dark=self.dark,

892 flat=self.flat,

893 deferredChargeCalib=self.cti,

894 crosstalk=self.crosstalk,

895 defects=self.defects,

896 ptc=self.ptc,

897 linearizer=self.linearizer,

898 bfKernel=self.bf_kernel,

899 )

900 self._check_applied_keys(result.exposure.metadata, isr_config)

901

902 mock_config = self.get_mock_config_no_signal()

903 mock_config.isTrimmed = False

904 mock_config.doAddDark = True

905 mock_config.doAddFlat = True

906 mock_config.doAddSky = True

907 mock_config.doAddSource = True

908 mock_config.sourceFlux = [75000.0]

909 mock_config.doAddBrighterFatter = False

910

911 mock = isrMockLSST.IsrMockLSST(config=mock_config)

912 input_truth = mock.run()

913

914 isr_config = self.get_isr_config_electronic_corrections()

915 isr_config.doBias = True

916 isr_config.doDark = True

917 isr_config.doFlat = True

918 isr_config.doBrighterFatter = False

919

920 isr_task = IsrTaskLSST(config=isr_config)

921 with self.assertNoLogs(level=logging.WARNING):

922 result_truth = isr_task.run(

923 input_truth.clone(),

924 bias=self.bias,

925 dark=self.dark,

926 flat=self.flat,

927 deferredChargeCalib=self.cti,

928 crosstalk=self.crosstalk,

929 defects=self.defects,

930 ptc=self.ptc,

931 linearizer=self.linearizer,

932 bfKernel=self.bf_kernel,

933 )

934

935 # Measure the source size in the BF-corrected image.

936 # The injected source is a Gaussian with 3.0px

937 image = galsim.ImageF(result.exposure.image.array)

938 image_truth = galsim.ImageF(result_truth.exposure.image.array)

939 source_centroid = galsim.PositionD(mock_config.sourceX[0], mock_config.sourceY[0])

940 hsm_result = galsim.hsm.FindAdaptiveMom(image, guess_centroid=source_centroid, strict=False)

941 hsm_result_truth = galsim.hsm.FindAdaptiveMom(image_truth, guess_centroid=source_centroid,

942 strict=False)

943 measured_sigma = hsm_result.moments_sigma

944 true_sigma = hsm_result_truth.moments_sigma

945 self.assertFloatsAlmostEqual(measured_sigma, true_sigma, rtol=3e-3)

946

947 # Check that the variance in an amp far away from the

948 # source is expected. The source is in amp 0; this will

949 # check the variation in neighboring amp 1

950 test_amp_bbox = result.exposure.detector.getAmplifiers()[1].getBBox()

951 n_pixels = test_amp_bbox.getArea()

952 stdev = np.std(result.exposure[test_amp_bbox].image.array)

953 stdev_truth = np.std(result_truth.exposure[test_amp_bbox].image.array)

954 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

955

956 # Check that the variance in the amp with a defect is

957 # unchanged as a result of applying the BF correction after

958 # interpolating. The defect was added to amplifier 2.

959 test_amp_bbox = result.exposure.detector.getAmplifiers()[2].getBBox()

960 good_pixels = self.get_non_defect_pixels(result.exposure[test_amp_bbox].mask)

961 stdev = np.nanstd(result.exposure[test_amp_bbox].image.array[good_pixels])

962 stdev_truth = np.nanstd(result_truth.exposure[test_amp_bbox].image.array[good_pixels])

963 self.assertFloatsAlmostEqual(stdev, stdev_truth, atol=3*stdev_truth/np.sqrt(n_pixels))

964

965 # Check that BF has converged in the expected number of iterations.

966 metadata = result.exposure.metadata

967 key = "LSST ISR BF ITERS"

968 self.assertIn(key, metadata)

969 self.assertEqual(metadata[key], 2)

970

971 def test_isrSkyImage(self):

972 """Test processing of a sky image."""

973 mock_config = self.get_mock_config_no_signal()

974 mock_config.doAddDark = True

975 mock_config.doAddFlat = True

976 # Set this to False until we have fringe correction.

977 mock_config.doAddFringe = False

978 mock_config.doAddSky = True

979 mock_config.doAddSource = True

980

981 mock = isrMockLSST.IsrMockLSST(config=mock_config)

982 input_exp = mock.run()

983

984 isr_config = self.get_isr_config_electronic_corrections()

985 isr_config.doBias = True

986 isr_config.doDark = True

987 isr_config.doFlat = True

988

989 isr_task = IsrTaskLSST(config=isr_config)

990 with self.assertNoLogs(level=logging.WARNING):

991 result = isr_task.run(

992 input_exp.clone(),

993 bias=self.bias,

994 dark=self.dark,

995 flat=self.flat,

996 crosstalk=self.crosstalk,

997 defects=self.defects,

998 ptc=self.ptc,

999 linearizer=self.linearizer,

1000 deferredChargeCalib=self.cti,

1001 )

1002 self._check_applied_keys(result.exposure.metadata, isr_config)

1003

1004 # Confirm that the output has the defect line as bad.

1005 sat_val = 2**result.exposure.mask.getMaskPlane("BAD")

1006 for defect in self.defects:

1007 np.testing.assert_array_equal(

1008 result.exposure.mask[defect.getBBox()].array & sat_val,

1009 sat_val,

1010 )

1011

1012 clean_mock_config = self.get_mock_config_clean()

1013 # We want the dark noise for more direct comparison.

1014 clean_mock_config.doAddDarkNoiseOnly = True

1015 clean_mock_config.doAddSky = True

1016 clean_mock_config.doAddSource = True

1017

1018 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1019 clean_exp = clean_mock.run()

1020

1021 delta = result.exposure.image.array - clean_exp.image.array

1022

1023 good_pixels = self.get_non_defect_pixels(result.exposure.mask)

1024

1025 # We compare the good pixels in the entirety.

1026 self.assertLess(np.std(delta[good_pixels]), 5.0)

1027 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1028

1029 # Make sure the corrected image is overall consistent with the

1030 # straight image.

1031 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1032

1033 # And overall where the interpolation is a bit worse but

1034 # the statistics are still fine.

1035 self.assertLess(np.std(delta), 5.5)

1036

1037 metadata = result.exposure.metadata

1038

1039 key = "LSST ISR BOOTSTRAP"

1040 self.assertIn(key, metadata)

1041 self.assertEqual(metadata[key], False)

1042

1043 key = "LSST ISR UNITS"

1044 self.assertIn(key, metadata)

1045 self.assertEqual(metadata[key], "electron")

1046

1047 for amp in self.detector:

1048 amp_name = amp.getName()

1049 key = f"LSST ISR GAIN {amp_name}"

1050 self.assertIn(key, metadata)

1051 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1052 key = f"LSST ISR READNOISE {amp_name}"

1053 self.assertIn(key, metadata)

1054 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1055 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1056 self.assertIn(key, metadata)

1057 self.assertEqual(metadata[key], self.saturation_adu * gain)

1058 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1059 self.assertIn(key, metadata)

1060 self.assertEqual(metadata[key], self.saturation_adu * gain)

1061

1062 # Test the variance plane in the case of electron units.

1063 # The expected variance starts with the image array.

1064 expected_variance = result.exposure.image.clone()

1065 # We have to remove the flat-fielding from the image pixels.

1066 expected_variance.array *= self.flat.image.array

1067 # And add in the bias variance.

1068 expected_variance.array += self.bias.variance.array

1069 # And add in the scaled dark variance.

1070 scale = result.exposure.visitInfo.darkTime / self.dark.visitInfo.darkTime

1071 expected_variance.array += scale**2. * self.dark.variance.array

1072 # And add the read noise (in electrons) per amp.

1073 for amp in self.detector:

1074 gain = self.ptc.gain[amp.getName()]

1075 read_noise = self.ptc.noise[amp.getName()]

1076

1077 # The image, read noise, and variance plane should all have

1078 # units of electrons, electrons, and electrons^2.

1079 expected_variance[amp.getBBox()].array += read_noise**2.

1080

1081 # And apply the full formula for dividing by the flat with variance.

1082 # See https://github.com/lsst/afw/blob/efa07fa68475fbe12f8f16df245a99ba3042166d/src/image/MaskedImage.cc#L353-L358 # noqa: E501, W505

1083 unflat_image_array = result.exposure.image.array * self.flat.image.array

1084 expected_variance.array = ((unflat_image_array**2. * self.flat.variance.array

1085 + self.flat.image.array**2. * expected_variance.array)

1086 / self.flat.image.array**4.)

1087

1088 self.assertFloatsAlmostEqual(

1089 result.exposure.variance.array[good_pixels],

1090 expected_variance.array[good_pixels],

1091 rtol=1e-6,

1092 )

1093

1094 def test_isrSkyImageSaturated(self):

1095 """Test processing of a sky image.

1096

1097 This variation uses saturated pixels instead of defects.

1098

1099 This additionally tests the gain config override.

1100 """

1101 mock_config = self.get_mock_config_no_signal()

1102 mock_config.doAddDark = True

1103 mock_config.doAddFlat = True

1104 # Set this to False until we have fringe correction.

1105 mock_config.doAddFringe = False

1106 mock_config.doAddSky = True

1107 mock_config.doAddSource = True

1108 mock_config.brightDefectLevel = 170_000.0 # Above saturation.

1109

1110 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1111 input_exp = mock.run()

1112

1113 isr_config = self.get_isr_config_electronic_corrections()

1114 isr_config.doBias = True

1115 isr_config.doDark = True

1116 isr_config.doFlat = True

1117 # We turn off defect masking to test the saturation code.

1118 # However, the same pixels below should be masked/interpolated.

1119 isr_config.doDefect = False

1120

1121 # Use a config override saturation value, confirm it is picked up.

1122 saturation_level = self.saturation_adu * 1.05

1123

1124 # This code will set the gain of one amp to the same as the ptc

1125 # value, and we will check that it is logged and used but the

1126 # results should be the same.

1127 detectorConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector)

1128 detectorConfig.defaultAmpConfig.saturation = saturation_level

1129 overscanAmpConfig = copy.copy(detectorConfig.defaultAmpConfig)

1130 overscanAmpConfig.gain = self.ptc.gain[self.detector[1].getName()]

1131 detectorConfig.ampRules[self.detector[1].getName()] = overscanAmpConfig

1132

1133 isr_task = IsrTaskLSST(config=isr_config)

1134 with self.assertLogs(level=logging.WARNING) as cm:

1135 result = isr_task.run(

1136 input_exp.clone(),

1137 bias=self.bias,

1138 dark=self.dark,

1139 flat=self.flat,

1140 deferredChargeCalib=self.cti,

1141 crosstalk=self.crosstalk,

1142 defects=self.defects,

1143 ptc=self.ptc,

1144 linearizer=self.linearizer,

1145 )

1146 self.assertIn("Overriding gain", cm.output[0])

1147 self._check_applied_keys(result.exposure.metadata, isr_config)

1148

1149 # Confirm that the output has the defect line as saturated.

1150 sat_val = 2**result.exposure.mask.getMaskPlane("SAT")

1151 for defect in self.defects:

1152 np.testing.assert_array_equal(

1153 result.exposure.mask[defect.getBBox()].array & sat_val,

1154 sat_val,

1155 )

1156

1157 clean_mock_config = self.get_mock_config_clean()

1158 # We want the dark noise for more direct comparison.

1159 clean_mock_config.doAddDarkNoiseOnly = True

1160 clean_mock_config.doAddSky = True

1161 clean_mock_config.doAddSource = True

1162

1163 clean_mock = isrMockLSST.IsrMockLSST(config=clean_mock_config)

1164 clean_exp = clean_mock.run()

1165

1166 delta = result.exposure.image.array - clean_exp.image.array

1167

1168 bad_val = 2**result.exposure.mask.getMaskPlane("BAD")

1169 good_pixels = np.where((result.exposure.mask.array & (sat_val | bad_val)) == 0)

1170

1171 # We compare the good pixels in the entirety.

1172 self.assertLess(np.std(delta[good_pixels]), 5.0)

1173 # This is sensitive to parallel overscan masking.

1174 self.assertLess(np.max(np.abs(delta[good_pixels])), 5.0*7)

1175

1176 # Make sure the corrected image is overall consistent with the

1177 # straight image.

1178 self.assertLess(np.abs(np.median(delta[good_pixels])), 0.51)

1179

1180 # And overall where the interpolation is a bit worse but

1181 # the statistics are still fine. Note that this is worse than

1182 # the defect case because of the widening of the saturation

1183 # trail.

1184 self.assertLess(np.std(delta), 7.0)

1185

1186 metadata = result.exposure.metadata

1187

1188 for amp in self.detector:

1189 amp_name = amp.getName()

1190 key = f"LSST ISR GAIN {amp_name}"

1191 self.assertIn(key, metadata)

1192 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name])

1193 key = f"LSST ISR READNOISE {amp_name}"

1194 self.assertIn(key, metadata)

1195 self.assertEqual(metadata[key], self.ptc.noise[amp_name])

1196 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1197 self.assertIn(key, metadata)

1198 self.assertEqual(metadata[key], saturation_level * gain)

1199

1200 def test_isrFlatVignette(self):

1201 """Test ISR when the flat has a validPolygon and vignetted region."""

1202

1203 # We use a flat frame for this test for convenience.

1204 mock_config = self.get_mock_config_no_signal()

1205 mock_config.doAddDark = True

1206 mock_config.doAddFlat = True

1207 # The doAddSky option adds the equivalent of flat-field flux.

1208 mock_config.doAddSky = True

1209

1210 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1211 input_exp = mock.run()

1212

1213 isr_config = self.get_isr_config_electronic_corrections()

1214 isr_config.doBias = True

1215 isr_config.doDark = True

1216 isr_config.doFlat = True

1217 isr_config.doDefect = True

1218

1219 flat = self.flat.clone()

1220 bbox = geom.Box2D(

1221 corner=geom.Point2D(0, 0),

1222 dimensions=geom.Extent2D(50, 50),

1223 )

1224 polygon = afwGeom.Polygon(bbox)

1225 flat.info.setValidPolygon(polygon)

1226 maskVignettedRegion(flat, polygon, vignetteValue=0.0)

1227

1228 isr_task = IsrTaskLSST(config=isr_config)

1229 with self.assertNoLogs(level=logging.WARNING):

1230 result = isr_task.run(

1231 input_exp.clone(),

1232 bias=self.bias,

1233 dark=self.dark,

1234 flat=flat,

1235 crosstalk=self.crosstalk,

1236 ptc=self.ptc,

1237 linearizer=self.linearizer,

1238 defects=self.defects,

1239 deferredChargeCalib=self.cti,

1240 )

1241

1242 self.assertEqual(result.exposure.info.getValidPolygon(), polygon)

1243

1244 noDataFlat = (flat.mask.array & flat.mask.getPlaneBitMask("NO_DATA")) > 0

1245 noDataExp = (result.exposure.mask.array & result.exposure.mask.getPlaneBitMask("NO_DATA")) > 0

1246 np.testing.assert_array_equal(noDataExp, noDataFlat)

1247 np.testing.assert_array_equal(result.exposure.image.array[noDataExp], 0.0)

1248 np.testing.assert_array_equal(result.exposure.variance.array[noDataExp], 0.0)

1249 self.assertFalse(np.any(~np.isfinite(result.exposure.image.array)))

1250 self.assertFalse(np.any(~np.isfinite(result.exposure.variance.array)))

1251

1252 def test_isrFloodedSaturatedE2V(self):

1253 """Test ISR when the amps are completely saturated.

1254

1255 This version tests what happens when the parallel overscan

1256 region is flooded like E2V detectors, where the saturation

1257 spreads evenly, but at a greater level than the saturation

1258 value.

1259 """

1260 # We are simulating a flat field.

1261 # Note that these aren't very important because we are replacing

1262 # the flux, but we may as well.

1263 mock_config = self.get_mock_config_no_signal()

1264 mock_config.doAddDark = True

1265 mock_config.doAddFlat = True

1266 # The doAddSky option adds the equivalent of flat-field flux.

1267 mock_config.doAddSky = True

1268

1269 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1270 input_exp = mock.run()

1271

1272 isr_config = self.get_isr_config_minimal_corrections()

1273 isr_config.doBootstrap = True

1274 isr_config.doApplyGains = False

1275 isr_config.doBias = True

1276 isr_config.doDark = True

1277 isr_config.doFlat = False

1278 # Tun off saturation masking to simulate a PTC flat.

1279 isr_config.doSaturation = False

1280 isr_config.doE2VEdgeBleedMask = False

1281 isr_config.doITLEdgeBleedMask = False

1282

1283 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1284 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1285

1286 detector = input_exp.getDetector()

1287 for i, amp in enumerate(detector):

1288 # For half of the amps we are testing what happens when the

1289 # parallel overscan region is above the configured saturation

1290 # level; for the other half we are testing the other branch

1291 # when it saturates below this level (which is a priori

1292 # unknown).

1293 if i < len(detector) // 2:

1294 data_level = (parallel_overscan_saturation * 1.05

1295 + mock_config.biasLevel

1296 + mock_config.clockInjectedOffsetLevel)

1297 parallel_overscan_level = (parallel_overscan_saturation * 1.1

1298 + mock_config.biasLevel

1299 + mock_config.clockInjectedOffsetLevel)

1300 else:

1301 data_level = (parallel_overscan_saturation * 0.7

1302 + mock_config.biasLevel

1303 + mock_config.clockInjectedOffsetLevel)

1304 parallel_overscan_level = (parallel_overscan_saturation * 0.75

1305 + mock_config.biasLevel

1306 + mock_config.clockInjectedOffsetLevel)

1307

1308 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1309 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1310

1311 isr_task = IsrTaskLSST(config=isr_config)

1312 with self.assertLogs(level=logging.WARNING) as cm:

1313 result = isr_task.run(

1314 input_exp.clone(),

1315 bias=self.bias_adu,

1316 dark=self.dark_adu,

1317 )

1318 self.assertEqual(len(cm.records), len(detector))

1319

1320 n_all = 0

1321 n_level = 0

1322 for record in cm.records:

1323 if "All overscan pixels masked" in record.message:

1324 n_all += 1

1325 if "The level in the overscan region" in record.message:

1326 n_level += 1

1327

1328 self.assertEqual(n_all, len(detector) // 2)

1329 self.assertEqual(n_level, len(detector) // 2)

1330

1331 # And confirm that the post-ISR levels are high for each amp.

1332 for amp in detector:

1333 med = np.median(result.exposure[amp.getBBox()].image.array)

1334 self.assertGreater(med, parallel_overscan_saturation*0.8)

1335

1336 def test_isrFloodedSaturatedITL(self):

1337 """Test ISR when the amps are completely saturated.

1338

1339 This version tests what happens when the parallel overscan

1340 region is flooded like ITL detectors, where the saturation

1341 is at a lower level than the imaging region, and also

1342 spreads partly into the serial/parallel region.

1343 """

1344 # We are simulating a flat field.

1345 # Note that these aren't very important because we are replacing

1346 # the flux, but we may as well.

1347 mock_config = self.get_mock_config_no_signal()

1348 mock_config.doAddDark = True

1349 mock_config.doAddFlat = True

1350 # The doAddSky option adds the equivalent of flat-field flux.

1351 mock_config.doAddSky = True

1352

1353 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1354 input_exp = mock.run()

1355

1356 isr_config = self.get_isr_config_minimal_corrections()

1357 isr_config.doBootstrap = True

1358 isr_config.doApplyGains = False

1359 isr_config.doBias = True

1360 isr_config.doDark = True

1361 isr_config.doFlat = False

1362 # Tun off saturation masking to simulate a PTC flat.

1363 isr_config.doSaturation = False

1364 isr_config.doE2VEdgeBleedMask = False

1365 isr_config.doITLEdgeBleedMask = False

1366

1367 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1368 parallel_overscan_saturation = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1369

1370 detector = input_exp.getDetector()

1371 for i, amp in enumerate(detector):

1372 # For half of the amps we are testing what happens when the

1373 # parallel overscan region is above the configured saturation

1374 # level; for the other half we are testing the other branch

1375 # when it saturates below this level (which is a priori

1376 # unknown).

1377 if i < len(detector) // 2:

1378 data_level = (parallel_overscan_saturation * 1.1

1379 + mock_config.biasLevel

1380 + mock_config.clockInjectedOffsetLevel)

1381 parallel_overscan_level = (parallel_overscan_saturation * 1.05

1382 + mock_config.biasLevel

1383 + mock_config.clockInjectedOffsetLevel)

1384 else:

1385 data_level = (parallel_overscan_saturation * 0.75

1386 + mock_config.biasLevel

1387 + mock_config.clockInjectedOffsetLevel)

1388 parallel_overscan_level = (parallel_overscan_saturation * 0.7

1389 + mock_config.biasLevel

1390 + mock_config.clockInjectedOffsetLevel)

1391

1392 input_exp[amp.getRawDataBBox()].image.array[:, :] = data_level

1393 input_exp[amp.getRawParallelOverscanBBox()].image.array[:, :] = parallel_overscan_level

1394 # The serial/parallel region for the test camera looks like this:

1395 serial_overscan_bbox = amp.getRawSerialOverscanBBox()

1396 parallel_overscan_bbox = amp.getRawParallelOverscanBBox()

1397

1398 overscan_corner_bbox = geom.Box2I(

1399 geom.Point2I(

1400 serial_overscan_bbox.getMinX(),

1401 parallel_overscan_bbox.getMinY(),

1402 ),

1403 geom.Extent2I(

1404 serial_overscan_bbox.getWidth(),

1405 parallel_overscan_bbox.getHeight(),

1406 ),

1407 )

1408 input_exp[overscan_corner_bbox].image.array[-2:, :] = parallel_overscan_level

1409

1410 isr_task = IsrTaskLSST(config=isr_config)

1411 with self.assertLogs(level=logging.WARNING) as cm:

1412 result = isr_task.run(

1413 input_exp.clone(),

1414 bias=self.bias_adu,

1415 dark=self.dark_adu,

1416 )

1417 self.assertEqual(len(cm.records), len(detector))

1418

1419 n_all = 0

1420 n_level = 0

1421 for record in cm.records:

1422 if "All overscan pixels masked" in record.message:

1423 n_all += 1

1424 if "The level in the overscan region" in record.message:

1425 n_level += 1

1426

1427 self.assertEqual(n_all, len(detector) // 2)

1428 self.assertEqual(n_level, len(detector) // 2)

1429

1430 # And confirm that the post-ISR levels are high for each amp.

1431 for amp in detector:

1432 med = np.median(result.exposure[amp.getBBox()].image.array)

1433 self.assertGreater(med, parallel_overscan_saturation*0.8)

1434

1435 def test_isrBadParallelOverscanColumnsBootstrap(self):

1436 """Test processing a bias when we have a bad parallel overscan column.

1437

1438 This tests in bootstrap mode.

1439 """

1440 # We base this on the bootstrap bias, and make sure

1441 # that the bad column remains.

1442 mock_config = self.get_mock_config_no_signal()

1443 isr_config = self.get_isr_config_minimal_corrections()

1444 isr_config.doSaturation = False

1445 isr_config.doE2VEdgeBleedMask = False

1446 isr_config.doITLEdgeBleedMask = False

1447 isr_config.doBootstrap = True

1448 isr_config.doApplyGains = False

1449

1450 amp_config = isr_config.overscanCamera.defaultDetectorConfig.defaultAmpConfig

1451 overscan_sat_level_adu = amp_config.parallelOverscanConfig.parallelOverscanSaturationLevel

1452 # The defect is in amp 2.

1453 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1454 overscan_sat_level = amp_gain * overscan_sat_level_adu

1455 # The expected defect level is in adu for the bootstrap bias.

1456 expected_defect_level = mock_config.brightDefectLevel / amp_gain

1457

1458 # The levels are set in electron units.

1459 # We test 3 levels:

1460 # * 10.0, a very low outlier, to test median smoothing detection

1461 # code. This value is given by gain*threshold + cushion.

1462 # * 575.0, a lowish but outlier level, given by gain*threshold +

1463 # 100.0 (average of the parallel overscan offset) + 10.0

1464 # (an additional cushion).

1465 # * 1.05*saturation.

1466 # Note that the default parallel overscan saturation level for

1467 # bootstrap (pre-saturation-measure) analysis is very low, in

1468 # order to capture all types of amps, even with low saturation.

1469 # Therefore, we only need to test above this saturation level.

1470 # (c.f. test_isrBadParallelOverscanColumns).

1471 levels = np.array([10.0, 575.0, 1.05*overscan_sat_level])

1472

1473 for level in levels:

1474 mock_config.badParallelOverscanColumnLevel = level

1475 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1476 input_exp = mock.run()

1477

1478 isr_task = IsrTaskLSST(config=isr_config)

1479 with self.assertNoLogs(level=logging.WARNING):

1480 result = isr_task.run(input_exp.clone())

1481

1482 for defect in self.defects:

1483 bbox = defect.getBBox()

1484 defect_image = result.exposure[bbox].image.array

1485

1486 # Check that the defect is the correct level

1487 # (not subtracted away).

1488 defect_median = np.median(defect_image)

1489 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1490

1491 # Check that the neighbors aren't over-subtracted.

1492 for neighbor in [-1, 1]:

1493 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1494 neighbor_image = result.exposure[bbox_neighbor].image.array

1495

1496 neighbor_median = np.median(neighbor_image)

1497 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1498

1499 def test_isrBadParallelOverscanColumns(self):

1500 """Test processing a bias when we have a bad parallel overscan column.

1501

1502 This test uses regular non-bootstrap processing.

1503 """

1504 mock_config = self.get_mock_config_no_signal()

1505 isr_config = self.get_isr_config_electronic_corrections()

1506 # We do not do defect correction when processing biases.

1507 isr_config.doDefect = False

1508

1509 # The defect is in amp 2.

1510 sat_level_adu = self.ptc.ptcTurnoff[self.detector[2].getName()]

1511 amp_gain = mock_config.gainDict[self.detector[2].getName()]

1512 sat_level = amp_gain * sat_level_adu

1513 # The expected defect level is in electron for the full bias.

1514 expected_defect_level = mock_config.brightDefectLevel

1515

1516 # The levels are set in electron units.

1517 # We test 4 levels:

1518 # * 10.0, a very low outlier, to test median smoothing detection

1519 # code. This value is given by gain*threshold + cushion.

1520 # * 575.0, a lowish but outlier level, given by gain*threshold +

1521 # 100.0 (average of the parallel overscan offset) + 10.0

1522 # (an additional cushion).

1523 # * 0.9*saturation, following ITL-style parallel overscan bleeds.

1524 # * 1.05*saturation, following E2V-style parallel overscan bleeds.

1525 levels = np.array([10.0, 575.0, 0.9*sat_level, 1.1*sat_level])

1526

1527 for level in levels:

1528 mock_config.badParallelOverscanColumnLevel = level

1529 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1530 input_exp = mock.run()

1531

1532 isr_task = IsrTaskLSST(config=isr_config)

1533 with self.assertNoLogs(level=logging.WARNING):

1534 result = isr_task.run(

1535 input_exp.clone(),

1536 crosstalk=self.crosstalk,

1537 ptc=self.ptc,

1538 linearizer=self.linearizer,

1539 deferredChargeCalib=self.cti,

1540 )

1541

1542 for defect in self.defects:

1543 bbox = defect.getBBox()

1544 defect_image = result.exposure[bbox].image.array

1545

1546 # Check that the defect is the correct level

1547 # (not subtracted away).

1548 defect_median = np.median(defect_image)

1549 self.assertFloatsAlmostEqual(defect_median, expected_defect_level, rtol=1e-4)

1550

1551 # Check that the neighbors aren't over-subtracted.

1552 for neighbor in [-1, 1]:

1553 bbox_neighbor = bbox.shiftedBy(geom.Extent2I(neighbor, 0))

1554 neighbor_image = result.exposure[bbox_neighbor].image.array

1555

1556 neighbor_median = np.median(neighbor_image)

1557 self.assertFloatsAlmostEqual(neighbor_median, 0.0, atol=7.0)

1558

1559 def test_isrBadPtcGain(self):

1560 """Test processing when an amp has a bad (nan) PTC gain.

1561 """

1562 # We use a flat frame for this test for convenience.

1563 mock_config = self.get_mock_config_no_signal()

1564 mock_config.doAddDark = True

1565 mock_config.doAddFlat = True

1566 # The doAddSky option adds the equivalent of flat-field flux.

1567 mock_config.doAddSky = True

1568

1569 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1570 input_exp = mock.run()

1571

1572 isr_config = self.get_isr_config_electronic_corrections()

1573 isr_config.doBias = True

1574 isr_config.doDark = True

1575 isr_config.doFlat = False

1576 isr_config.doDefect = True

1577

1578 # Set a bad amplifier to a nan gain.

1579 bad_amp = self.detector[0].getName()

1580

1581 ptc = copy.copy(self.ptc)

1582 ptc.gain[bad_amp] = np.nan

1583

1584 # We also want non-zero (but very small) crosstalk values

1585 # to ensure that these don't propagate nans.

1586 crosstalk = copy.copy(self.crosstalk)

1587 for i in range(len(self.detector)):

1588 for j in range(len(self.detector)):

1589 if i == j:

1590 continue

1591 if crosstalk.coeffs[i, j] == 0:

1592 crosstalk.coeffs[i, j] = 1e-10

1593

1594 isr_task = IsrTaskLSST(config=isr_config)

1595 with self.assertLogs(level=logging.WARNING) as cm:

1596 result = isr_task.run(

1597 input_exp.clone(),

1598 bias=self.bias,

1599 dark=self.dark,

1600 crosstalk=crosstalk,

1601 ptc=ptc,

1602 linearizer=self.linearizer,

1603 defects=self.defects,

1604 deferredChargeCalib=self.cti,

1605 )

1606 self.assertIn(f"Amplifier {bad_amp} is bad (non-finite gain)", cm.output[0])

1607

1608 # Confirm that the bad_amp is marked bad and the other amps are not.

1609 # We have to special case the amp with the defect.

1610 mask = result.exposure.mask

1611

1612 for amp in self.detector:

1613 bbox = amp.getBBox()

1614 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1615

1616 if amp.getName() == bad_amp:

1617 self.assertTrue(np.all(bad_in_amp))

1618 elif amp.getName() == "C:0,2":

1619 # This is the amp with the defect.

1620 self.assertEqual(np.sum(bad_in_amp), 51)

1621 else:

1622 self.assertTrue(np.all(~bad_in_amp))

1623

1624 def test_saturationModes(self):

1625 """Test the different saturation modes."""

1626 # Use a simple bias run for these.

1627 mock_config = self.get_mock_config_no_signal()

1628

1629 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1630 input_exp = mock.run()

1631

1632 isr_config = self.get_isr_config_electronic_corrections()

1633 isr_config.doSaturation = True

1634 isr_config.maskNegativeVariance = False

1635 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1636 amp_config = copy.copy(detector_config.defaultAmpConfig)

1637

1638 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1639 isr_config.defaultSaturationSource = mode

1640

1641 # Reset the PTC.

1642 ptc = copy.copy(self.ptc)

1643 # Reset the detector config.

1644 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1645 if mode == "NONE":

1646 # We must use the config.

1647 sat_level = 1.2 * self.saturation_adu

1648 amp_config_new = copy.copy(amp_config)

1649 amp_config_new.saturation = sat_level

1650 detector_config_new = copy.copy(detector_config)

1651 detector_config_new.defaultAmpConfig = amp_config_new

1652 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1653 elif mode == "CAMERAMODEL":

1654 sat_level = input_exp.getDetector()[0].getSaturation()

1655 elif mode == "PTCTURNOFF":

1656 sat_level = 1.3 * self.saturation_adu

1657 for amp_name in ptc.ampNames:

1658 ptc.ptcTurnoff[amp_name] = sat_level

1659

1660 isr_task = IsrTaskLSST(config=isr_config)

1661 with self.assertNoLogs(level=logging.WARNING):

1662 result = isr_task.run(

1663 input_exp.clone(),

1664 bias=self.bias,

1665 crosstalk=self.crosstalk,

1666 ptc=self.ptc,

1667 linearizer=self.linearizer,

1668 deferredChargeCalib=self.cti,

1669 defects=self.defects,

1670 )

1671

1672 metadata = result.exposure.metadata

1673

1674 for amp in self.detector:

1675 amp_name = amp.getName()

1676 key = f"LSST ISR GAIN {amp_name}"

1677 self.assertIn(key, metadata, msg=mode)

1678 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1679 key = f"LSST ISR SATURATION LEVEL {amp_name}"

1680 self.assertIn(key, metadata, msg=mode)

1681 self.assertEqual(metadata[key], sat_level * gain, msg=mode)

1682

1683 def test_noPTC(self):

1684 """Test if we do not supply a PTC."""

1685 mock_config = self.get_mock_config_no_signal()

1686

1687 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1688 input_exp = mock.run()

1689

1690 isr_config = self.get_isr_config_minimal_corrections()

1691 isr_task = IsrTaskLSST(config=isr_config)

1692

1693 with self.assertRaises(RuntimeError) as cm:

1694 _ = isr_task.run(input_exp.clone())

1695 self.assertIn("A PTC must be supplied", cm.exception.args[0])

1696

1697 def test_suspectModes(self):

1698 """Test the different suspect modes."""

1699 # Use a simple bias run for these.

1700 mock_config = self.get_mock_config_no_signal()

1701

1702 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1703 input_exp = mock.run()

1704

1705 isr_config = self.get_isr_config_electronic_corrections()

1706 isr_config.doSaturation = True

1707 isr_config.maskNegativeVariance = False

1708 detector_config = copy.copy(isr_config.overscanCamera.defaultDetectorConfig)

1709 amp_config = copy.copy(detector_config.defaultAmpConfig)

1710

1711 for mode in ["NONE", "CAMERAMODEL", "PTCTURNOFF"]:

1712 isr_config.defaultSuspectSource = mode

1713

1714 # Reset the PTC.

1715 ptc = copy.copy(self.ptc)

1716 # Reset the detector config.

1717 isr_config.overscanCamera.defaultDetectorConfig = detector_config

1718 if mode == "NONE":

1719 # We must use the config.

1720 suspect_level = 1.2 * self.saturation_adu

1721 amp_config_new = copy.copy(amp_config)

1722 amp_config_new.suspectLevel = suspect_level

1723 detector_config_new = copy.copy(detector_config)

1724 detector_config_new.defaultAmpConfig = amp_config_new

1725 isr_config.overscanCamera.defaultDetectorConfig = detector_config_new

1726 elif mode == "CAMERAMODEL":

1727 suspect_level = input_exp.getDetector()[0].getSuspectLevel()

1728 elif mode == "PTCTURNOFF":

1729 suspect_level = 1.3 * self.saturation_adu

1730 for amp_name in ptc.ampNames:

1731 ptc.ptcTurnoff[amp_name] = suspect_level

1732

1733 isr_task = IsrTaskLSST(config=isr_config)

1734 with self.assertNoLogs(level=logging.WARNING):

1735 result = isr_task.run(

1736 input_exp.clone(),

1737 bias=self.bias,

1738 crosstalk=self.crosstalk,

1739 ptc=self.ptc,

1740 linearizer=self.linearizer,

1741 deferredChargeCalib=self.cti,

1742 defects=self.defects,

1743 )

1744

1745 metadata = result.exposure.metadata

1746

1747 for amp in self.detector:

1748 amp_name = amp.getName()

1749 key = f"LSST ISR GAIN {amp_name}"

1750 self.assertIn(key, metadata, msg=mode)

1751 self.assertEqual(metadata[key], gain := self.ptc.gain[amp_name], msg=mode)

1752 key = f"LSST ISR SUSPECT LEVEL {amp_name}"

1753 self.assertIn(key, metadata, msg=mode)

1754 self.assertEqual(metadata[key], suspect_level * gain, msg=mode)

1755

1756 def test_sequencerMismatches(self):

1757 """Test with a pile of sequencer mismatches."""

1758 mock_config = self.get_mock_config_no_signal()

1759 mock_config.doAddDark = True

1760 mock_config.doAddFlat = True

1761 # Set this to False until we have fringe correction.

1762 mock_config.doAddFringe = False

1763 mock_config.doAddSky = True

1764 mock_config.doAddSource = True

1765

1766 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1767 input_exp = mock.run()

1768 input_exp.metadata["SEQFILE"] = "a_sequencer"

1769

1770 isr_config = self.get_isr_config_electronic_corrections()

1771 isr_config.doBias = True

1772 isr_config.doDark = True

1773 isr_config.doFlat = True

1774 isr_config.cameraKeywordsToCompare = ["SEQFILE"]

1775

1776 bias = self.bias.clone()

1777 bias.metadata["SEQFILE"] = "b_sequencer"

1778 dark = self.dark.clone()

1779 dark.metadata["SEQFILE"] = "b_sequencer"

1780 flat = self.flat.clone()

1781 flat.metadata["SEQFILE"] = "b_sequencer"

1782 crosstalk = copy.copy(self.crosstalk)

1783 crosstalk.metadata["SEQFILE"] = "b_sequencer"

1784 defects = copy.copy(self.defects)

1785 defects.metadata["SEQFILE"] = "b_sequencer"

1786 ptc = copy.copy(self.ptc)

1787 ptc.metadata["SEQFILE"] = "b_sequencer"

1788 linearizer = copy.copy(self.linearizer)

1789 linearizer.metadata["SEQFILE"] = "b_sequencer"

1790 cti = copy.copy(self.cti)

1791 cti.metadata["SEQFILE"] = "b_sequencer"

1792

1793 isr_task = IsrTaskLSST(config=isr_config)

1794 with self.assertLogs(level=logging.WARNING):

1795 result = isr_task.run(

1796 input_exp.clone(),

1797 bias=bias,

1798 dark=dark,

1799 flat=flat,

1800 crosstalk=crosstalk,

1801 defects=defects,

1802 ptc=ptc,

1803 linearizer=linearizer,

1804 deferredChargeCalib=cti,

1805 )

1806

1807 for ctype in ["BIAS", "DARK", "FLAT", "CROSSTALK", "DEFECTS", "PTC", "LINEARIZER", "CTI"]:

1808 self.assertTrue(result.exposure.metadata[f"ISR {ctype} SEQUENCER MISMATCH"])

1809

1810 def test_highPtcNoiseAmps(self):

1811 """Test for masking of high noise amps (in PTC)."""

1812 # We use a flat frame for this test for convenience.

1813 mock_config = self.get_mock_config_no_signal()

1814 mock_config.doAddDark = True

1815 mock_config.doAddFlat = True

1816 # The doAddSky option adds the equivalent of flat-field flux.

1817 mock_config.doAddSky = True

1818

1819 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1820 input_exp = mock.run()

1821

1822 isr_config = self.get_isr_config_electronic_corrections()

1823 isr_config.doBias = True

1824 isr_config.doDark = True

1825 isr_config.doFlat = False

1826 isr_config.doDefect = True

1827

1828 # Set a bad amplifier to a high noise.

1829 bad_amp = self.detector[0].getName()

1830

1831 ptc = copy.copy(self.ptc)

1832 ptc.noise[bad_amp] = 50.0

1833

1834 isr_task = IsrTaskLSST(config=isr_config)

1835

1836 # With the PTC this should not warn.

1837 with self.assertNoLogs(level=logging.WARNING):

1838 result = isr_task.run(

1839 input_exp.clone(),

1840 bias=self.bias,

1841 dark=self.dark,

1842 crosstalk=self.crosstalk,

1843 ptc=ptc,

1844 linearizer=self.linearizer,

1845 defects=self.defects,

1846 deferredChargeCalib=self.cti,

1847 )

1848

1849 # Confirm that the bad_amp is marked bad and the other amps are not.

1850 # We have to special case the amp with the defect.

1851 mask = result.exposure.mask

1852

1853 for amp in self.detector:

1854 bbox = amp.getBBox()

1855 bad_in_amp = ((mask[bbox].array & 2**mask.getMaskPlaneDict()["BAD"]) > 0)

1856

1857 if amp.getName() == bad_amp:

1858 self.assertTrue(np.all(bad_in_amp))

1859 elif amp.getName() == "C:0,2":

1860 # This is the amp with the defect.

1861 self.assertEqual(np.sum(bad_in_amp), 51)

1862 else:

1863 self.assertTrue(np.all(~bad_in_amp))

1864

1865 def test_changedOverscanAmps(self):

1866 """Tests for masking of amps where the overscan level changed."""

1867

1868 # We use a flat frame for this test for convenience.

1869 mock_config = self.get_mock_config_no_signal()

1870 mock_config.doAddDark = True

1871 mock_config.doAddFlat = True

1872 # The doAddSky option adds the equivalent of flat-field flux.

1873 mock_config.doAddSky = True

1874

1875 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1876 input_exp = mock.run()

1877

1878 # Offset one amp with a constant value.

1879 bad_amp = "C:0,0"

1880

1881 input_exp2 = input_exp.clone()

1882 input_exp2.image[self.detector[bad_amp].getRawBBox()].array[:, :] -= 2000.0

1883

1884 isr_config = self.get_isr_config_electronic_corrections()

1885 isr_config.doBias = True

1886 isr_config.doDark = True

1887 isr_config.doFlat = False

1888 isr_config.doDefect = True

1889 isr_config.doInterpolate = False

1890 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

1891

1892 isr_task = IsrTaskLSST(config=isr_config)

1893 with self.assertLogs(level=logging.WARNING) as cm:

1894 _ = isr_task.run(

1895 input_exp2,

1896 bias=self.bias,

1897 dark=self.dark,

1898 crosstalk=self.crosstalk,

1899 ptc=self.ptc,

1900 linearizer=self.linearizer,

1901 defects=self.defects,

1902 deferredChargeCalib=self.cti,

1903 )

1904 self.assertEqual(len(cm.output), 1)

1905 self.assertIn(f"Amplifier {bad_amp} has an overscan level", cm.output[0])

1906

1907 # Offset all amps to see that it is now unprocessable.

1908

1909 input_exp2 = input_exp.clone()

1910 input_exp2.image.array[:, :] -= 2000.0

1911

1912 with self.assertLogs(level=logging.WARNING):

1913 with self.assertRaises(UnprocessableDataError):

1914 _ = isr_task.run(

1915 input_exp2,

1916 bias=self.bias,

1917 dark=self.dark,

1918 crosstalk=self.crosstalk,

1919 ptc=self.ptc,

1920 linearizer=self.linearizer,

1921 defects=self.defects,

1922 deferredChargeCalib=self.cti,

1923 )

1924

1925 # Remove the values in the PTC and turn off check.

1926 # This should run without warnings.

1927 ptc = copy.copy(self.ptc)

1928 for amp in self.detector:

1929 ptc.overscanMedian[amp.getName()] = np.nan

1930

1931 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

1932

1933 isr_task = IsrTaskLSST(config=isr_config)

1934 with self.assertNoLogs(level=logging.WARNING):

1935 _ = isr_task.run(

1936 input_exp.clone(),

1937 bias=self.bias,

1938 dark=self.dark,

1939 crosstalk=self.crosstalk,

1940 ptc=ptc,

1941 linearizer=self.linearizer,

1942 defects=self.defects,

1943 deferredChargeCalib=self.cti,

1944 )

1945

1946 # Turn the check back on; this should have 1 warning.

1947 isr_config.serialOverscanMedianShiftSigmaThreshold = 100.0

1948

1949 isr_task = IsrTaskLSST(config=isr_config)

1950 with self.assertLogs(level=logging.WARNING) as cm:

1951 _ = isr_task.run(

1952 input_exp.clone(),

1953 bias=self.bias,

1954 dark=self.dark,

1955 crosstalk=self.crosstalk,

1956 ptc=ptc,

1957 linearizer=self.linearizer,

1958 defects=self.defects,

1959 deferredChargeCalib=self.cti,

1960 )

1961 self.assertEqual(len(cm.output), 1)

1962 self.assertIn("No PTC overscan information", cm.output[0])

1963

1964 def test_highOverscanNoiseAmps(self):

1965 """Test for masking of high noise amps (in overscan)."""

1966

1967 # We use a flat frame for this test for convenience.

1968 mock_config = self.get_mock_config_no_signal()

1969 mock_config.doAddDark = True

1970 mock_config.doAddFlat = True

1971 # The doAddSky option adds the equivalent of flat-field flux.

1972 mock_config.doAddSky = True

1973 mock_config.readNoise = 50.0

1974

1975 mock = isrMockLSST.IsrMockLSST(config=mock_config)

1976 input_exp = mock.run()

1977

1978 isr_config = self.get_isr_config_electronic_corrections()

1979 isr_config.doBias = True

1980 isr_config.doDark = True

1981 isr_config.doFlat = False

1982 isr_config.doDefect = True

1983 isr_config.doInterpolate = False

1984 # Let all the amps fail to check the logging.

1985 isr_config.doCheckUnprocessableData = False

1986

1987 isr_task = IsrTaskLSST(config=isr_config)

1988 with self.assertLogs(level=logging.WARNING) as cm:

1989 result = isr_task.run(

1990 input_exp.clone(),

1991 bias=self.bias,

1992 dark=self.dark,

1993 crosstalk=self.crosstalk,

1994 ptc=self.ptc,

1995 linearizer=self.linearizer,

1996 defects=self.defects,

1997 deferredChargeCalib=self.cti,

1998 )

1999 self.assertEqual(len(cm.output), len(self.detector))

2000

2001 # All pixels should be BAD

2002 bad_value = result.exposure.mask.getPlaneBitMask("BAD")

2003 np.testing.assert_array_equal(result.exposure.mask.array & bad_value, bad_value)

2004

2005 # And run again to check the UnprocessableDataError.

2006 isr_config.doCheckUnprocessableData = True

2007 isr_task = IsrTaskLSST(config=isr_config)

2008

2009 with self.assertRaises(UnprocessableDataError):

2010 with self.assertLogs(level=logging.WARNING):

2011 result = isr_task.run(

2012 input_exp.clone(),

2013 bias=self.bias,

2014 dark=self.dark,

2015 crosstalk=self.crosstalk,

2016 ptc=self.ptc,

2017 linearizer=self.linearizer,

2018 defects=self.defects,

2019 deferredChargeCalib=self.cti,

2020 )

2021

2022 def test_bssVoltageChecks(self):

2023 """Test the BSS voltage checks."""

2024 # We use a flat frame for this test for convenience.

2025 mock_config = self.get_mock_config_no_signal()

2026 mock_config.doAddDark = True

2027 mock_config.doAddFlat = True

2028 # The doAddSky option adds the equivalent of flat-field flux.

2029 mock_config.doAddSky = True

2030

2031 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2032 input_exp = mock.run()

2033

2034 isr_config = self.get_isr_config_electronic_corrections()

2035 isr_config.doBias = True

2036 isr_config.doDark = True

2037 isr_config.doFlat = False

2038 isr_config.doDefect = True

2039

2040 # Set the voltage.

2041 input_exp.metadata["BSSVBS"] = 0.25

2042

2043 # Check that processing runs with checks turned off.

2044 isr_config.doCheckUnprocessableData = False

2045

2046 isr_task = IsrTaskLSST(config=isr_config)

2047 with self.assertNoLogs(level=logging.WARNING):

2048 _ = isr_task.run(

2049 input_exp.clone(),

2050 bias=self.bias,

2051 dark=self.dark,

2052 crosstalk=self.crosstalk,

2053 ptc=self.ptc,

2054 linearizer=self.linearizer,

2055 defects=self.defects,

2056 deferredChargeCalib=self.cti,

2057 )

2058

2059 # Check that processing runs with other way of turning checks off.

2060 isr_config.doCheckUnprocessableData = True

2061 isr_config.bssVoltageMinimum = 0.0

2062

2063 isr_task = IsrTaskLSST(config=isr_config)

2064 with self.assertNoLogs(level=logging.WARNING):

2065 _ = isr_task.run(

2066 input_exp.clone(),

2067 bias=self.bias,

2068 dark=self.dark,

2069 crosstalk=self.crosstalk,

2070 ptc=self.ptc,

2071 linearizer=self.linearizer,

2072 defects=self.defects,

2073 deferredChargeCalib=self.cti,

2074 )

2075

2076 # Check that processing runs but warns if header keyword is None.

2077 isr_config.doCheckUnprocessableData = True

2078 isr_config.bssVoltageMinimum = 10.0

2079

2080 input_exp2 = input_exp.clone()

2081 input_exp2.metadata["BSSVBS"] = None

2082

2083 isr_task = IsrTaskLSST(config=isr_config)

2084 with self.assertLogs(level=logging.WARNING) as cm:

2085 _ = isr_task.run(

2086 input_exp2,

2087 bias=self.bias,

2088 dark=self.dark,

2089 crosstalk=self.crosstalk,

2090 ptc=self.ptc,

2091 linearizer=self.linearizer,

2092 defects=self.defects,

2093 deferredChargeCalib=self.cti,

2094 )

2095 self.assertEqual(len(cm.output), 1)

2096 self.assertIn("Back-side bias voltage BSSVBS not found", cm.output[0])

2097

2098 # Check that it raises.

2099 isr_config.doCheckUnprocessableData = True

2100 isr_config.bssVoltageMinimum = 10.0

2101

2102 isr_task = IsrTaskLSST(config=isr_config)

2103 with self.assertRaises(UnprocessableDataError):

2104 _ = isr_task.run(

2105 input_exp.clone(),

2106 bias=self.bias,

2107 dark=self.dark,

2108 crosstalk=self.crosstalk,

2109 ptc=self.ptc,

2110 linearizer=self.linearizer,

2111 defects=self.defects,

2112 deferredChargeCalib=self.cti,

2113 )

2114

2115 def test_overrideMaskBadAmp(self):

2116 """Test overriding config to mask an amp."""

2117 # We use a flat frame for this test for convenience.

2118 mock_config = self.get_mock_config_no_signal()

2119 mock_config.doAddDark = True

2120 mock_config.doAddFlat = True

2121 # The doAddSky option adds the equivalent of flat-field flux.

2122 mock_config.doAddSky = True

2123

2124 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2125 input_exp = mock.run()

2126

2127 isr_config = self.get_isr_config_electronic_corrections()

2128 isr_config.doBias = True

2129 isr_config.doDark = True

2130 isr_config.doFlat = False

2131 isr_config.doDefect = True

2132

2133 bad_amps = ["C:0,0", "C:0,1"]

2134

2135 detector_name = self.detector.getName()

2136 isr_config.badAmps = [f"{detector_name}_{bad_amp}" for bad_amp in bad_amps]

2137

2138 isr_task = IsrTaskLSST(config=isr_config)

2139 with self.assertNoLogs(level=logging.WARNING):

2140 result = isr_task.run(

2141 input_exp.clone(),

2142 bias=self.bias,

2143 dark=self.dark,

2144 crosstalk=self.crosstalk,

2145 ptc=self.ptc,

2146 linearizer=self.linearizer,

2147 defects=self.defects,

2148 deferredChargeCalib=self.cti,

2149 )

2150

2151 for amp_name in bad_amps:

2152 mask_array = result.exposure[self.detector[amp_name].getBBox()].mask.array

2153 bad_mask = result.exposure.mask.getPlaneBitMask("BAD")

2154 self.assertTrue(np.all((mask_array & bad_mask) > 0))

2155

2156 # Make sure it didn't obliterate everything.

2157 mask_array = result.exposure.mask.array

2158 self.assertFalse(np.all((mask_array & bad_mask) > 0))

2159

2160 def test_hvBiasChecks(self):

2161 """Test the HVBIAS checks."""

2162 # We use a flat frame for this test for convenience.

2163 mock_config = self.get_mock_config_no_signal()

2164 mock_config.doAddDark = True

2165 mock_config.doAddFlat = True

2166 # The doAddSky option adds the equivalent of flat-field flux.

2167 mock_config.doAddSky = True

2168

2169 mock = isrMockLSST.IsrMockLSST(config=mock_config)

2170 input_exp = mock.run()

2171

2172 isr_config = self.get_isr_config_electronic_corrections()

2173 isr_config.doBias = True

2174 isr_config.doDark = True

2175 isr_config.doFlat = False

2176 isr_config.doDefect = True

2177

2178 # Set the voltage.

2179 input_exp.metadata["HVBIAS"] = "OFF"

2180

2181 # Check that processing runs with checks turned off.

2182 isr_config.doCheckUnprocessableData = False

2183

2184 isr_task = IsrTaskLSST(config=isr_config)

2185 with self.assertNoLogs(level=logging.WARNING):

2186 _ = isr_task.run(

2187 input_exp.clone(),

2188 bias=self.bias,

2189 dark=self.dark,

2190 crosstalk=self.crosstalk,

2191 ptc=self.ptc,

2192 linearizer=self.linearizer,

2193 defects=self.defects,

2194 deferredChargeCalib=self.cti,

2195 )

2196

2197 # Check that processing runs with other way of turning checks off.

2198 isr_config.doCheckUnprocessableData = True

2199 isr_config.bssVoltageMinimum = 0.0

2200

2201 isr_task = IsrTaskLSST(config=isr_config)

2202 with self.assertNoLogs(level=logging.WARNING):

2203 _ = isr_task.run(

2204 input_exp.clone(),

2205 bias=self.bias,

2206 dark=self.dark,

2207 crosstalk=self.crosstalk,

2208 ptc=self.ptc,

2209 linearizer=self.linearizer,

2210 defects=self.defects,

2211 deferredChargeCalib=self.cti,

2212 )

2213

2214 # Check that processing runs but warns if header keyword is None.

2215 isr_config.doCheckUnprocessableData = True

2216 isr_config.bssVoltageMinimum = 10.0

2217

2218 input_exp2 = input_exp.clone()

2219 input_exp2.metadata["HVBIAS"] = None

2220

2221 isr_task = IsrTaskLSST(config=isr_config)

2222 with self.assertLogs(level=logging.WARNING) as cm:

2223 _ = isr_task.run(

2224 input_exp2,

2225 bias=self.bias,

2226 dark=self.dark,

2227 crosstalk=self.crosstalk,

2228 ptc=self.ptc,

2229 linearizer=self.linearizer,

2230 defects=self.defects,

2231 deferredChargeCalib=self.cti,

2232 )

2233 self.assertEqual(len(cm.output), 1)

2234 self.assertIn("HV bias on HVBIAS not found in metadata", cm.output[0])

2235

2236 # Check that it raises.

2237 isr_config.doCheckUnprocessableData = True

2238 isr_config.bssVoltageMinimum = 10.0

2239

2240 isr_task = IsrTaskLSST(config=isr_config)

2241 with self.assertRaises(UnprocessableDataError):

2242 _ = isr_task.run(

2243 input_exp.clone(),

2244 bias=self.bias,

2245 dark=self.dark,

2246 crosstalk=self.crosstalk,

2247 ptc=self.ptc,

2248 linearizer=self.linearizer,

2249 defects=self.defects,

2250 deferredChargeCalib=self.cti,

2251 )

2252

2253 def get_mock_config_no_signal(self):

2254 """Get an IsrMockLSSTConfig with all signal set to False.

2255

2256 This will have all the electronic effects turned on (including

2257 2D bias).

2258 """

2259 mock_config = isrMockLSST.IsrMockLSSTConfig()

2260 mock_config.isTrimmed = False

2261 mock_config.doAddDark = False

2262 mock_config.doAddFlat = False

2263 mock_config.doAddFringe = False

2264 mock_config.doAddSky = False

2265 mock_config.doAddSource = False

2266

2267 mock_config.doAdd2DBias = True

2268 mock_config.doAddBias = True

2269 mock_config.doAddCrosstalk = True

2270 mock_config.doAddDeferredCharge = True

2271 mock_config.doAddBrightDefects = True

2272 mock_config.doAddClockInjectedOffset = True

2273 mock_config.doAddParallelOverscanRamp = True

2274 mock_config.doAddSerialOverscanRamp = True

2275 mock_config.doAddHighSignalNonlinearity = True

2276 mock_config.doApplyGain = True

2277 mock_config.doRoundAdu = True

2278

2279 # We always want to generate the image with these configs.

2280 mock_config.doGenerateImage = True

2281

2282 return mock_config

2283

2284 def get_mock_config_clean(self):

2285 """Get an IsrMockLSSTConfig trimmed with all electronic signatures

2286 turned off.

2287 """

2288 mock_config = isrMockLSST.IsrMockLSSTConfig()

2289 mock_config.doAddBias = False

2290 mock_config.doAdd2DBias = False

2291 mock_config.doAddClockInjectedOffset = False

2292 mock_config.doAddDark = False

2293 mock_config.doAddDarkNoiseOnly = False

2294 mock_config.doAddFlat = False

2295 mock_config.doAddFringe = False

2296 mock_config.doAddSky = False

2297 mock_config.doAddSource = False

2298 mock_config.doRoundAdu = False

2299 mock_config.doAddHighSignalNonlinearity = False

2300 mock_config.doApplyGain = False

2301 mock_config.doAddCrosstalk = False

2302 mock_config.doAddBrightDefects = False

2303 mock_config.doAddParallelOverscanRamp = False

2304 mock_config.doAddSerialOverscanRamp = False

2305

2306 mock_config.isTrimmed = True

2307 mock_config.doGenerateImage = True

2308

2309 return mock_config

2310

2311 def get_isr_config_minimal_corrections(self):

2312 """Get an IsrTaskLSSTConfig with minimal corrections.

2313 """

2314 isr_config = IsrTaskLSSTConfig()

2315 isr_config.bssVoltageMinimum = 0.0

2316 isr_config.ampNoiseThreshold = np.inf

2317 isr_config.serialOverscanMedianShiftSigmaThreshold = np.inf

2318 isr_config.doBias = False

2319 isr_config.doDark = False

2320 isr_config.doDeferredCharge = False

2321 isr_config.doLinearize = False

2322 isr_config.doCorrectGains = False

2323 isr_config.doCrosstalk = False

2324 isr_config.doDefect = False

2325 isr_config.doBrighterFatter = False

2326 isr_config.doFlat = False

2327 isr_config.doSaturation = False

2328 isr_config.doE2VEdgeBleedMask = False

2329 isr_config.doITLEdgeBleedMask = False

2330 isr_config.doSuspect = False

2331 # We override the leading/trailing to skip here because of the limited

2332 # size of the test camera overscan regions.

2333 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2334 defaultAmpConfig.doSerialOverscan = True

2335 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2336 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2337 defaultAmpConfig.doParallelOverscan = True

2338 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2339 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2340 # Our strong overscan slope in the tests requires an override.

2341 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2342

2343 isr_config.doAssembleCcd = True

2344 isr_config.crosstalk.doSubtrahendMasking = True

2345 isr_config.crosstalk.minPixelToMask = 1.0

2346

2347 return isr_config

2348

2349 def get_isr_config_electronic_corrections(self):

2350 """Get an IsrTaskLSSTConfig with electronic corrections.

2351

2352 This tests all the corrections that we support in the mocks/ISR.

2353 """

2354 isr_config = IsrTaskLSSTConfig()

2355 # We add these as appropriate in the tests.

2356 isr_config.doBias = False

2357 isr_config.doDark = False

2358 isr_config.doFlat = False

2359

2360 # These are the electronic effects the tests support (in addition

2361 # to overscan).

2362 isr_config.doCrosstalk = True

2363 isr_config.doDefect = True

2364 isr_config.doLinearize = True

2365 isr_config.doDeferredCharge = True

2366

2367 # This is False because it is only used in a single test case

2368 # as it takes a while to solve

2369 isr_config.doBrighterFatter = False

2370

2371 # These are the electronic effects we do not support in tests yet.

2372 isr_config.doCorrectGains = False

2373

2374 # We override the leading/trailing to skip here because of the limited

2375 # size of the test camera overscan regions.

2376 defaultAmpConfig = isr_config.overscanCamera.getOverscanDetectorConfig(self.detector).defaultAmpConfig

2377 defaultAmpConfig.doSerialOverscan = True

2378 defaultAmpConfig.serialOverscanConfig.leadingToSkip = 0

2379 defaultAmpConfig.serialOverscanConfig.trailingToSkip = 0

2380 defaultAmpConfig.doParallelOverscan = True

2381 defaultAmpConfig.parallelOverscanConfig.leadingToSkip = 0

2382 defaultAmpConfig.parallelOverscanConfig.trailingToSkip = 0

2383 # Our strong overscan slope in the tests requires an override.

2384 defaultAmpConfig.parallelOverscanConfig.maxDeviation = 300.0

2385

2386 isr_config.doAssembleCcd = True

2387 isr_config.crosstalk.doSubtrahendMasking = True

2388 isr_config.crosstalk.minPixelToMask = 1.0

2389

2390 return isr_config

2391

2392 def get_non_defect_pixels(self, mask_origin):

2393 """Get the non-defect pixels to compare.

2394

2395 Parameters

2396 ----------

2397 mask_origin : `lsst.afw.image.MaskX`

2398 The origin mask (for shape and type).

2399

2400 Returns

2401 -------

2402 pix_x, pix_y : `tuple` [`np.ndarray`]

2403 x and y values of good pixels.

2404 """

2405 mask_temp = mask_origin.clone()

2406 mask_temp[:, :] = 0

2407

2408 for defect in self.defects:

2409 mask_temp[defect.getBBox()] = 1

2410

2411 return np.where(mask_temp.array == 0)

2412

2413 def _check_bad_column_crosstalk_correction(

2414 self,

2415 exp,

2416 nsigma_cut=5.0,

2417 ):

2418 """Test bad column crosstalk correction.

2419

2420 This includes possible provblems from parallel overscan

2421 crosstalk and gain mismatches.

2422

2423 The target amp is self.detector[0], "C:0,0".

2424

2425 Parameters

2426 ----------

2427 exp : `lsst.afw.image.Exposure`

2428 Input exposure.

2429 nsigma_cut : `float`, optional

2430 Number of sigma to check for outliers.

2431 """

2432 amp = self.detector[0]

2433 amp_image = exp[amp.getBBox()].image.array

2434 sigma = median_abs_deviation(amp_image.ravel(), scale="normal")

2435

2436 med = np.median(amp_image.ravel())

2437 self.assertLess(amp_image.max(), med + nsigma_cut*sigma)

2438 self.assertGreater(amp_image.min(), med - nsigma_cut*sigma)

2439

2440

2441class MemoryTester(lsst.utils.tests.MemoryTestCase):

2442 pass

2443

2444

2445def setup_module(module):

2446 lsst.utils.tests.init()

2447

2448

2449if __name__ == "__main__": 2449 ↛ 2450line 2449 didn't jump to line 2450 because the condition on line 2449 was never true

2450 lsst.utils.tests.init()

2451 unittest.main()