Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

550

551

552

553

554

555

556

557

558

559

560

561

562

563

564

565

566

567

568

569

570

571

572

573

574

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626

627

628

629

630

631

632

633

634

635

636

637

638

639

640

641

642

643

644

645

646

647

648

649

650

651

652

653

654

655

656

657

658

659

660

661

662

663

664

665

666

667

668

669

670

671

672

673

674

675

676

677

678

679

680

681

682

683

684

685

686

687

688

689

690

691

692

693

694

695

696

697

698

699

700

701

702

703

704

705

706

707

708

709

710

711

712

713

714

715

716

717

718

719

720

721

722

723

724

725

726

727

728

729

730

731

732

733

734

735

736

737

738

739

740

741

742

743

744

745

746

747

748

749

750

751

752

753

754

755

756

757

758

759

760

761

762

763

764

765

766

767

768

769

770

771

772

773

774

775

776

777

778

779

780

781

782

783

784

785

786

787

788

789

790

791

792

793

794

795

796

797

798

799

800

801

802

803

804

805

806

807

808

809

810

811

812

813

814

815

816

817

818

819

820

821

822

823

824

825

826

827

828

829

830

831

832

833

834

835

836

837

838

839

840

841

842

843

844

845

846

847

848

849

850

851

852

853

854

855

856

857

858

859

860

861

862

863

864

865

866

867

868

869

870

871

872

873

874

875

876

877

878

879

880

881

882

883

884

885

886

887

888

889

890

891

892

893

894

895

896

897

898

899

900

901

902

903

904

905

906

907

908

909

910

911

912

913

914

915

916

917

918

919

920

921

922

923

924

925

926

927

928

929

930

931

932

933

934

935

936

937

938

939

940

941

942

943

944

945

946

947

948

949

950

951

952

953

954

955

956

957

958

959

960

961

962

963

964

965

966

967

968

969

970

971

972

973

974

975

976

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

993

994

995

996

997

998

999

1000

1001

1002

1003

1004

1005

1006

1007

1008

1009

1010

1011

1012

1013

1014

1015

1016

1017

1018

1019

1020

1021

1022

1023

1024

1025

1026

1027

1028

1029

1030

1031

1032

1033

1034

1035

1036

1037

1038

1039

1040

1041

1042

1043

1044

1045

1046

1047

1048

1049

1050

1051

1052

1053

1054

1055

1056

1057

1058

1059

1060

1061

1062

1063

1064

1065

1066

1067

1068

1069

1070

1071

1072

1073

1074

1075

1076

1077

1078

1079

1080

1081

1082

1083

1084

1085

1086

1087

1088

1089

1090

1091

1092

1093

1094

1095

1096

1097

1098

1099

1100

1101

1102

1103

1104

1105

1106

1107

1108

1109

1110

1111

1112

1113

1114

1115

1116

1117

1118

1119

1120

1121

1122

1123

1124

1125

1126

1127

1128

1129

1130

1131

1132

1133

1134

1135

1136

1137

1138

1139

1140

1141

1142

1143

1144

1145

1146

1147

1148

1149

1150

1151

1152

1153

1154

1155

1156

1157

1158

1159

1160

1161

1162

1163

1164

1165

1166

1167

1168

1169

1170

1171

1172

1173

1174

1175

1176

1177

1178

1179

1180

1181

1182

1183

1184

1185

1186

1187

1188

1189

1190

1191

1192

1193

1194

1195

1196

1197

1198

1199

1200

1201

1202

1203

1204

1205

1206

1207

1208

1209

1210

1211

1212

1213

1214

1215

1216

1217

1218

1219

1220

1221

1222

1223

1224

1225

1226

1227

1228

1229

1230

1231

1232

1233

1234

1235

1236

1237

1238

1239

1240

1241

1242

1243

1244

1245

1246

1247

1248

1249

1250

1251

1252

1253

1254

1255

1256

1257

1258

1259

1260

1261

1262

1263

1264

1265

1266

1267

1268

1269

1270

1271

1272

1273

1274

1275

1276

1277

1278

1279

1280

1281

1282

1283

1284

1285

1286

1287

1288

1289

1290

1291

1292

# 

# LSST Data Management System 

# Copyright 2008-2016 AURA/LSST. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

import math 

import numpy 

 

import lsst.afw.geom as afwGeom 

import lsst.afw.image as afwImage 

import lsst.meas.algorithms as measAlg 

import lsst.pex.config as pexConfig 

import lsst.pipe.base as pipeBase 

import lsst.afw.math as afwMath 

from lsst.daf.persistence import ButlerDataRef 

from lsstDebug import getDebugFrame 

from lsst.afw.display import getDisplay 

from . import isrFunctions 

from .assembleCcdTask import AssembleCcdTask 

from .fringe import FringeTask 

from lsst.afw.geom import Polygon 

from lsst.afw.geom.wcsUtils import makeDistortedTanWcs 

from lsst.afw.cameraGeom import PIXELS, FOCAL_PLANE, FIELD_ANGLE, NullLinearityType 

from contextlib import contextmanager 

from .isr import maskNans 

from .crosstalk import CrosstalkTask 

 

 

class IsrTaskConfig(pexConfig.Config): 

doBias = pexConfig.Field( 

dtype=bool, 

doc="Apply bias frame correction?", 

default=True, 

) 

doDark = pexConfig.Field( 

dtype=bool, 

doc="Apply dark frame correction?", 

default=True, 

) 

doFlat = pexConfig.Field( 

dtype=bool, 

doc="Apply flat field correction?", 

default=True, 

) 

doFringe = pexConfig.Field( 

dtype=bool, 

doc="Apply fringe correction?", 

default=True, 

) 

doDefect = pexConfig.Field( 

dtype=bool, 

doc="Apply correction for CCD defects, e.g. hot pixels?", 

default=True, 

) 

doAddDistortionModel = pexConfig.Field( 

dtype=bool, 

doc="Apply a distortion model based on camera geometry to the WCS?", 

default=True, 

) 

doWrite = pexConfig.Field( 

dtype=bool, 

doc="Persist postISRCCD?", 

default=True, 

) 

biasDataProductName = pexConfig.Field( 

dtype=str, 

doc="Name of the bias data product", 

default="bias", 

) 

darkDataProductName = pexConfig.Field( 

dtype=str, 

doc="Name of the dark data product", 

default="dark", 

) 

flatDataProductName = pexConfig.Field( 

dtype=str, 

doc="Name of the flat data product", 

default="flat", 

) 

assembleCcd = pexConfig.ConfigurableField( 

target=AssembleCcdTask, 

doc="CCD assembly task", 

) 

gain = pexConfig.Field( 

dtype=float, 

doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)", 

default=float("NaN"), 

) 

readNoise = pexConfig.Field( 

dtype=float, 

doc="The read noise to use if no Detector is present in the Exposure", 

default=0.0, 

) 

saturation = pexConfig.Field( 

dtype=float, 

doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)", 

default=float("NaN"), 

) 

fringeAfterFlat = pexConfig.Field( 

dtype=bool, 

doc="Do fringe subtraction after flat-fielding?", 

default=True, 

) 

fringe = pexConfig.ConfigurableField( 

target=FringeTask, 

doc="Fringe subtraction task", 

) 

fwhm = pexConfig.Field( 

dtype=float, 

doc="FWHM of PSF (arcsec)", 

default=1.0, 

) 

saturatedMaskName = pexConfig.Field( 

dtype=str, 

doc="Name of mask plane to use in saturation detection and interpolation", 

default="SAT", 

) 

suspectMaskName = pexConfig.Field( 

dtype=str, 

doc="Name of mask plane to use for suspect pixels", 

default="SUSPECT", 

) 

flatScalingType = pexConfig.ChoiceField( 

dtype=str, 

doc="The method for scaling the flat on the fly.", 

default='USER', 

allowed={ 

"USER": "Scale by flatUserScale", 

"MEAN": "Scale by the inverse of the mean", 

"MEDIAN": "Scale by the inverse of the median", 

}, 

) 

flatUserScale = pexConfig.Field( 

dtype=float, 

doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise", 

default=1.0, 

) 

overscanFitType = pexConfig.ChoiceField( 

dtype=str, 

doc="The method for fitting the overscan bias level.", 

default='MEDIAN', 

allowed={ 

"POLY": "Fit ordinary polynomial to the longest axis of the overscan region", 

"CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region", 

"LEG": "Fit Legendre polynomial to the longest axis of the overscan region", 

"NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region", 

"CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region", 

"AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region", 

"MEAN": "Correct using the mean of the overscan region", 

"MEANCLIP": "Correct using a clipped mean of the overscan region", 

"MEDIAN": "Correct using the median of the overscan region", 

}, 

) 

overscanOrder = pexConfig.Field( 

dtype=int, 

doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " + 

"or number of spline knots if overscan fit type is a spline."), 

default=1, 

) 

overscanNumSigmaClip = pexConfig.Field( 

dtype=float, 

doc="Rejection threshold (sigma) for collapsing overscan before fit", 

default=3.0, 

) 

 

overscanNumLeadingColumnsToSkip = pexConfig.Field( 

dtype=int, 

doc="Number of columns to skip in overscan, i.e. those closest to amplifier", 

default=0, 

) 

overscanNumTrailingColumnsToSkip = pexConfig.Field( 

dtype=int, 

doc="Number of columns to skip in overscan, i.e. those farthest from amplifier", 

default=0, 

) 

growSaturationFootprintSize = pexConfig.Field( 

dtype=int, 

doc="Number of pixels by which to grow the saturation footprints", 

default=1, 

) 

doSaturationInterpolation = pexConfig.Field( 

dtype=bool, 

doc="Perform interpolation over pixels masked as saturated?", 

default=True, 

) 

doNanInterpAfterFlat = pexConfig.Field( 

dtype=bool, 

doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we " 

"also have to interpolate them before flat-fielding."), 

default=False, 

) 

fluxMag0T1 = pexConfig.Field( 

dtype=float, 

doc="The approximate flux of a zero-magnitude object in a one-second exposure", 

default=1e10, 

) 

keysToRemoveFromAssembledCcd = pexConfig.ListField( 

dtype=str, 

doc="fields to remove from the metadata of the assembled ccd.", 

default=[], 

) 

doAssembleIsrExposures = pexConfig.Field( 

dtype=bool, 

default=False, 

doc="Assemble amp-level calibration exposures into ccd-level exposure?" 

) 

doAssembleCcd = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Assemble amp-level exposures into a ccd-level exposure?" 

) 

expectWcs = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Expect input science images to have a WCS (set False for e.g. spectrographs)" 

) 

doLinearize = pexConfig.Field( 

dtype=bool, 

doc="Correct for nonlinearity of the detector's response?", 

default=True, 

) 

doCrosstalk = pexConfig.Field( 

dtype=bool, 

doc="Apply intra-CCD crosstalk correction?", 

default=False, 

) 

crosstalk = pexConfig.ConfigurableField( 

target=CrosstalkTask, 

doc="Intra-CCD crosstalk correction", 

) 

doBrighterFatter = pexConfig.Field( 

dtype=bool, 

default=False, 

doc="Apply the brighter fatter correction" 

) 

brighterFatterLevel = pexConfig.ChoiceField( 

doc="The level at which to correct for brighter-fatter", 

dtype=str, default="DETECTOR", 

allowed={ 

"AMP": "Every amplifier treated separately", 

"DETECTOR": "One kernel per detector", 

} 

) 

brighterFatterKernelFile = pexConfig.Field( 

dtype=str, 

default='', 

doc="Kernel file used for the brighter fatter correction" 

) 

brighterFatterMaxIter = pexConfig.Field( 

dtype=int, 

default=10, 

doc="Maximum number of iterations for the brighter fatter correction" 

) 

brighterFatterThreshold = pexConfig.Field( 

dtype=float, 

default=1000, 

doc="Threshold used to stop iterating the brighter fatter correction. It is the " 

" absolute value of the difference between the current corrected image and the one" 

" from the previous iteration summed over all the pixels." 

) 

brighterFatterApplyGain = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Should the gain be applied when applying the brighter fatter correction?" 

) 

datasetType = pexConfig.Field( 

dtype=str, 

doc="Dataset type for input data; users will typically leave this alone, " 

"but camera-specific ISR tasks will override it", 

default="raw", 

) 

fallbackFilterName = pexConfig.Field(dtype=str, 

doc="Fallback default filter name for calibrations", optional=True) 

doAttachTransmissionCurve = pexConfig.Field( 

dtype=bool, 

default=False, 

doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?" 

) 

doUseOpticsTransmission = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?" 

) 

doUseFilterTransmission = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?" 

) 

doUseSensorTransmission = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?" 

) 

doUseAtmosphereTransmission = pexConfig.Field( 

dtype=bool, 

default=True, 

doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?" 

) 

doEmpiricalReadNoise = pexConfig.Field( 

dtype=bool, 

default=False, 

doc="Calculate empirical read noise instead of value from AmpInfo data?" 

) 

 

## @addtogroup LSST_task_documentation 

## @{ 

## @page IsrTask 

## @ref IsrTask_ "IsrTask" 

## @copybrief IsrTask 

## @} 

 

 

class IsrTask(pipeBase.CmdLineTask): 

r"""! 

@anchor IsrTask_ 

 

@brief Apply common instrument signature correction algorithms to a raw frame. 

 

@section ip_isr_isr_Contents Contents 

 

- @ref ip_isr_isr_Purpose 

- @ref ip_isr_isr_Initialize 

- @ref ip_isr_isr_IO 

- @ref ip_isr_isr_Config 

- @ref ip_isr_isr_Debug 

 

 

@section ip_isr_isr_Purpose Description 

 

The process for correcting imaging data is very similar from camera to camera. 

This task provides a vanilla implementation of doing these corrections, including 

the ability to turn certain corrections off if they are not needed. 

The inputs to the primary method, run, are a raw exposure to be corrected and the 

calibration data products. The raw input is a single chip sized mosaic of all amps 

including overscans and other non-science pixels. 

The method runDataRef() is intended for use by a lsst.pipe.base.cmdLineTask.CmdLineTask 

and takes as input only a daf.persistence.butlerSubset.ButlerDataRef. 

This task may not meet all needs and it is expected that it will be subclassed for 

specific applications. 

 

@section ip_isr_isr_Initialize Task initialization 

 

@copydoc \_\_init\_\_ 

 

@section ip_isr_isr_IO Inputs/Outputs to the run method 

 

@copydoc run 

 

@section ip_isr_isr_Config Configuration parameters 

 

See @ref IsrTaskConfig 

 

@section ip_isr_isr_Debug Debug variables 

 

The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a 

flag @c --debug, @c -d to import @b debug.py from your @c PYTHONPATH; see <a 

href="http://lsst-web.ncsa.illinois.edu/~buildbot/doxygen/x_masterDoxyDoc/base_debug.html"> 

Using lsstDebug to control debugging output</a> for more about @b debug.py files. 

 

The available variables in IsrTask are: 

<DL> 

<DT> @c display 

<DD> A dictionary containing debug point names as keys with frame number as value. Valid keys are: 

<DL> 

<DT> postISRCCD 

<DD> display exposure after ISR has been applied 

</DL> 

</DL> 

 

For example, put something like 

@code{.py} 

import lsstDebug 

def DebugInfo(name): 

di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively 

if name == "lsst.ip.isr.isrTask": 

di.display = {'postISRCCD':2} 

return di 

lsstDebug.Info = DebugInfo 

@endcode 

into your debug.py file and run the commandline task with the @c --debug flag. 

 

<HR> 

""" 

ConfigClass = IsrTaskConfig 

_DefaultName = "isr" 

 

def __init__(self, *args, **kwargs): 

'''!Constructor for IsrTask 

@param[in] *args a list of positional arguments passed on to the Task constructor 

@param[in] **kwargs a dictionary of keyword arguments passed on to the Task constructor 

Call the lsst.pipe.base.task.Task.__init__ method 

Then setup the assembly and fringe correction subtasks 

''' 

pipeBase.Task.__init__(self, *args, **kwargs) 

self.makeSubtask("assembleCcd") 

self.makeSubtask("fringe") 

self.makeSubtask("crosstalk") 

 

def readIsrData(self, dataRef, rawExposure): 

"""!Retrieve necessary frames for instrument signature removal 

@param[in] dataRef a daf.persistence.butlerSubset.ButlerDataRef 

of the detector data to be processed 

@param[in] rawExposure a reference raw exposure that will later be 

corrected with the retrieved calibration data; 

should not be modified in this method. 

@return a pipeBase.Struct with fields containing kwargs expected by run() 

- bias: exposure of bias frame 

- dark: exposure of dark frame 

- flat: exposure of flat field 

- defects: list of detects 

- fringeStruct: a pipeBase.Struct with field fringes containing 

exposure of fringe frame or list of fringe exposure 

""" 

ccd = rawExposure.getDetector() 

 

biasExposure = self.getIsrExposure(dataRef, self.config.biasDataProductName) \ 

if self.config.doBias else None 

# immediate=True required for functors and linearizers are functors; see ticket DM-6515 

linearizer = dataRef.get("linearizer", immediate=True) if self.doLinearize(ccd) else None 

darkExposure = self.getIsrExposure(dataRef, self.config.darkDataProductName) \ 

if self.config.doDark else None 

flatExposure = self.getIsrExposure(dataRef, self.config.flatDataProductName) \ 

if self.config.doFlat else None 

brighterFatterKernel = dataRef.get("brighterFatterKernel") if self.config.doBrighterFatter else None 

defectList = dataRef.get("defects") if self.config.doDefect else None 

 

if self.config.doCrosstalk: 

crosstalkSources = self.crosstalk.prepCrosstalk(dataRef) 

else: 

crosstalkSources = None 

 

if self.config.doFringe and self.fringe.checkFilter(rawExposure): 

fringeStruct = self.fringe.readFringes(dataRef, assembler=self.assembleCcd 

if self.config.doAssembleIsrExposures else None) 

else: 

fringeStruct = pipeBase.Struct(fringes=None) 

 

if self.config.doAttachTransmissionCurve: 

opticsTransmission = (dataRef.get("transmission_optics") 

if self.config.doUseOpticsTransmission else None) 

filterTransmission = (dataRef.get("transmission_filter") 

if self.config.doUseFilterTransmission else None) 

sensorTransmission = (dataRef.get("transmission_sensor") 

if self.config.doUseSensorTransmission else None) 

atmosphereTransmission = (dataRef.get("transmission_atmosphere") 

if self.config.doUseAtmosphereTransmission else None) 

else: 

opticsTransmission = None 

filterTransmission = None 

sensorTransmission = None 

atmosphereTransmission = None 

 

# Struct should include only kwargs to run() 

return pipeBase.Struct(bias=biasExposure, 

linearizer=linearizer, 

dark=darkExposure, 

flat=flatExposure, 

defects=defectList, 

fringes=fringeStruct, 

bfKernel=brighterFatterKernel, 

opticsTransmission=opticsTransmission, 

filterTransmission=filterTransmission, 

sensorTransmission=sensorTransmission, 

atmosphereTransmission=atmosphereTransmission, 

crosstalkSources=crosstalkSources, 

) 

 

@pipeBase.timeMethod 

def run(self, ccdExposure, bias=None, linearizer=None, dark=None, flat=None, defects=None, 

fringes=None, bfKernel=None, camera=None, 

opticsTransmission=None, filterTransmission=None, 

sensorTransmission=None, atmosphereTransmission=None, 

crosstalkSources=None): 

"""!Perform instrument signature removal on an exposure 

 

Steps include: 

- Detect saturation, apply overscan correction, bias, dark and flat 

- Perform CCD assembly 

- Interpolate over defects, saturated pixels and all NaNs 

 

@param[in] ccdExposure lsst.afw.image.exposure of detector data 

@param[in] bias exposure of bias frame 

@param[in] linearizer linearizing functor; a subclass of lsst.ip.isrFunctions.LinearizeBase 

@param[in] dark exposure of dark frame 

@param[in] flat exposure of flatfield 

@param[in] defects list of detects 

@param[in] fringes a pipeBase.Struct with field fringes containing 

exposure of fringe frame or list of fringe exposure 

@param[in] bfKernel kernel for brighter-fatter correction, an 

lsst.cp.pipe.makeBrighterFatterKernel.BrighterFatterKernel object 

@param[in] camera camera geometry, an lsst.afw.cameraGeom.Camera; 

used by addDistortionModel 

@param[in] opticsTransmission a TransmissionCurve for the optics 

@param[in] filterTransmission a TransmissionCurve for the filter 

@param[in] sensorTransmission a TransmissionCurve for the sensor 

@param[in] atmosphereTransmission a TransmissionCurve for the atmosphere 

@param[in] crosstalkSources a defaultdict used for DECam inter-CCD crosstalk 

 

@return a pipeBase.Struct with field: 

- exposure 

""" 

# parseAndRun expects to be able to call run() with a dataRef; see DM-6640 

if isinstance(ccdExposure, ButlerDataRef): 

return self.runDataRef(ccdExposure) 

 

ccd = ccdExposure.getDetector() 

 

# Validate Input 

if self.config.doBias and bias is None: 

raise RuntimeError("Must supply a bias exposure if config.doBias True") 

if self.doLinearize(ccd) and linearizer is None: 

raise RuntimeError("Must supply a linearizer if config.doLinearize True") 

if self.config.doDark and dark is None: 

raise RuntimeError("Must supply a dark exposure if config.doDark True") 

if self.config.doFlat and flat is None: 

raise RuntimeError("Must supply a flat exposure if config.doFlat True") 

if self.config.doBrighterFatter and bfKernel is None: 

raise RuntimeError("Must supply a kernel if config.doBrighterFatter True") 

if fringes is None: 

fringes = pipeBase.Struct(fringes=None) 

if self.config.doFringe and not isinstance(fringes, pipeBase.Struct): 

raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct") 

if self.config.doDefect and defects is None: 

raise RuntimeError("Must supply defects if config.doDefect True") 

if self.config.doAddDistortionModel and camera is None: 

raise RuntimeError("Must supply camera if config.doAddDistortionModel True") 

 

ccdExposure = self.convertIntToFloat(ccdExposure) 

 

if not ccd: 

assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd" 

ccd = [FakeAmp(ccdExposure, self.config)] 

 

overscans = [] 

for amp in ccd: 

# if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

if ccdExposure.getBBox().contains(amp.getBBox()): 

self.saturationDetection(ccdExposure, amp) 

self.suspectDetection(ccdExposure, amp) 

overscanResults = self.overscanCorrection(ccdExposure, amp) 

overscans.append(overscanResults.overscanImage if overscanResults is not None else None) 

else: 

overscans.append(None) 

 

if self.config.doCrosstalk: 

self.crosstalk.run(ccdExposure, crosstalkSources) 

 

if self.config.doAssembleCcd: 

ccdExposure = self.assembleCcd.assembleCcd(ccdExposure) 

if self.config.expectWcs and not ccdExposure.getWcs(): 

self.log.warn("No WCS found in input exposure") 

 

if self.config.doBias: 

self.biasCorrection(ccdExposure, bias) 

 

if self.doLinearize(ccd): 

linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log) 

 

assert len(ccd) == len(overscans) 

for amp, overscanImage in zip(ccd, overscans): 

# if ccdExposure is one amp, check for coverage to prevent performing ops multiple times 

if ccdExposure.getBBox().contains(amp.getBBox()): 

ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox()) 

self.updateVariance(ampExposure, amp, overscanImage) 

 

interpolationDone = False 

 

if self.config.doBrighterFatter: 

 

# We need to apply flats and darks before we can interpolate, and we 

# need to interpolate before we do B-F, but we do B-F without the 

# flats and darks applied so we can work in units of electrons or holes. 

# This context manager applies and then removes the darks and flats. 

with self.flatContext(ccdExposure, flat, dark): 

if self.config.doDefect: 

self.maskAndInterpDefect(ccdExposure, defects) 

if self.config.doSaturationInterpolation: 

self.saturationInterpolation(ccdExposure) 

self.maskAndInterpNan(ccdExposure) 

interpolationDone = True 

 

if self.config.brighterFatterLevel == 'DETECTOR': 

kernelElement = bfKernel.kernel[ccdExposure.getDetector().getId()] 

else: 

# TODO: DM-15631 for implementing this 

raise NotImplementedError("per-amplifier brighter-fatter correction not yet implemented") 

self.brighterFatterCorrection(ccdExposure, kernelElement, 

self.config.brighterFatterMaxIter, 

self.config.brighterFatterThreshold, 

self.config.brighterFatterApplyGain, 

) 

 

if self.config.doDark: 

self.darkCorrection(ccdExposure, dark) 

 

if self.config.doFringe and not self.config.fringeAfterFlat: 

self.fringe.run(ccdExposure, **fringes.getDict()) 

 

if self.config.doFlat: 

self.flatCorrection(ccdExposure, flat) 

 

if not interpolationDone: 

if self.config.doDefect: 

self.maskAndInterpDefect(ccdExposure, defects) 

if self.config.doSaturationInterpolation: 

self.saturationInterpolation(ccdExposure) 

if not interpolationDone or self.config.doNanInterpAfterFlat: 

self.maskAndInterpNan(ccdExposure) 

 

if self.config.doFringe and self.config.fringeAfterFlat: 

self.fringe.run(ccdExposure, **fringes.getDict()) 

 

exposureTime = ccdExposure.getInfo().getVisitInfo().getExposureTime() 

ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1*exposureTime) 

 

if self.config.doAddDistortionModel: 

self.addDistortionModel(exposure=ccdExposure, camera=camera) 

 

if self.config.doAttachTransmissionCurve: 

self.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission, 

filterTransmission=filterTransmission, 

sensorTransmission=sensorTransmission, 

atmosphereTransmission=atmosphereTransmission) 

 

frame = getDebugFrame(self._display, "postISRCCD") 

if frame: 

display = getDisplay(frame) 

display.scale('asinh', 'zscale') 

display.mtv(ccdExposure) 

 

return pipeBase.Struct( 

exposure=ccdExposure, 

) 

 

@pipeBase.timeMethod 

def runDataRef(self, sensorRef): 

"""Perform instrument signature removal on a ButlerDataRef of a Sensor 

 

- Read in necessary detrending/isr/calibration data 

- Process raw exposure in run() 

- Persist the ISR-corrected exposure as "postISRCCD" if config.doWrite is True 

 

Parameters 

---------- 

sensorRef : `daf.persistence.butlerSubset.ButlerDataRef` 

DataRef of the detector data to be processed 

 

Returns 

------- 

result : `pipeBase.Struct` 

Struct contains field "exposure," which is the exposure after application of ISR 

""" 

self.log.info("Performing ISR on sensor %s" % (sensorRef.dataId)) 

ccdExposure = sensorRef.get('raw') 

camera = sensorRef.get("camera") 

if camera is None and self.config.doAddDistortionModel: 

raise RuntimeError("config.doAddDistortionModel is True " 

"but could not get a camera from the butler") 

isrData = self.readIsrData(sensorRef, ccdExposure) 

 

result = self.run(ccdExposure, camera=camera, **isrData.getDict()) 

 

if self.config.doWrite: 

sensorRef.put(result.exposure, "postISRCCD") 

 

return result 

 

def convertIntToFloat(self, exposure): 

"""Convert an exposure from uint16 to float, set variance plane to 1 and mask plane to 0 

""" 

if isinstance(exposure, afwImage.ExposureF): 

# Nothing to be done 

return exposure 

if not hasattr(exposure, "convertF"): 

raise RuntimeError("Unable to convert exposure (%s) to float" % type(exposure)) 

 

newexposure = exposure.convertF() 

newexposure.variance[:] = 1 

newexposure.mask[:] = 0x0 

 

return newexposure 

 

def biasCorrection(self, exposure, biasExposure): 

"""!Apply bias correction in place 

 

@param[in,out] exposure exposure to process 

@param[in] biasExposure bias exposure of same size as exposure 

""" 

isrFunctions.biasCorrection(exposure.getMaskedImage(), biasExposure.getMaskedImage()) 

 

def darkCorrection(self, exposure, darkExposure, invert=False): 

"""!Apply dark correction in place 

 

@param[in,out] exposure exposure to process 

@param[in] darkExposure dark exposure of same size as exposure 

@param[in] invert if True, remove the dark from an already-corrected image 

""" 

expScale = exposure.getInfo().getVisitInfo().getDarkTime() 

if math.isnan(expScale): 

raise RuntimeError("Exposure darktime is NAN") 

darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime() 

if math.isnan(darkScale): 

raise RuntimeError("Dark calib darktime is NAN") 

isrFunctions.darkCorrection( 

maskedImage=exposure.getMaskedImage(), 

darkMaskedImage=darkExposure.getMaskedImage(), 

expScale=expScale, 

darkScale=darkScale, 

invert=invert 

) 

 

def doLinearize(self, detector): 

"""!Is linearization wanted for this detector? 

 

Checks config.doLinearize and the linearity type of the first amplifier. 

 

@param[in] detector detector information (an lsst.afw.cameraGeom.Detector) 

""" 

return self.config.doLinearize and \ 

detector.getAmpInfoCatalog()[0].getLinearityType() != NullLinearityType 

 

def updateVariance(self, ampExposure, amp, overscanImage=None): 

"""Set the variance plane using the amplifier gain and read noise 

 

The read noise is calculated from the ``overscanImage`` if the 

``doEmpiricalReadNoise`` option is set in the configuration; otherwise 

the value from the amplifier data is used. 

 

Parameters 

---------- 

ampExposure : `lsst.afw.image.Exposure` 

Exposure to process. 

amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp` 

Amplifier detector data. 

overscanImage : `lsst.afw.image.MaskedImage`, optional. 

Image of overscan, required only for empirical read noise. 

""" 

maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName] 

gain = amp.getGain() 

if not math.isnan(gain): 

if gain <= 0: 

patchedGain = 1.0 

self.log.warn("Gain for amp %s == %g <= 0; setting to %f" % 

(amp.getName(), gain, patchedGain)) 

gain = patchedGain 

 

if self.config.doEmpiricalReadNoise and overscanImage is not None: 

stats = afwMath.StatisticsControl() 

stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes)) 

readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue() 

self.log.info("Calculated empirical read noise for amp %s: %f", amp.getName(), readNoise) 

else: 

readNoise = amp.getReadNoise() 

 

isrFunctions.updateVariance( 

maskedImage=ampExposure.getMaskedImage(), 

gain=gain, 

readNoise=readNoise, 

) 

 

def flatCorrection(self, exposure, flatExposure, invert=False): 

"""!Apply flat correction in place 

 

@param[in,out] exposure exposure to process 

@param[in] flatExposure flatfield exposure same size as exposure 

@param[in] invert if True, unflatten an already-flattened image instead. 

""" 

isrFunctions.flatCorrection( 

maskedImage=exposure.getMaskedImage(), 

flatMaskedImage=flatExposure.getMaskedImage(), 

scalingType=self.config.flatScalingType, 

userScale=self.config.flatUserScale, 

invert=invert 

) 

 

def getIsrExposure(self, dataRef, datasetType, immediate=True): 

"""!Retrieve a calibration dataset for removing instrument signature 

 

@param[in] dataRef data reference for exposure 

@param[in] datasetType type of dataset to retrieve (e.g. 'bias', 'flat') 

@param[in] immediate if True, disable butler proxies to enable error 

handling within this routine 

@return exposure 

""" 

try: 

exp = dataRef.get(datasetType, immediate=immediate) 

except Exception as exc1: 

if not self.config.fallbackFilterName: 

raise RuntimeError("Unable to retrieve %s for %s: %s" % (datasetType, dataRef.dataId, exc1)) 

try: 

exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate) 

except Exception as exc2: 

raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s" % 

(datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2)) 

self.log.warn("Using fallback calibration from filter %s" % self.config.fallbackFilterName) 

 

if self.config.doAssembleIsrExposures: 

exp = self.assembleCcd.assembleCcd(exp) 

return exp 

 

def saturationDetection(self, exposure, amp): 

"""!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place 

 

@param[in,out] exposure exposure to process; only the amp DataSec is processed 

@param[in] amp amplifier device data 

""" 

if not math.isnan(amp.getSaturation()): 

maskedImage = exposure.getMaskedImage() 

dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

isrFunctions.makeThresholdMask( 

maskedImage=dataView, 

threshold=amp.getSaturation(), 

growFootprints=0, 

maskName=self.config.saturatedMaskName, 

) 

 

def saturationInterpolation(self, ccdExposure): 

"""!Interpolate over saturated pixels, in place 

 

@param[in,out] ccdExposure exposure to process 

 

@warning: 

- Call saturationDetection first, so that saturated pixels have been identified in the "SAT" mask. 

- Call this after CCD assembly, since saturated regions may cross amplifier boundaries 

""" 

isrFunctions.interpolateFromMask( 

maskedImage=ccdExposure.getMaskedImage(), 

fwhm=self.config.fwhm, 

growFootprints=self.config.growSaturationFootprintSize, 

maskName=self.config.saturatedMaskName, 

) 

 

def suspectDetection(self, exposure, amp): 

"""!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place 

 

Suspect pixels are pixels whose value is greater than amp.getSuspectLevel(). 

This is intended to indicate pixels that may be affected by unknown systematics; 

for example if non-linearity corrections above a certain level are unstable 

then that would be a useful value for suspectLevel. A value of `nan` indicates 

that no such level exists and no pixels are to be masked as suspicious. 

 

@param[in,out] exposure exposure to process; only the amp DataSec is processed 

@param[in] amp amplifier device data 

""" 

suspectLevel = amp.getSuspectLevel() 

if math.isnan(suspectLevel): 

return 

 

maskedImage = exposure.getMaskedImage() 

dataView = maskedImage.Factory(maskedImage, amp.getRawBBox()) 

isrFunctions.makeThresholdMask( 

maskedImage=dataView, 

threshold=suspectLevel, 

growFootprints=0, 

maskName=self.config.suspectMaskName, 

) 

 

def maskAndInterpDefect(self, ccdExposure, defectBaseList): 

"""!Mask defects using mask plane "BAD" and interpolate over them, in place 

 

@param[in,out] ccdExposure exposure to process 

@param[in] defectBaseList a list of defects to mask and interpolate 

 

@warning: call this after CCD assembly, since defects may cross amplifier boundaries 

""" 

maskedImage = ccdExposure.getMaskedImage() 

defectList = [] 

for d in defectBaseList: 

bbox = d.getBBox() 

nd = measAlg.Defect(bbox) 

defectList.append(nd) 

isrFunctions.maskPixelsFromDefectList(maskedImage, defectList, maskName='BAD') 

isrFunctions.interpolateDefectList( 

maskedImage=maskedImage, 

defectList=defectList, 

fwhm=self.config.fwhm, 

) 

 

def maskAndInterpNan(self, exposure): 

"""!Mask NaNs using mask plane "UNMASKEDNAN" and interpolate over them, in place 

 

We mask and interpolate over all NaNs, including those 

that are masked with other bits (because those may or may 

not be interpolated over later, and we want to remove all 

NaNs). Despite this behaviour, the "UNMASKEDNAN" mask plane 

is used to preserve the historical name. 

 

@param[in,out] exposure exposure to process 

""" 

maskedImage = exposure.getMaskedImage() 

 

# Find and mask NaNs 

maskedImage.getMask().addMaskPlane("UNMASKEDNAN") 

maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN") 

numNans = maskNans(maskedImage, maskVal) 

self.metadata.set("NUMNANS", numNans) 

 

# Interpolate over these previously-unmasked NaNs 

if numNans > 0: 

self.log.warn("There were %i unmasked NaNs", numNans) 

nanDefectList = isrFunctions.getDefectListFromMask( 

maskedImage=maskedImage, 

maskName='UNMASKEDNAN', 

) 

isrFunctions.interpolateDefectList( 

maskedImage=exposure.getMaskedImage(), 

defectList=nanDefectList, 

fwhm=self.config.fwhm, 

) 

 

def overscanCorrection(self, exposure, amp): 

"""Apply overscan correction, in-place 

 

Parameters 

---------- 

exposure : `lsst.afw.image.Exposure` 

Exposure to process; must include both data and bias regions. 

amp : `lsst.afw.table.AmpInfoRecord` 

Amplifier device data. 

 

Results 

------- 

result : `lsst.pipe.base.Struct` or `NoneType` 

`None` if there is no overscan; otherwise, this is a 

result struct with components: 

 

- ``imageFit``: Value(s) removed from image (scalar or 

`lsst.afw.image.Image`). 

- ``overscanFit``: Value(s) removed from overscan (scalar or 

`lsst.afw.image.Image`). 

- ``overscanImage``: Image of the overscan, post-subtraction 

(`lsst.afw.image.Image`). 

""" 

if not amp.getHasRawInfo(): 

raise RuntimeError("This method must be executed on an amp with raw information.") 

 

if amp.getRawHorizontalOverscanBBox().isEmpty(): 

self.log.info("No Overscan region. Not performing Overscan Correction.") 

return None 

 

oscanBBox = amp.getRawHorizontalOverscanBBox() 

 

# afw.cameraGeom.assembleImage.makeUpdatedDetector doesn't update readoutCorner; DM-15559 

x0, x1 = oscanBBox.getBeginX(), oscanBBox.getEndX() 

 

prescanBBox = amp.getRawPrescanBBox() 

if oscanBBox.getBeginX() > prescanBBox.getBeginX(): # amp is at the right 

x0 += self.config.overscanNumLeadingColumnsToSkip 

x1 -= self.config.overscanNumTrailingColumnsToSkip 

else: 

x0 += self.config.overscanNumTrailingColumnsToSkip 

x1 -= self.config.overscanNumLeadingColumnsToSkip 

 

oscanBBox = afwGeom.BoxI(afwGeom.PointI(x0, oscanBBox.getBeginY()), 

afwGeom.PointI(x1 - 1, oscanBBox.getEndY() - 1)) 

 

maskedImage = exposure.maskedImage 

dataView = maskedImage[amp.getRawDataBBox()] 

overscanImage = maskedImage[oscanBBox] 

 

sctrl = afwMath.StatisticsControl() 

sctrl.setNumSigmaClip(self.config.overscanNumSigmaClip) 

 

results = isrFunctions.overscanCorrection( 

ampMaskedImage=dataView, 

overscanImage=overscanImage, 

fitType=self.config.overscanFitType, 

order=self.config.overscanOrder, 

statControl=sctrl, 

) 

results.overscanImage = overscanImage 

return results 

 

def addDistortionModel(self, exposure, camera): 

"""!Update the WCS in exposure with a distortion model based on camera geometry 

 

Add a model for optical distortion based on geometry found in `camera` 

and the `exposure`'s detector. The raw input exposure is assumed 

have a TAN WCS that has no compensation for optical distortion. 

Two other possibilities are: 

- The raw input exposure already has a model for optical distortion, 

as is the case for raw DECam data. 

In that case you should set config.doAddDistortionModel False. 

- The raw input exposure has a model for distortion, but it has known 

deficiencies severe enough to be worth fixing (e.g. because they 

cause problems for fitting a better WCS). In that case you should 

override this method with a version suitable for your raw data. 

 

@param[in,out] exposure exposure to process; must include a Detector and a WCS; 

the WCS of the exposure is modified in place 

@param[in] camera camera geometry; an lsst.afw.cameraGeom.Camera 

""" 

self.log.info("Adding a distortion model to the WCS") 

wcs = exposure.getWcs() 

if wcs is None: 

raise RuntimeError("exposure has no WCS") 

if camera is None: 

raise RuntimeError("camera is None") 

detector = exposure.getDetector() 

if detector is None: 

raise RuntimeError("exposure has no Detector") 

pixelToFocalPlane = detector.getTransform(PIXELS, FOCAL_PLANE) 

focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE) 

distortedWcs = makeDistortedTanWcs(wcs, pixelToFocalPlane, focalPlaneToFieldAngle) 

exposure.setWcs(distortedWcs) 

 

def setValidPolygonIntersect(self, ccdExposure, fpPolygon): 

"""!Set the valid polygon as the intersection of fpPolygon and the ccd corners 

 

@param[in,out] ccdExposure exposure to process 

@param[in] fpPolygon Polygon in focal plane coordinates 

""" 

# Get ccd corners in focal plane coordinates 

ccd = ccdExposure.getDetector() 

fpCorners = ccd.getCorners(FOCAL_PLANE) 

ccdPolygon = Polygon(fpCorners) 

 

# Get intersection of ccd corners with fpPolygon 

intersect = ccdPolygon.intersectionSingle(fpPolygon) 

 

# Transform back to pixel positions and build new polygon 

ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS) 

validPolygon = Polygon(ccdPoints) 

ccdExposure.getInfo().setValidPolygon(validPolygon) 

 

def brighterFatterCorrection(self, exposure, kernel, maxIter, threshold, applyGain): 

"""Apply brighter fatter correction in place for the image 

 

This correction takes a kernel that has been derived from flat field images to 

redistribute the charge. The gradient of the kernel is the deflection 

field due to the accumulated charge. 

 

Given the original image I(x) and the kernel K(x) we can compute the corrected image Ic(x) 

using the following equation: 

 

Ic(x) = I(x) + 0.5*d/dx(I(x)*d/dx(int( dy*K(x-y)*I(y)))) 

 

To evaluate the derivative term we expand it as follows: 

 

0.5 * ( d/dx(I(x))*d/dx(int(dy*K(x-y)*I(y))) + I(x)*d^2/dx^2(int(dy* K(x-y)*I(y))) ) 

 

Because we use the measured counts instead of the incident counts we apply the correction 

iteratively to reconstruct the original counts and the correction. We stop iterating when the 

summed difference between the current corrected image and the one from the previous iteration 

is below the threshold. We do not require convergence because the number of iterations is 

too large a computational cost. How we define the threshold still needs to be evaluated, the 

current default was shown to work reasonably well on a small set of images. For more information 

on the method see DocuShare Document-19407. 

 

The edges as defined by the kernel are not corrected because they have spurious values 

due to the convolution. 

""" 

self.log.info("Applying brighter fatter correction") 

 

image = exposure.getMaskedImage().getImage() 

 

# The image needs to be units of electrons/holes 

with self.gainContext(exposure, image, applyGain): 

 

kLx = numpy.shape(kernel)[0] 

kLy = numpy.shape(kernel)[1] 

kernelImage = afwImage.ImageD(kLx, kLy) 

kernelImage.getArray()[:, :] = kernel 

tempImage = image.clone() 

 

nanIndex = numpy.isnan(tempImage.getArray()) 

tempImage.getArray()[nanIndex] = 0. 

 

outImage = afwImage.ImageF(image.getDimensions()) 

corr = numpy.zeros_like(image.getArray()) 

prev_image = numpy.zeros_like(image.getArray()) 

convCntrl = afwMath.ConvolutionControl(False, True, 1) 

fixedKernel = afwMath.FixedKernel(kernelImage) 

 

# Define boundary by convolution region. The region that the correction will be 

# calculated for is one fewer in each dimension because of the second derivative terms. 

# NOTE: these need to use integer math, as we're using start:end as numpy index ranges. 

startX = kLx//2 

endX = -kLx//2 

startY = kLy//2 

endY = -kLy//2 

 

for iteration in range(maxIter): 

 

afwMath.convolve(outImage, tempImage, fixedKernel, convCntrl) 

tmpArray = tempImage.getArray() 

outArray = outImage.getArray() 

 

with numpy.errstate(invalid="ignore", over="ignore"): 

# First derivative term 

gradTmp = numpy.gradient(tmpArray[startY:endY, startX:endX]) 

gradOut = numpy.gradient(outArray[startY:endY, startX:endX]) 

first = (gradTmp[0]*gradOut[0] + gradTmp[1]*gradOut[1])[1:-1, 1:-1] 

 

# Second derivative term 

diffOut20 = numpy.diff(outArray, 2, 0)[startY:endY, startX + 1:endX - 1] 

diffOut21 = numpy.diff(outArray, 2, 1)[startY + 1:endY - 1, startX:endX] 

second = tmpArray[startY + 1:endY - 1, startX + 1:endX - 1]*(diffOut20 + diffOut21) 

 

corr[startY + 1:endY - 1, startX + 1:endX - 1] = 0.5*(first + second) 

 

tmpArray[:, :] = image.getArray()[:, :] 

tmpArray[nanIndex] = 0. 

tmpArray[startY:endY, startX:endX] += corr[startY:endY, startX:endX] 

 

if iteration > 0: 

diff = numpy.sum(numpy.abs(prev_image - tmpArray)) 

 

if diff < threshold: 

break 

prev_image[:, :] = tmpArray[:, :] 

 

if iteration == maxIter - 1: 

self.log.warn("Brighter fatter correction did not converge, final difference %f" % diff) 

 

self.log.info("Finished brighter fatter in %d iterations" % (iteration + 1)) 

image.getArray()[startY + 1:endY - 1, startX + 1:endX - 1] += \ 

corr[startY + 1:endY - 1, startX + 1:endX - 1] 

 

def attachTransmissionCurve(self, exposure, opticsTransmission=None, filterTransmission=None, 

sensorTransmission=None, atmosphereTransmission=None): 

"""Attach a TransmissionCurve to an Exposure, given separate curves for 

different components. 

 

Parameters 

---------- 

exposure : `lsst.afw.image.Exposure` 

Exposure object to modify by attaching the product of all given 

``TransmissionCurves`` in post-assembly trimmed detector 

coordinates. Must have a valid ``Detector`` attached that matches 

the detector associated with sensorTransmission. 

opticsTransmission : `lsst.afw.image.TransmissionCurve` 

A ``TransmissionCurve`` that represents the throughput of the 

optics, to be evaluated in focal-plane coordinates. 

filterTransmission : `lsst.afw.image.TransmissionCurve` 

A ``TransmissionCurve`` that represents the throughput of the 

filter itself, to be evaluated in focal-plane coordinates. 

sensorTransmission : `lsst.afw.image.TransmissionCurve` 

A ``TransmissionCurve`` that represents the throughput of the 

sensor itself, to be evaluated in post-assembly trimmed detector 

coordinates. 

atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

A ``TransmissionCurve`` that represents the throughput of the 

atmosphere, assumed to be spatially constant. 

 

All ``TransmissionCurve`` arguments are optional; if none are provided, 

the attached ``TransmissionCurve`` will have unit transmission 

everywhere. 

 

Returns 

------- 

combined : ``lsst.afw.image.TransmissionCurve`` 

The TransmissionCurve attached to the exposure. 

""" 

return isrFunctions.attachTransmissionCurve(exposure, opticsTransmission=opticsTransmission, 

filterTransmission=filterTransmission, 

sensorTransmission=sensorTransmission, 

atmosphereTransmission=atmosphereTransmission) 

 

@contextmanager 

def gainContext(self, exp, image, apply): 

"""Context manager that applies and removes gain 

""" 

if apply: 

ccd = exp.getDetector() 

for amp in ccd: 

sim = image.Factory(image, amp.getBBox()) 

sim *= amp.getGain() 

 

try: 

yield exp 

finally: 

if apply: 

ccd = exp.getDetector() 

for amp in ccd: 

sim = image.Factory(image, amp.getBBox()) 

sim /= amp.getGain() 

 

@contextmanager 

def flatContext(self, exp, flat, dark=None): 

"""Context manager that applies and removes flats and darks, 

if the task is configured to apply them. 

""" 

if self.config.doDark and dark is not None: 

self.darkCorrection(exp, dark) 

if self.config.doFlat: 

self.flatCorrection(exp, flat) 

try: 

yield exp 

finally: 

if self.config.doFlat: 

self.flatCorrection(exp, flat, invert=True) 

if self.config.doDark and dark is not None: 

self.darkCorrection(exp, dark, invert=True) 

 

 

class FakeAmp(object): 

"""A Detector-like object that supports returning gain and saturation level""" 

 

def __init__(self, exposure, config): 

self._bbox = exposure.getBBox(afwImage.LOCAL) 

self._RawHorizontalOverscanBBox = afwGeom.Box2I() 

self._gain = config.gain 

self._readNoise = config.readNoise 

self._saturation = config.saturation 

 

def getBBox(self): 

return self._bbox 

 

def getRawBBox(self): 

return self._bbox 

 

def getHasRawInfo(self): 

return True # but see getRawHorizontalOverscanBBox() 

 

def getRawHorizontalOverscanBBox(self): 

return self._RawHorizontalOverscanBBox 

 

def getGain(self): 

return self._gain 

 

def getReadNoise(self): 

return self._readNoise 

 

def getSaturation(self): 

return self._saturation 

 

def getSuspectLevel(self): 

return float("NaN") 

 

 

class RunIsrConfig(pexConfig.Config): 

isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal") 

 

## @addtogroup LSST_task_documentation 

## @{ 

## @page RunIsrTask 

## @ref RunIsrTask_ "RunIsrTask" 

## @copybrief RunIsrTask 

## @} 

 

 

class RunIsrTask(pipeBase.CmdLineTask): 

"""Task to wrap the default IsrTask to allow it to be retargeted. 

 

The standard IsrTask can be called directly from a command line 

program, but doing so removes the ability of the task to be 

retargeted. As most cameras override some set of the IsrTask 

methods, this would remove those data-specific methods in the 

output post-ISR images. This wrapping class fixes the issue, 

allowing identical post-ISR images to be generated by both the 

processCcd and isrTask code. 

""" 

ConfigClass = RunIsrConfig 

_DefaultName = "runIsr" 

 

def __init__(self, *args, **kwargs): 

super().__init__(*args, **kwargs) 

self.makeSubtask("isr") 

 

def runDataRef(self, dataRef): 

""" 

Parameters 

---------- 

dataRef : `lsst.daf.persistence.ButlerDataRef` 

data reference of the detector data to be processed 

 

Returns 

------- 

result : `pipeBase.Struct` 

Result struct with component: 

 

- exposure : `lsst.afw.image.Exposure` 

Post-ISR processed exposure. 

""" 

return self.isr.runDataRef(dataRef)