Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of cp_pipe. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21# 

22"""Calibration products production task code.""" 

23from __future__ import absolute_import, division, print_function 

24 

25import os 

26import glob 

27import sys 

28 

29import lsst.pex.config as pexConfig 

30import lsst.pipe.base as pipeBase 

31import lsst.log as lsstLog 

32import lsst.eotest.sensor as sensorTest 

33 

34 

35class RunEotestConfig(pexConfig.Config): 

36 """Config class for the calibration products production (CP) task.""" 

37 

38 ccdKey = pexConfig.Field( 

39 dtype=str, 

40 doc="The key by which to pull a detector from a dataId, e.g. 'ccd' or 'detector'", 

41 default='ccd', 

42 ) 

43 fe55 = pexConfig.ConfigurableField( 

44 target=sensorTest.Fe55Task, 

45 doc="The Fe55 analysis task.", 

46 ) 

47 doFe55 = pexConfig.Field( 

48 dtype=bool, 

49 doc="Measure gains using Fe55?", 

50 default=True, 

51 ) 

52 readNoise = pexConfig.ConfigurableField( 

53 target=sensorTest.ReadNoiseTask, 

54 doc="The read noise task.", 

55 ) 

56 doReadNoise = pexConfig.Field( 

57 dtype=bool, 

58 doc="Measure the read-noise?", 

59 default=True, 

60 ) 

61 brightPixels = pexConfig.ConfigurableField( 

62 target=sensorTest.BrightPixelsTask, 

63 doc="The bright pixel/column finding task.", 

64 ) 

65 doBrightPixels = pexConfig.Field( 

66 dtype=bool, 

67 doc="Find bright pixels?", 

68 default=True, 

69 ) 

70 darkPixels = pexConfig.ConfigurableField( 

71 target=sensorTest.DarkPixelsTask, 

72 doc="The dark pixel/column finding task.", 

73 ) 

74 doDarkPixels = pexConfig.Field( 

75 dtype=bool, 

76 doc="Find dark pixels?", 

77 default=True, 

78 ) 

79 traps = pexConfig.ConfigurableField( 

80 target=sensorTest.TrapTask, 

81 doc="The trap-finding task.", 

82 ) 

83 doTraps = pexConfig.Field( 

84 dtype=bool, 

85 doc="Find traps using pocket-pumping exposures?", 

86 default=True, 

87 ) 

88 cte = pexConfig.ConfigurableField( 

89 target=sensorTest.CteTask, 

90 doc="The CTE analysis task.", 

91 ) 

92 doCTE = pexConfig.Field( 

93 dtype=bool, 

94 doc="Measure the charge transfer efficiency?", 

95 default=True, 

96 ) 

97 ptc = pexConfig.ConfigurableField( 

98 target=sensorTest.PtcTask, 

99 doc="The PTC analysis task.", 

100 ) 

101 doPTC = pexConfig.Field( 

102 dtype=bool, 

103 doc="Measure the photon transfer curve?", 

104 default=True, 

105 ) 

106 flatPair = pexConfig.ConfigurableField( 

107 target=sensorTest.FlatPairTask, 

108 doc="The flat-pair analysis task.", 

109 ) 

110 doFlatPair = pexConfig.Field( 

111 dtype=bool, 

112 doc="Measure the detector response vs incident flux using flat pairs?", 

113 default=True, 

114 ) 

115 eotestOutputPath = pexConfig.Field( 

116 dtype=str, 

117 doc="Path to which to write the eotest output results. Madatory runtime arg for running eotest.", 

118 default='', 

119 ) 

120 requireAllEOTests = pexConfig.Field( 

121 dtype=bool, 

122 doc="If True, all tests are required to be runnable, and will Raise if data is missing. If False, " 

123 "processing will continue if a previous part failed due to the input dataset being incomplete.", 

124 default=True, 

125 ) 

126 flatPairMaxPdFracDev = pexConfig.Field( 

127 dtype=float, 

128 doc="Maximum allowed fractional deviation between photodiode currents for the eotest flatPair task. " 

129 "This value is passed to the task's run() method at runtime rather than being stored in the task's" 

130 "own pexConfig field.", 

131 default=0.05, 

132 ) 

133 

134 def setDefaults(self): 

135 """Set default config options for the subTasks.""" 

136 # TODO: Set to proper values - DM-12939 

137 self.fe55.temp_set_point = -100 

138 self.fe55.temp_set_point_tol = 20 

139 

140 # TODO: Set to proper values - DM-12939 

141 self.readNoise.temp_set_point = -100 

142 self.readNoise.temp_set_point_tol = 20 

143 

144 # TODO: make this work - DM-12939 

145 self.brightPixels.temp_set_point = -100 

146 self.brightPixels.temp_set_point_tol = 20 

147 

148 # TODO: find the proper settings for flatPairTask to work. This will mean either using the expTime, 

149 # or working out what's wrong with the MONDIODE values (debug that anyway as a large difference might 

150 # indicate something else going on). Note that this task doesn't really use much in its config class, 

151 # but passes in things at runtime param to its run() method, hence putting in a slot for them here. 

152 # DM-12939 

153 self.flatPairMaxPdFracDev = 0.99 

154 

155 def validate(self): 

156 """Override of the valiate() method. 

157 

158 The pexConfigs of the subTasks here cannot be validated in the normal way, as they are the configs 

159 for eotest, which does illegal things, and this would require an upstream PR to fix. Therefore, we 

160 override the validate() method here, and use it to set the output directory for each of the tasks 

161 based on the legal pexConfig parameter for the main task. 

162 """ 

163 log = lsstLog.Log.getLogger("cp.pipe.runEotestConfig") 

164 if not self.eotestOutputPath: 

165 raise RuntimeError("Must supply an output path for eotest data. " 

166 "Please set config.eotestOutputPath.") 

167 

168 taskList = ['fe55', 'brightPixels', 'darkPixels', 'readNoise', 'traps', 'cte', 'flatPair', 'ptc'] 

169 for task in taskList: 

170 if getattr(self, task).output_dir != '.': 

171 # Being thorough here: '.' is the eotest default. If this is not the value then the user has 

172 # specified something, and we're going to clobber it, so raise a warning. Unlike to happen. 

173 log.warn("OVERWRITING: Found a user defined output path of %s for %sTask. " 

174 "This has been overwritten with %s, as individually specified output paths for " 

175 "subTasks are not supported at present" % (getattr(self, task).output_dir, 

176 task, self.eotestOutputPath)) 

177 getattr(self, task).output_dir = self.eotestOutputPath 

178 

179 

180class RunEotestTask(pipeBase.CmdLineTask): 

181 """ 

182 Task to run test stand data through eotest using a butler. 

183 

184 This task is used to produce an eotest report (the project's sensor 

185 acceptance testing package) 

186 Examples of some of its operations are as follows: 

187 * Given a set of flat-field images, find the dark pixels and columns. 

188 * Given a set of darks, find the bright pixels and columns. 

189 * Given a set of Fe55 exposures, calulate the gain of the readout chain, 

190 in e-/ADU 

191 * Given a set of Fe55 exposures, calulate the instrinsic PSF of the silicon, 

192 and the degradation of 

193 * the PSF due to CTE. 

194 * Given a set of flat-pairs, measure the photon transfer curve (PTC). 

195 * Given a set of bias frames, calculate the read noise of the system in e-. 

196 * Given a set of pocket-pumping exposures, find charge-traps in the silicon. 

197 

198 The RunEotestTask.runEotestDirect() is only applicable to LSST sensors, and 

199 only for a specific type of dataset. This method takes a 

200 dafPersistance.Butler corresponding to a repository in which a full eotest 

201 run has been taken and ingested, and runs each of the tasks in eotest 

202 directly, allowing for bitwise comparison with results given by the camera 

203 team. 

204 

205 See http://ls.st/ldm-151 Chapter 4, Calibration Products Production for 

206 further details regarding the inputs and outputs. 

207 """ 

208 

209 ConfigClass = RunEotestConfig 

210 _DefaultName = "runEotest" 

211 

212 def __init__(self, *args, **kwargs): 

213 """Constructor for the RunEotestTask.""" 

214 if 'lsst.eotest.sensor' not in sys.modules: # check we have eotest before going further 

215 raise RuntimeError('eotest failed to import') 

216 

217 pipeBase.CmdLineTask.__init__(self, *args, **kwargs) 

218 

219 # Note - we can't currently call validate on the subTask configs, as they are NOT valid 

220 # due to state of eotest. However, we override validate() and call it here 

221 # and use it to set the output dir config parameter in the subTasks. 

222 self.config.validate() 

223 self.config.freeze() 

224 

225 self.makeSubtask("fe55") 

226 self.makeSubtask("readNoise") 

227 self.makeSubtask("brightPixels") 

228 self.makeSubtask("darkPixels") 

229 self.makeSubtask("traps") 

230 self.makeSubtask("cte") 

231 self.makeSubtask("flatPair") 

232 self.makeSubtask("ptc") 

233 

234 def _getMaskFiles(self, path, ccd): 

235 """Get all available eotest mask files for a given ccd. 

236 

237 Each stage of the processing generates more mask files, so this allows each to be picked up 

238 as more and more tests run, and saves having to have clever logic for if some tasks fail. 

239 

240 Parameters 

241 ---------- 

242 path : `str` 

243 Path on which to find the mask files 

244 ccd : `string` or `int` 

245 Name/identifier of the CCD 

246 

247 Returns 

248 ------- 

249 maskFiles : iterable of `str` 

250 List of mask files, or an empty tuple if none are found 

251 """ 

252 pattern = '*' + str(ccd) + '*mask*' # the cast to str supports obs_auxTel 

253 maskFiles = glob.glob(os.path.join(path, pattern)) 

254 return maskFiles if len(maskFiles) > 0 else () # eotest wants an empty tuple here 

255 

256 def _cleanupEotest(self, path): 

257 """Delete all the medianed files left behind after eotest has run. 

258 

259 Running eotest generates a lot of interim medianed files, so this just cleans them up. 

260 

261 Parameters 

262 ---------- 

263 path : `str` 

264 Path on which to delete all the eotest medianed files. 

265 """ 

266 for filename in glob.glob(os.path.join(path, '*_median_*.fits')): 

267 os.remove(filename) 

268 

269 def makeEotestReport(self, butler): 

270 """After running eotest, generate pdf(s) of the results. 

271 

272 Generate a sensor test report from the output data in config.eotestOutputPath, one for each CCD. 

273 The pdf file(s), along with the .tex file(s) and the individual plots are written 

274 to the eotestOutputPath. 

275 .pdf generation requires a TeX distro including pdflatex to be installed. 

276 """ 

277 ccds = butler.queryMetadata('raw', self.config.ccdKey) 

278 for ccd in ccds: 

279 self.log.info("Starting test report generation for %s"%ccd) 

280 try: 

281 plotPath = os.path.join(self.config.eotestOutputPath, 'plots') 

282 if not os.path.exists(plotPath): 

283 os.makedirs(plotPath) 

284 plots = sensorTest.EOTestPlots(ccd, self.config.eotestOutputPath, plotPath) 

285 eoTestReport = sensorTest.EOTestReport(plots, wl_dir='') 

286 eoTestReport.make_figures() 

287 eoTestReport.make_pdf() 

288 except Exception as e: 

289 self.log.warn("Failed to make eotest report for %s: %s"%(ccd, e)) 

290 self.log.info("Finished test report generation.") 

291 

292 @pipeBase.timeMethod 

293 def runEotestDirect(self, butler, run=None): 

294 """ 

295 Generate calibration products using eotest algorithms. 

296 

297 Generate all calibration products possible using the vanilla eotest implementation, 

298 given a butler for a TS8 (raft-test) repo. It can contain multiple runs, but must correspond to 

299 only a single raft/RTM. 

300 

301 - Run all eotest tasks possible, using the butler to gather the data 

302 - Write outputs in eotest format 

303 

304 In order to replicate the canonical eotest analysis, the tasks should be run in a specific order. 

305 This is given/defined in the "Steps" section here: 

306 http://lsst-camera.slac.stanford.edu/eTraveler/exp/LSST-CAMERA/displayProcess.jsp?processPath=1179 

307 

308 But is replicated here for conveniece: 

309 * 55Fe Analysis 

310 * CCD Read Noise Analysis 

311 * Bright Defects Analysis 

312 * Dark Defects Analysis 

313 * Traps Finding 

314 * Dark Current X - will not be implemented here 

315 * Charge Transfer Efficiencies 

316 * Photo-response analysis X - will not be implemented here 

317 * Flat Pairs Analysis 

318 * Photon Transfer Curve 

319 * Quantum Efficiency X - will not be implemented here 

320 

321 List of tasks that exist in the eotest package but aren't mentioned on the above link: 

322 * linearityTask() 

323 * fe55CteTask() 

324 * eperTask() 

325 * crosstalkTask() 

326 * persistenceTask() 

327 

328 # TODO: For each eotest task, find out what the standard raft testing does for the optional params. 

329 i.e. many have optional params for gains, bias-frames etc - if we want bitwise identicallity then we 

330 need to know what is typically provided to these tasks when the camera team runs this code. 

331 This can probably be worked out from https://github.com/lsst-camera-dh/lcatr-harness 

332 but it sounds like Jim Chiang doesn't recommend trying to do that. 

333 DM-12939 

334 

335 Parameters 

336 ---------- 

337 butler : `lsst.daf.persistence.butler` 

338 Butler for the repo containg the eotest data to be used 

339 run : `str` or `int` 

340 Optional run number, to be used for repos containing multiple runs 

341 """ 

342 self.log.info("Running eotest routines direct") 

343 

344 # Input testing to check that run is in the repo 

345 runs = butler.queryMetadata('raw', ['run']) 

346 if run is None: 

347 if len(runs) == 1: 

348 run = runs[0] 

349 else: 

350 raise RuntimeError("Butler query found %s for runs. eotest datasets must have a run number," 

351 "and you must specify which run to use if a respoitory contains several." 

352 % runs) 

353 else: 

354 run = str(run) 

355 if run not in runs: 

356 raise RuntimeError("Butler query found %s for runs, but the run specified (%s) " 

357 "was not among them." % (runs, run)) 

358 del runs # we have run defined now, so remove this to avoid potential confusion later 

359 

360 if not os.path.exists(self.config.eotestOutputPath): 

361 os.makedirs(self.config.eotestOutputPath) 

362 

363 ccds = butler.queryMetadata('raw', self.config.ccdKey) 

364 imTypes = butler.queryMetadata('raw', ['imageType']) 

365 testTypes = butler.queryMetadata('raw', ['testType']) 

366 

367 ################################ 

368 ################################ 

369 # Run the Fe55 task 

370 if self.config.doFe55: 

371 fe55TaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'FE55'} 

372 self.log.info("Starting Fe55 pixel task") 

373 for ccd in ccds: 

374 if 'FE55' not in testTypes: 

375 msg = "No Fe55 tests found. Available data: %s" % testTypes 

376 if self.config.requireAllEOTests: 

377 raise RuntimeError(msg) 

378 else: 

379 self.log.warn(msg + "\nSkipping Fe55 task") 

380 break 

381 fe55Filenames = [butler.get('raw_filename', dataId={'visit': visit, 

382 self.config.ccdKey: ccd})[0][:-3] 

383 for visit in butler.queryMetadata('raw', ['visit'], dataId=fe55TaskDataId)] 

384 self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(fe55Filenames))) 

385 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

386 gains = self.fe55.run(sensor_id=ccd, infiles=fe55Filenames, mask_files=maskFiles) 

387 # gainsPropSet = dafBase.PropertySet() 

388 # for amp, gain in gains.items(): # there is no propSet.fromDict() method so make like this 

389 # gainsPropSet.addDouble(str(amp), gain) 

390 butler.put(gains, 'eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

391 del fe55TaskDataId 

392 

393 # TODO: validate the results above, and/or change code to (be able to) always run 

394 # over all files instead of stopping at the "required accuracy" 

395 # This will require making changes to the eotest code. 

396 # DM-12939 

397 

398 ################################ 

399 ################################ 

400 # Run the Noise task 

401 if self.config.doReadNoise: 

402 # note that LCA-10103 defines the Fe55 bias frames as the ones to use here 

403 self.log.info("Starting readNoise task") 

404 noiseTaskDataId = {'run': run, 'testType': 'FE55', 'imageType': 'BIAS'} 

405 for ccd in ccds: 

406 if ('FE55' not in testTypes) or ('BIAS' not in imTypes): 

407 msg = "Required data for readNoise unavailable. Available data:\ 

408 \ntestTypes: %s\nimageTypes: %s" % (testTypes, imTypes) 

409 if self.config.requireAllEOTests: 

410 raise RuntimeError(msg) 

411 else: 

412 self.log.warn(msg + "\nSkipping noise task") 

413 noiseFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

414 self.config.ccdKey: ccd})[0][:-3] 

415 for visit in butler.queryMetadata('raw', ['visit'], 

416 dataId=noiseTaskDataId)] 

417 self.log.trace("Fe55Task: Processing %s with %s files" % (ccd, len(noiseFilenames))) 

418 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

419 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

420 self.readNoise.run(sensor_id=ccd, bias_files=noiseFilenames, 

421 gains=gains, mask_files=maskFiles) 

422 del noiseTaskDataId 

423 

424 ################################ 

425 ################################ 

426 # Run the bright task 

427 if self.config.doBrightPixels: 

428 self.log.info("Starting bright pixel task") 

429 brightTaskDataId = {'run': run, 'testType': 'DARK', 'imageType': 'DARK'} 

430 for ccd in ccds: 

431 if 'DARK' not in testTypes: 

432 msg = "No dark tests found. Available data: %s" % testTypes 

433 if self.config.requireAllEOTests: 

434 raise RuntimeError(msg) 

435 else: 

436 self.log.warn(msg + "\nSkipping bright pixel task") 

437 break 

438 darkFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

439 self.config.ccdKey: ccd})[0][:-3] 

440 for visit in butler.queryMetadata('raw', ['visit'], 

441 dataId=brightTaskDataId)] 

442 self.log.trace("BrightTask: Processing %s with %s files" % (ccd, len(darkFilenames))) 

443 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

444 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

445 self.brightPixels.run(sensor_id=ccd, dark_files=darkFilenames, 

446 mask_files=maskFiles, gains=gains) 

447 del brightTaskDataId 

448 

449 ################################ 

450 ################################ 

451 # Run the dark task 

452 if self.config.doDarkPixels: 

453 self.log.info("Starting dark pixel task") 

454 darkTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'} 

455 for ccd in ccds: 

456 if 'SFLAT_500' not in testTypes: 

457 msg = "No superflats found. Available data: %s" % testTypes 

458 if self.config.requireAllEOTests: 

459 raise RuntimeError(msg) 

460 else: 

461 self.log.warn(msg + "\nSkipping dark pixel task") 

462 break 

463 sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

464 self.config.ccdKey: ccd})[0][:-3] 

465 for visit in butler.queryMetadata('raw', ['visit'], 

466 dataId=darkTaskDataId)] 

467 self.log.trace("DarkTask: Processing %s with %s files" % (ccd, len(sflatFilenames))) 

468 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

469 self.darkPixels.run(sensor_id=ccd, sflat_files=sflatFilenames, mask_files=maskFiles) 

470 del darkTaskDataId 

471 

472 ################################ 

473 ################################ 

474 # Run the trap task 

475 if self.config.doTraps: 

476 self.log.info("Starting trap task") 

477 trapTaskDataId = {'run': run, 'testType': 'TRAP', 'imageType': 'PPUMP'} 

478 for ccd in ccds: 

479 if ('TRAP' not in testTypes) and ('PPUMP' not in imTypes): 

480 msg = "No pocket pumping exposures found. Available data: %s" % testTypes 

481 if self.config.requireAllEOTests: 

482 raise RuntimeError(msg) 

483 else: 

484 self.log.warn(msg + "\nSkipping trap task") 

485 break 

486 trapFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

487 self.config.ccdKey: ccd})[0][:-3] 

488 for visit in butler.queryMetadata('raw', ['visit'], dataId=trapTaskDataId)] 

489 if len(trapFilenames) != 1: # eotest can't handle more than one 

490 msg = "Trap Task: Found more than one ppump trap file: %s" % trapFilenames 

491 msg += " Running using only the first one found." 

492 self.log.warn(msg) 

493 self.log.trace("Trap Task: Processing %s with %s files" % (ccd, len(trapFilenames))) 

494 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

495 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

496 self.traps.run(sensor_id=ccd, pocket_pumped_file=trapFilenames[0], 

497 mask_files=maskFiles, gains=gains) 

498 del trapTaskDataId 

499 

500 ################################ 

501 ################################ 

502 # Run the CTE task 

503 if self.config.doCTE: 

504 self.log.info("Starting CTE task") 

505 cteTaskDataId = {'run': run, 'testType': 'SFLAT_500', 'imageType': 'FLAT'} 

506 for ccd in ccds: 

507 if 'SFLAT_500' not in testTypes: 

508 msg = "No superflats found. Available data: %s" % testTypes 

509 if self.config.requireAllEOTests: 

510 raise RuntimeError(msg) 

511 else: 

512 self.log.warn(msg + "\nSkipping CTE task") 

513 break 

514 sflatFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

515 self.config.ccdKey: ccd})[0][:-3] 

516 for visit in butler.queryMetadata('raw', ['visit'], dataId=cteTaskDataId)] 

517 self.log.trace("CTETask: Processing %s with %s files" % (ccd, len(sflatFilenames))) 

518 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

519 self.cte.run(sensor_id=ccd, superflat_files=sflatFilenames, mask_files=maskFiles) 

520 del cteTaskDataId 

521 

522 ################################ 

523 ################################ 

524 # Run the flatPair task 

525 if self.config.doFlatPair: 

526 self.log.info("Starting flatPair task") 

527 flatPairDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'} 

528 for ccd in ccds: 

529 if 'FLAT' not in testTypes: 

530 msg = "No dataset for flat_pairs found. Available data: %s" % testTypes 

531 if self.config.requireAllEOTests: 

532 raise RuntimeError(msg) 

533 else: 

534 self.log.warn(msg + "\nSkipping flatPair task") 

535 break 

536 flatPairFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

537 self.config.ccdKey: ccd})[0][:-3] 

538 for visit in butler.queryMetadata('raw', ['visit'], 

539 dataId=flatPairDataId)] 

540 # Note that eotest needs the original filename as written by the test-stand data acquisition 

541 # system, as that is the only place the flat pair-number is recorded, so we have to resolve 

542 # sym-links and pass in the *original* paths/filenames here :( 

543 # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here 

544 # so we need to filter these for only the pair acquisitions (as the eotest code looks like it 

545 # isn't totally thorough on rejecting the wrong types of data here) 

546 # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done 

547 # by the butler instead of here. DM-12939 

548 flatPairFilenames = [os.path.realpath(f) for f in flatPairFilenames if 

549 os.path.realpath(f).find('flat1') != -1 

550 or os.path.realpath(f).find('flat2') != -1] 

551 if not flatPairFilenames: 

552 raise RuntimeError("No flatPair files found.") 

553 self.log.trace("FlatPairTask: Processing %s with %s files" % (ccd, len(flatPairFilenames))) 

554 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

555 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

556 self.flatPair.run(sensor_id=ccd, infiles=flatPairFilenames, mask_files=maskFiles, 

557 gains=gains, max_pd_frac_dev=self.config.flatPairMaxPdFracDev) 

558 del flatPairDataId 

559 

560 ################################ 

561 ################################ 

562 # Run the PTC task 

563 if self.config.doPTC: 

564 self.log.info("Starting PTC task") 

565 ptcDataId = {'run': run, 'testType': 'FLAT', 'imageType': 'FLAT'} 

566 for ccd in ccds: 

567 if 'FLAT' not in testTypes: 

568 msg = "No dataset for flat_pairs found. Available data: %s" % testTypes 

569 if self.config.requireAllEOTests: 

570 raise RuntimeError(msg) 

571 else: 

572 self.log.warn(msg + "\nSkipping PTC task") 

573 break 

574 ptcFilenames = [butler.get('raw_filename', dataId={'visit': visit, 

575 self.config.ccdKey: ccd})[0][:-3] 

576 for visit in butler.queryMetadata('raw', ['visit'], dataId=ptcDataId)] 

577 # Note that eotest needs the original filename as written by the test-stand data acquisition 

578 # system, as that is the only place the flat pair-number is recorded, so we have to resolve 

579 # sym-links and pass in the *original* paths/filenames here :( 

580 # Also, there is no "flat-pair" test type, so all FLAT/FLAT imType/testType will appear here 

581 # so we need to filter these for only the pair acquisitions (as the eotest code looks like it 

582 # isn't totally thorough on rejecting the wrong types of data here) 

583 # TODO: adding a translator to obs_comCam and ingesting this would allow this to be done 

584 # by the butler instead of here. DM-12939 

585 ptcFilenames = [os.path.realpath(f) for f in ptcFilenames if 

586 os.path.realpath(f).find('flat1') != -1 

587 or os.path.realpath(f).find('flat2') != -1] 

588 if not ptcFilenames: 

589 raise RuntimeError("No flatPair files found") 

590 self.log.trace("PTCTask: Processing %s with %s files" % (ccd, len(ptcFilenames))) 

591 maskFiles = self._getMaskFiles(self.config.eotestOutputPath, ccd) 

592 gains = butler.get('eotest_gain', dataId={self.config.ccdKey: ccd, 'run': run}) 

593 self.ptc.run(sensor_id=ccd, infiles=ptcFilenames, mask_files=maskFiles, gains=gains) 

594 del ptcDataId 

595 

596 self._cleanupEotest(self.config.eotestOutputPath) 

597 self.log.info("Finished running EOTest")