Coverage for python/lsst/ap/verify/testPipeline.py: 49%

97 statements  

« prev     ^ index     » next       coverage.py v6.4.1, created at 2022-06-16 12:35 +0000

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24 

25# These classes exist only to be included in a mock pipeline, and don't need 

26# to be public for that. 

27__all__ = [] 

28 

29 

30import numpy as np 

31import pandas 

32 

33import lsst.geom as geom 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.afw.table as afwTable 

37import lsst.obs.base as obsBase 

38from lsst.pipe.base import PipelineTask, Struct 

39from lsst.ip.isr import IsrTaskConfig 

40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig 

41from lsst.pipe.tasks.calibrate import CalibrateConfig 

42from lsst.pipe.tasks.imageDifference import ImageDifferenceConfig 

43from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig 

44 

45 

46class MockIsrTask(PipelineTask): 

47 """A do-nothing substitute for IsrTask. 

48 """ 

49 ConfigClass = IsrTaskConfig 

50 _DefaultName = "notIsr" 

51 

52 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

53 crosstalk=None, crosstalkSources=None, 

54 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

55 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

56 sensorTransmission=None, atmosphereTransmission=None, 

57 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

58 isGen3=False, 

59 ): 

60 """Accept ISR inputs, and produce ISR outputs with no processing. 

61 

62 Parameters 

63 ---------- 

64 ccdExposure : `lsst.afw.image.Exposure` 

65 The raw exposure that is to be run through ISR. The 

66 exposure is modified by this method. 

67 camera : `lsst.afw.cameraGeom.Camera`, optional 

68 The camera geometry for this exposure. Required if 

69 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

70 ``flat`` does not have an associated detector. 

71 bias : `lsst.afw.image.Exposure`, optional 

72 Bias calibration frame. 

73 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

74 Functor for linearization. 

75 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

76 Calibration for crosstalk. 

77 crosstalkSources : `list`, optional 

78 List of possible crosstalk sources. 

79 dark : `lsst.afw.image.Exposure`, optional 

80 Dark calibration frame. 

81 flat : `lsst.afw.image.Exposure`, optional 

82 Flat calibration frame. 

83 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

84 Photon transfer curve dataset, with, e.g., gains 

85 and read noise. 

86 bfKernel : `numpy.ndarray`, optional 

87 Brighter-fatter kernel. 

88 bfGains : `dict` of `float`, optional 

89 Gains used to override the detector's nominal gains for the 

90 brighter-fatter correction. A dict keyed by amplifier name for 

91 the detector in question. 

92 defects : `lsst.ip.isr.Defects`, optional 

93 List of defects. 

94 fringes : `lsst.pipe.base.Struct`, optional 

95 Struct containing the fringe correction data, with 

96 elements: 

97 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

98 - ``seed``: random seed derived from the ccdExposureId for random 

99 number generator (`uint32`) 

100 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

101 A ``TransmissionCurve`` that represents the throughput of the 

102 optics, to be evaluated in focal-plane coordinates. 

103 filterTransmission : `lsst.afw.image.TransmissionCurve` 

104 A ``TransmissionCurve`` that represents the throughput of the 

105 filter itself, to be evaluated in focal-plane coordinates. 

106 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

107 A ``TransmissionCurve`` that represents the throughput of the 

108 sensor itself, to be evaluated in post-assembly trimmed detector 

109 coordinates. 

110 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

111 A ``TransmissionCurve`` that represents the throughput of the 

112 atmosphere, assumed to be spatially constant. 

113 detectorNum : `int`, optional 

114 The integer number for the detector to process. 

115 isGen3 : bool, optional 

116 Flag this call to run() as using the Gen3 butler environment. 

117 strayLightData : `object`, optional 

118 Opaque object containing calibration information for stray-light 

119 correction. If `None`, no correction will be performed. 

120 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

121 Illumination correction image. 

122 

123 Returns 

124 ------- 

125 result : `lsst.pipe.base.Struct` 

126 Result struct with components: 

127 

128 ``exposure`` 

129 The fully ISR corrected exposure (`afw.image.Exposure`). 

130 ``outputExposure`` 

131 An alias for ``exposure`` (`afw.image.Exposure`). 

132 ``ossThumb`` 

133 Thumbnail image of the exposure after overscan subtraction 

134 (`numpy.ndarray`). 

135 ``flattenedThumb`` 

136 Thumbnail image of the exposure after flat-field correction 

137 (`numpy.ndarray`). 

138 """ 

139 return Struct(exposure=afwImage.ExposureF(), 

140 outputExposure=afwImage.ExposureF(), 

141 ossThumb=np.empty((1, 1)), 

142 flattenedThumb=np.empty((1, 1)), 

143 ) 

144 

145 

146class MockCharacterizeImageTask(PipelineTask): 

147 """A do-nothing substitute for CharacterizeImageTask. 

148 """ 

149 ConfigClass = CharacterizeImageConfig 

150 _DefaultName = "notCharacterizeImage" 

151 

152 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

153 super().__init__(**kwargs) 

154 self.outputSchema = afwTable.SourceCatalog() 

155 

156 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

157 inputs = butlerQC.get(inputRefs) 

158 if 'exposureIdInfo' not in inputs.keys(): 

159 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId( 

160 butlerQC.quantum.dataId, "visit_detector") 

161 outputs = self.run(**inputs) 

162 butlerQC.put(outputs, outputRefs) 

163 

164 def run(self, exposure, exposureIdInfo=None, background=None): 

165 """Produce characterization outputs with no processing. 

166 

167 Parameters 

168 ---------- 

169 exposure : `lsst.afw.image.Exposure` 

170 Exposure to characterize. 

171 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

172 ID info for exposure. 

173 background : `lsst.afw.math.BackgroundList` 

174 Initial model of background already subtracted from exposure. 

175 

176 Returns 

177 ------- 

178 result : `lsst.pipe.base.Struct` 

179 Struct containing these fields: 

180 

181 ``characterized`` 

182 Characterized exposure (`lsst.afw.image.Exposure`). 

183 ``sourceCat`` 

184 Detected sources (`lsst.afw.table.SourceCatalog`). 

185 ``backgroundModel`` 

186 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`) 

187 ``psfCellSet`` 

188 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`) 

189 """ 

190 # Can't persist empty BackgroundList; DM-33714 

191 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

192 afwImage.MaskedImageF(16, 16)) 

193 return Struct(characterized=exposure, 

194 sourceCat=afwTable.SourceCatalog(), 

195 backgroundModel=afwMath.BackgroundList(bg), 

196 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10), 

197 ) 

198 

199 

200class MockCalibrateTask(PipelineTask): 

201 """A do-nothing substitute for CalibrateTask. 

202 """ 

203 ConfigClass = CalibrateConfig 

204 _DefaultName = "notCalibrate" 

205 

206 def __init__(self, butler=None, astromRefObjLoader=None, 

207 photoRefObjLoader=None, icSourceSchema=None, 

208 initInputs=None, **kwargs): 

209 super().__init__(**kwargs) 

210 self.outputSchema = afwTable.SourceCatalog() 

211 

212 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

213 inputs = butlerQC.get(inputRefs) 

214 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId( 

215 butlerQC.quantum.dataId, "visit_detector") 

216 

217 if self.config.doAstrometry: 

218 inputs.pop('astromRefCat') 

219 if self.config.doPhotoCal: 

220 inputs.pop('photoRefCat') 

221 

222 outputs = self.run(**inputs) 

223 

224 if self.config.doWriteMatches and self.config.doAstrometry: 

225 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

226 if self.config.doWriteMatchesDenormalized: 

227 outputs.matchesDenormalized = outputs.astromMatches 

228 outputs.matches = normalizedMatches 

229 butlerQC.put(outputs, outputRefs) 

230 

231 def run(self, exposure, exposureIdInfo=None, background=None, 

232 icSourceCat=None): 

233 """Produce calibration outputs with no processing. 

234 

235 Parameters 

236 ---------- 

237 exposure : `lsst.afw.image.Exposure` 

238 Exposure to calibrate. 

239 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

240 ID info for exposure. 

241 background : `lsst.afw.math.BackgroundList` 

242 Background model already subtracted from exposure. 

243 icSourceCat : `lsst.afw.table.SourceCatalog` 

244 A SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

245 

246 Returns 

247 ------- 

248 result : `lsst.pipe.base.Struct` 

249 Struct containing these fields: 

250 

251 ``outputExposure`` 

252 Calibrated science exposure with refined WCS and PhotoCalib 

253 (`lsst.afw.image.Exposure`). 

254 ``outputBackground`` 

255 Model of background subtracted from exposure 

256 (`lsst.afw.math.BackgroundList`). 

257 ``outputCat`` 

258 Catalog of measured sources (`lsst.afw.table.SourceCatalog`). 

259 ``astromMatches`` 

260 List of source/refObj matches from the astrometry solver 

261 (`list` [`lsst.afw.table.ReferenceMatch`]). 

262 """ 

263 # Can't persist empty BackgroundList; DM-33714 

264 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

265 afwImage.MaskedImageF(16, 16)) 

266 return Struct(outputExposure=exposure, 

267 outputBackground=afwMath.BackgroundList(bg), 

268 outputCat=afwTable.SourceCatalog(), 

269 astromMatches=[], 

270 ) 

271 

272 

273class MockImageDifferenceTask(PipelineTask): 

274 """A do-nothing substitute for ImageDifferenceTask. 

275 """ 

276 ConfigClass = ImageDifferenceConfig 

277 _DefaultName = "notImageDifference" 

278 

279 def __init__(self, butler=None, **kwargs): 

280 super().__init__(**kwargs) 

281 self.outputSchema = afwTable.SourceCatalog() 

282 

283 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

284 inputs = butlerQC.get(inputRefs) 

285 outputs = self.run(exposure=inputs['exposure'], 

286 templateExposure=afwImage.ExposureF(), 

287 idFactory=obsBase.ExposureIdInfo(8, 4).makeSourceIdFactory()) 

288 butlerQC.put(outputs, outputRefs) 

289 

290 def run(self, exposure=None, selectSources=None, templateExposure=None, templateSources=None, 

291 idFactory=None, calexpBackgroundExposure=None, subtractedExposure=None): 

292 """Produce differencing outputs with no processing. 

293 

294 Parameters 

295 ---------- 

296 exposure : `lsst.afw.image.ExposureF`, optional 

297 The science exposure, the minuend in the image subtraction. 

298 Can be None only if ``config.doSubtract==False``. 

299 selectSources : `lsst.afw.table.SourceCatalog`, optional 

300 Identified sources on the science exposure. This catalog is used to 

301 select sources in order to perform the AL PSF matching on stamp images 

302 around them. The selection steps depend on config options and whether 

303 ``templateSources`` and ``matchingSources`` specified. 

304 templateExposure : `lsst.afw.image.ExposureF`, optional 

305 The template to be subtracted from ``exposure`` in the image subtraction. 

306 ``templateExposure`` is modified in place if ``config.doScaleTemplateVariance==True``. 

307 The template exposure should cover the same sky area as the science exposure. 

308 It is either a stich of patches of a coadd skymap image or a calexp 

309 of the same pointing as the science exposure. Can be None only 

310 if ``config.doSubtract==False`` and ``subtractedExposure`` is not None. 

311 templateSources : `lsst.afw.table.SourceCatalog`, optional 

312 Identified sources on the template exposure. 

313 idFactory : `lsst.afw.table.IdFactory` 

314 Generator object to assign ids to detected sources in the difference image. 

315 calexpBackgroundExposure : `lsst.afw.image.ExposureF`, optional 

316 Background exposure to be added back to the science exposure 

317 if ``config.doAddCalexpBackground==True`` 

318 subtractedExposure : `lsst.afw.image.ExposureF`, optional 

319 If ``config.doSubtract==False`` and ``config.doDetection==True``, 

320 performs the post subtraction source detection only on this exposure. 

321 Otherwise should be None. 

322 

323 Returns 

324 ------- 

325 results : `lsst.pipe.base.Struct` 

326 

327 ``subtractedExposure`` : `lsst.afw.image.ExposureF` 

328 Difference image. 

329 ``scoreExposure`` : `lsst.afw.image.ExposureF` or `None` 

330 The zogy score exposure, if calculated. 

331 ``matchedExposure`` : `lsst.afw.image.ExposureF` 

332 The matched PSF exposure. 

333 ``warpedExposure`` : `lsst.afw.image.ExposureF` 

334 The warped PSF exposure. 

335 ``subtractRes`` : `lsst.pipe.base.Struct` 

336 The returned result structure of the ImagePsfMatchTask subtask. 

337 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

338 The catalog of detected sources. 

339 ``selectSources`` : `lsst.afw.table.SourceCatalog` 

340 The input source catalog with optionally added Qa information. 

341 """ 

342 return Struct( 

343 subtractedExposure=afwImage.ExposureF(), 

344 scoreExposure=afwImage.ExposureF(), 

345 warpedExposure=afwImage.ExposureF(), 

346 matchedExposure=afwImage.ExposureF(), 

347 subtractRes=Struct(), 

348 diaSources=afwTable.SourceCatalog(), 

349 selectSources=afwTable.SourceCatalog(), 

350 ) 

351 

352 

353class MockTransformDiaSourceCatalogTask(PipelineTask): 

354 """A do-nothing substitute for TransformDiaSourceCatalogTask. 

355 """ 

356 ConfigClass = TransformDiaSourceCatalogConfig 

357 _DefaultName = "notTransformDiaSourceCatalog" 

358 

359 def __init__(self, initInputs, **kwargs): 

360 super().__init__(**kwargs) 

361 

362 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

363 inputs = butlerQC.get(inputRefs) 

364 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

365 returnMaxBits=True) 

366 inputs["ccdVisitId"] = expId 

367 inputs["band"] = butlerQC.quantum.dataId["band"] 

368 

369 outputs = self.run(**inputs) 

370 

371 butlerQC.put(outputs, outputRefs) 

372 

373 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None): 

374 """Produce transformation outputs with no processing. 

375 

376 Parameters 

377 ---------- 

378 diaSourceCat : `lsst.afw.table.SourceCatalog` 

379 The catalog to transform. 

380 diffIm : `lsst.afw.image.Exposure` 

381 An image, to provide supplementary information. 

382 band : `str` 

383 The band in which the sources were observed. 

384 ccdVisitId : `int` 

385 The exposure ID in which the sources were observed. 

386 funcs 

387 Unused. 

388 

389 Returns 

390 ------- 

391 results : `lsst.pipe.base.Struct` 

392 Results struct with components: 

393 

394 ``diaSourceTable`` 

395 Catalog of DiaSources (`pandas.DataFrame`). 

396 """ 

397 return Struct(diaSourceTable=pandas.DataFrame(), 

398 ) 

399 

400 

401class MockDiaPipelineTask(PipelineTask): 

402 """A do-nothing substitute for DiaPipelineTask. 

403 """ 

404 ConfigClass = DiaPipelineConfig 

405 _DefaultName = "notDiaPipe" 

406 

407 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

408 inputs = butlerQC.get(inputRefs) 

409 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

410 returnMaxBits=True) 

411 inputs["ccdExposureIdBits"] = expBits 

412 inputs["band"] = butlerQC.quantum.dataId["band"] 

413 if not self.config.doSolarSystemAssociation: 

414 inputs["solarSystemObjectTable"] = None 

415 

416 outputs = self.run(**inputs) 

417 

418 butlerQC.put(outputs, outputRefs) 

419 

420 def run(self, 

421 diaSourceTable, 

422 solarSystemObjectTable, 

423 diffIm, 

424 exposure, 

425 warpedExposure, 

426 ccdExposureIdBits, 

427 band): 

428 """Produce DiaSource and DiaObject outputs with no processing. 

429 

430 Parameters 

431 ---------- 

432 diaSourceTable : `pandas.DataFrame` 

433 Newly detected DiaSources. 

434 solarSystemObjectTable : `pandas.DataFrame` 

435 Expected solar system objects in the field of view. 

436 diffIm : `lsst.afw.image.ExposureF` 

437 Difference image exposure in which the sources in ``diaSourceCat`` 

438 were detected. 

439 exposure : `lsst.afw.image.ExposureF` 

440 Calibrated exposure differenced with a template to create 

441 ``diffIm``. 

442 warpedExposure : `lsst.afw.image.ExposureF` 

443 Template exposure used to create diffIm. 

444 ccdExposureIdBits : `int` 

445 Number of bits used for a unique ``ccdVisitId``. 

446 band : `str` 

447 The band in which the new DiaSources were detected. 

448 

449 Returns 

450 ------- 

451 results : `lsst.pipe.base.Struct` 

452 Results struct with components: 

453 

454 ``apdbMarker`` 

455 Marker dataset to store in the Butler indicating that this 

456 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`). 

457 ``associatedDiaSources`` 

458 Catalog of newly associated DiaSources (`pandas.DataFrame`). 

459 """ 

460 return Struct(apdbMarker=self.config.apdb.value, 

461 associatedDiaSources=pandas.DataFrame(), 

462 )