Coverage for python/lsst/ap/verify/testPipeline.py: 50%

115 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-31 12:14 +0000

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24 

25# These classes exist only to be included in a mock pipeline, and don't need 

26# to be public for that. 

27__all__ = [] 

28 

29 

30import numpy as np 

31import pandas 

32 

33import lsst.geom as geom 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.afw.table as afwTable 

37import lsst.obs.base as obsBase 

38from lsst.pipe.base import PipelineTask, Struct 

39from lsst.ip.isr import IsrTaskConfig 

40from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig 

41from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig 

42from lsst.pipe.tasks.calibrate import CalibrateConfig 

43from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig 

44 

45 

46class MockIsrTask(PipelineTask): 

47 """A do-nothing substitute for IsrTask. 

48 """ 

49 ConfigClass = IsrTaskConfig 

50 _DefaultName = "notIsr" 

51 

52 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

53 crosstalk=None, crosstalkSources=None, 

54 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

55 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

56 sensorTransmission=None, atmosphereTransmission=None, 

57 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

58 deferredCharge=None, 

59 ): 

60 """Accept ISR inputs, and produce ISR outputs with no processing. 

61 

62 Parameters 

63 ---------- 

64 ccdExposure : `lsst.afw.image.Exposure` 

65 The raw exposure that is to be run through ISR. The 

66 exposure is modified by this method. 

67 camera : `lsst.afw.cameraGeom.Camera`, optional 

68 The camera geometry for this exposure. Required if 

69 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

70 ``flat`` does not have an associated detector. 

71 bias : `lsst.afw.image.Exposure`, optional 

72 Bias calibration frame. 

73 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

74 Functor for linearization. 

75 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

76 Calibration for crosstalk. 

77 crosstalkSources : `list`, optional 

78 List of possible crosstalk sources. 

79 dark : `lsst.afw.image.Exposure`, optional 

80 Dark calibration frame. 

81 flat : `lsst.afw.image.Exposure`, optional 

82 Flat calibration frame. 

83 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

84 Photon transfer curve dataset, with, e.g., gains 

85 and read noise. 

86 bfKernel : `numpy.ndarray`, optional 

87 Brighter-fatter kernel. 

88 bfGains : `dict` of `float`, optional 

89 Gains used to override the detector's nominal gains for the 

90 brighter-fatter correction. A dict keyed by amplifier name for 

91 the detector in question. 

92 defects : `lsst.ip.isr.Defects`, optional 

93 List of defects. 

94 fringes : `lsst.pipe.base.Struct`, optional 

95 Struct containing the fringe correction data, with 

96 elements: 

97 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

98 - ``seed``: random seed derived from the ccdExposureId for random 

99 number generator (`uint32`) 

100 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

101 A ``TransmissionCurve`` that represents the throughput of the 

102 optics, to be evaluated in focal-plane coordinates. 

103 filterTransmission : `lsst.afw.image.TransmissionCurve` 

104 A ``TransmissionCurve`` that represents the throughput of the 

105 filter itself, to be evaluated in focal-plane coordinates. 

106 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

107 A ``TransmissionCurve`` that represents the throughput of the 

108 sensor itself, to be evaluated in post-assembly trimmed detector 

109 coordinates. 

110 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

111 A ``TransmissionCurve`` that represents the throughput of the 

112 atmosphere, assumed to be spatially constant. 

113 detectorNum : `int`, optional 

114 The integer number for the detector to process. 

115 isGen3 : bool, optional 

116 Flag this call to run() as using the Gen3 butler environment. 

117 strayLightData : `object`, optional 

118 Opaque object containing calibration information for stray-light 

119 correction. If `None`, no correction will be performed. 

120 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

121 Illumination correction image. 

122 

123 Returns 

124 ------- 

125 result : `lsst.pipe.base.Struct` 

126 Result struct with components: 

127 

128 ``exposure`` 

129 The fully ISR corrected exposure (`afw.image.Exposure`). 

130 ``outputExposure`` 

131 An alias for ``exposure`` (`afw.image.Exposure`). 

132 ``ossThumb`` 

133 Thumbnail image of the exposure after overscan subtraction 

134 (`numpy.ndarray`). 

135 ``flattenedThumb`` 

136 Thumbnail image of the exposure after flat-field correction 

137 (`numpy.ndarray`). 

138 - ``outputStatistics`` : mapping [`str`] 

139 Values of the additional statistics calculated. 

140 """ 

141 return Struct(exposure=afwImage.ExposureF(), 

142 outputExposure=afwImage.ExposureF(), 

143 ossThumb=np.empty((1, 1)), 

144 flattenedThumb=np.empty((1, 1)), 

145 preInterpExposure=afwImage.ExposureF(), 

146 outputOssThumbnail=np.empty((1, 1)), 

147 outputFlattenedThumbnail=np.empty((1, 1)), 

148 outputStatistics={}, 

149 ) 

150 

151 

152class MockCharacterizeImageTask(PipelineTask): 

153 """A do-nothing substitute for CharacterizeImageTask. 

154 """ 

155 ConfigClass = CharacterizeImageConfig 

156 _DefaultName = "notCharacterizeImage" 

157 

158 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

159 super().__init__(**kwargs) 

160 self.outputSchema = afwTable.SourceCatalog() 

161 

162 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

163 inputs = butlerQC.get(inputRefs) 

164 if 'exposureIdInfo' not in inputs.keys(): 

165 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId( 

166 butlerQC.quantum.dataId, "visit_detector") 

167 outputs = self.run(**inputs) 

168 butlerQC.put(outputs, outputRefs) 

169 

170 def run(self, exposure, exposureIdInfo=None, background=None): 

171 """Produce characterization outputs with no processing. 

172 

173 Parameters 

174 ---------- 

175 exposure : `lsst.afw.image.Exposure` 

176 Exposure to characterize. 

177 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

178 ID info for exposure. 

179 background : `lsst.afw.math.BackgroundList` 

180 Initial model of background already subtracted from exposure. 

181 

182 Returns 

183 ------- 

184 result : `lsst.pipe.base.Struct` 

185 Struct containing these fields: 

186 

187 ``characterized`` 

188 Characterized exposure (`lsst.afw.image.Exposure`). 

189 ``sourceCat`` 

190 Detected sources (`lsst.afw.table.SourceCatalog`). 

191 ``backgroundModel`` 

192 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`) 

193 ``psfCellSet`` 

194 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`) 

195 """ 

196 # Can't persist empty BackgroundList; DM-33714 

197 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

198 afwImage.MaskedImageF(16, 16)) 

199 return Struct(characterized=exposure, 

200 sourceCat=afwTable.SourceCatalog(), 

201 backgroundModel=afwMath.BackgroundList(bg), 

202 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10), 

203 ) 

204 

205 

206class MockCalibrateTask(PipelineTask): 

207 """A do-nothing substitute for CalibrateTask. 

208 """ 

209 ConfigClass = CalibrateConfig 

210 _DefaultName = "notCalibrate" 

211 

212 def __init__(self, butler=None, astromRefObjLoader=None, 

213 photoRefObjLoader=None, icSourceSchema=None, 

214 initInputs=None, **kwargs): 

215 super().__init__(**kwargs) 

216 self.outputSchema = afwTable.SourceCatalog() 

217 

218 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

219 inputs = butlerQC.get(inputRefs) 

220 inputs['exposureIdInfo'] = obsBase.ExposureIdInfo.fromDataId( 

221 butlerQC.quantum.dataId, "visit_detector") 

222 

223 if self.config.doAstrometry: 

224 inputs.pop('astromRefCat') 

225 if self.config.doPhotoCal: 

226 inputs.pop('photoRefCat') 

227 

228 outputs = self.run(**inputs) 

229 

230 if self.config.doWriteMatches and self.config.doAstrometry: 

231 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

232 if self.config.doWriteMatchesDenormalized: 

233 outputs.matchesDenormalized = outputs.astromMatches 

234 outputs.matches = normalizedMatches 

235 butlerQC.put(outputs, outputRefs) 

236 

237 def run(self, exposure, exposureIdInfo=None, background=None, 

238 icSourceCat=None): 

239 """Produce calibration outputs with no processing. 

240 

241 Parameters 

242 ---------- 

243 exposure : `lsst.afw.image.Exposure` 

244 Exposure to calibrate. 

245 exposureIdInfo : `lsst.obs.base.ExposureIdInfo` 

246 ID info for exposure. 

247 background : `lsst.afw.math.BackgroundList` 

248 Background model already subtracted from exposure. 

249 icSourceCat : `lsst.afw.table.SourceCatalog` 

250 A SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

251 

252 Returns 

253 ------- 

254 result : `lsst.pipe.base.Struct` 

255 Struct containing these fields: 

256 

257 ``outputExposure`` 

258 Calibrated science exposure with refined WCS and PhotoCalib 

259 (`lsst.afw.image.Exposure`). 

260 ``outputBackground`` 

261 Model of background subtracted from exposure 

262 (`lsst.afw.math.BackgroundList`). 

263 ``outputCat`` 

264 Catalog of measured sources (`lsst.afw.table.SourceCatalog`). 

265 ``astromMatches`` 

266 List of source/refObj matches from the astrometry solver 

267 (`list` [`lsst.afw.table.ReferenceMatch`]). 

268 """ 

269 # Can't persist empty BackgroundList; DM-33714 

270 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

271 afwImage.MaskedImageF(16, 16)) 

272 return Struct(outputExposure=exposure, 

273 outputBackground=afwMath.BackgroundList(bg), 

274 outputCat=afwTable.SourceCatalog(), 

275 astromMatches=[], 

276 ) 

277 

278 

279class MockGetTemplateTask(PipelineTask): 

280 """A do-nothing substitute for GetTemplateTask. 

281 """ 

282 ConfigClass = GetTemplateConfig 

283 _DefaultName = "notGetTemplate" 

284 

285 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

286 inputs = butlerQC.get(inputRefs) 

287 # Mock GetTemplateTask.getOverlappingExposures 

288 results = Struct(coaddExposures=[], 

289 dataIds=[], 

290 ) 

291 inputs["coaddExposures"] = results.coaddExposures 

292 inputs["dataIds"] = results.dataIds 

293 outputs = self.run(**inputs) 

294 butlerQC.put(outputs, outputRefs) 

295 

296 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs): 

297 """Warp coadds from multiple tracts to form a template for image diff. 

298 

299 Where the tracts overlap, the resulting template image is averaged. 

300 The PSF on the template is created by combining the CoaddPsf on each 

301 template image into a meta-CoaddPsf. 

302 

303 Parameters 

304 ---------- 

305 coaddExposures : `list` of `lsst.afw.image.Exposure` 

306 Coadds to be mosaicked 

307 bbox : `lsst.geom.Box2I` 

308 Template Bounding box of the detector geometry onto which to 

309 resample the coaddExposures 

310 wcs : `lsst.afw.geom.SkyWcs` 

311 Template WCS onto which to resample the coaddExposures 

312 dataIds : `list` of `lsst.daf.butler.DataCoordinate` 

313 Record of the tract and patch of each coaddExposure. 

314 **kwargs 

315 Any additional keyword parameters. 

316 

317 Returns 

318 ------- 

319 result : `lsst.pipe.base.Struct` containing 

320 - ``template`` : a template coadd exposure assembled out of patches 

321 """ 

322 return Struct(template=afwImage.ExposureF(), 

323 ) 

324 

325 

326class MockAlardLuptonSubtractTask(PipelineTask): 

327 """A do-nothing substitute for AlardLuptonSubtractTask. 

328 """ 

329 ConfigClass = AlardLuptonSubtractConfig 

330 _DefaultName = "notAlardLuptonSubtract" 

331 

332 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None): 

333 """PSF match, subtract, and decorrelate two images. 

334 

335 Parameters 

336 ---------- 

337 template : `lsst.afw.image.ExposureF` 

338 Template exposure, warped to match the science exposure. 

339 science : `lsst.afw.image.ExposureF` 

340 Science exposure to subtract from the template. 

341 sources : `lsst.afw.table.SourceCatalog` 

342 Identified sources on the science exposure. This catalog is used to 

343 select sources in order to perform the AL PSF matching on stamp 

344 images around them. 

345 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

346 Exposure catalog with finalized psf models and aperture correction 

347 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

348 the detector id for the catalog id, sorted on id for fast lookup. 

349 

350 Returns 

351 ------- 

352 results : `lsst.pipe.base.Struct` 

353 ``difference`` : `lsst.afw.image.ExposureF` 

354 Result of subtracting template and science. 

355 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

356 Warped and PSF-matched template exposure. 

357 ``backgroundModel`` : `lsst.afw.math.Function2D` 

358 Background model that was fit while solving for the PSF-matching kernel 

359 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

360 Kernel used to PSF-match the convolved image. 

361 """ 

362 return Struct(difference=afwImage.ExposureF(), 

363 matchedTemplate=afwImage.ExposureF(), 

364 backgroundModel=afwMath.NullFunction2D(), 

365 psfMatchingKernel=afwMath.FixedKernel(), 

366 ) 

367 

368 

369class MockDetectAndMeasureTask(PipelineTask): 

370 """A do-nothing substitute for DetectAndMeasureTask. 

371 """ 

372 ConfigClass = DetectAndMeasureConfig 

373 _DefaultName = "notDetectAndMeasure" 

374 

375 def __init__(self, **kwargs): 

376 super().__init__(**kwargs) 

377 self.outputSchema = afwTable.SourceCatalog() 

378 

379 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

380 inputs = butlerQC.get(inputRefs) 

381 idFactory = obsBase.ExposureIdInfo(8, 4).makeSourceIdFactory() 

382 

383 outputs = self.run(inputs['science'], 

384 inputs['matchedTemplate'], 

385 inputs['difference'], 

386 inputs['selectSources'], 

387 idFactory=idFactory) 

388 butlerQC.put(outputs, outputRefs) 

389 

390 def run(self, science, matchedTemplate, difference, selectSources, 

391 idFactory=None): 

392 """Detect and measure sources on a difference image. 

393 

394 Parameters 

395 ---------- 

396 science : `lsst.afw.image.ExposureF` 

397 Science exposure that the template was subtracted from. 

398 matchedTemplate : `lsst.afw.image.ExposureF` 

399 Warped and PSF-matched template that was used produce the 

400 difference image. 

401 difference : `lsst.afw.image.ExposureF` 

402 Result of subtracting template from the science image. 

403 selectSources : `lsst.afw.table.SourceCatalog` 

404 Identified sources on the science exposure. 

405 idFactory : `lsst.afw.table.IdFactory`, optional 

406 Generator object to assign ids to detected sources in the difference image. 

407 

408 Returns 

409 ------- 

410 results : `lsst.pipe.base.Struct` 

411 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

412 Subtracted exposure with detection mask applied. 

413 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

414 The catalog of detected sources. 

415 """ 

416 return Struct(subtractedMeasuredExposure=difference, 

417 diaSources=afwTable.SourceCatalog(), 

418 ) 

419 

420 

421class MockTransformDiaSourceCatalogTask(PipelineTask): 

422 """A do-nothing substitute for TransformDiaSourceCatalogTask. 

423 """ 

424 ConfigClass = TransformDiaSourceCatalogConfig 

425 _DefaultName = "notTransformDiaSourceCatalog" 

426 

427 def __init__(self, initInputs, **kwargs): 

428 super().__init__(**kwargs) 

429 

430 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

431 inputs = butlerQC.get(inputRefs) 

432 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

433 returnMaxBits=True) 

434 inputs["ccdVisitId"] = expId 

435 inputs["band"] = butlerQC.quantum.dataId["band"] 

436 

437 outputs = self.run(**inputs) 

438 

439 butlerQC.put(outputs, outputRefs) 

440 

441 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None): 

442 """Produce transformation outputs with no processing. 

443 

444 Parameters 

445 ---------- 

446 diaSourceCat : `lsst.afw.table.SourceCatalog` 

447 The catalog to transform. 

448 diffIm : `lsst.afw.image.Exposure` 

449 An image, to provide supplementary information. 

450 band : `str` 

451 The band in which the sources were observed. 

452 ccdVisitId : `int` 

453 The exposure ID in which the sources were observed. 

454 funcs 

455 Unused. 

456 

457 Returns 

458 ------- 

459 results : `lsst.pipe.base.Struct` 

460 Results struct with components: 

461 

462 ``diaSourceTable`` 

463 Catalog of DiaSources (`pandas.DataFrame`). 

464 """ 

465 return Struct(diaSourceTable=pandas.DataFrame(), 

466 ) 

467 

468 

469class MockDiaPipelineTask(PipelineTask): 

470 """A do-nothing substitute for DiaPipelineTask. 

471 """ 

472 ConfigClass = DiaPipelineConfig 

473 _DefaultName = "notDiaPipe" 

474 

475 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

476 inputs = butlerQC.get(inputRefs) 

477 expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", 

478 returnMaxBits=True) 

479 inputs["ccdExposureIdBits"] = expBits 

480 inputs["band"] = butlerQC.quantum.dataId["band"] 

481 if not self.config.doSolarSystemAssociation: 

482 inputs["solarSystemObjectTable"] = None 

483 

484 outputs = self.run(**inputs) 

485 

486 butlerQC.put(outputs, outputRefs) 

487 

488 def run(self, 

489 diaSourceTable, 

490 solarSystemObjectTable, 

491 diffIm, 

492 exposure, 

493 template, 

494 ccdExposureIdBits, 

495 band): 

496 """Produce DiaSource and DiaObject outputs with no processing. 

497 

498 Parameters 

499 ---------- 

500 diaSourceTable : `pandas.DataFrame` 

501 Newly detected DiaSources. 

502 solarSystemObjectTable : `pandas.DataFrame` 

503 Expected solar system objects in the field of view. 

504 diffIm : `lsst.afw.image.ExposureF` 

505 Difference image exposure in which the sources in ``diaSourceCat`` 

506 were detected. 

507 exposure : `lsst.afw.image.ExposureF` 

508 Calibrated exposure differenced with a template to create 

509 ``diffIm``. 

510 template : `lsst.afw.image.ExposureF` 

511 Template exposure used to create diffIm. 

512 ccdExposureIdBits : `int` 

513 Number of bits used for a unique ``ccdVisitId``. 

514 band : `str` 

515 The band in which the new DiaSources were detected. 

516 

517 Returns 

518 ------- 

519 results : `lsst.pipe.base.Struct` 

520 Results struct with components: 

521 

522 ``apdbMarker`` 

523 Marker dataset to store in the Butler indicating that this 

524 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`). 

525 ``associatedDiaSources`` 

526 Catalog of newly associated DiaSources (`pandas.DataFrame`). 

527 """ 

528 return Struct(apdbMarker=self.config.apdb.value, 

529 associatedDiaSources=pandas.DataFrame(), 

530 )