Coverage for python/lsst/ap/verify/testPipeline.py: 44%

122 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-12 03:17 -0700

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24 

25# These classes exist only to be included in a mock pipeline, and don't need 

26# to be public for that. 

27__all__ = [] 

28 

29 

30import numpy as np 

31import pandas 

32 

33import lsst.geom as geom 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.afw.table as afwTable 

37from lsst.pipe.base import PipelineTask, Struct 

38from lsst.ip.isr import IsrTaskConfig 

39from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig 

40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig 

41from lsst.pipe.tasks.calibrate import CalibrateConfig 

42from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig 

43 

44 

45class MockIsrTask(PipelineTask): 

46 """A do-nothing substitute for IsrTask. 

47 """ 

48 ConfigClass = IsrTaskConfig 

49 _DefaultName = "notIsr" 

50 

51 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

52 crosstalk=None, crosstalkSources=None, 

53 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

54 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

55 sensorTransmission=None, atmosphereTransmission=None, 

56 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

57 deferredCharge=None, 

58 ): 

59 """Accept ISR inputs, and produce ISR outputs with no processing. 

60 

61 Parameters 

62 ---------- 

63 ccdExposure : `lsst.afw.image.Exposure` 

64 The raw exposure that is to be run through ISR. The 

65 exposure is modified by this method. 

66 camera : `lsst.afw.cameraGeom.Camera`, optional 

67 The camera geometry for this exposure. Required if 

68 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

69 ``flat`` does not have an associated detector. 

70 bias : `lsst.afw.image.Exposure`, optional 

71 Bias calibration frame. 

72 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

73 Functor for linearization. 

74 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

75 Calibration for crosstalk. 

76 crosstalkSources : `list`, optional 

77 List of possible crosstalk sources. 

78 dark : `lsst.afw.image.Exposure`, optional 

79 Dark calibration frame. 

80 flat : `lsst.afw.image.Exposure`, optional 

81 Flat calibration frame. 

82 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

83 Photon transfer curve dataset, with, e.g., gains 

84 and read noise. 

85 bfKernel : `numpy.ndarray`, optional 

86 Brighter-fatter kernel. 

87 bfGains : `dict` of `float`, optional 

88 Gains used to override the detector's nominal gains for the 

89 brighter-fatter correction. A dict keyed by amplifier name for 

90 the detector in question. 

91 defects : `lsst.ip.isr.Defects`, optional 

92 List of defects. 

93 fringes : `lsst.pipe.base.Struct`, optional 

94 Struct containing the fringe correction data, with 

95 elements: 

96 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

97 - ``seed``: random seed derived from the ccdExposureId for random 

98 number generator (`uint32`) 

99 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

100 A ``TransmissionCurve`` that represents the throughput of the 

101 optics, to be evaluated in focal-plane coordinates. 

102 filterTransmission : `lsst.afw.image.TransmissionCurve` 

103 A ``TransmissionCurve`` that represents the throughput of the 

104 filter itself, to be evaluated in focal-plane coordinates. 

105 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

106 A ``TransmissionCurve`` that represents the throughput of the 

107 sensor itself, to be evaluated in post-assembly trimmed detector 

108 coordinates. 

109 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

110 A ``TransmissionCurve`` that represents the throughput of the 

111 atmosphere, assumed to be spatially constant. 

112 detectorNum : `int`, optional 

113 The integer number for the detector to process. 

114 isGen3 : bool, optional 

115 Flag this call to run() as using the Gen3 butler environment. 

116 strayLightData : `object`, optional 

117 Opaque object containing calibration information for stray-light 

118 correction. If `None`, no correction will be performed. 

119 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

120 Illumination correction image. 

121 

122 Returns 

123 ------- 

124 result : `lsst.pipe.base.Struct` 

125 Result struct with components: 

126 

127 ``exposure`` 

128 The fully ISR corrected exposure (`afw.image.Exposure`). 

129 ``outputExposure`` 

130 An alias for ``exposure`` (`afw.image.Exposure`). 

131 ``ossThumb`` 

132 Thumbnail image of the exposure after overscan subtraction 

133 (`numpy.ndarray`). 

134 ``flattenedThumb`` 

135 Thumbnail image of the exposure after flat-field correction 

136 (`numpy.ndarray`). 

137 - ``outputStatistics`` : mapping [`str`] 

138 Values of the additional statistics calculated. 

139 """ 

140 return Struct(exposure=afwImage.ExposureF(), 

141 outputExposure=afwImage.ExposureF(), 

142 ossThumb=np.empty((1, 1)), 

143 flattenedThumb=np.empty((1, 1)), 

144 preInterpExposure=afwImage.ExposureF(), 

145 outputOssThumbnail=np.empty((1, 1)), 

146 outputFlattenedThumbnail=np.empty((1, 1)), 

147 outputStatistics={}, 

148 ) 

149 

150 

151class MockCharacterizeImageTask(PipelineTask): 

152 """A do-nothing substitute for CharacterizeImageTask. 

153 """ 

154 ConfigClass = CharacterizeImageConfig 

155 _DefaultName = "notCharacterizeImage" 

156 

157 def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs): 

158 super().__init__(**kwargs) 

159 self.outputSchema = afwTable.SourceCatalog() 

160 

161 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

162 inputs = butlerQC.get(inputRefs) 

163 if 'idGenerator' not in inputs.keys(): 

164 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

165 outputs = self.run(**inputs) 

166 butlerQC.put(outputs, outputRefs) 

167 

168 def run(self, exposure, exposureIdInfo=None, background=None, idGenerator=None): 

169 """Produce characterization outputs with no processing. 

170 

171 Parameters 

172 ---------- 

173 exposure : `lsst.afw.image.Exposure` 

174 Exposure to characterize. 

175 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional 

176 ID info for exposure. Deprecated in favor of ``idGenerator``, and 

177 ignored if that is provided. 

178 background : `lsst.afw.math.BackgroundList`, optional 

179 Initial model of background already subtracted from exposure. 

180 idGenerator : `lsst.meas.base.IdGenerator`, optional 

181 Object that generates source IDs and provides random number 

182 generator seeds. 

183 

184 Returns 

185 ------- 

186 result : `lsst.pipe.base.Struct` 

187 Struct containing these fields: 

188 

189 ``characterized`` 

190 Characterized exposure (`lsst.afw.image.Exposure`). 

191 ``sourceCat`` 

192 Detected sources (`lsst.afw.table.SourceCatalog`). 

193 ``backgroundModel`` 

194 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`) 

195 ``psfCellSet`` 

196 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`) 

197 """ 

198 # Can't persist empty BackgroundList; DM-33714 

199 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

200 afwImage.MaskedImageF(16, 16)) 

201 return Struct(characterized=exposure, 

202 sourceCat=afwTable.SourceCatalog(), 

203 backgroundModel=afwMath.BackgroundList(bg), 

204 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10), 

205 ) 

206 

207 

208class MockCalibrateTask(PipelineTask): 

209 """A do-nothing substitute for CalibrateTask. 

210 """ 

211 ConfigClass = CalibrateConfig 

212 _DefaultName = "notCalibrate" 

213 

214 def __init__(self, butler=None, astromRefObjLoader=None, 

215 photoRefObjLoader=None, icSourceSchema=None, 

216 initInputs=None, **kwargs): 

217 super().__init__(**kwargs) 

218 self.outputSchema = afwTable.SourceCatalog() 

219 

220 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

221 inputs = butlerQC.get(inputRefs) 

222 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

223 

224 if self.config.doAstrometry: 

225 inputs.pop('astromRefCat') 

226 if self.config.doPhotoCal: 

227 inputs.pop('photoRefCat') 

228 

229 outputs = self.run(**inputs) 

230 

231 if self.config.doWriteMatches and self.config.doAstrometry: 

232 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

233 if self.config.doWriteMatchesDenormalized: 

234 outputs.matchesDenormalized = outputs.astromMatches 

235 outputs.matches = normalizedMatches 

236 butlerQC.put(outputs, outputRefs) 

237 

238 def run(self, exposure, exposureIdInfo=None, background=None, 

239 icSourceCat=None, idGenerator=None): 

240 """Produce calibration outputs with no processing. 

241 

242 Parameters 

243 ---------- 

244 exposure : `lsst.afw.image.Exposure` 

245 Exposure to calibrate. 

246 exposureIdInfo : `lsst.obs.base.ExposureIdInfo`, optional 

247 ID info for exposure. Deprecated in favor of ``idGenerator``, and 

248 ignored if that is provided. 

249 background : `lsst.afw.math.BackgroundList`, optional 

250 Background model already subtracted from exposure. 

251 icSourceCat : `lsst.afw.table.SourceCatalog`, optional 

252 A SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

253 idGenerator : `lsst.meas.base.IdGenerator`, optional 

254 Object that generates source IDs and provides random number 

255 generator seeds. 

256 

257 Returns 

258 ------- 

259 result : `lsst.pipe.base.Struct` 

260 Struct containing these fields: 

261 

262 ``outputExposure`` 

263 Calibrated science exposure with refined WCS and PhotoCalib 

264 (`lsst.afw.image.Exposure`). 

265 ``outputBackground`` 

266 Model of background subtracted from exposure 

267 (`lsst.afw.math.BackgroundList`). 

268 ``outputCat`` 

269 Catalog of measured sources (`lsst.afw.table.SourceCatalog`). 

270 ``astromMatches`` 

271 List of source/refObj matches from the astrometry solver 

272 (`list` [`lsst.afw.table.ReferenceMatch`]). 

273 """ 

274 # Can't persist empty BackgroundList; DM-33714 

275 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

276 afwImage.MaskedImageF(16, 16)) 

277 return Struct(outputExposure=exposure, 

278 outputBackground=afwMath.BackgroundList(bg), 

279 outputCat=afwTable.SourceCatalog(), 

280 astromMatches=[], 

281 ) 

282 

283 

284class MockGetTemplateTask(PipelineTask): 

285 """A do-nothing substitute for GetTemplateTask. 

286 """ 

287 ConfigClass = GetTemplateConfig 

288 _DefaultName = "notGetTemplate" 

289 

290 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

291 inputs = butlerQC.get(inputRefs) 

292 # Mock GetTemplateTask.getOverlappingExposures 

293 results = Struct(coaddExposures=[], 

294 dataIds=[], 

295 ) 

296 inputs["coaddExposures"] = results.coaddExposures 

297 inputs["dataIds"] = results.dataIds 

298 outputs = self.run(**inputs) 

299 butlerQC.put(outputs, outputRefs) 

300 

301 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs): 

302 """Warp coadds from multiple tracts to form a template for image diff. 

303 

304 Where the tracts overlap, the resulting template image is averaged. 

305 The PSF on the template is created by combining the CoaddPsf on each 

306 template image into a meta-CoaddPsf. 

307 

308 Parameters 

309 ---------- 

310 coaddExposures : `list` of `lsst.afw.image.Exposure` 

311 Coadds to be mosaicked 

312 bbox : `lsst.geom.Box2I` 

313 Template Bounding box of the detector geometry onto which to 

314 resample the coaddExposures 

315 wcs : `lsst.afw.geom.SkyWcs` 

316 Template WCS onto which to resample the coaddExposures 

317 dataIds : `list` of `lsst.daf.butler.DataCoordinate` 

318 Record of the tract and patch of each coaddExposure. 

319 **kwargs 

320 Any additional keyword parameters. 

321 

322 Returns 

323 ------- 

324 result : `lsst.pipe.base.Struct` containing 

325 - ``template`` : a template coadd exposure assembled out of patches 

326 """ 

327 return Struct(template=afwImage.ExposureF(), 

328 ) 

329 

330 

331class MockAlardLuptonSubtractTask(PipelineTask): 

332 """A do-nothing substitute for AlardLuptonSubtractTask. 

333 """ 

334 ConfigClass = AlardLuptonSubtractConfig 

335 _DefaultName = "notAlardLuptonSubtract" 

336 

337 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None): 

338 """PSF match, subtract, and decorrelate two images. 

339 

340 Parameters 

341 ---------- 

342 template : `lsst.afw.image.ExposureF` 

343 Template exposure, warped to match the science exposure. 

344 science : `lsst.afw.image.ExposureF` 

345 Science exposure to subtract from the template. 

346 sources : `lsst.afw.table.SourceCatalog` 

347 Identified sources on the science exposure. This catalog is used to 

348 select sources in order to perform the AL PSF matching on stamp 

349 images around them. 

350 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

351 Exposure catalog with finalized psf models and aperture correction 

352 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses 

353 the detector id for the catalog id, sorted on id for fast lookup. 

354 

355 Returns 

356 ------- 

357 results : `lsst.pipe.base.Struct` 

358 ``difference`` : `lsst.afw.image.ExposureF` 

359 Result of subtracting template and science. 

360 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

361 Warped and PSF-matched template exposure. 

362 ``backgroundModel`` : `lsst.afw.math.Function2D` 

363 Background model that was fit while solving for the PSF-matching kernel 

364 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

365 Kernel used to PSF-match the convolved image. 

366 """ 

367 return Struct(difference=afwImage.ExposureF(), 

368 matchedTemplate=afwImage.ExposureF(), 

369 backgroundModel=afwMath.NullFunction2D(), 

370 psfMatchingKernel=afwMath.FixedKernel(), 

371 ) 

372 

373 

374class MockDetectAndMeasureConfig(DetectAndMeasureConfig): 

375 

376 def setDefaults(self): 

377 super().setDefaults() 

378 # Avoid delegating to lsst.obs.base.Instrument specialization for the 

379 # data ID packing algorithm to use, since test code often does not use a 

380 # real Instrument in its data IDs. 

381 self.idGenerator.packer.name = "observation" 

382 

383 

384class MockDetectAndMeasureTask(PipelineTask): 

385 """A do-nothing substitute for DetectAndMeasureTask. 

386 """ 

387 ConfigClass = MockDetectAndMeasureConfig 

388 _DefaultName = "notDetectAndMeasure" 

389 

390 def __init__(self, **kwargs): 

391 super().__init__(**kwargs) 

392 self.outputSchema = afwTable.SourceCatalog() 

393 

394 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

395 inputs = butlerQC.get(inputRefs) 

396 idFactory = afwTable.IdFactory.makeSimple() 

397 

398 outputs = self.run(inputs['science'], 

399 inputs['matchedTemplate'], 

400 inputs['difference'], 

401 idFactory=idFactory) 

402 butlerQC.put(outputs, outputRefs) 

403 

404 def run(self, science, matchedTemplate, difference, 

405 idFactory=None): 

406 """Detect and measure sources on a difference image. 

407 

408 Parameters 

409 ---------- 

410 science : `lsst.afw.image.ExposureF` 

411 Science exposure that the template was subtracted from. 

412 matchedTemplate : `lsst.afw.image.ExposureF` 

413 Warped and PSF-matched template that was used produce the 

414 difference image. 

415 difference : `lsst.afw.image.ExposureF` 

416 Result of subtracting template from the science image. 

417 idFactory : `lsst.afw.table.IdFactory`, optional 

418 Generator object to assign ids to detected sources in the difference image. 

419 

420 Returns 

421 ------- 

422 results : `lsst.pipe.base.Struct` 

423 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

424 Subtracted exposure with detection mask applied. 

425 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

426 The catalog of detected sources. 

427 """ 

428 return Struct(subtractedMeasuredExposure=difference, 

429 diaSources=afwTable.SourceCatalog(), 

430 ) 

431 

432 

433class MockTransformDiaSourceCatalogTask(PipelineTask): 

434 """A do-nothing substitute for TransformDiaSourceCatalogTask. 

435 """ 

436 ConfigClass = TransformDiaSourceCatalogConfig 

437 _DefaultName = "notTransformDiaSourceCatalog" 

438 

439 def __init__(self, initInputs, **kwargs): 

440 super().__init__(**kwargs) 

441 

442 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

443 inputs = butlerQC.get(inputRefs) 

444 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

445 inputs["ccdVisitId"] = idGenerator.catalog_id 

446 inputs["band"] = butlerQC.quantum.dataId["band"] 

447 

448 outputs = self.run(**inputs) 

449 

450 butlerQC.put(outputs, outputRefs) 

451 

452 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None): 

453 """Produce transformation outputs with no processing. 

454 

455 Parameters 

456 ---------- 

457 diaSourceCat : `lsst.afw.table.SourceCatalog` 

458 The catalog to transform. 

459 diffIm : `lsst.afw.image.Exposure` 

460 An image, to provide supplementary information. 

461 band : `str` 

462 The band in which the sources were observed. 

463 ccdVisitId : `int` 

464 The exposure ID in which the sources were observed. 

465 funcs, optional 

466 Unused. 

467 

468 Returns 

469 ------- 

470 results : `lsst.pipe.base.Struct` 

471 Results struct with components: 

472 

473 ``diaSourceTable`` 

474 Catalog of DiaSources (`pandas.DataFrame`). 

475 """ 

476 return Struct(diaSourceTable=pandas.DataFrame(), 

477 ) 

478 

479 

480class MockDiaPipelineConfig(DiaPipelineConfig): 

481 

482 def setDefaults(self): 

483 super().setDefaults() 

484 # Avoid delegating to lsst.obs.base.Instrument specialization for the 

485 # data ID packing algorithm to use, since test code often does not use a 

486 # real Instrument in its data IDs. 

487 self.idGenerator.packer.name = "observation" 

488 

489 

490class MockDiaPipelineTask(PipelineTask): 

491 """A do-nothing substitute for DiaPipelineTask. 

492 """ 

493 ConfigClass = MockDiaPipelineConfig 

494 _DefaultName = "notDiaPipe" 

495 

496 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

497 inputs = butlerQC.get(inputRefs) 

498 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

499 # Need to set ccdExposureIdBits (now deprecated) to None and pass it, 

500 # since there are non-optional positional arguments after it. 

501 inputs["ccdExposureIdBits"] = None 

502 inputs["band"] = butlerQC.quantum.dataId["band"] 

503 if not self.config.doSolarSystemAssociation: 

504 inputs["solarSystemObjectTable"] = None 

505 

506 outputs = self.run(**inputs) 

507 

508 butlerQC.put(outputs, outputRefs) 

509 

510 def run(self, 

511 diaSourceTable, 

512 solarSystemObjectTable, 

513 diffIm, 

514 exposure, 

515 template, 

516 ccdExposureIdBits, 

517 band, 

518 idGenerator=None): 

519 """Produce DiaSource and DiaObject outputs with no processing. 

520 

521 Parameters 

522 ---------- 

523 diaSourceTable : `pandas.DataFrame` 

524 Newly detected DiaSources. 

525 solarSystemObjectTable : `pandas.DataFrame` 

526 Expected solar system objects in the field of view. 

527 diffIm : `lsst.afw.image.ExposureF` 

528 Difference image exposure in which the sources in ``diaSourceCat`` 

529 were detected. 

530 exposure : `lsst.afw.image.ExposureF` 

531 Calibrated exposure differenced with a template to create 

532 ``diffIm``. 

533 template : `lsst.afw.image.ExposureF` 

534 Template exposure used to create diffIm. 

535 ccdExposureIdBits : `int` 

536 Number of bits used for a unique ``ccdVisitId``. Deprecated in 

537 favor of ``idGenerator``, and ignored if that is present. Pass 

538 `None` explicitly to avoid a deprecation warning (a default is 

539 impossible given that later positional arguments are not 

540 defaulted). 

541 band : `str` 

542 The band in which the new DiaSources were detected. 

543 idGenerator : `lsst.meas.base.IdGenerator`, optional 

544 Object that generates source IDs and random number generator seeds. 

545 Will be required after ``ccdExposureIdBits`` is removed. 

546 

547 Returns 

548 ------- 

549 results : `lsst.pipe.base.Struct` 

550 Results struct with components: 

551 

552 ``apdbMarker`` 

553 Marker dataset to store in the Butler indicating that this 

554 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`). 

555 ``associatedDiaSources`` 

556 Catalog of newly associated DiaSources (`pandas.DataFrame`). 

557 """ 

558 return Struct(apdbMarker=self.config.apdb.value, 

559 associatedDiaSources=pandas.DataFrame(), 

560 diaForcedSources=pandas.DataFrame(), 

561 diaObjects=pandas.DataFrame(), 

562 )