Coverage for python/lsst/ap/verify/testPipeline.py: 44%

122 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-25 13:58 +0000

1# 

2# This file is part of ap_verify. 

3# 

4# Developed for the LSST Data Management System. 

5# This product includes software developed by the LSST Project 

6# (http://www.lsst.org). 

7# See the COPYRIGHT file at the top-level directory of this distribution 

8# for details of code ownership. 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the GNU General Public License 

21# along with this program. If not, see <http://www.gnu.org/licenses/>. 

22# 

23 

24 

25# These classes exist only to be included in a mock pipeline, and don't need 

26# to be public for that. 

27__all__ = [] 

28 

29 

30import numpy as np 

31import pandas 

32 

33import lsst.geom as geom 

34import lsst.afw.image as afwImage 

35import lsst.afw.math as afwMath 

36import lsst.afw.table as afwTable 

37from lsst.pipe.base import PipelineTask, Struct 

38from lsst.ip.isr import IsrTaskConfig 

39from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig 

40from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig 

41from lsst.pipe.tasks.calibrate import CalibrateConfig 

42from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig 

43 

44 

45class MockIsrTask(PipelineTask): 

46 """A do-nothing substitute for IsrTask. 

47 """ 

48 ConfigClass = IsrTaskConfig 

49 _DefaultName = "notIsr" 

50 

51 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None, 

52 crosstalk=None, crosstalkSources=None, 

53 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, 

54 fringes=Struct(fringes=None), opticsTransmission=None, filterTransmission=None, 

55 sensorTransmission=None, atmosphereTransmission=None, 

56 detectorNum=None, strayLightData=None, illumMaskedImage=None, 

57 deferredCharge=None, 

58 ): 

59 """Accept ISR inputs, and produce ISR outputs with no processing. 

60 

61 Parameters 

62 ---------- 

63 ccdExposure : `lsst.afw.image.Exposure` 

64 The raw exposure that is to be run through ISR. The 

65 exposure is modified by this method. 

66 camera : `lsst.afw.cameraGeom.Camera`, optional 

67 The camera geometry for this exposure. Required if 

68 one or more of ``ccdExposure``, ``bias``, ``dark``, or 

69 ``flat`` does not have an associated detector. 

70 bias : `lsst.afw.image.Exposure`, optional 

71 Bias calibration frame. 

72 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional 

73 Functor for linearization. 

74 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional 

75 Calibration for crosstalk. 

76 crosstalkSources : `list`, optional 

77 List of possible crosstalk sources. 

78 dark : `lsst.afw.image.Exposure`, optional 

79 Dark calibration frame. 

80 flat : `lsst.afw.image.Exposure`, optional 

81 Flat calibration frame. 

82 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional 

83 Photon transfer curve dataset, with, e.g., gains 

84 and read noise. 

85 bfKernel : `numpy.ndarray`, optional 

86 Brighter-fatter kernel. 

87 bfGains : `dict` of `float`, optional 

88 Gains used to override the detector's nominal gains for the 

89 brighter-fatter correction. A dict keyed by amplifier name for 

90 the detector in question. 

91 defects : `lsst.ip.isr.Defects`, optional 

92 List of defects. 

93 fringes : `lsst.pipe.base.Struct`, optional 

94 Struct containing the fringe correction data, with 

95 elements: 

96 - ``fringes``: fringe calibration frame (`afw.image.Exposure`) 

97 - ``seed``: random seed derived from the ccdExposureId for random 

98 number generator (`uint32`) 

99 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional 

100 A ``TransmissionCurve`` that represents the throughput of the 

101 optics, to be evaluated in focal-plane coordinates. 

102 filterTransmission : `lsst.afw.image.TransmissionCurve` 

103 A ``TransmissionCurve`` that represents the throughput of the 

104 filter itself, to be evaluated in focal-plane coordinates. 

105 sensorTransmission : `lsst.afw.image.TransmissionCurve` 

106 A ``TransmissionCurve`` that represents the throughput of the 

107 sensor itself, to be evaluated in post-assembly trimmed detector 

108 coordinates. 

109 atmosphereTransmission : `lsst.afw.image.TransmissionCurve` 

110 A ``TransmissionCurve`` that represents the throughput of the 

111 atmosphere, assumed to be spatially constant. 

112 detectorNum : `int`, optional 

113 The integer number for the detector to process. 

114 isGen3 : bool, optional 

115 Flag this call to run() as using the Gen3 butler environment. 

116 strayLightData : `object`, optional 

117 Opaque object containing calibration information for stray-light 

118 correction. If `None`, no correction will be performed. 

119 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional 

120 Illumination correction image. 

121 

122 Returns 

123 ------- 

124 result : `lsst.pipe.base.Struct` 

125 Result struct with components: 

126 

127 ``exposure`` 

128 The fully ISR corrected exposure (`afw.image.Exposure`). 

129 ``outputExposure`` 

130 An alias for ``exposure`` (`afw.image.Exposure`). 

131 ``ossThumb`` 

132 Thumbnail image of the exposure after overscan subtraction 

133 (`numpy.ndarray`). 

134 ``flattenedThumb`` 

135 Thumbnail image of the exposure after flat-field correction 

136 (`numpy.ndarray`). 

137 - ``outputStatistics`` : mapping [`str`] 

138 Values of the additional statistics calculated. 

139 """ 

140 return Struct(exposure=afwImage.ExposureF(), 

141 outputExposure=afwImage.ExposureF(), 

142 ossThumb=np.empty((1, 1)), 

143 flattenedThumb=np.empty((1, 1)), 

144 preInterpExposure=afwImage.ExposureF(), 

145 outputOssThumbnail=np.empty((1, 1)), 

146 outputFlattenedThumbnail=np.empty((1, 1)), 

147 outputStatistics={}, 

148 ) 

149 

150 

151class MockCharacterizeImageTask(PipelineTask): 

152 """A do-nothing substitute for CharacterizeImageTask. 

153 """ 

154 ConfigClass = CharacterizeImageConfig 

155 _DefaultName = "notCharacterizeImage" 

156 

157 def __init__(self, refObjLoader=None, schema=None, **kwargs): 

158 super().__init__(**kwargs) 

159 self.outputSchema = afwTable.SourceCatalog() 

160 

161 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

162 inputs = butlerQC.get(inputRefs) 

163 if 'idGenerator' not in inputs.keys(): 

164 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

165 outputs = self.run(**inputs) 

166 butlerQC.put(outputs, outputRefs) 

167 

168 def run(self, exposure, background=None, idGenerator=None): 

169 """Produce characterization outputs with no processing. 

170 

171 Parameters 

172 ---------- 

173 exposure : `lsst.afw.image.Exposure` 

174 Exposure to characterize. 

175 background : `lsst.afw.math.BackgroundList`, optional 

176 Initial model of background already subtracted from exposure. 

177 idGenerator : `lsst.meas.base.IdGenerator`, optional 

178 Object that generates source IDs and provides random number 

179 generator seeds. 

180 

181 Returns 

182 ------- 

183 result : `lsst.pipe.base.Struct` 

184 Struct containing these fields: 

185 

186 ``characterized`` 

187 Characterized exposure (`lsst.afw.image.Exposure`). 

188 ``sourceCat`` 

189 Detected sources (`lsst.afw.table.SourceCatalog`). 

190 ``backgroundModel`` 

191 Model of background subtracted from exposure (`lsst.afw.math.BackgroundList`) 

192 ``psfCellSet`` 

193 Spatial cells of PSF candidates (`lsst.afw.math.SpatialCellSet`) 

194 """ 

195 # Can't persist empty BackgroundList; DM-33714 

196 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

197 afwImage.MaskedImageF(16, 16)) 

198 return Struct(characterized=exposure, 

199 sourceCat=afwTable.SourceCatalog(), 

200 backgroundModel=afwMath.BackgroundList(bg), 

201 psfCellSet=afwMath.SpatialCellSet(exposure.getBBox(), 10), 

202 ) 

203 

204 

205class MockCalibrateTask(PipelineTask): 

206 """A do-nothing substitute for CalibrateTask. 

207 """ 

208 ConfigClass = CalibrateConfig 

209 _DefaultName = "notCalibrate" 

210 

211 def __init__(self, astromRefObjLoader=None, 

212 photoRefObjLoader=None, icSourceSchema=None, 

213 initInputs=None, **kwargs): 

214 super().__init__(**kwargs) 

215 self.outputSchema = afwTable.SourceCatalog() 

216 

217 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

218 inputs = butlerQC.get(inputRefs) 

219 inputs['idGenerator'] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

220 

221 if self.config.doAstrometry: 

222 inputs.pop('astromRefCat') 

223 if self.config.doPhotoCal: 

224 inputs.pop('photoRefCat') 

225 

226 outputs = self.run(**inputs) 

227 

228 if self.config.doWriteMatches and self.config.doAstrometry: 

229 normalizedMatches = afwTable.packMatches(outputs.astromMatches) 

230 if self.config.doWriteMatchesDenormalized: 

231 # Just need an empty BaseCatalog with a valid schema. 

232 outputs.matchesDenormalized = afwTable.BaseCatalog(outputs.outputCat.schema) 

233 outputs.matches = normalizedMatches 

234 butlerQC.put(outputs, outputRefs) 

235 

236 def run(self, exposure, background=None, 

237 icSourceCat=None, idGenerator=None): 

238 """Produce calibration outputs with no processing. 

239 

240 Parameters 

241 ---------- 

242 exposure : `lsst.afw.image.Exposure` 

243 Exposure to calibrate. 

244 background : `lsst.afw.math.BackgroundList`, optional 

245 Background model already subtracted from exposure. 

246 icSourceCat : `lsst.afw.table.SourceCatalog`, optional 

247 A SourceCatalog from CharacterizeImageTask from which we can copy some fields. 

248 idGenerator : `lsst.meas.base.IdGenerator`, optional 

249 Object that generates source IDs and provides random number 

250 generator seeds. 

251 

252 Returns 

253 ------- 

254 result : `lsst.pipe.base.Struct` 

255 Struct containing these fields: 

256 

257 ``outputExposure`` 

258 Calibrated science exposure with refined WCS and PhotoCalib 

259 (`lsst.afw.image.Exposure`). 

260 ``outputBackground`` 

261 Model of background subtracted from exposure 

262 (`lsst.afw.math.BackgroundList`). 

263 ``outputCat`` 

264 Catalog of measured sources (`lsst.afw.table.SourceCatalog`). 

265 ``astromMatches`` 

266 List of source/refObj matches from the astrometry solver 

267 (`list` [`lsst.afw.table.ReferenceMatch`]). 

268 """ 

269 # Can't persist empty BackgroundList; DM-33714 

270 bg = afwMath.BackgroundMI(geom.Box2I(geom.Point2I(0, 0), geom.Point2I(16, 16)), 

271 afwImage.MaskedImageF(16, 16)) 

272 return Struct(outputExposure=exposure, 

273 outputBackground=afwMath.BackgroundList(bg), 

274 outputCat=afwTable.SourceCatalog(), 

275 astromMatches=[], 

276 ) 

277 

278 

279class MockGetTemplateTask(PipelineTask): 

280 """A do-nothing substitute for GetTemplateTask. 

281 """ 

282 ConfigClass = GetTemplateConfig 

283 _DefaultName = "notGetTemplate" 

284 

285 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

286 inputs = butlerQC.get(inputRefs) 

287 # Mock GetTemplateTask.getOverlappingExposures 

288 results = Struct(coaddExposures=[], 

289 dataIds=[], 

290 ) 

291 inputs["coaddExposures"] = results.coaddExposures 

292 inputs["dataIds"] = results.dataIds 

293 outputs = self.run(**inputs) 

294 butlerQC.put(outputs, outputRefs) 

295 

296 def run(self, coaddExposures, bbox, wcs, dataIds, **kwargs): 

297 """Warp coadds from multiple tracts to form a template for image diff. 

298 

299 Where the tracts overlap, the resulting template image is averaged. 

300 The PSF on the template is created by combining the CoaddPsf on each 

301 template image into a meta-CoaddPsf. 

302 

303 Parameters 

304 ---------- 

305 coaddExposures : `list` of `lsst.afw.image.Exposure` 

306 Coadds to be mosaicked 

307 bbox : `lsst.geom.Box2I` 

308 Template Bounding box of the detector geometry onto which to 

309 resample the coaddExposures 

310 wcs : `lsst.afw.geom.SkyWcs` 

311 Template WCS onto which to resample the coaddExposures 

312 dataIds : `list` of `lsst.daf.butler.DataCoordinate` 

313 Record of the tract and patch of each coaddExposure. 

314 **kwargs 

315 Any additional keyword parameters. 

316 

317 Returns 

318 ------- 

319 result : `lsst.pipe.base.Struct` containing 

320 - ``template`` : a template coadd exposure assembled out of patches 

321 """ 

322 return Struct(template=afwImage.ExposureF(), 

323 ) 

324 

325 

326class MockAlardLuptonSubtractTask(PipelineTask): 

327 """A do-nothing substitute for AlardLuptonSubtractTask. 

328 """ 

329 ConfigClass = AlardLuptonSubtractConfig 

330 _DefaultName = "notAlardLuptonSubtract" 

331 

332 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None): 

333 """PSF match, subtract, and decorrelate two images. 

334 

335 Parameters 

336 ---------- 

337 template : `lsst.afw.image.ExposureF` 

338 Template exposure, warped to match the science exposure. 

339 science : `lsst.afw.image.ExposureF` 

340 Science exposure to subtract from the template. 

341 sources : `lsst.afw.table.SourceCatalog` 

342 Identified sources on the science exposure. This catalog is used to 

343 select sources in order to perform the AL PSF matching on stamp 

344 images around them. 

345 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional 

346 Exposure catalog with finalized psf models and aperture correction 

347 maps to be applied if config.doApplyFinalizedPsf=True. Catalog 

348 uses the detector id for the catalog id, sorted on id for fast 

349 lookup. Deprecated in favor of ``visitSummary``, and will be 

350 removed after v26. 

351 visitSummary : `lsst.afw.table.ExposureCatalog`, optional 

352 Exposure catalog with external calibrations to be applied. Catalog 

353 uses the detector id for the catalog id, sorted on id for fast 

354 lookup. Ignored (for temporary backwards compatibility) if 

355 ``finalizedPsfApCorrCatalog`` is provided. 

356 

357 Returns 

358 ------- 

359 results : `lsst.pipe.base.Struct` 

360 ``difference`` : `lsst.afw.image.ExposureF` 

361 Result of subtracting template and science. 

362 ``matchedTemplate`` : `lsst.afw.image.ExposureF` 

363 Warped and PSF-matched template exposure. 

364 ``backgroundModel`` : `lsst.afw.math.Function2D` 

365 Background model that was fit while solving for the 

366 PSF-matching kernel 

367 ``psfMatchingKernel`` : `lsst.afw.math.Kernel` 

368 Kernel used to PSF-match the convolved image. 

369 """ 

370 return Struct(difference=afwImage.ExposureF(), 

371 matchedTemplate=afwImage.ExposureF(), 

372 backgroundModel=afwMath.NullFunction2D(), 

373 psfMatchingKernel=afwMath.FixedKernel(), 

374 ) 

375 

376 

377class MockDetectAndMeasureConfig(DetectAndMeasureConfig): 

378 

379 def setDefaults(self): 

380 super().setDefaults() 

381 # Avoid delegating to lsst.obs.base.Instrument specialization for the 

382 # data ID packing algorithm to use, since test code often does not use a 

383 # real Instrument in its data IDs. 

384 self.idGenerator.packer.name = "observation" 

385 

386 

387class MockDetectAndMeasureTask(PipelineTask): 

388 """A do-nothing substitute for DetectAndMeasureTask. 

389 """ 

390 ConfigClass = MockDetectAndMeasureConfig 

391 _DefaultName = "notDetectAndMeasure" 

392 

393 def __init__(self, **kwargs): 

394 super().__init__(**kwargs) 

395 self.outputSchema = afwTable.SourceCatalog() 

396 

397 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

398 inputs = butlerQC.get(inputRefs) 

399 idFactory = afwTable.IdFactory.makeSimple() 

400 

401 outputs = self.run(inputs['science'], 

402 inputs['matchedTemplate'], 

403 inputs['difference'], 

404 idFactory=idFactory) 

405 butlerQC.put(outputs, outputRefs) 

406 

407 def run(self, science, matchedTemplate, difference, 

408 idFactory=None): 

409 """Detect and measure sources on a difference image. 

410 

411 Parameters 

412 ---------- 

413 science : `lsst.afw.image.ExposureF` 

414 Science exposure that the template was subtracted from. 

415 matchedTemplate : `lsst.afw.image.ExposureF` 

416 Warped and PSF-matched template that was used produce the 

417 difference image. 

418 difference : `lsst.afw.image.ExposureF` 

419 Result of subtracting template from the science image. 

420 idFactory : `lsst.afw.table.IdFactory`, optional 

421 Generator object to assign ids to detected sources in the difference image. 

422 

423 Returns 

424 ------- 

425 results : `lsst.pipe.base.Struct` 

426 ``subtractedMeasuredExposure`` : `lsst.afw.image.ExposureF` 

427 Subtracted exposure with detection mask applied. 

428 ``diaSources`` : `lsst.afw.table.SourceCatalog` 

429 The catalog of detected sources. 

430 """ 

431 return Struct(subtractedMeasuredExposure=difference, 

432 diaSources=afwTable.SourceCatalog(), 

433 ) 

434 

435 

436class MockTransformDiaSourceCatalogTask(PipelineTask): 

437 """A do-nothing substitute for TransformDiaSourceCatalogTask. 

438 """ 

439 ConfigClass = TransformDiaSourceCatalogConfig 

440 _DefaultName = "notTransformDiaSourceCatalog" 

441 

442 def __init__(self, initInputs, **kwargs): 

443 super().__init__(**kwargs) 

444 

445 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

446 inputs = butlerQC.get(inputRefs) 

447 idGenerator = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

448 inputs["ccdVisitId"] = idGenerator.catalog_id 

449 inputs["band"] = butlerQC.quantum.dataId["band"] 

450 

451 outputs = self.run(**inputs) 

452 

453 butlerQC.put(outputs, outputRefs) 

454 

455 def run(self, diaSourceCat, diffIm, band, ccdVisitId, funcs=None): 

456 """Produce transformation outputs with no processing. 

457 

458 Parameters 

459 ---------- 

460 diaSourceCat : `lsst.afw.table.SourceCatalog` 

461 The catalog to transform. 

462 diffIm : `lsst.afw.image.Exposure` 

463 An image, to provide supplementary information. 

464 band : `str` 

465 The band in which the sources were observed. 

466 ccdVisitId : `int` 

467 The exposure ID in which the sources were observed. 

468 funcs, optional 

469 Unused. 

470 

471 Returns 

472 ------- 

473 results : `lsst.pipe.base.Struct` 

474 Results struct with components: 

475 

476 ``diaSourceTable`` 

477 Catalog of DiaSources (`pandas.DataFrame`). 

478 """ 

479 return Struct(diaSourceTable=pandas.DataFrame(), 

480 ) 

481 

482 

483class MockDiaPipelineConfig(DiaPipelineConfig): 

484 

485 def setDefaults(self): 

486 super().setDefaults() 

487 # Avoid delegating to lsst.obs.base.Instrument specialization for the 

488 # data ID packing algorithm to use, since test code often does not use a 

489 # real Instrument in its data IDs. 

490 self.idGenerator.packer.name = "observation" 

491 

492 

493class MockDiaPipelineTask(PipelineTask): 

494 """A do-nothing substitute for DiaPipelineTask. 

495 """ 

496 ConfigClass = MockDiaPipelineConfig 

497 _DefaultName = "notDiaPipe" 

498 

499 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

500 inputs = butlerQC.get(inputRefs) 

501 inputs["idGenerator"] = self.config.idGenerator.apply(butlerQC.quantum.dataId) 

502 # Need to set ccdExposureIdBits (now deprecated) to None and pass it, 

503 # since there are non-optional positional arguments after it. 

504 inputs["ccdExposureIdBits"] = None 

505 inputs["band"] = butlerQC.quantum.dataId["band"] 

506 if not self.config.doSolarSystemAssociation: 

507 inputs["solarSystemObjectTable"] = None 

508 

509 outputs = self.run(**inputs) 

510 

511 butlerQC.put(outputs, outputRefs) 

512 

513 def run(self, 

514 diaSourceTable, 

515 solarSystemObjectTable, 

516 diffIm, 

517 exposure, 

518 template, 

519 ccdExposureIdBits, 

520 band, 

521 idGenerator=None): 

522 """Produce DiaSource and DiaObject outputs with no processing. 

523 

524 Parameters 

525 ---------- 

526 diaSourceTable : `pandas.DataFrame` 

527 Newly detected DiaSources. 

528 solarSystemObjectTable : `pandas.DataFrame` 

529 Expected solar system objects in the field of view. 

530 diffIm : `lsst.afw.image.ExposureF` 

531 Difference image exposure in which the sources in ``diaSourceCat`` 

532 were detected. 

533 exposure : `lsst.afw.image.ExposureF` 

534 Calibrated exposure differenced with a template to create 

535 ``diffIm``. 

536 template : `lsst.afw.image.ExposureF` 

537 Template exposure used to create diffIm. 

538 ccdExposureIdBits : `int` 

539 Number of bits used for a unique ``ccdVisitId``. Deprecated in 

540 favor of ``idGenerator``, and ignored if that is present. Pass 

541 `None` explicitly to avoid a deprecation warning (a default is 

542 impossible given that later positional arguments are not 

543 defaulted). 

544 band : `str` 

545 The band in which the new DiaSources were detected. 

546 idGenerator : `lsst.meas.base.IdGenerator`, optional 

547 Object that generates source IDs and random number generator seeds. 

548 Will be required after ``ccdExposureIdBits`` is removed. 

549 

550 Returns 

551 ------- 

552 results : `lsst.pipe.base.Struct` 

553 Results struct with components: 

554 

555 ``apdbMarker`` 

556 Marker dataset to store in the Butler indicating that this 

557 ccdVisit has completed successfully (`lsst.dax.apdb.ApdbConfig`). 

558 ``associatedDiaSources`` 

559 Catalog of newly associated DiaSources (`pandas.DataFrame`). 

560 """ 

561 return Struct(apdbMarker=self.config.apdb.value, 

562 associatedDiaSources=pandas.DataFrame(), 

563 diaForcedSources=pandas.DataFrame(), 

564 diaObjects=pandas.DataFrame(), 

565 longTrailedSources=pandas.DataFrame(), 

566 )