Coverage for tests/assembleCoaddTestUtils.py: 24%

178 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-20 10:29 +0000

1# This file is part of pipe_tasks. 

2# 

3# LSST Data Management System 

4# This product includes software developed by the 

5# LSST Project (http://www.lsst.org/). 

6# See COPYRIGHT file at the top of the source tree. 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22"""Set up simulated test data and simplified APIs for AssembleCoaddTask 

23and its derived classes. 

24 

25This is not intended to test accessing data with the Butler and instead uses 

26mock Butler data references to pass in the simulated data. 

27""" 

28from astropy.time import Time 

29from astropy import units as u 

30from astropy.coordinates import SkyCoord, EarthLocation, Angle 

31import numpy as np 

32 

33from lsst.afw.cameraGeom.testUtils import DetectorWrapper 

34import lsst.afw.geom as afwGeom 

35import lsst.afw.image as afwImage 

36import lsst.daf.butler 

37import lsst.geom as geom 

38from lsst.geom import arcseconds, degrees 

39from lsst.meas.algorithms.testUtils import plantSources 

40from lsst.obs.base import MakeRawVisitInfoViaObsInfo 

41import lsst.pipe.base as pipeBase 

42from lsst.pipe.tasks.coaddInputRecorder import CoaddInputRecorderTask, CoaddInputRecorderConfig 

43 

44from astro_metadata_translator import makeObservationInfo 

45 

46__all__ = ["MockWarpReference", "makeMockSkyInfo", "MockCoaddTestData"] 

47 

48 

49class MockWarpReference(lsst.daf.butler.DeferredDatasetHandle): 

50 """Very simple object that looks like a Gen 3 data reference to a warped 

51 exposure. 

52 

53 Parameters 

54 ---------- 

55 exposure : `lsst.afw.image.Exposure` 

56 The exposure to be retrieved by the data reference. 

57 coaddName : `str` 

58 The type of coadd being produced. Typically 'deep'. 

59 patch : `int` 

60 Unique identifier for a subdivision of a tract. 

61 tract : `int` 

62 Unique identifier for a tract of a skyMap 

63 visit : `int` 

64 Unique identifier for an observation, 

65 potentially consisting of multiple ccds. 

66 """ 

67 def __init__(self, exposure, coaddName='deep', patch=42, tract=0, visit=100): 

68 self.coaddName = coaddName 

69 self.exposure = exposure 

70 self.tract = tract 

71 self.patch = patch 

72 self.visit = visit 

73 

74 def get(self, bbox=None, component=None, parameters=None, **kwargs): 

75 """Retrieve the specified dataset using the API of the Gen 3 Butler. 

76 

77 Parameters 

78 ---------- 

79 bbox : `lsst.geom.box.Box2I`, optional 

80 If supplied, retrieve only a subregion of the exposure. 

81 component : `str`, optional 

82 If supplied, return the named metadata of the exposure. 

83 parameters : `dict`, optional 

84 If supplied, use the parameters to modify the exposure, 

85 typically by taking a subset. 

86 **kwargs 

87 Additional keyword arguments such as `immediate=True` that would 

88 control internal butler behavior. 

89 

90 Returns 

91 ------- 

92 `lsst.afw.image.Exposure` or `lsst.afw.image.VisitInfo` 

93 or `lsst.meas.algorithms.SingleGaussianPsf` 

94 Either the exposure or its metadata, depending on the datasetType. 

95 """ 

96 if component == 'psf': 

97 return self.exposure.getPsf() 

98 elif component == 'visitInfo': 

99 return self.exposure.getInfo().getVisitInfo() 

100 if parameters is not None: 

101 if "bbox" in parameters: 

102 bbox = parameters["bbox"] 

103 exp = self.exposure.clone() 

104 if bbox is not None: 

105 return exp[bbox] 

106 else: 

107 return exp 

108 

109 @property 

110 def dataId(self): 

111 """Generate a valid data identifier. 

112 

113 Returns 

114 ------- 

115 dataId : `lsst.daf.butler.DataCoordinate` 

116 Data identifier dict for the patch. 

117 """ 

118 return lsst.daf.butler.DataCoordinate.standardize( 

119 tract=self.tract, 

120 patch=self.patch, 

121 visit=self.visit, 

122 instrument="DummyCam", 

123 skymap="Skymap", 

124 universe=lsst.daf.butler.DimensionUniverse(), 

125 ) 

126 

127 

128def makeMockSkyInfo(bbox, wcs, patch): 

129 """Construct a `Struct` containing the geometry of the patch to be coadded. 

130 

131 Parameters 

132 ---------- 

133 bbox : `lsst.geom.Box` 

134 Bounding box of the patch to be coadded. 

135 wcs : `lsst.afw.geom.SkyWcs` 

136 Coordinate system definition (wcs) for the exposure. 

137 

138 Returns 

139 ------- 

140 skyInfo : `lsst.pipe.base.Struct` 

141 Patch geometry information. 

142 """ 

143 def getIndex(): 

144 return patch 

145 patchInfo = pipeBase.Struct(getIndex=getIndex) 

146 skyInfo = pipeBase.Struct(bbox=bbox, wcs=wcs, patchInfo=patchInfo) 

147 return skyInfo 

148 

149 

150class MockCoaddTestData: 

151 """Generate repeatable simulated exposures with consistent metadata that 

152 are realistic enough to test the image coaddition algorithms. 

153 

154 Notes 

155 ----- 

156 The simple GaussianPsf used by lsst.meas.algorithms.testUtils.plantSources 

157 will always return an average position of (0, 0). 

158 The bounding box of the exposures MUST include (0, 0), or else the PSF will 

159 not be valid and `AssembleCoaddTask` will fail with the error 

160 'Could not find a valid average position for CoaddPsf'. 

161 

162 Parameters 

163 ---------- 

164 shape : `lsst.geom.Extent2I`, optional 

165 Size of the bounding box of the exposures to be simulated, in pixels. 

166 offset : `lsst.geom.Point2I`, optional 

167 Pixel coordinate of the lower left corner of the bounding box. 

168 backgroundLevel : `float`, optional 

169 Background value added to all pixels in the simulated images. 

170 seed : `int`, optional 

171 Seed value to initialize the random number generator. 

172 nSrc : `int`, optional 

173 Number of sources to simulate. 

174 fluxRange : `float`, optional 

175 Range in flux amplitude of the simulated sources. 

176 noiseLevel : `float`, optional 

177 Standard deviation of the noise to add to each pixel. 

178 sourceSigma : `float`, optional 

179 Average amplitude of the simulated sources, 

180 relative to ``noiseLevel`` 

181 minPsfSize : `float`, optional 

182 The smallest PSF width (sigma) to use, in pixels. 

183 maxPsfSize : `float`, optional 

184 The largest PSF width (sigma) to use, in pixels. 

185 pixelScale : `lsst.geom.Angle`, optional 

186 The plate scale of the simulated images. 

187 ra : `lsst.geom.Angle`, optional 

188 Right Ascension of the boresight of the camera for the observation. 

189 dec : `lsst.geom.Angle`, optional 

190 Declination of the boresight of the camera for the observation. 

191 ccd : `int`, optional 

192 CCD number to put in the metadata of the exposure. 

193 patch : `int`, optional 

194 Unique identifier for a subdivision of a tract. 

195 tract : `int`, optional 

196 Unique identifier for a tract of a skyMap. 

197 

198 Raises 

199 ------ 

200 ValueError 

201 If the bounding box does not contain the pixel coordinate (0, 0). 

202 This is due to `GaussianPsf` that is used by `lsst.meas.algorithms.testUtils.plantSources` 

203 lacking the option to specify the pixel origin. 

204 """ 

205 rotAngle = 0.*degrees 

206 "Rotation of the pixel grid on the sky, East from North (`lsst.geom.Angle`)." 

207 filterLabel = None 

208 """The filter definition, usually set in the current instruments' obs package. 

209 For these tests, a simple filter is defined without using an obs package (`lsst.afw.image.FilterLabel`). 

210 """ 

211 rngData = None 

212 """Pre-initialized random number generator for constructing the test images 

213 repeatably (`numpy.random.Generator`). 

214 """ 

215 rngMods = None 

216 """Pre-initialized random number generator for applying modifications to 

217 the test images for only some test cases (`numpy.random.Generator`). 

218 """ 

219 kernelSize = None 

220 "Width of the kernel used for simulating sources, in pixels." 

221 exposures = {} 

222 "The simulated test data, with variable PSF sizes (`dict` of `lsst.afw.image.Exposure`)" 

223 matchedExposures = {} 

224 """The simulated exposures, all with PSF width set to `maxPsfSize` 

225 (`dict` of `lsst.afw.image.Exposure`). 

226 """ 

227 photoCalib = afwImage.makePhotoCalibFromCalibZeroPoint(27, 10) 

228 "The photometric zero point to use for converting counts to flux units (`lsst.afw.image.PhotoCalib`)." 

229 badMaskPlanes = ["NO_DATA", "BAD"] 

230 "Mask planes that, if set, the associated pixel should not be included in the coaddTempExp." 

231 detector = None 

232 "Properties of the CCD for the exposure (`lsst.afw.cameraGeom.Detector`)." 

233 

234 def __init__(self, shape=geom.Extent2I(201, 301), offset=geom.Point2I(-123, -45), 

235 backgroundLevel=314.592, seed=42, nSrc=37, 

236 fluxRange=2., noiseLevel=5, sourceSigma=200., 

237 minPsfSize=1.5, maxPsfSize=3., 

238 pixelScale=0.2*arcseconds, ra=209.*degrees, dec=-20.25*degrees, 

239 ccd=37, patch=42, tract=0): 

240 self.ra = ra 

241 self.dec = dec 

242 self.pixelScale = pixelScale 

243 self.patch = patch 

244 self.tract = tract 

245 self.filterLabel = afwImage.FilterLabel(band="gTest", physical="gTest") 

246 self.rngData = np.random.default_rng(seed) 

247 self.rngMods = np.random.default_rng(seed + 1) 

248 self.bbox = geom.Box2I(offset, shape) 

249 if not self.bbox.contains(0, 0): 

250 raise ValueError(f"The bounding box must contain the coordinate (0, 0). {repr(self.bbox)}") 

251 self.wcs = self.makeDummyWcs() 

252 

253 # Set up properties of the simulations 

254 nSigmaForKernel = 5 

255 self.kernelSize = (int(maxPsfSize*nSigmaForKernel + 0.5)//2)*2 + 1 # make sure it is odd 

256 

257 bufferSize = self.kernelSize//2 

258 x0, y0 = self.bbox.getBegin() 

259 xSize, ySize = self.bbox.getDimensions() 

260 # Set the pixel coordinates and fluxes of the simulated sources. 

261 self.xLoc = self.rngData.random(nSrc)*(xSize - 2*bufferSize) + bufferSize + x0 

262 self.yLoc = self.rngData.random(nSrc)*(ySize - 2*bufferSize) + bufferSize + y0 

263 self.flux = (self.rngData.random(nSrc)*(fluxRange - 1.) + 1.)*sourceSigma*noiseLevel 

264 

265 self.backgroundLevel = backgroundLevel 

266 self.noiseLevel = noiseLevel 

267 self.minPsfSize = minPsfSize 

268 self.maxPsfSize = maxPsfSize 

269 self.detector = DetectorWrapper(name=f"detector {ccd}", id=ccd).detector 

270 

271 def setDummyCoaddInputs(self, exposure, expId): 

272 """Generate an `ExposureCatalog` as though the exposures had been 

273 processed using `warpAndPsfMatch`. 

274 

275 Parameters 

276 ---------- 

277 exposure : `lsst.afw.image.Exposure` 

278 The exposure to construct a `CoaddInputs` `ExposureCatalog` for. 

279 expId : `int` 

280 A unique identifier for the visit. 

281 """ 

282 badPixelMask = afwImage.Mask.getPlaneBitMask(self.badMaskPlanes) 

283 nGoodPix = np.sum(exposure.getMask().getArray() & badPixelMask == 0) 

284 

285 config = CoaddInputRecorderConfig() 

286 inputRecorder = CoaddInputRecorderTask(config=config, name="inputRecorder") 

287 tempExpInputRecorder = inputRecorder.makeCoaddTempExpRecorder(expId, num=1) 

288 tempExpInputRecorder.addCalExp(exposure, expId, nGoodPix) 

289 tempExpInputRecorder.finish(exposure, nGoodPix=nGoodPix) 

290 

291 def makeCoaddTempExp(self, rawExposure, visitInfo, expId): 

292 """Add the metadata required by `AssembleCoaddTask` to an exposure. 

293 

294 Parameters 

295 ---------- 

296 rawExposure : `lsst.afw.image.Exposure` 

297 The simulated exposure. 

298 visitInfo : `lsst.afw.image.VisitInfo` 

299 VisitInfo containing metadata for the exposure. 

300 expId : `int` 

301 A unique identifier for the visit. 

302 

303 Returns 

304 ------- 

305 tempExp : `lsst.afw.image.Exposure` 

306 The exposure, with all of the metadata needed for coaddition. 

307 """ 

308 tempExp = rawExposure.clone() 

309 tempExp.setWcs(self.wcs) 

310 

311 tempExp.setFilter(self.filterLabel) 

312 tempExp.setPhotoCalib(self.photoCalib) 

313 tempExp.getInfo().setVisitInfo(visitInfo) 

314 tempExp.getInfo().setDetector(self.detector) 

315 self.setDummyCoaddInputs(tempExp, expId) 

316 return tempExp 

317 

318 def makeDummyWcs(self, rotAngle=None, pixelScale=None, crval=None, flipX=True): 

319 """Make a World Coordinate System object for testing. 

320 

321 Parameters 

322 ---------- 

323 rotAngle : `lsst.geom.Angle` 

324 Rotation of the CD matrix, East from North 

325 pixelScale : `lsst.geom.Angle` 

326 Pixel scale of the projection. 

327 crval : `lsst.afw.geom.SpherePoint` 

328 Coordinates of the reference pixel of the wcs. 

329 flipX : `bool`, optional 

330 Flip the direction of increasing Right Ascension. 

331 

332 Returns 

333 ------- 

334 wcs : `lsst.afw.geom.skyWcs.SkyWcs` 

335 A wcs that matches the inputs. 

336 """ 

337 if rotAngle is None: 

338 rotAngle = self.rotAngle 

339 if pixelScale is None: 

340 pixelScale = self.pixelScale 

341 if crval is None: 

342 crval = geom.SpherePoint(self.ra, self.dec) 

343 crpix = geom.Box2D(self.bbox).getCenter() 

344 cdMatrix = afwGeom.makeCdMatrix(scale=pixelScale, orientation=rotAngle, flipX=flipX) 

345 wcs = afwGeom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

346 return wcs 

347 

348 def makeDummyVisitInfo(self, exposureId, randomizeTime=False): 

349 """Make a self-consistent visitInfo object for testing. 

350 

351 Parameters 

352 ---------- 

353 exposureId : `int`, optional 

354 Unique integer identifier for this observation. 

355 randomizeTime : `bool`, optional 

356 Add a random offset within a 6 hour window to the observation time. 

357 

358 Returns 

359 ------- 

360 visitInfo : `lsst.afw.image.VisitInfo` 

361 VisitInfo for the exposure. 

362 """ 

363 lsstLat = -30.244639*u.degree 

364 lsstLon = -70.749417*u.degree 

365 lsstAlt = 2663.*u.m 

366 lsstTemperature = 20.*u.Celsius 

367 lsstHumidity = 40. # in percent 

368 lsstPressure = 73892.*u.pascal 

369 loc = EarthLocation(lat=lsstLat, 

370 lon=lsstLon, 

371 height=lsstAlt) 

372 

373 time = Time(2000.0, format="jyear", scale="tt") 

374 if randomizeTime: 

375 # Pick a random time within a 6 hour window 

376 time += 6*u.hour*(self.rngMods.random() - 0.5) 

377 radec = SkyCoord(dec=self.dec.asDegrees(), ra=self.ra.asDegrees(), 

378 unit='deg', obstime=time, frame='icrs', location=loc) 

379 airmass = float(1.0/np.sin(radec.altaz.alt)) 

380 obsInfo = makeObservationInfo(location=loc, 

381 detector_exposure_id=exposureId, 

382 datetime_begin=time, 

383 datetime_end=time, 

384 boresight_airmass=airmass, 

385 boresight_rotation_angle=Angle(0.*u.degree), 

386 boresight_rotation_coord='sky', 

387 temperature=lsstTemperature, 

388 pressure=lsstPressure, 

389 relative_humidity=lsstHumidity, 

390 tracking_radec=radec, 

391 altaz_begin=radec.altaz, 

392 observation_type='science', 

393 ) 

394 visitInfo = MakeRawVisitInfoViaObsInfo.observationInfo2visitInfo(obsInfo) 

395 return visitInfo 

396 

397 def makeTestImage(self, expId, noiseLevel=None, psfSize=None, backgroundLevel=None, 

398 detectionSigma=5., badRegionBox=None): 

399 """Make a reproduceable PSF-convolved masked image for testing. 

400 

401 Parameters 

402 ---------- 

403 expId : `int` 

404 A unique identifier to use to refer to the visit. 

405 noiseLevel : `float`, optional 

406 Standard deviation of the noise to add to each pixel. 

407 psfSize : `float`, optional 

408 Width of the PSF of the simulated sources, in pixels. 

409 backgroundLevel : `float`, optional 

410 Background value added to all pixels in the simulated images. 

411 detectionSigma : `float`, optional 

412 Threshold amplitude of the image to set the "DETECTED" mask. 

413 badRegionBox : `lsst.geom.Box2I`, optional 

414 Add a bad region bounding box (set to "BAD"). 

415 """ 

416 if backgroundLevel is None: 

417 backgroundLevel = self.backgroundLevel 

418 if noiseLevel is None: 

419 noiseLevel = 5. 

420 visitInfo = self.makeDummyVisitInfo(expId, randomizeTime=True) 

421 

422 if psfSize is None: 

423 psfSize = self.rngMods.random()*(self.maxPsfSize - self.minPsfSize) + self.minPsfSize 

424 nSrc = len(self.flux) 

425 sigmas = [psfSize for src in range(nSrc)] 

426 sigmasPsfMatched = [self.maxPsfSize for src in range(nSrc)] 

427 coordList = list(zip(self.xLoc, self.yLoc, self.flux, sigmas)) 

428 coordListPsfMatched = list(zip(self.xLoc, self.yLoc, self.flux, sigmasPsfMatched)) 

429 xSize, ySize = self.bbox.getDimensions() 

430 model = plantSources(self.bbox, self.kernelSize, self.backgroundLevel, 

431 coordList, addPoissonNoise=False) 

432 modelPsfMatched = plantSources(self.bbox, self.kernelSize, self.backgroundLevel, 

433 coordListPsfMatched, addPoissonNoise=False) 

434 model.variance.array = np.abs(model.image.array) + noiseLevel 

435 modelPsfMatched.variance.array = np.abs(modelPsfMatched.image.array) + noiseLevel 

436 noise = self.rngData.random((ySize, xSize))*noiseLevel 

437 noise -= np.median(noise) 

438 model.image.array += noise 

439 modelPsfMatched.image.array += noise 

440 detectedMask = afwImage.Mask.getPlaneBitMask("DETECTED") 

441 detectionThreshold = self.backgroundLevel + detectionSigma*noiseLevel 

442 model.mask.array[model.image.array > detectionThreshold] += detectedMask 

443 

444 if badRegionBox is not None: 

445 model.mask[badRegionBox] = afwImage.Mask.getPlaneBitMask("BAD") 

446 

447 exposure = self.makeCoaddTempExp(model, visitInfo, expId) 

448 matchedExposure = self.makeCoaddTempExp(modelPsfMatched, visitInfo, expId) 

449 return exposure, matchedExposure 

450 

451 @staticmethod 

452 def makeDataRefList(exposures, matchedExposures, warpType, tract=0, patch=42, coaddName="deep"): 

453 """Make data references from the simulated exposures that can be 

454 retrieved using the Gen 3 Butler API. 

455 

456 Parameters 

457 ---------- 

458 warpType : `str` 

459 Either 'direct' or 'psfMatched'. 

460 tract : `int`, optional 

461 Unique identifier for a tract of a skyMap. 

462 patch : `int`, optional 

463 Unique identifier for a subdivision of a tract. 

464 coaddName : `str`, optional 

465 The type of coadd being produced. Typically 'deep'. 

466 

467 Returns 

468 ------- 

469 dataRefList : `list` of `MockWarpReference` 

470 The data references. 

471 

472 Raises 

473 ------ 

474 ValueError 

475 If an unknown `warpType` is supplied. 

476 """ 

477 dataRefList = [] 

478 for expId in exposures: 

479 if warpType == 'direct': 

480 exposure = exposures[expId] 

481 elif warpType == 'psfMatched': 

482 exposure = matchedExposures[expId] 

483 else: 

484 raise ValueError("warpType must be one of 'direct' or 'psfMatched'") 

485 dataRef = MockWarpReference(exposure, coaddName=coaddName, 

486 tract=tract, patch=patch, visit=expId) 

487 dataRefList.append(dataRef) 

488 return dataRefList