Coverage for python/lsst/pipe/tasks/mocks/mockCoadd.py: 90%

169 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-17 08:50 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008-2015 AURA/LSST. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <https://www.lsstcorp.org/LegalNotices/>. 

21# 

22import lsst.afw.image 

23import lsst.geom 

24import lsst.pex.config 

25import lsst.afw.table 

26import lsst.pipe.base 

27from lsst.pipe.tasks.makeSkyMap import MakeSkyMapTask 

28from lsst.pipe.tasks.makeCoaddTempExp import MakeCoaddTempExpTask 

29from lsst.pipe.tasks.assembleCoadd import (AssembleCoaddTask, SafeClipAssembleCoaddTask, 

30 CompareWarpAssembleCoaddTask) 

31from .mockObject import MockObjectTask 

32from .mockObservation import MockObservationTask 

33from .mockSelect import MockSelectImagesTask 

34 

35 

36class MockCoaddConfig(lsst.pex.config.Config): 

37 makeSkyMap = lsst.pex.config.ConfigurableField( 

38 doc="SkyMap builder subtask", 

39 target=MakeSkyMapTask 

40 ) 

41 mockObject = lsst.pex.config.ConfigurableField( 

42 doc="Subtask that generates and draws the objects/sources in the mock images", 

43 target=MockObjectTask 

44 ) 

45 mockObservation = lsst.pex.config.ConfigurableField( 

46 doc="Subtask that generates the Wcs, Psf, PhotoCalib, etc. of mock images", 

47 target=MockObservationTask 

48 ) 

49 coaddName = lsst.pex.config.Field( 

50 doc="Coadd name used as a prefix for other datasets", 

51 dtype=str, 

52 optional=False, 

53 default="deep" 

54 ) 

55 nObservations = lsst.pex.config.Field( 

56 doc="Number of mock observations to generate.", 

57 dtype=int, 

58 optional=False, 

59 default=12 

60 ) 

61 edgeBuffer = lsst.pex.config.Field( 

62 doc=("Number of pixels by which to grow object bounding boxes when determining whether they land " 

63 " completely on a generated image"), 

64 dtype=int, 

65 optional=False, 

66 default=5 

67 ) 

68 

69 def setupSkyMapPatches(self, nPatches=2, patchSize=400, pixelScale=0.2*lsst.geom.arcseconds): 

70 """ 

71 Set the nested [discrete] skymap config parameters such that the full tract 

72 has nPatches x nPatches patches of the given size and pixel scale. 

73 """ 

74 self.makeSkyMap.skyMap['discrete'].patchInnerDimensions = [patchSize, patchSize] 

75 self.makeSkyMap.skyMap['discrete'].pixelScale = pixelScale.asArcseconds() 

76 # multiply by 0.5 because we want a half-width; subtract 0.49 to ensure that we get the right 

77 # number after skyMap.TractInfo rounds up. 

78 radius = (0.5 * nPatches - 0.49) * patchSize * pixelScale.asDegrees() 

79 self.makeSkyMap.skyMap['discrete'].radiusList = [radius] 

80 

81 def setDefaults(self): 

82 self.makeSkyMap.skyMap.name = 'discrete' 

83 self.makeSkyMap.skyMap['discrete'].raList = [90.0] 

84 self.makeSkyMap.skyMap['discrete'].decList = [0.0] 

85 self.makeSkyMap.skyMap['discrete'].patchBorder = 10 

86 self.makeSkyMap.skyMap['discrete'].projection = "TAN" 

87 self.makeSkyMap.skyMap['discrete'].tractOverlap = 0.0 

88 self.setupSkyMapPatches() 

89 

90 

91class MockCoaddTask(lsst.pipe.base.CmdLineTask): 

92 """MockCoaddTask is a driver task for creating mock coadds. As opposed to more realistic 

93 simulations, MockCoadd generates and uses extremely simple "toy" data that can be used to more 

94 rigorously test the behavior of high-level task code because the expected results are 

95 more easily predicted. In particular, calexps are generated directly from the truth catalog, 

96 and contain only zero-noise stars that are created using the same Psf, PhotoCalib, and Wcs that will 

97 be attached to the mock calexp. 

98 

99 In addition to creating the mock calexps and truth catalogs, MockCoadd also contains driver 

100 code to run the MakeSkyMap, MakeCoaddTempExp, and AssembleCoadd tasks on the mock calexps, 

101 and code to directly create a mock coadd image using CoaddPsf, which can be compared to the 

102 output of the regular coadd tasks to check that the coadd code and CoaddPsf are consistent. 

103 

104 Note that aside from MakeSkyMapTask, the coadd tasks are *not* subtasks of MockCoaddTasks, 

105 and their configs are not part of MockCoaddConfig; these are created locally within 

106 MockCoaddTask methods when needed, as not all coadd task config options are appropriate 

107 for the mock data generated by MockCoadd. 

108 """ 

109 

110 ConfigClass = MockCoaddConfig 

111 

112 _DefaultName = "MockCoadd" 

113 

114 def __init__(self, **kwds): 

115 """Construct a MockCoaddTask and the subtasks used for generating skymaps, objects, 

116 and observations (i.e. calexp parameters). 

117 """ 

118 lsst.pipe.base.CmdLineTask.__init__(self, **kwds) 

119 self.makeSubtask("makeSkyMap") 

120 self.makeSubtask("mockObject") 

121 self.makeSubtask("mockObservation") 

122 self.schema = lsst.afw.table.SimpleTable.makeMinimalSchema() 

123 self.objectIdKey = self.schema.addField("objectId", type="L", doc="foreign key to truth catalog") 

124 self.exposureIdKey = self.schema.addField("exposureId", type="L", 

125 doc="foreign key to observation catalog") 

126 self.centroidInBBoxKey = self.schema.addField( 

127 "centroidInBBox", type="Flag", 

128 doc="set if this source's center position is inside the generated image's bbox" 

129 ) 

130 self.partialOverlapKey = self.schema.addField( 

131 "partialOverlap", type="Flag", 

132 doc="set if this source was not completely inside the generated image" 

133 ) 

134 

135 def buildSkyMap(self, butler): 

136 """Build the skymap for the mock dataset.""" 

137 return self.makeSkyMap.runDataRef(butler.dataRef(self.config.coaddName + "Coadd_skyMap")).skyMap 

138 

139 def buildTruthCatalog(self, butler=None, skyMap=None, tract=0): 

140 """Create and save (if butler is not None) a truth catalog containing all the mock objects. 

141 

142 Must be run after buildSkyMap. 

143 

144 Most of the work is delegated to the mockObject subtask. 

145 """ 

146 if skyMap is None: 146 ↛ 147line 146 didn't jump to line 147, because the condition on line 146 was never true

147 skyMap = butler.get(self.config.coaddName + "Coadd_skyMap") 

148 catalog = self.mockObject.run(tractInfo=skyMap[tract]) 

149 if butler is not None: 149 ↛ 151line 149 didn't jump to line 151, because the condition on line 149 was never false

150 butler.put(catalog, "truth", tract=tract) 

151 return catalog 

152 

153 def buildObservationCatalog(self, butler=None, skyMap=None, tract=0, camera=None): 

154 """Create and save (if butler is not None) an ExposureCatalog of simulated observations, 

155 containing the Psfs, Wcss, PhotoCalibs, etc. of the calexps to be simulated. 

156 

157 Must be run after buildSkyMap. 

158 

159 Most of the work is delegated to the mockObservation subtask. 

160 """ 

161 if skyMap is None: 161 ↛ 162line 161 didn't jump to line 162, because the condition on line 161 was never true

162 skyMap = butler.get(self.config.coaddName + "Coadd_skyMap") 

163 if camera is None: 163 ↛ 165line 163 didn't jump to line 165, because the condition on line 163 was never false

164 camera = butler.get("camera") 

165 catalog = self.mockObservation.run(butler=butler, 

166 n=self.config.nObservations, camera=camera, 

167 tractInfo=skyMap[tract]) 

168 catalog.sort() 

169 if butler is not None: 169 ↛ 171line 169 didn't jump to line 171, because the condition on line 169 was never false

170 butler.put(catalog, "observations", tract=tract) 

171 return catalog 

172 

173 def buildInputImages(self, butler, obsCatalog=None, truthCatalog=None, tract=0): 

174 """Use the truth catalog and observation catalog to create and save (if butler is not None) 

175 mock calexps and an ExposureCatalog ('simsrc') that contains information about which objects 

176 appear partially or fully in each exposure. 

177 

178 Must be run after buildTruthCatalog and buildObservationCatalog. 

179 """ 

180 if obsCatalog is None: 180 ↛ 181line 180 didn't jump to line 181, because the condition on line 180 was never true

181 obsCatalog = butler.get("observations", tract=tract) 

182 if truthCatalog is None: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true

183 truthCatalog = butler.get("truth", tract=tract) 

184 ccdKey = obsCatalog.getSchema().find("ccd").key 

185 visitKey = obsCatalog.getSchema().find("visit").key 

186 simSrcCatalog = lsst.afw.table.SimpleCatalog(self.schema) 

187 for obsRecord in obsCatalog: 

188 ccd = obsRecord.getI(ccdKey) 

189 visit = obsRecord.getI(visitKey) 

190 self.log.info("Generating image for visit={visit}, ccd={ccd}".format(ccd=ccd, visit=visit)) 

191 exposure = lsst.afw.image.ExposureF(obsRecord.getBBox()) 

192 # Apply a tiny offset to the images, so that they have non-zero background. 

193 # If the image background is identically zero, the calculated variance will be NaN. 

194 exposure.maskedImage.image.array += 1e-8 

195 exposure.setPhotoCalib(obsRecord.getPhotoCalib()) 

196 exposure.setWcs(obsRecord.getWcs()) 

197 exposure.setPsf(obsRecord.getPsf()) 

198 exposure.getInfo().setApCorrMap(obsRecord.getApCorrMap()) 

199 exposure.getInfo().setTransmissionCurve(obsRecord.getTransmissionCurve()) 

200 for truthRecord in truthCatalog: 

201 status = self.mockObject.drawSource(truthRecord, exposure, buffer=self.config.edgeBuffer) 

202 if status: 

203 simSrcRecord = simSrcCatalog.addNew() 

204 simSrcRecord.setCoord(truthRecord.getCoord()) 

205 simSrcRecord.setL(self.objectIdKey, truthRecord.getId()) 

206 simSrcRecord.setL(self.exposureIdKey, obsRecord.getId()) 

207 simSrcRecord.setFlag(self.centroidInBBoxKey, obsRecord.contains(truthRecord.getCoord())) 

208 simSrcRecord.setFlag(self.partialOverlapKey, status == 1) 

209 self.log.info(" added object {id}".format(id=truthRecord.getId())) 

210 exposure.getMaskedImage().getVariance().set(1.0) 

211 if butler is not None: 211 ↛ 187line 211 didn't jump to line 187, because the condition on line 211 was never false

212 butler.put(exposure, "calexp", ccd=ccd, visit=visit) 

213 if butler is not None: 213 ↛ 215line 213 didn't jump to line 215, because the condition on line 213 was never false

214 butler.put(simSrcCatalog, "simsrc", tract=tract) 

215 return simSrcCatalog 

216 

217 def buildAllInputs(self, butler): 

218 """Convenience function that calls buildSkyMap, buildObservationCatalog, buildTruthCatalog, 

219 and buildInputImages. 

220 """ 

221 skyMap = self.buildSkyMap(butler) 

222 observations = self.buildObservationCatalog(butler, skyMap=skyMap) 

223 truth = self.buildTruthCatalog(butler, skyMap=skyMap) 

224 self.buildInputImages(butler, obsCatalog=observations, truthCatalog=truth) 

225 

226 def makeCoaddTask(self, cls, assemblePsfMatched=False): 

227 """Helper function to create a Coadd task with configuration appropriate for the simulations. 

228 

229 MockCoaddTask does not include MakeCoaddTempExpTask or AssembleCoaddTask as subtasks, because 

230 we want explicit control over their configs, rather than leaving this up to the user. 

231 However, we have to install our own SelectImages task for both of these, so it made sense 

232 to have a single method that would create one of these two tasks, set the config values we 

233 want, and install the custom SelectImagesTask. 

234 """ 

235 config = cls.ConfigClass() 

236 config.coaddName = self.config.coaddName 

237 config.select.retarget(MockSelectImagesTask) 

238 if cls == MakeCoaddTempExpTask: 

239 config.bgSubtracted = True 

240 config.makeDirect = True 

241 config.makePsfMatched = True 

242 config.modelPsf.defaultFwhm = 9 

243 config.modelPsf.addWing = False 

244 config.warpAndPsfMatch.psfMatch.kernel['AL'].scaleByFwhm = False 

245 config.warpAndPsfMatch.psfMatch.kernel['AL'].kernelSize = 25 

246 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellX = 64 

247 config.warpAndPsfMatch.psfMatch.kernel['AL'].sizeCellY = 64 

248 

249 elif cls in [AssembleCoaddTask, SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask]: 249 ↛ 257line 249 didn't jump to line 257, because the condition on line 249 was never false

250 if assemblePsfMatched: 

251 config.warpType = 'psfMatched' 

252 if cls != AssembleCoaddTask: 

253 config.doWrite = False 

254 if cls == CompareWarpAssembleCoaddTask: 

255 config.assembleStaticSkyModel.select.retarget(MockSelectImagesTask) 

256 config.doAttachTransmissionCurve = True 

257 return cls(config=config) 

258 

259 def iterPatchRefs(self, butler, tractInfo): 

260 """Generator that iterates over the patches in a tract, yielding dataRefs. 

261 """ 

262 nPatchX, nPatchY = tractInfo.getNumPatches() 

263 for iPatchX in range(nPatchX): 

264 for iPatchY in range(nPatchY): 

265 patchRef = butler.dataRef(self.config.coaddName + "Coadd", 

266 tract=tractInfo.getId(), patch="%d,%d" % (iPatchX, iPatchY), 

267 filter='r') 

268 yield patchRef 

269 

270 def buildCoadd(self, butler, skyMap=None, tract=0): 

271 """Run the coadd tasks (MakeCoaddTempExp and AssembleCoadd) on the mock data. 

272 

273 Must be run after buildInputImages. 

274 Makes both direct and PSF-matched coadds 

275 """ 

276 if skyMap is None: 276 ↛ 278line 276 didn't jump to line 278, because the condition on line 276 was never false

277 skyMap = butler.get(self.config.coaddName + "Coadd_skyMap") 

278 tractInfo = skyMap[tract] 

279 makeCoaddTempExpTask = self.makeCoaddTask(MakeCoaddTempExpTask) 

280 directCoaddTaskList = [] 

281 for coaddTask in [SafeClipAssembleCoaddTask, CompareWarpAssembleCoaddTask, AssembleCoaddTask]: 

282 directCoaddTaskList.append(self.makeCoaddTask(coaddTask)) 

283 assemblePsfMatchedCoaddTask = self.makeCoaddTask(AssembleCoaddTask, assemblePsfMatched=True) 

284 for patchRef in self.iterPatchRefs(butler, tractInfo): 

285 makeCoaddTempExpTask.runDataRef(patchRef) 

286 for patchRef in self.iterPatchRefs(butler, tractInfo): 

287 for directCoaddTask in directCoaddTaskList: 

288 directCoaddTask.runDataRef(patchRef) 

289 assemblePsfMatchedCoaddTask.runDataRef(patchRef) 

290 

291 def buildMockCoadd(self, butler, truthCatalog=None, skyMap=None, tract=0): 

292 """Directly create a simulation of the coadd, using the CoaddPsf (and ModelPsf) 

293 of the direct (and psfMatched) coadd exposure and the truth catalog. 

294 

295 Must be run after buildCoadd. 

296 """ 

297 if truthCatalog is None: 297 ↛ 299line 297 didn't jump to line 299, because the condition on line 297 was never false

298 truthCatalog = butler.get("truth", tract=tract) 

299 if skyMap is None: 299 ↛ 301line 299 didn't jump to line 301, because the condition on line 299 was never false

300 skyMap = butler.get(self.config.coaddName + "Coadd_skyMap") 

301 tractInfo = skyMap[tract] 

302 for patchRef in self.iterPatchRefs(butler, tractInfo): 

303 for dataProduct in ["Coadd", "CoaddPsfMatched"]: 

304 exposure = patchRef.get(self.config.coaddName + dataProduct) 

305 exposure.getMaskedImage().getImage().set(0.0) 

306 coaddPsf = lsst.meas.algorithms.CoaddPsf( 

307 exposure.getInfo().getCoaddInputs().ccds, exposure.getWcs() 

308 ) 

309 exposure.setPsf(coaddPsf) 

310 for truthRecord in truthCatalog: 

311 self.mockObject.drawSource(truthRecord, exposure, buffer=0) 

312 patchRef.put(exposure, self.config.coaddName + dataProduct + "_mock") 

313 

314 

315def run(root): 

316 """Convenience function to create and run MockCoaddTask with default settings. 

317 """ 

318 from .simpleMapper import makeDataRepo 

319 butler = makeDataRepo(root=root) 

320 task = MockCoaddTask() 

321 task.buildAllInputs(butler) 

322 task.buildCoadd(butler) 

323 task.buildMockCoadd(butler)