Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from lsst.ip.diffim.dcrModel import DcrModel 

30 

31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

33 

34 

35class GetCoaddAsTemplateConfig(pexConfig.Config): 

36 templateBorderSize = pexConfig.Field( 

37 dtype=int, 

38 default=10, 

39 doc="Number of pixels to grow the requested template image to account for warping" 

40 ) 

41 coaddName = pexConfig.Field( 

42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

43 dtype=str, 

44 default="deep", 

45 ) 

46 numSubfilters = pexConfig.Field( 

47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'", 

48 dtype=int, 

49 default=3, 

50 ) 

51 warpType = pexConfig.Field( 

52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

53 dtype=str, 

54 default="direct", 

55 ) 

56 

57 

58class GetCoaddAsTemplateTask(pipeBase.Task): 

59 """Subtask to retrieve coadd for use as an image difference template. 

60 

61 This is the default getTemplate Task to be run as a subtask by 

62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

63 ``runGen3()``. 

64 

65 Notes 

66 ----- 

67 From the given skymap, the closest tract is selected; multiple tracts are 

68 not supported. The assembled template inherits the WCS of the selected 

69 skymap tract and the resolution of the template exposures. Overlapping box 

70 regions of the input template patches are pixel by pixel copied into the 

71 assembled template image. There is no warping or pixel resampling. 

72 

73 Pixels with no overlap of any available input patches are set to ``nan`` value 

74 and ``NO_DATA`` flagged. 

75 """ 

76 

77 ConfigClass = GetCoaddAsTemplateConfig 

78 _DefaultName = "GetCoaddAsTemplateTask" 

79 

80 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

82 that overlaps the science exposure. 

83 

84 Parameters 

85 ---------- 

86 exposure: `lsst.afw.image.Exposure` 

87 an exposure for which to generate an overlapping template 

88 sensorRef : TYPE 

89 a Butler data reference that can be used to obtain coadd data 

90 templateIdList : TYPE, optional 

91 list of data ids, unused here, in the case of coadd template 

92 

93 Returns 

94 ------- 

95 result : `lsst.pipe.base.Struct` 

96 - ``exposure`` : `lsst.afw.image.ExposureF` 

97 a template coadd exposure assembled out of patches 

98 - ``sources`` : None for this subtask 

99 """ 

100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

102 

103 availableCoaddRefs = dict() 

104 for patchInfo in patchList: 

105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

106 patchArgDict = dict( 

107 datasetType=self.getCoaddDatasetName() + "_sub", 

108 bbox=patchInfo.getOuterBBox(), 

109 tract=tractInfo.getId(), 

110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

111 subfilter=0, 

112 numSubfilters=self.config.numSubfilters, 

113 ) 

114 

115 if sensorRef.datasetExists(**patchArgDict): 

116 self.log.info("Reading patch %s" % patchArgDict) 

117 availableCoaddRefs[patchNumber] = patchArgDict 

118 

119 templateExposure = self.run( 

120 tractInfo, patchList, skyCorners, availableCoaddRefs, 

121 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

122 ) 

123 return pipeBase.Struct(exposure=templateExposure, sources=None) 

124 

125 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

126 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

127 that overlaps the science exposure. 

128 

129 Parameters 

130 ---------- 

131 exposure : `lsst.afw.image.Exposure` 

132 The science exposure to define the sky region of the template coadd. 

133 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

134 Butler like object that supports getting data by DatasetRef. 

135 skyMapRef : `lsst.daf.butler.DatasetRef` 

136 Reference to SkyMap object that corresponds to the template coadd. 

137 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

138 Iterable of references to the available template coadd patches. 

139 

140 Returns 

141 ------- 

142 result : `lsst.pipe.base.Struct` 

143 - ``exposure`` : `lsst.afw.image.ExposureF` 

144 a template coadd exposure assembled out of patches 

145 - ``sources`` : `None` for this subtask 

146 """ 

147 skyMap = butlerQC.get(skyMapRef) 

148 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

149 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList) 

150 

151 availableCoaddRefs = dict() 

152 for coaddRef in coaddExposureRefs: 

153 dataId = coaddRef.datasetRef.dataId 

154 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter: 

155 if self.config.coaddName == 'dcr': 

156 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" % 

157 (tractInfo.getId(), dataId['patch'], dataId['subfilter'])) 

158 if dataId['patch'] in availableCoaddRefs: 

159 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef)) 

160 else: 

161 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ] 

162 else: 

163 self.log.info("Using template input tract=%s, patch=%s" % 

164 (tractInfo.getId(), dataId['patch'])) 

165 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef) 

166 

167 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs, 

168 visitInfo=exposure.getInfo().getVisitInfo()) 

169 return pipeBase.Struct(exposure=templateExposure, sources=None) 

170 

171 def getOverlapPatchList(self, exposure, skyMap): 

172 """Select the relevant tract and its patches that overlap with the science exposure. 

173 

174 Parameters 

175 ---------- 

176 exposure : `lsst.afw.image.Exposure` 

177 The science exposure to define the sky region of the template coadd. 

178 

179 skyMap : `lsst.skymap.BaseSkyMap` 

180 SkyMap object that corresponds to the template coadd. 

181 

182 Returns 

183 ------- 

184 result : `tuple` of 

185 - ``tractInfo`` : `lsst.skymap.TractInfo` 

186 The selected tract. 

187 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

188 List of all overlap patches of the selected tract. 

189 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

190 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

191 """ 

192 expWcs = exposure.getWcs() 

193 expBoxD = geom.Box2D(exposure.getBBox()) 

194 expBoxD.grow(self.config.templateBorderSize) 

195 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

196 tractInfo = skyMap.findTract(ctrSkyPos) 

197 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

198 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

199 patchList = tractInfo.findPatchList(skyCorners) 

200 

201 if not patchList: 

202 raise RuntimeError("No suitable tract found") 

203 

204 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

205 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

206 

207 return (tractInfo, patchList, skyCorners) 

208 

209 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

210 sensorRef=None, visitInfo=None): 

211 """Gen2 and gen3 shared code: determination of exposure dimensions and 

212 copying of pixels from overlapping patch regions. 

213 

214 Parameters 

215 ---------- 

216 skyMap : `lsst.skymap.BaseSkyMap` 

217 SkyMap object that corresponds to the template coadd. 

218 tractInfo : `lsst.skymap.TractInfo` 

219 The selected tract. 

220 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

221 Patches to consider for making the template exposure. 

222 skyCorners : list of `lsst.geom.SpherePoint` 

223 Sky corner coordinates to be covered by the template exposure. 

224 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3) 

225 `dict` (Gen2) 

226 Dictionary of spatially relevant retrieved coadd patches, 

227 indexed by their sequential patch number. In Gen3 mode, .get() is called, 

228 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd. 

229 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

230 Butler data reference to get coadd data. 

231 Must be `None` for Gen3. 

232 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

233 VisitInfo to make dcr model. 

234 

235 Returns 

236 ------- 

237 templateExposure: `lsst.afw.image.ExposureF` 

238 The created template exposure. 

239 """ 

240 coaddWcs = tractInfo.getWcs() 

241 

242 # compute coadd bbox 

243 coaddBBox = geom.Box2D() 

244 for skyPos in skyCorners: 

245 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

246 coaddBBox = geom.Box2I(coaddBBox) 

247 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

248 

249 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

250 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

251 nPatchesFound = 0 

252 coaddFilter = None 

253 coaddPsf = None 

254 coaddPhotoCalib = None 

255 for patchInfo in patchList: 

256 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

257 patchSubBBox = patchInfo.getOuterBBox() 

258 patchSubBBox.clip(coaddBBox) 

259 if patchSubBBox.isEmpty(): 

260 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, " 

261 f"patch={patchNumber}; no overlapping pixels") 

262 continue 

263 

264 if self.config.coaddName == 'dcr': 

265 patchInnerBBox = patchInfo.getInnerBBox() 

266 patchInnerBBox.clip(coaddBBox) 

267 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

268 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." 

269 % availableCoaddRefs[patchNumber]) 

270 continue 

271 self.log.info("Constructing DCR-matched template for patch %s" 

272 % availableCoaddRefs[patchNumber]) 

273 

274 if sensorRef: 

275 dcrModel = DcrModel.fromDataRef(sensorRef, **availableCoaddRefs[patchNumber]) 

276 else: 

277 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber]) 

278 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

279 # Each patch has significant overlap, and the contaminated edge pixels in 

280 # a new patch will overwrite good pixels in the overlap region from 

281 # previous patches. 

282 # Shrink the BBox to remove the contaminated pixels, 

283 # but make sure it is only the overlap region that is reduced. 

284 dcrBBox = geom.Box2I(patchSubBBox) 

285 dcrBBox.grow(-self.config.templateBorderSize) 

286 dcrBBox.include(patchInnerBBox) 

287 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

288 wcs=coaddWcs, 

289 visitInfo=visitInfo) 

290 else: 

291 if sensorRef is None: 

292 # Gen3 

293 coaddPatch = availableCoaddRefs[patchNumber].get() 

294 else: 

295 # Gen2 

296 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

297 nPatchesFound += 1 

298 

299 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

300 # calculation from caller code. Gen3 also does not do this. 

301 overlapBox = coaddPatch.getBBox() 

302 overlapBox.clip(coaddBBox) 

303 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

304 

305 if coaddFilter is None: 

306 coaddFilter = coaddPatch.getFilter() 

307 

308 # Retrieve the PSF for this coadd tract, if not already retrieved 

309 if coaddPsf is None and coaddPatch.hasPsf(): 

310 coaddPsf = coaddPatch.getPsf() 

311 

312 # Retrieve the calibration for this coadd tract, if not already retrieved 

313 if coaddPhotoCalib is None: 

314 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

315 

316 if coaddPhotoCalib is None: 

317 raise RuntimeError("No coadd PhotoCalib found!") 

318 if nPatchesFound == 0: 

319 raise RuntimeError("No patches found!") 

320 if coaddPsf is None: 

321 raise RuntimeError("No coadd Psf found!") 

322 

323 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

324 coaddExposure.setPsf(coaddPsf) 

325 coaddExposure.setFilter(coaddFilter) 

326 return coaddExposure 

327 

328 def getCoaddDatasetName(self): 

329 """Return coadd name for given task config 

330 

331 Returns 

332 ------- 

333 CoaddDatasetName : `string` 

334 

335 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

336 """ 

337 warpType = self.config.warpType 

338 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

339 return self.config.coaddName + "Coadd" + suffix 

340 

341 

342class GetCalexpAsTemplateConfig(pexConfig.Config): 

343 doAddCalexpBackground = pexConfig.Field( 

344 dtype=bool, 

345 default=True, 

346 doc="Add background to calexp before processing it." 

347 ) 

348 

349 

350class GetCalexpAsTemplateTask(pipeBase.Task): 

351 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

352 for use as an image difference template. Only gen2 supported. 

353 

354 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

355 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

356 This code was originally part of Winter2013ImageDifferenceTask. 

357 """ 

358 

359 ConfigClass = GetCalexpAsTemplateConfig 

360 _DefaultName = "GetCalexpAsTemplateTask" 

361 

362 def run(self, exposure, sensorRef, templateIdList): 

363 """Return a calexp exposure with based on input sensorRef. 

364 

365 Construct a dataId based on the sensorRef.dataId combined 

366 with the specifications from the first dataId in templateIdList 

367 

368 Parameters 

369 ---------- 

370 exposure : `lsst.afw.image.Exposure` 

371 exposure (unused) 

372 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

373 Data reference of the calexp(s) to subtract from. 

374 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

375 Data reference of the template calexp to be subtraced. 

376 Can be incomplete, fields are initialized from `sensorRef`. 

377 If there are multiple items, only the first one is used. 

378 

379 Returns 

380 ------- 

381 result : `struct` 

382 

383 return a pipeBase.Struct: 

384 

385 - ``exposure`` : a template calexp 

386 - ``sources`` : source catalog measured on the template 

387 """ 

388 

389 if len(templateIdList) == 0: 

390 raise RuntimeError("No template data reference supplied.") 

391 if len(templateIdList) > 1: 

392 self.log.warn("Multiple template data references supplied. Using the first one only.") 

393 

394 templateId = sensorRef.dataId.copy() 

395 templateId.update(templateIdList[0]) 

396 

397 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

398 

399 butler = sensorRef.getButler() 

400 template = butler.get(datasetType="calexp", dataId=templateId) 

401 if self.config.doAddCalexpBackground: 

402 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

403 mi = template.getMaskedImage() 

404 mi += templateBg.getImage() 

405 

406 if not template.hasPsf(): 

407 raise pipeBase.TaskError("Template has no psf") 

408 

409 templateSources = butler.get(datasetType="src", dataId=templateId) 

410 return pipeBase.Struct(exposure=template, 

411 sources=templateSources) 

412 

413 def runDataRef(self, *args, **kwargs): 

414 return self.run(*args, **kwargs) 

415 

416 def runQuantum(self, **kwargs): 

417 raise NotImplementedError("Calexp template is not supported with gen3 middleware")