Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from lsst.ip.diffim.dcrModel import DcrModel 

30 

31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

33 

34 

35class GetCoaddAsTemplateConfig(pexConfig.Config): 

36 templateBorderSize = pexConfig.Field( 

37 dtype=int, 

38 default=10, 

39 doc="Number of pixels to grow the requested template image to account for warping" 

40 ) 

41 coaddName = pexConfig.Field( 

42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

43 dtype=str, 

44 default="deep", 

45 ) 

46 numSubfilters = pexConfig.Field( 

47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'", 

48 dtype=int, 

49 default=3, 

50 ) 

51 warpType = pexConfig.Field( 

52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

53 dtype=str, 

54 default="direct", 

55 ) 

56 

57 

58class GetCoaddAsTemplateTask(pipeBase.Task): 

59 """Subtask to retrieve coadd for use as an image difference template. 

60 

61 This is the default getTemplate Task to be run as a subtask by 

62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

63 ``runGen3()``. 

64 

65 Notes 

66 ----- 

67 From the given skymap, the closest tract is selected; multiple tracts are 

68 not supported. The assembled template inherits the WCS of the selected 

69 skymap tract and the resolution of the template exposures. Overlapping box 

70 regions of the input template patches are pixel by pixel copied into the 

71 assembled template image. There is no warping or pixel resampling. 

72 

73 Pixels with no overlap of any available input patches are set to ``nan`` value 

74 and ``NO_DATA`` flagged. 

75 """ 

76 

77 ConfigClass = GetCoaddAsTemplateConfig 

78 _DefaultName = "GetCoaddAsTemplateTask" 

79 

80 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

82 that overlaps the science exposure. 

83 

84 Parameters 

85 ---------- 

86 exposure: `lsst.afw.image.Exposure` 

87 an exposure for which to generate an overlapping template 

88 sensorRef : TYPE 

89 a Butler data reference that can be used to obtain coadd data 

90 templateIdList : TYPE, optional 

91 list of data ids, unused here, in the case of coadd template 

92 

93 Returns 

94 ------- 

95 result : `lsst.pipe.base.Struct` 

96 - ``exposure`` : `lsst.afw.image.ExposureF` 

97 a template coadd exposure assembled out of patches 

98 - ``sources`` : None for this subtask 

99 """ 

100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

102 

103 availableCoaddRefs = dict() 

104 for patchInfo in patchList: 

105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

106 patchArgDict = dict( 

107 datasetType=self.getCoaddDatasetName() + "_sub", 

108 bbox=patchInfo.getOuterBBox(), 

109 tract=tractInfo.getId(), 

110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

111 numSubfilters=self.config.numSubfilters, 

112 ) 

113 

114 if self.config.coaddName != 'dcr' and sensorRef.datasetExists(**patchArgDict): 

115 self.log.info("Reading patch %s" % patchArgDict) 

116 availableCoaddRefs[patchNumber] = patchArgDict 

117 

118 templateExposure = self.run( 

119 tractInfo, patchList, skyCorners, availableCoaddRefs, 

120 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

121 ) 

122 return pipeBase.Struct(exposure=templateExposure, sources=None) 

123 

124 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

125 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

126 that overlaps the science exposure. 

127 

128 Parameters 

129 ---------- 

130 exposure : `lsst.afw.image.Exposure` 

131 The science exposure to define the sky region of the template coadd. 

132 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

133 Butler like object that supports getting data by DatasetRef. 

134 skyMapRef : `lsst.daf.butler.DatasetRef` 

135 Reference to SkyMap object that corresponds to the template coadd. 

136 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

137 Iterable of references to the available template coadd patches. 

138 

139 Returns 

140 ------- 

141 result : `lsst.pipe.base.Struct` 

142 - ``exposure`` : `lsst.afw.image.ExposureF` 

143 a template coadd exposure assembled out of patches 

144 - ``sources`` : `None` for this subtask 

145 """ 

146 skyMap = butlerQC.get(skyMapRef) 

147 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

148 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList) 

149 

150 availableCoaddRefs = dict() 

151 for coaddRef in coaddExposureRefs: 

152 dataId = coaddRef.datasetRef.dataId 

153 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter: 

154 self.log.info("Using template input tract=%s, patch=%s" % 

155 (tractInfo.getId(), dataId['patch'])) 

156 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef) 

157 

158 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs) 

159 return pipeBase.Struct(exposure=templateExposure, sources=None) 

160 

161 def getOverlapPatchList(self, exposure, skyMap): 

162 """Select the relevant tract and its patches that overlap with the science exposure. 

163 

164 Parameters 

165 ---------- 

166 exposure : `lsst.afw.image.Exposure` 

167 The science exposure to define the sky region of the template coadd. 

168 

169 skyMap : `lsst.skymap.BaseSkyMap` 

170 SkyMap object that corresponds to the template coadd. 

171 

172 Returns 

173 ------- 

174 result : `tuple` of 

175 - ``tractInfo`` : `lsst.skymap.TractInfo` 

176 The selected tract. 

177 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

178 List of all overlap patches of the selected tract. 

179 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

180 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

181 """ 

182 expWcs = exposure.getWcs() 

183 expBoxD = geom.Box2D(exposure.getBBox()) 

184 expBoxD.grow(self.config.templateBorderSize) 

185 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

186 tractInfo = skyMap.findTract(ctrSkyPos) 

187 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

188 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

189 patchList = tractInfo.findPatchList(skyCorners) 

190 

191 if not patchList: 

192 raise RuntimeError("No suitable tract found") 

193 

194 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

195 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

196 

197 return (tractInfo, patchList, skyCorners) 

198 

199 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

200 sensorRef=None, visitInfo=None): 

201 """Gen2 and gen3 shared code: determination of exposure dimensions and 

202 copying of pixels from overlapping patch regions. 

203 

204 Parameters 

205 ---------- 

206 skyMap : `lsst.skymap.BaseSkyMap` 

207 SkyMap object that corresponds to the template coadd. 

208 tractInfo : `lsst.skymap.TractInfo` 

209 The selected tract. 

210 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

211 Patches to consider for making the template exposure. 

212 skyCorners : list of `lsst.geom.SpherePoint` 

213 Sky corner coordinates to be covered by the template exposure. 

214 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3) 

215 `dict` (Gen2) 

216 Dictionary of spatially relevant retrieved coadd patches, 

217 indexed by their sequential patch number. In Gen3 mode, .get() is called, 

218 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd. 

219 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

220 TODO DM-22952 Butler data reference to get coadd data. 

221 Must be `None` for Gen3. 

222 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

223 TODO DM-22952 VisitInfo to make dcr model. 

224 

225 Returns 

226 ------- 

227 templateExposure: `lsst.afw.image.ExposureF` 

228 The created template exposure. 

229 """ 

230 coaddWcs = tractInfo.getWcs() 

231 

232 # compute coadd bbox 

233 coaddBBox = geom.Box2D() 

234 for skyPos in skyCorners: 

235 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

236 coaddBBox = geom.Box2I(coaddBBox) 

237 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

238 

239 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

240 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

241 nPatchesFound = 0 

242 coaddFilter = None 

243 coaddPsf = None 

244 coaddPhotoCalib = None 

245 for patchInfo in patchList: 

246 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

247 patchSubBBox = patchInfo.getOuterBBox() 

248 patchSubBBox.clip(coaddBBox) 

249 patchArgDict = dict( 

250 datasetType=self.getCoaddDatasetName() + "_sub", 

251 bbox=patchSubBBox, 

252 tract=tractInfo.getId(), 

253 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

254 numSubfilters=self.config.numSubfilters, 

255 ) 

256 if patchSubBBox.isEmpty(): 

257 self.log.info(f"skip tract={patchArgDict['tract']}, " 

258 f"patch={patchNumber}; no overlapping pixels") 

259 continue 

260 

261 # TODO DM-22952 

262 # Dcr coadd support is gen2 only 

263 # Under gen3, sensorRef will be None and execution is stopped in runQuantum if dcr coadd is 

264 # configured 

265 if self.config.coaddName == 'dcr': 

266 if not sensorRef.datasetExists(subfilter=0, **patchArgDict): 

267 self.log.warn("%(datasetType)s, tract=%(tract)s, patch=%(patch)s," 

268 " numSubfilters=%(numSubfilters)s, subfilter=0 does not exist" 

269 % patchArgDict) 

270 continue 

271 patchInnerBBox = patchInfo.getInnerBBox() 

272 patchInnerBBox.clip(coaddBBox) 

273 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

274 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." % patchArgDict) 

275 continue 

276 self.log.info("Constructing DCR-matched template for patch %s" % patchArgDict) 

277 

278 dcrModel = DcrModel.fromDataRef(sensorRef, **patchArgDict) 

279 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

280 # Each patch has significant overlap, and the contaminated edge pixels in 

281 # a new patch will overwrite good pixels in the overlap region from 

282 # previous patches. 

283 # Shrink the BBox to remove the contaminated pixels, 

284 # but make sure it is only the overlap region that is reduced. 

285 dcrBBox = geom.Box2I(patchSubBBox) 

286 dcrBBox.grow(-self.config.templateBorderSize) 

287 dcrBBox.include(patchInnerBBox) 

288 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

289 wcs=coaddWcs, 

290 visitInfo=visitInfo) 

291 else: 

292 if patchNumber not in availableCoaddRefs: 

293 self.log.warn(f"{patchArgDict['datasetType']}, " 

294 f"tract={patchArgDict['tract']}, patch={patchNumber} does not exist") 

295 continue 

296 if sensorRef is None: 

297 # Gen3 

298 coaddPatch = availableCoaddRefs[patchNumber].get() 

299 else: 

300 # Gen2 

301 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

302 nPatchesFound += 1 

303 

304 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

305 # calculation from caller code. Gen3 also does not do this. 

306 overlapBox = coaddPatch.getBBox() 

307 overlapBox.clip(coaddBBox) 

308 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

309 

310 if coaddFilter is None: 

311 coaddFilter = coaddPatch.getFilter() 

312 

313 # Retrieve the PSF for this coadd tract, if not already retrieved 

314 if coaddPsf is None and coaddPatch.hasPsf(): 

315 coaddPsf = coaddPatch.getPsf() 

316 

317 # Retrieve the calibration for this coadd tract, if not already retrieved 

318 if coaddPhotoCalib is None: 

319 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

320 

321 if coaddPhotoCalib is None: 

322 raise RuntimeError("No coadd PhotoCalib found!") 

323 if nPatchesFound == 0: 

324 raise RuntimeError("No patches found!") 

325 if coaddPsf is None: 

326 raise RuntimeError("No coadd Psf found!") 

327 

328 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

329 coaddExposure.setPsf(coaddPsf) 

330 coaddExposure.setFilter(coaddFilter) 

331 return coaddExposure 

332 

333 def getCoaddDatasetName(self): 

334 """Return coadd name for given task config 

335 

336 Returns 

337 ------- 

338 CoaddDatasetName : `string` 

339 

340 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

341 """ 

342 warpType = self.config.warpType 

343 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

344 return self.config.coaddName + "Coadd" + suffix 

345 

346 

347class GetCalexpAsTemplateConfig(pexConfig.Config): 

348 doAddCalexpBackground = pexConfig.Field( 

349 dtype=bool, 

350 default=True, 

351 doc="Add background to calexp before processing it." 

352 ) 

353 

354 

355class GetCalexpAsTemplateTask(pipeBase.Task): 

356 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

357 for use as an image difference template. Only gen2 supported. 

358 

359 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

360 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

361 This code was originally part of Winter2013ImageDifferenceTask. 

362 """ 

363 

364 ConfigClass = GetCalexpAsTemplateConfig 

365 _DefaultName = "GetCalexpAsTemplateTask" 

366 

367 def run(self, exposure, sensorRef, templateIdList): 

368 """Return a calexp exposure with based on input sensorRef. 

369 

370 Construct a dataId based on the sensorRef.dataId combined 

371 with the specifications from the first dataId in templateIdList 

372 

373 Parameters 

374 ---------- 

375 exposure : `lsst.afw.image.Exposure` 

376 exposure (unused) 

377 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

378 Data reference of the calexp(s) to subtract from. 

379 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

380 Data reference of the template calexp to be subtraced. 

381 Can be incomplete, fields are initialized from `sensorRef`. 

382 If there are multiple items, only the first one is used. 

383 

384 Returns 

385 ------- 

386 result : `struct` 

387 

388 return a pipeBase.Struct: 

389 

390 - ``exposure`` : a template calexp 

391 - ``sources`` : source catalog measured on the template 

392 """ 

393 

394 if len(templateIdList) == 0: 

395 raise RuntimeError("No template data reference supplied.") 

396 if len(templateIdList) > 1: 

397 self.log.warn("Multiple template data references supplied. Using the first one only.") 

398 

399 templateId = sensorRef.dataId.copy() 

400 templateId.update(templateIdList[0]) 

401 

402 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

403 

404 butler = sensorRef.getButler() 

405 template = butler.get(datasetType="calexp", dataId=templateId) 

406 if self.config.doAddCalexpBackground: 

407 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

408 mi = template.getMaskedImage() 

409 mi += templateBg.getImage() 

410 

411 if not template.hasPsf(): 

412 raise pipeBase.TaskError("Template has no psf") 

413 

414 templateSources = butler.get(datasetType="src", dataId=templateId) 

415 return pipeBase.Struct(exposure=template, 

416 sources=templateSources) 

417 

418 def runDataRef(self, *args, **kwargs): 

419 return self.run(*args, **kwargs) 

420 

421 def runQuantum(self, **kwargs): 

422 raise NotImplementedError("Calexp template is not supported with gen3 middleware")