Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from lsst.ip.diffim.dcrModel import DcrModel 

30 

31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

33 

34 

35class GetCoaddAsTemplateConfig(pexConfig.Config): 

36 templateBorderSize = pexConfig.Field( 

37 dtype=int, 

38 default=10, 

39 doc="Number of pixels to grow the requested template image to account for warping" 

40 ) 

41 coaddName = pexConfig.Field( 

42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

43 dtype=str, 

44 default="deep", 

45 ) 

46 numSubfilters = pexConfig.Field( 

47 doc="Number of subfilters in the DcrCoadd, used only if ``coaddName``='dcr'", 

48 dtype=int, 

49 default=3, 

50 ) 

51 warpType = pexConfig.Field( 

52 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

53 dtype=str, 

54 default="direct", 

55 ) 

56 

57 

58class GetCoaddAsTemplateTask(pipeBase.Task): 

59 """Subtask to retrieve coadd for use as an image difference template. 

60 

61 This is the default getTemplate Task to be run as a subtask by 

62 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

63 ``runGen3()``. 

64 

65 Notes 

66 ----- 

67 From the given skymap, the closest tract is selected; multiple tracts are 

68 not supported. The assembled template inherits the WCS of the selected 

69 skymap tract and the resolution of the template exposures. Overlapping box 

70 regions of the input template patches are pixel by pixel copied into the 

71 assembled template image. There is no warping or pixel resampling. 

72 

73 Pixels with no overlap of any available input patches are set to ``nan`` value 

74 and ``NO_DATA`` flagged. 

75 """ 

76 

77 ConfigClass = GetCoaddAsTemplateConfig 

78 _DefaultName = "GetCoaddAsTemplateTask" 

79 

80 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

81 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

82 that overlaps the science exposure. 

83 

84 Parameters 

85 ---------- 

86 exposure: `lsst.afw.image.Exposure` 

87 an exposure for which to generate an overlapping template 

88 sensorRef : TYPE 

89 a Butler data reference that can be used to obtain coadd data 

90 templateIdList : TYPE, optional 

91 list of data ids, unused here, in the case of coadd template 

92 

93 Returns 

94 ------- 

95 result : `lsst.pipe.base.Struct` 

96 - ``exposure`` : `lsst.afw.image.ExposureF` 

97 a template coadd exposure assembled out of patches 

98 - ``sources`` : None for this subtask 

99 """ 

100 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

101 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

102 

103 availableCoaddRefs = dict() 

104 for patchInfo in patchList: 

105 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

106 patchArgDict = dict( 

107 datasetType=self.getCoaddDatasetName() + "_sub", 

108 bbox=patchInfo.getOuterBBox(), 

109 tract=tractInfo.getId(), 

110 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

111 numSubfilters=self.config.numSubfilters, 

112 ) 

113 

114 if sensorRef.datasetExists(**patchArgDict): 

115 self.log.info("Reading patch %s" % patchArgDict) 

116 availableCoaddRefs[patchNumber] = patchArgDict 

117 

118 templateExposure = self.run( 

119 tractInfo, patchList, skyCorners, availableCoaddRefs, 

120 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

121 ) 

122 return pipeBase.Struct(exposure=templateExposure, sources=None) 

123 

124 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

125 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

126 that overlaps the science exposure. 

127 

128 Parameters 

129 ---------- 

130 exposure : `lsst.afw.image.Exposure` 

131 The science exposure to define the sky region of the template coadd. 

132 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

133 Butler like object that supports getting data by DatasetRef. 

134 skyMapRef : `lsst.daf.butler.DatasetRef` 

135 Reference to SkyMap object that corresponds to the template coadd. 

136 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

137 Iterable of references to the available template coadd patches. 

138 

139 Returns 

140 ------- 

141 result : `lsst.pipe.base.Struct` 

142 - ``exposure`` : `lsst.afw.image.ExposureF` 

143 a template coadd exposure assembled out of patches 

144 - ``sources`` : `None` for this subtask 

145 """ 

146 skyMap = butlerQC.get(skyMapRef) 

147 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

148 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList) 

149 

150 availableCoaddRefs = dict() 

151 for coaddRef in coaddExposureRefs: 

152 dataId = coaddRef.datasetRef.dataId 

153 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter: 

154 if self.config.coaddName == 'dcr': 

155 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" % 

156 (tractInfo.getId(), dataId['patch'], dataId['subfilter'])) 

157 if dataId['patch'] in availableCoaddRefs: 

158 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef)) 

159 else: 

160 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ] 

161 else: 

162 self.log.info("Using template input tract=%s, patch=%s" % 

163 (tractInfo.getId(), dataId['patch'])) 

164 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef) 

165 

166 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs, 

167 visitInfo=exposure.getInfo().getVisitInfo()) 

168 return pipeBase.Struct(exposure=templateExposure, sources=None) 

169 

170 def getOverlapPatchList(self, exposure, skyMap): 

171 """Select the relevant tract and its patches that overlap with the science exposure. 

172 

173 Parameters 

174 ---------- 

175 exposure : `lsst.afw.image.Exposure` 

176 The science exposure to define the sky region of the template coadd. 

177 

178 skyMap : `lsst.skymap.BaseSkyMap` 

179 SkyMap object that corresponds to the template coadd. 

180 

181 Returns 

182 ------- 

183 result : `tuple` of 

184 - ``tractInfo`` : `lsst.skymap.TractInfo` 

185 The selected tract. 

186 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

187 List of all overlap patches of the selected tract. 

188 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

189 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

190 """ 

191 expWcs = exposure.getWcs() 

192 expBoxD = geom.Box2D(exposure.getBBox()) 

193 expBoxD.grow(self.config.templateBorderSize) 

194 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

195 tractInfo = skyMap.findTract(ctrSkyPos) 

196 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

197 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

198 patchList = tractInfo.findPatchList(skyCorners) 

199 

200 if not patchList: 

201 raise RuntimeError("No suitable tract found") 

202 

203 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

204 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

205 

206 return (tractInfo, patchList, skyCorners) 

207 

208 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

209 sensorRef=None, visitInfo=None): 

210 """Gen2 and gen3 shared code: determination of exposure dimensions and 

211 copying of pixels from overlapping patch regions. 

212 

213 Parameters 

214 ---------- 

215 skyMap : `lsst.skymap.BaseSkyMap` 

216 SkyMap object that corresponds to the template coadd. 

217 tractInfo : `lsst.skymap.TractInfo` 

218 The selected tract. 

219 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

220 Patches to consider for making the template exposure. 

221 skyCorners : list of `lsst.geom.SpherePoint` 

222 Sky corner coordinates to be covered by the template exposure. 

223 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3) 

224 `dict` (Gen2) 

225 Dictionary of spatially relevant retrieved coadd patches, 

226 indexed by their sequential patch number. In Gen3 mode, .get() is called, 

227 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd. 

228 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

229 TODO DM-22952 Butler data reference to get coadd data. 

230 Must be `None` for Gen3. 

231 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

232 TODO DM-22952 VisitInfo to make dcr model. 

233 

234 Returns 

235 ------- 

236 templateExposure: `lsst.afw.image.ExposureF` 

237 The created template exposure. 

238 """ 

239 coaddWcs = tractInfo.getWcs() 

240 

241 # compute coadd bbox 

242 coaddBBox = geom.Box2D() 

243 for skyPos in skyCorners: 

244 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

245 coaddBBox = geom.Box2I(coaddBBox) 

246 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

247 

248 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

249 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

250 nPatchesFound = 0 

251 coaddFilter = None 

252 coaddPsf = None 

253 coaddPhotoCalib = None 

254 for patchInfo in patchList: 

255 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

256 patchSubBBox = patchInfo.getOuterBBox() 

257 patchSubBBox.clip(coaddBBox) 

258 patchArgDict = dict( 

259 datasetType=self.getCoaddDatasetName() + "_sub", 

260 bbox=patchSubBBox, 

261 tract=tractInfo.getId(), 

262 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

263 numSubfilters=self.config.numSubfilters, 

264 ) 

265 if patchSubBBox.isEmpty(): 

266 self.log.info(f"skip tract={patchArgDict['tract']}, " 

267 f"patch={patchNumber}; no overlapping pixels") 

268 continue 

269 if patchNumber not in availableCoaddRefs: 

270 self.log.warn(f"{patchArgDict['datasetType']}, " 

271 f"tract={patchArgDict['tract']}, patch={patchNumber} does not exist") 

272 continue 

273 

274 if self.config.coaddName == 'dcr': 

275 if sensorRef and not sensorRef.datasetExists(subfilter=0, **patchArgDict): 

276 self.log.warn("%(datasetType)s, tract=%(tract)s, patch=%(patch)s," 

277 " numSubfilters=%(numSubfilters)s, subfilter=0 does not exist" 

278 % patchArgDict) 

279 continue 

280 patchInnerBBox = patchInfo.getInnerBBox() 

281 patchInnerBBox.clip(coaddBBox) 

282 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

283 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." % patchArgDict) 

284 continue 

285 self.log.info("Constructing DCR-matched template for patch %s" % patchArgDict) 

286 

287 if sensorRef: 

288 dcrModel = DcrModel.fromDataRef(sensorRef, **patchArgDict) 

289 else: 

290 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber]) 

291 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

292 # Each patch has significant overlap, and the contaminated edge pixels in 

293 # a new patch will overwrite good pixels in the overlap region from 

294 # previous patches. 

295 # Shrink the BBox to remove the contaminated pixels, 

296 # but make sure it is only the overlap region that is reduced. 

297 dcrBBox = geom.Box2I(patchSubBBox) 

298 dcrBBox.grow(-self.config.templateBorderSize) 

299 dcrBBox.include(patchInnerBBox) 

300 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

301 wcs=coaddWcs, 

302 visitInfo=visitInfo) 

303 else: 

304 if sensorRef is None: 

305 # Gen3 

306 coaddPatch = availableCoaddRefs[patchNumber].get() 

307 else: 

308 # Gen2 

309 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

310 nPatchesFound += 1 

311 

312 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

313 # calculation from caller code. Gen3 also does not do this. 

314 overlapBox = coaddPatch.getBBox() 

315 overlapBox.clip(coaddBBox) 

316 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

317 

318 if coaddFilter is None: 

319 coaddFilter = coaddPatch.getFilter() 

320 

321 # Retrieve the PSF for this coadd tract, if not already retrieved 

322 if coaddPsf is None and coaddPatch.hasPsf(): 

323 coaddPsf = coaddPatch.getPsf() 

324 

325 # Retrieve the calibration for this coadd tract, if not already retrieved 

326 if coaddPhotoCalib is None: 

327 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

328 

329 if coaddPhotoCalib is None: 

330 raise RuntimeError("No coadd PhotoCalib found!") 

331 if nPatchesFound == 0: 

332 raise RuntimeError("No patches found!") 

333 if coaddPsf is None: 

334 raise RuntimeError("No coadd Psf found!") 

335 

336 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

337 coaddExposure.setPsf(coaddPsf) 

338 coaddExposure.setFilter(coaddFilter) 

339 return coaddExposure 

340 

341 def getCoaddDatasetName(self): 

342 """Return coadd name for given task config 

343 

344 Returns 

345 ------- 

346 CoaddDatasetName : `string` 

347 

348 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

349 """ 

350 warpType = self.config.warpType 

351 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

352 return self.config.coaddName + "Coadd" + suffix 

353 

354 

355class GetCalexpAsTemplateConfig(pexConfig.Config): 

356 doAddCalexpBackground = pexConfig.Field( 

357 dtype=bool, 

358 default=True, 

359 doc="Add background to calexp before processing it." 

360 ) 

361 

362 

363class GetCalexpAsTemplateTask(pipeBase.Task): 

364 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

365 for use as an image difference template. Only gen2 supported. 

366 

367 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

368 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

369 This code was originally part of Winter2013ImageDifferenceTask. 

370 """ 

371 

372 ConfigClass = GetCalexpAsTemplateConfig 

373 _DefaultName = "GetCalexpAsTemplateTask" 

374 

375 def run(self, exposure, sensorRef, templateIdList): 

376 """Return a calexp exposure with based on input sensorRef. 

377 

378 Construct a dataId based on the sensorRef.dataId combined 

379 with the specifications from the first dataId in templateIdList 

380 

381 Parameters 

382 ---------- 

383 exposure : `lsst.afw.image.Exposure` 

384 exposure (unused) 

385 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

386 Data reference of the calexp(s) to subtract from. 

387 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

388 Data reference of the template calexp to be subtraced. 

389 Can be incomplete, fields are initialized from `sensorRef`. 

390 If there are multiple items, only the first one is used. 

391 

392 Returns 

393 ------- 

394 result : `struct` 

395 

396 return a pipeBase.Struct: 

397 

398 - ``exposure`` : a template calexp 

399 - ``sources`` : source catalog measured on the template 

400 """ 

401 

402 if len(templateIdList) == 0: 

403 raise RuntimeError("No template data reference supplied.") 

404 if len(templateIdList) > 1: 

405 self.log.warn("Multiple template data references supplied. Using the first one only.") 

406 

407 templateId = sensorRef.dataId.copy() 

408 templateId.update(templateIdList[0]) 

409 

410 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

411 

412 butler = sensorRef.getButler() 

413 template = butler.get(datasetType="calexp", dataId=templateId) 

414 if self.config.doAddCalexpBackground: 

415 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

416 mi = template.getMaskedImage() 

417 mi += templateBg.getImage() 

418 

419 if not template.hasPsf(): 

420 raise pipeBase.TaskError("Template has no psf") 

421 

422 templateSources = butler.get(datasetType="src", dataId=templateId) 

423 return pipeBase.Struct(exposure=template, 

424 sources=templateSources) 

425 

426 def runDataRef(self, *args, **kwargs): 

427 return self.run(*args, **kwargs) 

428 

429 def runQuantum(self, **kwargs): 

430 raise NotImplementedError("Calexp template is not supported with gen3 middleware")