Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.pex.config as pexConfig 

28import lsst.pipe.base as pipeBase 

29from lsst.ip.diffim.dcrModel import DcrModel 

30 

31__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

32 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

33 

34 

35class GetCoaddAsTemplateConfig(pexConfig.Config): 

36 templateBorderSize = pexConfig.Field( 

37 dtype=int, 

38 default=10, 

39 doc="Number of pixels to grow the requested template image to account for warping" 

40 ) 

41 coaddName = pexConfig.Field( 

42 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

43 dtype=str, 

44 default="deep", 

45 ) 

46 numSubfilters = pexConfig.Field( 

47 doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'", 

48 dtype=int, 

49 default=3, 

50 ) 

51 effectiveWavelength = pexConfig.Field( 

52 doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'", 

53 optional=True, 

54 dtype=float, 

55 ) 

56 bandwidth = pexConfig.Field( 

57 doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'", 

58 optional=True, 

59 dtype=float, 

60 ) 

61 warpType = pexConfig.Field( 

62 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

63 dtype=str, 

64 default="direct", 

65 ) 

66 

67 def validate(self): 

68 if self.coaddName == 'dcr': 

69 if self.effectiveWavelength is None or self.bandwidth is None: 

70 raise ValueError("The effective wavelength and bandwidth of the physical filter " 

71 "must be set in the getTemplate config for DCR coadds. " 

72 "Required until transmission curves are used in DM-13668.") 

73 

74 

75class GetCoaddAsTemplateTask(pipeBase.Task): 

76 """Subtask to retrieve coadd for use as an image difference template. 

77 

78 This is the default getTemplate Task to be run as a subtask by 

79 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

80 ``runGen3()``. 

81 

82 Notes 

83 ----- 

84 From the given skymap, the closest tract is selected; multiple tracts are 

85 not supported. The assembled template inherits the WCS of the selected 

86 skymap tract and the resolution of the template exposures. Overlapping box 

87 regions of the input template patches are pixel by pixel copied into the 

88 assembled template image. There is no warping or pixel resampling. 

89 

90 Pixels with no overlap of any available input patches are set to ``nan`` value 

91 and ``NO_DATA`` flagged. 

92 """ 

93 

94 ConfigClass = GetCoaddAsTemplateConfig 

95 _DefaultName = "GetCoaddAsTemplateTask" 

96 

97 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

98 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

99 that overlaps the science exposure. 

100 

101 Parameters 

102 ---------- 

103 exposure: `lsst.afw.image.Exposure` 

104 an exposure for which to generate an overlapping template 

105 sensorRef : TYPE 

106 a Butler data reference that can be used to obtain coadd data 

107 templateIdList : TYPE, optional 

108 list of data ids, unused here, in the case of coadd template 

109 

110 Returns 

111 ------- 

112 result : `lsst.pipe.base.Struct` 

113 - ``exposure`` : `lsst.afw.image.ExposureF` 

114 a template coadd exposure assembled out of patches 

115 - ``sources`` : None for this subtask 

116 """ 

117 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

118 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

119 

120 availableCoaddRefs = dict() 

121 for patchInfo in patchList: 

122 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

123 patchArgDict = dict( 

124 datasetType=self.getCoaddDatasetName() + "_sub", 

125 bbox=patchInfo.getOuterBBox(), 

126 tract=tractInfo.getId(), 

127 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

128 subfilter=0, 

129 numSubfilters=self.config.numSubfilters, 

130 ) 

131 

132 if sensorRef.datasetExists(**patchArgDict): 

133 self.log.info("Reading patch %s" % patchArgDict) 

134 availableCoaddRefs[patchNumber] = patchArgDict 

135 

136 templateExposure = self.run( 

137 tractInfo, patchList, skyCorners, availableCoaddRefs, 

138 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

139 ) 

140 return pipeBase.Struct(exposure=templateExposure, sources=None) 

141 

142 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

143 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

144 that overlaps the science exposure. 

145 

146 Parameters 

147 ---------- 

148 exposure : `lsst.afw.image.Exposure` 

149 The science exposure to define the sky region of the template coadd. 

150 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

151 Butler like object that supports getting data by DatasetRef. 

152 skyMapRef : `lsst.daf.butler.DatasetRef` 

153 Reference to SkyMap object that corresponds to the template coadd. 

154 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

155 Iterable of references to the available template coadd patches. 

156 

157 Returns 

158 ------- 

159 result : `lsst.pipe.base.Struct` 

160 - ``exposure`` : `lsst.afw.image.ExposureF` 

161 a template coadd exposure assembled out of patches 

162 - ``sources`` : `None` for this subtask 

163 """ 

164 skyMap = butlerQC.get(skyMapRef) 

165 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

166 patchNumFilter = frozenset(tractInfo.getSequentialPatchIndex(p) for p in patchList) 

167 

168 availableCoaddRefs = dict() 

169 for coaddRef in coaddExposureRefs: 

170 dataId = coaddRef.datasetRef.dataId 

171 if dataId['tract'] == tractInfo.getId() and dataId['patch'] in patchNumFilter: 

172 if self.config.coaddName == 'dcr': 

173 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" % 

174 (tractInfo.getId(), dataId['patch'], dataId['subfilter'])) 

175 if dataId['patch'] in availableCoaddRefs: 

176 availableCoaddRefs[dataId['patch']].append(butlerQC.get(coaddRef)) 

177 else: 

178 availableCoaddRefs[dataId['patch']] = [butlerQC.get(coaddRef), ] 

179 else: 

180 self.log.info("Using template input tract=%s, patch=%s" % 

181 (tractInfo.getId(), dataId['patch'])) 

182 availableCoaddRefs[dataId['patch']] = butlerQC.get(coaddRef) 

183 

184 templateExposure = self.run(tractInfo, patchList, skyCorners, availableCoaddRefs, 

185 visitInfo=exposure.getInfo().getVisitInfo()) 

186 return pipeBase.Struct(exposure=templateExposure, sources=None) 

187 

188 def getOverlapPatchList(self, exposure, skyMap): 

189 """Select the relevant tract and its patches that overlap with the science exposure. 

190 

191 Parameters 

192 ---------- 

193 exposure : `lsst.afw.image.Exposure` 

194 The science exposure to define the sky region of the template coadd. 

195 

196 skyMap : `lsst.skymap.BaseSkyMap` 

197 SkyMap object that corresponds to the template coadd. 

198 

199 Returns 

200 ------- 

201 result : `tuple` of 

202 - ``tractInfo`` : `lsst.skymap.TractInfo` 

203 The selected tract. 

204 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

205 List of all overlap patches of the selected tract. 

206 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

207 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

208 """ 

209 expWcs = exposure.getWcs() 

210 expBoxD = geom.Box2D(exposure.getBBox()) 

211 expBoxD.grow(self.config.templateBorderSize) 

212 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

213 tractInfo = skyMap.findTract(ctrSkyPos) 

214 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

215 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

216 patchList = tractInfo.findPatchList(skyCorners) 

217 

218 if not patchList: 

219 raise RuntimeError("No suitable tract found") 

220 

221 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

222 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

223 

224 return (tractInfo, patchList, skyCorners) 

225 

226 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

227 sensorRef=None, visitInfo=None): 

228 """Gen2 and gen3 shared code: determination of exposure dimensions and 

229 copying of pixels from overlapping patch regions. 

230 

231 Parameters 

232 ---------- 

233 skyMap : `lsst.skymap.BaseSkyMap` 

234 SkyMap object that corresponds to the template coadd. 

235 tractInfo : `lsst.skymap.TractInfo` 

236 The selected tract. 

237 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

238 Patches to consider for making the template exposure. 

239 skyCorners : list of `lsst.geom.SpherePoint` 

240 Sky corner coordinates to be covered by the template exposure. 

241 availableCoaddRefs : `dict` of `int` : `lsst.daf.butler.DeferredDatasetHandle` (Gen3) 

242 `dict` (Gen2) 

243 Dictionary of spatially relevant retrieved coadd patches, 

244 indexed by their sequential patch number. In Gen3 mode, .get() is called, 

245 in Gen2 mode, sensorRef.get(**coaddef) is called to retrieve the coadd. 

246 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

247 Butler data reference to get coadd data. 

248 Must be `None` for Gen3. 

249 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

250 VisitInfo to make dcr model. 

251 

252 Returns 

253 ------- 

254 templateExposure: `lsst.afw.image.ExposureF` 

255 The created template exposure. 

256 """ 

257 coaddWcs = tractInfo.getWcs() 

258 

259 # compute coadd bbox 

260 coaddBBox = geom.Box2D() 

261 for skyPos in skyCorners: 

262 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

263 coaddBBox = geom.Box2I(coaddBBox) 

264 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

265 

266 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

267 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

268 nPatchesFound = 0 

269 coaddFilter = None 

270 coaddPsf = None 

271 coaddPhotoCalib = None 

272 for patchInfo in patchList: 

273 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

274 patchSubBBox = patchInfo.getOuterBBox() 

275 patchSubBBox.clip(coaddBBox) 

276 if patchNumber not in availableCoaddRefs: 

277 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd") 

278 continue 

279 if patchSubBBox.isEmpty(): 

280 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, " 

281 f"patch={patchNumber}; no overlapping pixels") 

282 continue 

283 

284 if self.config.coaddName == 'dcr': 

285 patchInnerBBox = patchInfo.getInnerBBox() 

286 patchInnerBBox.clip(coaddBBox) 

287 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

288 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." 

289 % availableCoaddRefs[patchNumber]) 

290 continue 

291 self.log.info("Constructing DCR-matched template for patch %s" 

292 % availableCoaddRefs[patchNumber]) 

293 

294 if sensorRef: 

295 dcrModel = DcrModel.fromDataRef(sensorRef, 

296 self.config.effectiveWavelength, 

297 self.config.bandwidth, 

298 **availableCoaddRefs[patchNumber]) 

299 else: 

300 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber], 

301 self.config.effectiveWavelength, 

302 self.config.bandwidth) 

303 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

304 # Each patch has significant overlap, and the contaminated edge pixels in 

305 # a new patch will overwrite good pixels in the overlap region from 

306 # previous patches. 

307 # Shrink the BBox to remove the contaminated pixels, 

308 # but make sure it is only the overlap region that is reduced. 

309 dcrBBox = geom.Box2I(patchSubBBox) 

310 dcrBBox.grow(-self.config.templateBorderSize) 

311 dcrBBox.include(patchInnerBBox) 

312 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

313 wcs=coaddWcs, 

314 visitInfo=visitInfo) 

315 else: 

316 if sensorRef is None: 

317 # Gen3 

318 coaddPatch = availableCoaddRefs[patchNumber].get() 

319 else: 

320 # Gen2 

321 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

322 nPatchesFound += 1 

323 

324 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

325 # calculation from caller code. Gen3 also does not do this. 

326 overlapBox = coaddPatch.getBBox() 

327 overlapBox.clip(coaddBBox) 

328 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

329 

330 if coaddFilter is None: 

331 coaddFilter = coaddPatch.getFilter() 

332 

333 # Retrieve the PSF for this coadd tract, if not already retrieved 

334 if coaddPsf is None and coaddPatch.hasPsf(): 

335 coaddPsf = coaddPatch.getPsf() 

336 

337 # Retrieve the calibration for this coadd tract, if not already retrieved 

338 if coaddPhotoCalib is None: 

339 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

340 

341 if coaddPhotoCalib is None: 

342 raise RuntimeError("No coadd PhotoCalib found!") 

343 if nPatchesFound == 0: 

344 raise RuntimeError("No patches found!") 

345 if coaddPsf is None: 

346 raise RuntimeError("No coadd Psf found!") 

347 

348 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

349 coaddExposure.setPsf(coaddPsf) 

350 coaddExposure.setFilter(coaddFilter) 

351 return coaddExposure 

352 

353 def getCoaddDatasetName(self): 

354 """Return coadd name for given task config 

355 

356 Returns 

357 ------- 

358 CoaddDatasetName : `string` 

359 

360 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

361 """ 

362 warpType = self.config.warpType 

363 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

364 return self.config.coaddName + "Coadd" + suffix 

365 

366 

367class GetCalexpAsTemplateConfig(pexConfig.Config): 

368 doAddCalexpBackground = pexConfig.Field( 

369 dtype=bool, 

370 default=True, 

371 doc="Add background to calexp before processing it." 

372 ) 

373 

374 

375class GetCalexpAsTemplateTask(pipeBase.Task): 

376 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

377 for use as an image difference template. Only gen2 supported. 

378 

379 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

380 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

381 This code was originally part of Winter2013ImageDifferenceTask. 

382 """ 

383 

384 ConfigClass = GetCalexpAsTemplateConfig 

385 _DefaultName = "GetCalexpAsTemplateTask" 

386 

387 def run(self, exposure, sensorRef, templateIdList): 

388 """Return a calexp exposure with based on input sensorRef. 

389 

390 Construct a dataId based on the sensorRef.dataId combined 

391 with the specifications from the first dataId in templateIdList 

392 

393 Parameters 

394 ---------- 

395 exposure : `lsst.afw.image.Exposure` 

396 exposure (unused) 

397 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

398 Data reference of the calexp(s) to subtract from. 

399 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

400 Data reference of the template calexp to be subtraced. 

401 Can be incomplete, fields are initialized from `sensorRef`. 

402 If there are multiple items, only the first one is used. 

403 

404 Returns 

405 ------- 

406 result : `struct` 

407 

408 return a pipeBase.Struct: 

409 

410 - ``exposure`` : a template calexp 

411 - ``sources`` : source catalog measured on the template 

412 """ 

413 

414 if len(templateIdList) == 0: 

415 raise RuntimeError("No template data reference supplied.") 

416 if len(templateIdList) > 1: 

417 self.log.warn("Multiple template data references supplied. Using the first one only.") 

418 

419 templateId = sensorRef.dataId.copy() 

420 templateId.update(templateIdList[0]) 

421 

422 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

423 

424 butler = sensorRef.getButler() 

425 template = butler.get(datasetType="calexp", dataId=templateId) 

426 if self.config.doAddCalexpBackground: 

427 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

428 mi = template.getMaskedImage() 

429 mi += templateBg.getImage() 

430 

431 if not template.hasPsf(): 

432 raise pipeBase.TaskError("Template has no psf") 

433 

434 templateSources = butler.get(datasetType="src", dataId=templateId) 

435 return pipeBase.Struct(exposure=template, 

436 sources=templateSources) 

437 

438 def runDataRef(self, *args, **kwargs): 

439 return self.run(*args, **kwargs) 

440 

441 def runQuantum(self, **kwargs): 

442 raise NotImplementedError("Calexp template is not supported with gen3 middleware")