Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.sphgeom as sphgeom 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30from lsst.ip.diffim.dcrModel import DcrModel 

31 

32__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

33 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

34 

35 

36class GetCoaddAsTemplateConfig(pexConfig.Config): 

37 templateBorderSize = pexConfig.Field( 

38 dtype=int, 

39 default=10, 

40 doc="Number of pixels to grow the requested template image to account for warping" 

41 ) 

42 coaddName = pexConfig.Field( 

43 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

44 dtype=str, 

45 default="deep", 

46 ) 

47 numSubfilters = pexConfig.Field( 

48 doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'", 

49 dtype=int, 

50 default=3, 

51 ) 

52 effectiveWavelength = pexConfig.Field( 

53 doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'", 

54 optional=True, 

55 dtype=float, 

56 ) 

57 bandwidth = pexConfig.Field( 

58 doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'", 

59 optional=True, 

60 dtype=float, 

61 ) 

62 warpType = pexConfig.Field( 

63 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

64 dtype=str, 

65 default="direct", 

66 ) 

67 

68 def validate(self): 

69 if self.coaddName == 'dcr': 

70 if self.effectiveWavelength is None or self.bandwidth is None: 

71 raise ValueError("The effective wavelength and bandwidth of the physical filter " 

72 "must be set in the getTemplate config for DCR coadds. " 

73 "Required until transmission curves are used in DM-13668.") 

74 

75 

76class GetCoaddAsTemplateTask(pipeBase.Task): 

77 """Subtask to retrieve coadd for use as an image difference template. 

78 

79 This is the default getTemplate Task to be run as a subtask by 

80 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

81 ``runGen3()``. 

82 

83 Notes 

84 ----- 

85 From the given skymap, the closest tract is selected; multiple tracts are 

86 not supported. The assembled template inherits the WCS of the selected 

87 skymap tract and the resolution of the template exposures. Overlapping box 

88 regions of the input template patches are pixel by pixel copied into the 

89 assembled template image. There is no warping or pixel resampling. 

90 

91 Pixels with no overlap of any available input patches are set to ``nan`` value 

92 and ``NO_DATA`` flagged. 

93 """ 

94 

95 ConfigClass = GetCoaddAsTemplateConfig 

96 _DefaultName = "GetCoaddAsTemplateTask" 

97 

98 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

99 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

100 that overlaps the science exposure. 

101 

102 Parameters 

103 ---------- 

104 exposure: `lsst.afw.image.Exposure` 

105 an exposure for which to generate an overlapping template 

106 sensorRef : TYPE 

107 a Butler data reference that can be used to obtain coadd data 

108 templateIdList : TYPE, optional 

109 list of data ids, unused here, in the case of coadd template 

110 

111 Returns 

112 ------- 

113 result : `lsst.pipe.base.Struct` 

114 - ``exposure`` : `lsst.afw.image.ExposureF` 

115 a template coadd exposure assembled out of patches 

116 - ``sources`` : None for this subtask 

117 """ 

118 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

119 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

120 

121 availableCoaddRefs = dict() 

122 for patchInfo in patchList: 

123 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

124 patchArgDict = dict( 

125 datasetType=self.getCoaddDatasetName() + "_sub", 

126 bbox=patchInfo.getOuterBBox(), 

127 tract=tractInfo.getId(), 

128 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

129 subfilter=0, 

130 numSubfilters=self.config.numSubfilters, 

131 ) 

132 

133 if sensorRef.datasetExists(**patchArgDict): 

134 self.log.info("Reading patch %s" % patchArgDict) 

135 availableCoaddRefs[patchNumber] = patchArgDict 

136 

137 templateExposure = self.run( 

138 tractInfo, patchList, skyCorners, availableCoaddRefs, 

139 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

140 ) 

141 return pipeBase.Struct(exposure=templateExposure, sources=None) 

142 

143 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

144 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

145 that overlaps the science exposure. 

146 

147 Parameters 

148 ---------- 

149 exposure : `lsst.afw.image.Exposure` 

150 The science exposure to define the sky region of the template coadd. 

151 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

152 Butler like object that supports getting data by DatasetRef. 

153 skyMapRef : `lsst.daf.butler.DatasetRef` 

154 Reference to SkyMap object that corresponds to the template coadd. 

155 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

156 Iterable of references to the available template coadd patches. 

157 

158 Returns 

159 ------- 

160 result : `lsst.pipe.base.Struct` 

161 - ``exposure`` : `lsst.afw.image.ExposureF` 

162 a template coadd exposure assembled out of patches 

163 - ``sources`` : `None` for this subtask 

164 """ 

165 skyMap = butlerQC.get(skyMapRef) 

166 coaddExposureRefs = butlerQC.get(coaddExposureRefs) 

167 tracts = [ref.dataId['tract'] for ref in coaddExposureRefs] 

168 if tracts.count(tracts[0]) == len(tracts): 

169 tractInfo = skyMap[tracts[0]] 

170 else: 

171 raise RuntimeError("Templates constructed from multiple Tracts not yet supported") 

172 

173 detectorBBox = exposure.getBBox() 

174 detectorWcs = exposure.getWcs() 

175 detectorCorners = detectorWcs.pixelToSky(geom.Box2D(detectorBBox).getCorners()) 

176 

177 availableCoaddRefs = dict() 

178 for coaddRef in coaddExposureRefs: 

179 dataId = coaddRef.dataId 

180 patchWcs = skyMap[dataId['tract']].getWcs() 

181 patchBBox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox() 

182 if self.bboxIntersectsCorners(patchBBox, patchWcs, detectorCorners): 

183 if self.config.coaddName == 'dcr': 

184 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" % 

185 (dataId['tract'], dataId['patch'], dataId['subfilter'])) 

186 if dataId['patch'] in availableCoaddRefs: 

187 availableCoaddRefs[dataId['patch']].append(coaddRef) 

188 else: 

189 availableCoaddRefs[dataId['patch']] = [coaddRef, ] 

190 else: 

191 self.log.info("Using template input tract=%s, patch=%s" % 

192 (dataId['tract'], dataId['patch'])) 

193 availableCoaddRefs[dataId['patch']] = coaddRef 

194 

195 patchList = [tractInfo[patch] for patch in availableCoaddRefs.keys()] 

196 templateExposure = self.run(tractInfo, patchList, detectorCorners, availableCoaddRefs, 

197 visitInfo=exposure.getInfo().getVisitInfo()) 

198 return pipeBase.Struct(exposure=templateExposure, sources=None) 

199 

200 def getOverlapPatchList(self, exposure, skyMap): 

201 """Select the relevant tract and its patches that overlap with the science exposure. 

202 

203 Parameters 

204 ---------- 

205 exposure : `lsst.afw.image.Exposure` 

206 The science exposure to define the sky region of the template coadd. 

207 

208 skyMap : `lsst.skymap.BaseSkyMap` 

209 SkyMap object that corresponds to the template coadd. 

210 

211 Returns 

212 ------- 

213 result : `tuple` of 

214 - ``tractInfo`` : `lsst.skymap.TractInfo` 

215 The selected tract. 

216 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

217 List of all overlap patches of the selected tract. 

218 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

219 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

220 """ 

221 expWcs = exposure.getWcs() 

222 expBoxD = geom.Box2D(exposure.getBBox()) 

223 expBoxD.grow(self.config.templateBorderSize) 

224 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

225 tractInfo = skyMap.findTract(ctrSkyPos) 

226 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

227 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

228 patchList = tractInfo.findPatchList(skyCorners) 

229 

230 if not patchList: 

231 raise RuntimeError("No suitable tract found") 

232 

233 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

234 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

235 

236 return (tractInfo, patchList, skyCorners) 

237 

238 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

239 sensorRef=None, visitInfo=None): 

240 """Gen2 and gen3 shared code: determination of exposure dimensions and 

241 copying of pixels from overlapping patch regions. 

242 

243 Parameters 

244 ---------- 

245 skyMap : `lsst.skymap.BaseSkyMap` 

246 SkyMap object that corresponds to the template coadd. 

247 tractInfo : `lsst.skymap.TractInfo` 

248 The selected tract. 

249 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

250 Patches to consider for making the template exposure. 

251 skyCorners : list of `lsst.geom.SpherePoint` 

252 Sky corner coordinates to be covered by the template exposure. 

253 availableCoaddRefs : `dict` [`int`] 

254 Dictionary of spatially relevant retrieved coadd patches, 

255 indexed by their sequential patch number. In Gen3 mode, values are 

256 `lsst.daf.butler.DeferredDatasetHandle` and ``.get()`` is called, 

257 in Gen2 mode, ``sensorRef.get(**coaddef)`` is called to retrieve the coadd. 

258 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

259 Butler data reference to get coadd data. 

260 Must be `None` for Gen3. 

261 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

262 VisitInfo to make dcr model. 

263 

264 Returns 

265 ------- 

266 templateExposure: `lsst.afw.image.ExposureF` 

267 The created template exposure. 

268 """ 

269 coaddWcs = tractInfo.getWcs() 

270 

271 # compute coadd bbox 

272 coaddBBox = geom.Box2D() 

273 for skyPos in skyCorners: 

274 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

275 coaddBBox = geom.Box2I(coaddBBox) 

276 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

277 

278 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

279 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

280 nPatchesFound = 0 

281 coaddFilterLabel = None 

282 coaddPsf = None 

283 coaddPhotoCalib = None 

284 for patchInfo in patchList: 

285 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

286 patchSubBBox = patchInfo.getOuterBBox() 

287 patchSubBBox.clip(coaddBBox) 

288 if patchNumber not in availableCoaddRefs: 

289 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd") 

290 continue 

291 if patchSubBBox.isEmpty(): 

292 self.log.info(f"skip tract={availableCoaddRefs[patchNumber]['tract']}, " 

293 f"patch={patchNumber}; no overlapping pixels") 

294 continue 

295 

296 if self.config.coaddName == 'dcr': 

297 patchInnerBBox = patchInfo.getInnerBBox() 

298 patchInnerBBox.clip(coaddBBox) 

299 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

300 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." 

301 % availableCoaddRefs[patchNumber]) 

302 continue 

303 self.log.info("Constructing DCR-matched template for patch %s" 

304 % availableCoaddRefs[patchNumber]) 

305 

306 if sensorRef: 

307 dcrModel = DcrModel.fromDataRef(sensorRef, 

308 self.config.effectiveWavelength, 

309 self.config.bandwidth, 

310 **availableCoaddRefs[patchNumber]) 

311 else: 

312 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber], 

313 self.config.effectiveWavelength, 

314 self.config.bandwidth) 

315 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

316 # Each patch has significant overlap, and the contaminated edge pixels in 

317 # a new patch will overwrite good pixels in the overlap region from 

318 # previous patches. 

319 # Shrink the BBox to remove the contaminated pixels, 

320 # but make sure it is only the overlap region that is reduced. 

321 dcrBBox = geom.Box2I(patchSubBBox) 

322 dcrBBox.grow(-self.config.templateBorderSize) 

323 dcrBBox.include(patchInnerBBox) 

324 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

325 wcs=coaddWcs, 

326 visitInfo=visitInfo) 

327 else: 

328 if sensorRef is None: 

329 # Gen3 

330 coaddPatch = availableCoaddRefs[patchNumber].get() 

331 else: 

332 # Gen2 

333 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

334 nPatchesFound += 1 

335 

336 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

337 # calculation from caller code. Gen3 also does not do this. 

338 overlapBox = coaddPatch.getBBox() 

339 overlapBox.clip(coaddBBox) 

340 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

341 

342 if coaddFilterLabel is None: 

343 coaddFilterLabel = coaddPatch.getFilterLabel() 

344 

345 # Retrieve the PSF for this coadd tract, if not already retrieved 

346 if coaddPsf is None and coaddPatch.hasPsf(): 

347 coaddPsf = coaddPatch.getPsf() 

348 

349 # Retrieve the calibration for this coadd tract, if not already retrieved 

350 if coaddPhotoCalib is None: 

351 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

352 

353 if coaddPhotoCalib is None: 

354 raise RuntimeError("No coadd PhotoCalib found!") 

355 if nPatchesFound == 0: 

356 raise RuntimeError("No patches found!") 

357 if coaddPsf is None: 

358 raise RuntimeError("No coadd Psf found!") 

359 

360 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

361 coaddExposure.setPsf(coaddPsf) 

362 coaddExposure.setFilterLabel(coaddFilterLabel) 

363 return coaddExposure 

364 

365 def getCoaddDatasetName(self): 

366 """Return coadd name for given task config 

367 

368 Returns 

369 ------- 

370 CoaddDatasetName : `string` 

371 

372 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

373 """ 

374 warpType = self.config.warpType 

375 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

376 return self.config.coaddName + "Coadd" + suffix 

377 

378 def bboxIntersectsCorners(self, bbox, wcs, otherCorners): 

379 """Returns true if the bbox with wcs intersects otherCorners 

380 

381 Parameters 

382 ---------- 

383 bbox : `lsst.geom.Box2I` 

384 specifying the bounding box of test region 

385 wcs : `lsst.afw.geom.SkyWcs` 

386 specifying the WCS of test region 

387 otherCorners : `list` of `lsst.geom.SpherePoint` 

388 ICRS coordinates specifying boundary of the other sky region 

389 

390 Returns 

391 ------- 

392 result: `bool` 

393 Does bbox/wcs intersect other corners? 

394 """ 

395 bboxCorners = wcs.pixelToSky(geom.Box2D(bbox).getCorners()) 

396 bboxPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in bboxCorners]) 

397 otherPolygon = sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in otherCorners]) 

398 return otherPolygon.intersects(bboxPolygon) 

399 

400 

401class GetCalexpAsTemplateConfig(pexConfig.Config): 

402 doAddCalexpBackground = pexConfig.Field( 

403 dtype=bool, 

404 default=True, 

405 doc="Add background to calexp before processing it." 

406 ) 

407 

408 

409class GetCalexpAsTemplateTask(pipeBase.Task): 

410 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

411 for use as an image difference template. Only gen2 supported. 

412 

413 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

414 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

415 This code was originally part of Winter2013ImageDifferenceTask. 

416 """ 

417 

418 ConfigClass = GetCalexpAsTemplateConfig 

419 _DefaultName = "GetCalexpAsTemplateTask" 

420 

421 def run(self, exposure, sensorRef, templateIdList): 

422 """Return a calexp exposure with based on input sensorRef. 

423 

424 Construct a dataId based on the sensorRef.dataId combined 

425 with the specifications from the first dataId in templateIdList 

426 

427 Parameters 

428 ---------- 

429 exposure : `lsst.afw.image.Exposure` 

430 exposure (unused) 

431 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

432 Data reference of the calexp(s) to subtract from. 

433 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

434 Data reference of the template calexp to be subtraced. 

435 Can be incomplete, fields are initialized from `sensorRef`. 

436 If there are multiple items, only the first one is used. 

437 

438 Returns 

439 ------- 

440 result : `struct` 

441 

442 return a pipeBase.Struct: 

443 

444 - ``exposure`` : a template calexp 

445 - ``sources`` : source catalog measured on the template 

446 """ 

447 

448 if len(templateIdList) == 0: 

449 raise RuntimeError("No template data reference supplied.") 

450 if len(templateIdList) > 1: 

451 self.log.warn("Multiple template data references supplied. Using the first one only.") 

452 

453 templateId = sensorRef.dataId.copy() 

454 templateId.update(templateIdList[0]) 

455 

456 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

457 

458 butler = sensorRef.getButler() 

459 template = butler.get(datasetType="calexp", dataId=templateId) 

460 if self.config.doAddCalexpBackground: 

461 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

462 mi = template.getMaskedImage() 

463 mi += templateBg.getImage() 

464 

465 if not template.hasPsf(): 

466 raise pipeBase.TaskError("Template has no psf") 

467 

468 templateSources = butler.get(datasetType="src", dataId=templateId) 

469 return pipeBase.Struct(exposure=template, 

470 sources=templateSources) 

471 

472 def runDataRef(self, *args, **kwargs): 

473 return self.run(*args, **kwargs) 

474 

475 def runQuantum(self, **kwargs): 

476 raise NotImplementedError("Calexp template is not supported with gen3 middleware")