Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22 

23import numpy as np 

24 

25import lsst.afw.image as afwImage 

26import lsst.geom as geom 

27import lsst.afw.geom as afwGeom 

28import lsst.pex.config as pexConfig 

29import lsst.pipe.base as pipeBase 

30from lsst.daf.butler import DeferredDatasetHandle 

31from lsst.ip.diffim.dcrModel import DcrModel 

32 

33__all__ = ["GetCoaddAsTemplateTask", "GetCoaddAsTemplateConfig", 

34 "GetCalexpAsTemplateTask", "GetCalexpAsTemplateConfig"] 

35 

36 

37class GetCoaddAsTemplateConfig(pexConfig.Config): 

38 templateBorderSize = pexConfig.Field( 

39 dtype=int, 

40 default=10, 

41 doc="Number of pixels to grow the requested template image to account for warping" 

42 ) 

43 coaddName = pexConfig.Field( 

44 doc="coadd name: typically one of 'deep', 'goodSeeing', or 'dcr'", 

45 dtype=str, 

46 default="deep", 

47 ) 

48 numSubfilters = pexConfig.Field( 

49 doc="Number of subfilters in the DcrCoadd. Used only if ``coaddName``='dcr'", 

50 dtype=int, 

51 default=3, 

52 ) 

53 effectiveWavelength = pexConfig.Field( 

54 doc="Effective wavelength of the filter. Used only if ``coaddName``='dcr'", 

55 optional=True, 

56 dtype=float, 

57 ) 

58 bandwidth = pexConfig.Field( 

59 doc="Bandwidth of the physical filter. Used only if ``coaddName``='dcr'", 

60 optional=True, 

61 dtype=float, 

62 ) 

63 warpType = pexConfig.Field( 

64 doc="Warp type of the coadd template: one of 'direct' or 'psfMatched'", 

65 dtype=str, 

66 default="direct", 

67 ) 

68 

69 def validate(self): 

70 if self.coaddName == 'dcr': 

71 if self.effectiveWavelength is None or self.bandwidth is None: 

72 raise ValueError("The effective wavelength and bandwidth of the physical filter " 

73 "must be set in the getTemplate config for DCR coadds. " 

74 "Required until transmission curves are used in DM-13668.") 

75 

76 

77class GetCoaddAsTemplateTask(pipeBase.Task): 

78 """Subtask to retrieve coadd for use as an image difference template. 

79 

80 This is the default getTemplate Task to be run as a subtask by 

81 ``pipe.tasks.ImageDifferenceTask``. The main methods are ``run()`` and 

82 ``runGen3()``. 

83 

84 Notes 

85 ----- 

86 From the given skymap, the closest tract is selected; multiple tracts are 

87 not supported. The assembled template inherits the WCS of the selected 

88 skymap tract and the resolution of the template exposures. Overlapping box 

89 regions of the input template patches are pixel by pixel copied into the 

90 assembled template image. There is no warping or pixel resampling. 

91 

92 Pixels with no overlap of any available input patches are set to ``nan`` value 

93 and ``NO_DATA`` flagged. 

94 """ 

95 

96 ConfigClass = GetCoaddAsTemplateConfig 

97 _DefaultName = "GetCoaddAsTemplateTask" 

98 

99 def runDataRef(self, exposure, sensorRef, templateIdList=None): 

100 """Gen2 task entry point. Retrieve and mosaic a template coadd exposure 

101 that overlaps the science exposure. 

102 

103 Parameters 

104 ---------- 

105 exposure: `lsst.afw.image.Exposure` 

106 an exposure for which to generate an overlapping template 

107 sensorRef : TYPE 

108 a Butler data reference that can be used to obtain coadd data 

109 templateIdList : TYPE, optional 

110 list of data ids, unused here, in the case of coadd template 

111 

112 Returns 

113 ------- 

114 result : `lsst.pipe.base.Struct` 

115 - ``exposure`` : `lsst.afw.image.ExposureF` 

116 a template coadd exposure assembled out of patches 

117 - ``sources`` : None for this subtask 

118 """ 

119 skyMap = sensorRef.get(datasetType=self.config.coaddName + "Coadd_skyMap") 

120 tractInfo, patchList, skyCorners = self.getOverlapPatchList(exposure, skyMap) 

121 

122 availableCoaddRefs = dict() 

123 for patchInfo in patchList: 

124 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

125 patchArgDict = dict( 

126 datasetType=self.getCoaddDatasetName() + "_sub", 

127 bbox=patchInfo.getOuterBBox(), 

128 tract=tractInfo.getId(), 

129 patch="%s,%s" % (patchInfo.getIndex()[0], patchInfo.getIndex()[1]), 

130 subfilter=0, 

131 numSubfilters=self.config.numSubfilters, 

132 ) 

133 

134 if sensorRef.datasetExists(**patchArgDict): 

135 self.log.info("Reading patch %s" % patchArgDict) 

136 availableCoaddRefs[patchNumber] = patchArgDict 

137 

138 templateExposure = self.run( 

139 tractInfo, patchList, skyCorners, availableCoaddRefs, 

140 sensorRef=sensorRef, visitInfo=exposure.getInfo().getVisitInfo() 

141 ) 

142 return pipeBase.Struct(exposure=templateExposure, sources=None) 

143 

144 def runQuantum(self, exposure, butlerQC, skyMapRef, coaddExposureRefs): 

145 """Gen3 task entry point. Retrieve and mosaic a template coadd exposure 

146 that overlaps the science exposure. 

147 

148 Parameters 

149 ---------- 

150 exposure : `lsst.afw.image.Exposure` 

151 The science exposure to define the sky region of the template coadd. 

152 butlerQC : `lsst.pipe.base.ButlerQuantumContext` 

153 Butler like object that supports getting data by DatasetRef. 

154 skyMapRef : `lsst.daf.butler.DatasetRef` 

155 Reference to SkyMap object that corresponds to the template coadd. 

156 coaddExposureRefs : iterable of `lsst.daf.butler.DeferredDatasetRef` 

157 Iterable of references to the available template coadd patches. 

158 

159 Returns 

160 ------- 

161 result : `lsst.pipe.base.Struct` 

162 - ``exposure`` : `lsst.afw.image.ExposureF` 

163 a template coadd exposure assembled out of patches 

164 - ``sources`` : `None` for this subtask 

165 """ 

166 skyMap = butlerQC.get(skyMapRef) 

167 coaddExposureRefs = butlerQC.get(coaddExposureRefs) 

168 tracts = [ref.dataId['tract'] for ref in coaddExposureRefs] 

169 if tracts.count(tracts[0]) == len(tracts): 

170 tractInfo = skyMap[tracts[0]] 

171 else: 

172 raise RuntimeError("Templates constructed from multiple Tracts not yet supported") 

173 

174 detectorBBox = exposure.getBBox() 

175 detectorWcs = exposure.getWcs() 

176 detectorCorners = detectorWcs.pixelToSky(geom.Box2D(detectorBBox).getCorners()) 

177 validPolygon = exposure.getInfo().getValidPolygon() 

178 detectorPolygon = validPolygon if validPolygon else geom.Box2D(detectorBBox) 

179 

180 availableCoaddRefs = dict() 

181 overlappingArea = 0 

182 for coaddRef in coaddExposureRefs: 

183 dataId = coaddRef.dataId 

184 patchWcs = skyMap[dataId['tract']].getWcs() 

185 patchBBox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox() 

186 patchCorners = patchWcs.pixelToSky(geom.Box2D(patchBBox).getCorners()) 

187 patchPolygon = afwGeom.Polygon(detectorWcs.skyToPixel(patchCorners)) 

188 if patchPolygon.intersection(detectorPolygon): 

189 overlappingArea += patchPolygon.intersectionSingle(detectorPolygon).calculateArea() 

190 if self.config.coaddName == 'dcr': 

191 self.log.info("Using template input tract=%s, patch=%s, subfilter=%s" % 

192 (dataId['tract'], dataId['patch'], dataId['subfilter'])) 

193 if dataId['patch'] in availableCoaddRefs: 

194 availableCoaddRefs[dataId['patch']].append(coaddRef) 

195 else: 

196 availableCoaddRefs[dataId['patch']] = [coaddRef, ] 

197 else: 

198 self.log.info("Using template input tract=%s, patch=%s" % 

199 (dataId['tract'], dataId['patch'])) 

200 availableCoaddRefs[dataId['patch']] = coaddRef 

201 

202 if overlappingArea == 0: 

203 templateExposure = None 

204 pixGood = 0 

205 self.log.warn("No overlapping template patches found") 

206 else: 

207 patchList = [tractInfo[patch] for patch in availableCoaddRefs.keys()] 

208 templateExposure = self.run(tractInfo, patchList, detectorCorners, availableCoaddRefs, 

209 visitInfo=exposure.getInfo().getVisitInfo()) 

210 # Count the number of pixels with the NO_DATA mask bit set 

211 # counting NaN pixels is insufficient because pixels without data are often intepolated over) 

212 pixNoData = np.count_nonzero(templateExposure.mask.array 

213 & templateExposure.mask.getPlaneBitMask('NO_DATA')) 

214 pixGood = templateExposure.getBBox().getArea() - pixNoData 

215 self.log.info("template has %d good pixels (%.1f%%)", pixGood, 

216 100*pixGood/templateExposure.getBBox().getArea()) 

217 return pipeBase.Struct(exposure=templateExposure, sources=None, area=pixGood) 

218 

219 def getOverlapPatchList(self, exposure, skyMap): 

220 """Select the relevant tract and its patches that overlap with the science exposure. 

221 

222 Parameters 

223 ---------- 

224 exposure : `lsst.afw.image.Exposure` 

225 The science exposure to define the sky region of the template coadd. 

226 

227 skyMap : `lsst.skymap.BaseSkyMap` 

228 SkyMap object that corresponds to the template coadd. 

229 

230 Returns 

231 ------- 

232 result : `tuple` of 

233 - ``tractInfo`` : `lsst.skymap.TractInfo` 

234 The selected tract. 

235 - ``patchList`` : `list` of `lsst.skymap.PatchInfo` 

236 List of all overlap patches of the selected tract. 

237 - ``skyCorners`` : `list` of `lsst.geom.SpherePoint` 

238 Corners of the exposure in the sky in the order given by `lsst.geom.Box2D.getCorners`. 

239 """ 

240 expWcs = exposure.getWcs() 

241 expBoxD = geom.Box2D(exposure.getBBox()) 

242 expBoxD.grow(self.config.templateBorderSize) 

243 ctrSkyPos = expWcs.pixelToSky(expBoxD.getCenter()) 

244 tractInfo = skyMap.findTract(ctrSkyPos) 

245 self.log.info("Using skyMap tract %s" % (tractInfo.getId(),)) 

246 skyCorners = [expWcs.pixelToSky(pixPos) for pixPos in expBoxD.getCorners()] 

247 patchList = tractInfo.findPatchList(skyCorners) 

248 

249 if not patchList: 

250 raise RuntimeError("No suitable tract found") 

251 

252 self.log.info("Assembling %s coadd patches" % (len(patchList),)) 

253 self.log.info("exposure dimensions=%s" % exposure.getDimensions()) 

254 

255 return (tractInfo, patchList, skyCorners) 

256 

257 def run(self, tractInfo, patchList, skyCorners, availableCoaddRefs, 

258 sensorRef=None, visitInfo=None): 

259 """Gen2 and gen3 shared code: determination of exposure dimensions and 

260 copying of pixels from overlapping patch regions. 

261 

262 Parameters 

263 ---------- 

264 skyMap : `lsst.skymap.BaseSkyMap` 

265 SkyMap object that corresponds to the template coadd. 

266 tractInfo : `lsst.skymap.TractInfo` 

267 The selected tract. 

268 patchList : iterable of `lsst.skymap.patchInfo.PatchInfo` 

269 Patches to consider for making the template exposure. 

270 skyCorners : list of `lsst.geom.SpherePoint` 

271 Sky corner coordinates to be covered by the template exposure. 

272 availableCoaddRefs : `dict` [`int`] 

273 Dictionary of spatially relevant retrieved coadd patches, 

274 indexed by their sequential patch number. In Gen3 mode, values are 

275 `lsst.daf.butler.DeferredDatasetHandle` and ``.get()`` is called, 

276 in Gen2 mode, ``sensorRef.get(**coaddef)`` is called to retrieve the coadd. 

277 sensorRef : `lsst.daf.persistence.ButlerDataRef`, Gen2 only 

278 Butler data reference to get coadd data. 

279 Must be `None` for Gen3. 

280 visitInfo : `lsst.afw.image.VisitInfo`, Gen2 only 

281 VisitInfo to make dcr model. 

282 

283 Returns 

284 ------- 

285 templateExposure: `lsst.afw.image.ExposureF` 

286 The created template exposure. 

287 """ 

288 coaddWcs = tractInfo.getWcs() 

289 

290 # compute coadd bbox 

291 coaddBBox = geom.Box2D() 

292 for skyPos in skyCorners: 

293 coaddBBox.include(coaddWcs.skyToPixel(skyPos)) 

294 coaddBBox = geom.Box2I(coaddBBox) 

295 self.log.info("coadd dimensions=%s" % coaddBBox.getDimensions()) 

296 

297 coaddExposure = afwImage.ExposureF(coaddBBox, coaddWcs) 

298 coaddExposure.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

299 nPatchesFound = 0 

300 coaddFilterLabel = None 

301 coaddPsf = None 

302 coaddPhotoCalib = None 

303 for patchInfo in patchList: 

304 patchNumber = tractInfo.getSequentialPatchIndex(patchInfo) 

305 patchSubBBox = patchInfo.getOuterBBox() 

306 patchSubBBox.clip(coaddBBox) 

307 if patchNumber not in availableCoaddRefs: 

308 self.log.warn(f"skip patch={patchNumber}; patch does not exist for this coadd") 

309 continue 

310 if patchSubBBox.isEmpty(): 

311 if isinstance(availableCoaddRefs[patchNumber], DeferredDatasetHandle): 

312 tract = availableCoaddRefs[patchNumber].dataId['tract'] 

313 else: 

314 tract = availableCoaddRefs[patchNumber]['tract'] 

315 self.log.info(f"skip tract={tract} patch={patchNumber}; no overlapping pixels") 

316 continue 

317 

318 if self.config.coaddName == 'dcr': 

319 patchInnerBBox = patchInfo.getInnerBBox() 

320 patchInnerBBox.clip(coaddBBox) 

321 if np.min(patchInnerBBox.getDimensions()) <= 2*self.config.templateBorderSize: 

322 self.log.info("skip tract=%(tract)s, patch=%(patch)s; too few pixels." 

323 % availableCoaddRefs[patchNumber]) 

324 continue 

325 self.log.info("Constructing DCR-matched template for patch %s" 

326 % availableCoaddRefs[patchNumber]) 

327 

328 if sensorRef: 

329 dcrModel = DcrModel.fromDataRef(sensorRef, 

330 self.config.effectiveWavelength, 

331 self.config.bandwidth, 

332 **availableCoaddRefs[patchNumber]) 

333 else: 

334 dcrModel = DcrModel.fromQuantum(availableCoaddRefs[patchNumber], 

335 self.config.effectiveWavelength, 

336 self.config.bandwidth) 

337 # The edge pixels of the DcrCoadd may contain artifacts due to missing data. 

338 # Each patch has significant overlap, and the contaminated edge pixels in 

339 # a new patch will overwrite good pixels in the overlap region from 

340 # previous patches. 

341 # Shrink the BBox to remove the contaminated pixels, 

342 # but make sure it is only the overlap region that is reduced. 

343 dcrBBox = geom.Box2I(patchSubBBox) 

344 dcrBBox.grow(-self.config.templateBorderSize) 

345 dcrBBox.include(patchInnerBBox) 

346 coaddPatch = dcrModel.buildMatchedExposure(bbox=dcrBBox, 

347 wcs=coaddWcs, 

348 visitInfo=visitInfo) 

349 else: 

350 if sensorRef is None: 

351 # Gen3 

352 coaddPatch = availableCoaddRefs[patchNumber].get() 

353 else: 

354 # Gen2 

355 coaddPatch = sensorRef.get(**availableCoaddRefs[patchNumber]) 

356 nPatchesFound += 1 

357 

358 # Gen2 get() seems to clip based on bbox kwarg but we removed bbox 

359 # calculation from caller code. Gen3 also does not do this. 

360 overlapBox = coaddPatch.getBBox() 

361 overlapBox.clip(coaddBBox) 

362 coaddExposure.maskedImage.assign(coaddPatch.maskedImage[overlapBox], overlapBox) 

363 

364 if coaddFilterLabel is None: 

365 coaddFilterLabel = coaddPatch.getFilterLabel() 

366 

367 # Retrieve the PSF for this coadd tract, if not already retrieved 

368 if coaddPsf is None and coaddPatch.hasPsf(): 

369 coaddPsf = coaddPatch.getPsf() 

370 

371 # Retrieve the calibration for this coadd tract, if not already retrieved 

372 if coaddPhotoCalib is None: 

373 coaddPhotoCalib = coaddPatch.getPhotoCalib() 

374 

375 if coaddPhotoCalib is None: 

376 raise RuntimeError("No coadd PhotoCalib found!") 

377 if nPatchesFound == 0: 

378 raise RuntimeError("No patches found!") 

379 if coaddPsf is None: 

380 raise RuntimeError("No coadd Psf found!") 

381 

382 coaddExposure.setPhotoCalib(coaddPhotoCalib) 

383 coaddExposure.setPsf(coaddPsf) 

384 coaddExposure.setFilterLabel(coaddFilterLabel) 

385 return coaddExposure 

386 

387 def getCoaddDatasetName(self): 

388 """Return coadd name for given task config 

389 

390 Returns 

391 ------- 

392 CoaddDatasetName : `string` 

393 

394 TODO: This nearly duplicates a method in CoaddBaseTask (DM-11985) 

395 """ 

396 warpType = self.config.warpType 

397 suffix = "" if warpType == "direct" else warpType[0].upper() + warpType[1:] 

398 return self.config.coaddName + "Coadd" + suffix 

399 

400 

401class GetCalexpAsTemplateConfig(pexConfig.Config): 

402 doAddCalexpBackground = pexConfig.Field( 

403 dtype=bool, 

404 default=True, 

405 doc="Add background to calexp before processing it." 

406 ) 

407 

408 

409class GetCalexpAsTemplateTask(pipeBase.Task): 

410 """Subtask to retrieve calexp of the same ccd number as the science image SensorRef 

411 for use as an image difference template. Only gen2 supported. 

412 

413 To be run as a subtask by pipe.tasks.ImageDifferenceTask. 

414 Intended for use with simulations and surveys that repeatedly visit the same pointing. 

415 This code was originally part of Winter2013ImageDifferenceTask. 

416 """ 

417 

418 ConfigClass = GetCalexpAsTemplateConfig 

419 _DefaultName = "GetCalexpAsTemplateTask" 

420 

421 def run(self, exposure, sensorRef, templateIdList): 

422 """Return a calexp exposure with based on input sensorRef. 

423 

424 Construct a dataId based on the sensorRef.dataId combined 

425 with the specifications from the first dataId in templateIdList 

426 

427 Parameters 

428 ---------- 

429 exposure : `lsst.afw.image.Exposure` 

430 exposure (unused) 

431 sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef` 

432 Data reference of the calexp(s) to subtract from. 

433 templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef` 

434 Data reference of the template calexp to be subtraced. 

435 Can be incomplete, fields are initialized from `sensorRef`. 

436 If there are multiple items, only the first one is used. 

437 

438 Returns 

439 ------- 

440 result : `struct` 

441 

442 return a pipeBase.Struct: 

443 

444 - ``exposure`` : a template calexp 

445 - ``sources`` : source catalog measured on the template 

446 """ 

447 

448 if len(templateIdList) == 0: 

449 raise RuntimeError("No template data reference supplied.") 

450 if len(templateIdList) > 1: 

451 self.log.warn("Multiple template data references supplied. Using the first one only.") 

452 

453 templateId = sensorRef.dataId.copy() 

454 templateId.update(templateIdList[0]) 

455 

456 self.log.info("Fetching calexp (%s) as template." % (templateId)) 

457 

458 butler = sensorRef.getButler() 

459 template = butler.get(datasetType="calexp", dataId=templateId) 

460 if self.config.doAddCalexpBackground: 

461 templateBg = butler.get(datasetType="calexpBackground", dataId=templateId) 

462 mi = template.getMaskedImage() 

463 mi += templateBg.getImage() 

464 

465 if not template.hasPsf(): 

466 raise pipeBase.TaskError("Template has no psf") 

467 

468 templateSources = butler.get(datasetType="src", dataId=templateId) 

469 return pipeBase.Struct(exposure=template, 

470 sources=templateSources) 

471 

472 def runDataRef(self, *args, **kwargs): 

473 return self.run(*args, **kwargs) 

474 

475 def runQuantum(self, **kwargs): 

476 raise NotImplementedError("Calexp template is not supported with gen3 middleware")