Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy as np 

23import lsst.sphgeom 

24import lsst.pex.config as pexConfig 

25import lsst.pex.exceptions as pexExceptions 

26import lsst.geom as geom 

27import lsst.pipe.base as pipeBase 

28 

29__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask", 

30 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask"] 

31 

32 

33class DatabaseSelectImagesConfig(pexConfig.Config): 

34 """Base configuration for subclasses of BaseSelectImagesTask that use a database""" 

35 host = pexConfig.Field( 

36 doc="Database server host name", 

37 dtype=str, 

38 ) 

39 port = pexConfig.Field( 

40 doc="Database server port", 

41 dtype=int, 

42 ) 

43 database = pexConfig.Field( 

44 doc="Name of database", 

45 dtype=str, 

46 ) 

47 maxExposures = pexConfig.Field( 

48 doc="maximum exposures to select; intended for debugging; ignored if None", 

49 dtype=int, 

50 optional=True, 

51 ) 

52 

53 

54class BaseExposureInfo(pipeBase.Struct): 

55 """Data about a selected exposure 

56 """ 

57 

58 def __init__(self, dataId, coordList): 

59 """Create exposure information that can be used to generate data references 

60 

61 The object has the following fields: 

62 - dataId: data ID of exposure (a dict) 

63 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

64 plus any others items that are desired 

65 """ 

66 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList) 

67 

68 

69class BaseSelectImagesTask(pipeBase.Task): 

70 """Base task for selecting images suitable for coaddition 

71 """ 

72 ConfigClass = pexConfig.Config 

73 _DefaultName = "selectImages" 

74 

75 @pipeBase.timeMethod 

76 def run(self, coordList): 

77 """Select images suitable for coaddition in a particular region 

78 

79 @param[in] coordList: list of coordinates defining region of interest; if None then select all images 

80 subclasses may add additional keyword arguments, as required 

81 

82 @return a pipeBase Struct containing: 

83 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo), 

84 which have at least the following fields: 

85 - dataId: data ID dictionary 

86 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

87 """ 

88 raise NotImplementedError() 

89 

90 def _runArgDictFromDataId(self, dataId): 

91 """Extract keyword arguments for run (other than coordList) from a data ID 

92 

93 @return keyword arguments for run (other than coordList), as a dict 

94 """ 

95 raise NotImplementedError() 

96 

97 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

98 """Run based on a data reference 

99 

100 This delegates to run() and _runArgDictFromDataId() to do the actual 

101 selection. In the event that the selectDataList is non-empty, this will 

102 be used to further restrict the selection, providing the user with 

103 additional control over the selection. 

104 

105 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass 

106 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky 

107 @param[in] makeDataRefList: if True, return dataRefList 

108 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection 

109 @return a pipeBase Struct containing: 

110 - exposureInfoList: a list of objects derived from ExposureInfo 

111 - dataRefList: a list of data references (None if makeDataRefList False) 

112 """ 

113 runArgDict = self._runArgDictFromDataId(dataRef.dataId) 

114 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList 

115 

116 if len(selectDataList) > 0 and len(exposureInfoList) > 0: 

117 # Restrict the exposure selection further 

118 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList) 

119 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys) 

120 inValues = set(inValues) 

121 newExposureInfoList = [] 

122 for info, ccdVal in zip(exposureInfoList, ccdValues): 

123 if ccdVal in inValues: 

124 newExposureInfoList.append(info) 

125 else: 

126 self.log.info("De-selecting exposure %s: not in selectDataList" % info.dataId) 

127 exposureInfoList = newExposureInfoList 

128 

129 if makeDataRefList: 

130 butler = dataRef.butlerSubset.butler 

131 dataRefList = [butler.dataRef(datasetType="calexp", 

132 dataId=expInfo.dataId, 

133 ) for expInfo in exposureInfoList] 

134 else: 

135 dataRefList = None 

136 

137 return pipeBase.Struct( 

138 dataRefList=dataRefList, 

139 exposureInfoList=exposureInfoList, 

140 ) 

141 

142 

143def _extractKeyValue(dataList, keys=None): 

144 """Extract the keys and values from a list of dataIds 

145 

146 The input dataList is a list of objects that have 'dataId' members. 

147 This allows it to be used for both a list of data references and a 

148 list of ExposureInfo 

149 """ 

150 assert len(dataList) > 0 

151 if keys is None: 

152 keys = sorted(dataList[0].dataId.keys()) 

153 keySet = set(keys) 

154 values = list() 

155 for data in dataList: 

156 thisKeys = set(data.dataId.keys()) 

157 if thisKeys != keySet: 

158 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys)) 

159 values.append(tuple(data.dataId[k] for k in keys)) 

160 return keys, values 

161 

162 

163class SelectStruct(pipeBase.Struct): 

164 """A container for data to be passed to the WcsSelectImagesTask""" 

165 

166 def __init__(self, dataRef, wcs, bbox): 

167 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox) 

168 

169 

170class WcsSelectImagesTask(BaseSelectImagesTask): 

171 """Select images using their Wcs""" 

172 

173 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

174 """Select images in the selectDataList that overlap the patch 

175 

176 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define 

177 polygons on the celestial sphere, and test the polygon of the 

178 patch for overlap with the polygon of the image. 

179 

180 We use "convexHull" instead of generating a ConvexPolygon 

181 directly because the standard for the inputs to ConvexPolygon 

182 are pretty high and we don't want to be responsible for reaching them. 

183 

184 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

185 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

186 @param makeDataRefList: Construct a list of data references? 

187 @param selectDataList: List of SelectStruct, to consider for selection 

188 """ 

189 dataRefList = [] 

190 exposureInfoList = [] 

191 

192 patchVertices = [coord.getVector() for coord in coordList] 

193 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices) 

194 

195 for data in selectDataList: 

196 dataRef = data.dataRef 

197 imageWcs = data.wcs 

198 imageBox = data.bbox 

199 

200 try: 

201 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()] 

202 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e: 

203 # Protecting ourselves from awful Wcs solutions in input images 

204 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataRef.dataId, e) 

205 continue 

206 

207 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners]) 

208 if imagePoly is None: 

209 self.log.debug("Unable to create polygon from image %s: deselecting", dataRef.dataId) 

210 continue 

211 if patchPoly.intersects(imagePoly): # "intersects" also covers "contains" or "is contained by" 

212 self.log.info("Selecting calexp %s" % dataRef.dataId) 

213 dataRefList.append(dataRef) 

214 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners)) 

215 

216 return pipeBase.Struct( 

217 dataRefList=dataRefList if makeDataRefList else None, 

218 exposureInfoList=exposureInfoList, 

219 ) 

220 

221 

222class PsfWcsSelectImagesConfig(pexConfig.Config): 

223 maxEllipResidual = pexConfig.Field( 

224 doc="Maximum median ellipticity residual", 

225 dtype=float, 

226 default=0.007, 

227 optional=True, 

228 ) 

229 maxSizeScatter = pexConfig.Field( 

230 doc="Maximum scatter in the size residuals", 

231 dtype=float, 

232 optional=True, 

233 ) 

234 maxScaledSizeScatter = pexConfig.Field( 

235 doc="Maximum scatter in the size residuals, scaled by the median size", 

236 dtype=float, 

237 default=0.009, 

238 optional=True, 

239 ) 

240 starSelection = pexConfig.Field( 

241 doc="select star with this field", 

242 dtype=str, 

243 default='calib_psf_used' 

244 ) 

245 starShape = pexConfig.Field( 

246 doc="name of star shape", 

247 dtype=str, 

248 default='base_SdssShape' 

249 ) 

250 psfShape = pexConfig.Field( 

251 doc="name of psf shape", 

252 dtype=str, 

253 default='base_SdssShape_psf' 

254 ) 

255 

256 

257def sigmaMad(array): 

258 "Return median absolute deviation scaled to normally distributed data" 

259 return 1.4826*np.median(np.abs(array - np.median(array))) 

260 

261 

262class PsfWcsSelectImagesTask(WcsSelectImagesTask): 

263 """Select images using their Wcs and cuts on the PSF properties""" 

264 

265 ConfigClass = PsfWcsSelectImagesConfig 

266 _DefaultName = "PsfWcsSelectImages" 

267 

268 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

269 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera. 

270 

271 The PSF quality criteria are based on the size and ellipticity residuals from the 

272 adaptive second moments of the star and the PSF. 

273 

274 The criteria are: 

275 - the median of the ellipticty residuals 

276 - the robust scatter of the size residuals (using the median absolute deviation) 

277 - the robust scatter of the size residuals scaled by the square of 

278 the median size 

279 

280 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

281 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

282 @param makeDataRefList: Construct a list of data references? 

283 @param selectDataList: List of SelectStruct, to consider for selection 

284 """ 

285 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList, 

286 selectDataList) 

287 

288 dataRefList = [] 

289 exposureInfoList = [] 

290 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

291 butler = dataRef.butlerSubset.butler 

292 srcCatalog = butler.get('src', dataRef.dataId) 

293 mask = srcCatalog[self.config.starSelection] 

294 

295 starXX = srcCatalog[self.config.starShape+'_xx'][mask] 

296 starYY = srcCatalog[self.config.starShape+'_yy'][mask] 

297 starXY = srcCatalog[self.config.starShape+'_xy'][mask] 

298 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask] 

299 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask] 

300 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask] 

301 

302 starSize = np.power(starXX*starYY - starXY**2, 0.25) 

303 starE1 = (starXX - starYY)/(starXX + starYY) 

304 starE2 = 2*starXY/(starXX + starYY) 

305 medianSize = np.median(starSize) 

306 

307 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25) 

308 psfE1 = (psfXX - psfYY)/(psfXX + psfYY) 

309 psfE2 = 2*psfXY/(psfXX + psfYY) 

310 

311 medianE1 = np.abs(np.median(starE1 - psfE1)) 

312 medianE2 = np.abs(np.median(starE2 - psfE2)) 

313 medianE = np.sqrt(medianE1**2 + medianE2**2) 

314 

315 scatterSize = sigmaMad(starSize - psfSize) 

316 scaledScatterSize = scatterSize/medianSize**2 

317 

318 valid = True 

319 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: 

320 self.log.info("Removing visit %s because median e residual too large: %f vs %f" % 

321 (dataRef.dataId, medianE, self.config.maxEllipResidual)) 

322 valid = False 

323 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: 

324 self.log.info("Removing visit %s because size scatter is too large: %f vs %f" % 

325 (dataRef.dataId, scatterSize, self.config.maxSizeScatter)) 

326 valid = False 

327 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: 

328 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f" % 

329 (dataRef.dataId, scaledScatterSize, self.config.maxScaledSizeScatter)) 

330 valid = False 

331 

332 if valid is False: 

333 continue 

334 

335 dataRefList.append(dataRef) 

336 exposureInfoList.append(exposureInfo) 

337 

338 return pipeBase.Struct( 

339 dataRefList=dataRefList, 

340 exposureInfoList=exposureInfoList, 

341 ) 

342 

343 

344class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass): 

345 """Base configuration for BestSeeingSelectImagesTask. 

346 """ 

347 nImagesMax = pexConfig.Field( 

348 dtype=int, 

349 doc="Maximum number of images to select", 

350 default=5) 

351 maxPsfFwhm = pexConfig.Field( 

352 dtype=float, 

353 doc="Maximum PSF FWHM (in arcseconds) to select", 

354 default=1.5, 

355 optional=True) 

356 minPsfFwhm = pexConfig.Field( 

357 dtype=float, 

358 doc="Minimum PSF FWHM (in arcseconds) to select", 

359 default=0., 

360 optional=True) 

361 

362 

363class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask): 

364 """Select up to a maximum number of the best-seeing images using their Wcs. 

365 """ 

366 ConfigClass = BestSeeingWcsSelectImageConfig 

367 

368 def runDataRef(self, dataRef, coordList, makeDataRefList=True, 

369 selectDataList=None): 

370 """Select the best-seeing images in the selectDataList that overlap the patch. 

371 

372 Parameters 

373 ---------- 

374 dataRef : `lsst.daf.persistence.ButlerDataRef` 

375 Data reference for coadd/tempExp (with tract, patch) 

376 coordList : `list` of `lsst.geom.SpherePoint` 

377 List of ICRS sky coordinates specifying boundary of patch 

378 makeDataRefList : `boolean`, optional 

379 Construct a list of data references? 

380 selectDataList : `list` of `SelectStruct` 

381 List of SelectStruct, to consider for selection 

382 

383 Returns 

384 ------- 

385 result : `lsst.pipe.base.Struct` 

386 Result struct with components: 

387 - ``exposureList``: the selected exposures 

388 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`). 

389 - ``dataRefList``: the optional data references corresponding to 

390 each element of ``exposureList`` 

391 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`). 

392 """ 

393 if self.config.nImagesMax <= 0: 

394 raise RuntimeError(f"nImagesMax must be greater than zero: {self.config.nImagesMax}") 

395 

396 psfSizes = [] 

397 dataRefList = [] 

398 exposureInfoList = [] 

399 

400 if selectDataList is None: 

401 selectDataList = [] 

402 

403 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList) 

404 

405 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

406 cal = dataRef.get("calexp", immediate=True) 

407 

408 # if min/max PSF values are defined, remove images out of bounds 

409 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds() 

410 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds 

411 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.)) 

412 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm: 

413 continue 

414 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm: 

415 continue 

416 psfSizes.append(psfSize) 

417 dataRefList.append(dataRef) 

418 exposureInfoList.append(exposureInfo) 

419 

420 if len(psfSizes) > self.config.nImagesMax: 

421 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax] 

422 filteredDataRefList = [dataRefList[i] for i in sortedIndices] 

423 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices] 

424 self.log.info(f"{len(sortedIndices)} images selected with FWHM " 

425 f"range of {psfSizes[sortedIndices[0]]}--{psfSizes[sortedIndices[-1]]} arcseconds") 

426 

427 else: 

428 if len(psfSizes) == 0: 

429 self.log.warn("0 images selected.") 

430 else: 

431 self.log.debug(f"{len(psfSizes)} images selected with FWHM range " 

432 f"of {psfSizes[0]}--{psfSizes[-1]} arcseconds") 

433 filteredDataRefList = dataRefList 

434 filteredExposureInfoList = exposureInfoList 

435 

436 return pipeBase.Struct( 

437 dataRefList=filteredDataRefList if makeDataRefList else None, 

438 exposureInfoList=filteredExposureInfoList, 

439 )