Coverage for python/lsst/pipe/tasks/selectImages.py: 19%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

318 statements  

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy as np 

23import lsst.sphgeom 

24import lsst.utils as utils 

25import lsst.pex.config as pexConfig 

26import lsst.pex.exceptions as pexExceptions 

27import lsst.geom as geom 

28import lsst.pipe.base as pipeBase 

29from lsst.skymap import BaseSkyMap 

30from lsst.daf.base import DateTime 

31 

32__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask", 

33 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask", 

34 "BestSeeingQuantileSelectVisitsTask"] 

35 

36 

37class DatabaseSelectImagesConfig(pexConfig.Config): 

38 """Base configuration for subclasses of BaseSelectImagesTask that use a database""" 

39 host = pexConfig.Field( 

40 doc="Database server host name", 

41 dtype=str, 

42 ) 

43 port = pexConfig.Field( 

44 doc="Database server port", 

45 dtype=int, 

46 ) 

47 database = pexConfig.Field( 

48 doc="Name of database", 

49 dtype=str, 

50 ) 

51 maxExposures = pexConfig.Field( 

52 doc="maximum exposures to select; intended for debugging; ignored if None", 

53 dtype=int, 

54 optional=True, 

55 ) 

56 

57 

58class BaseExposureInfo(pipeBase.Struct): 

59 """Data about a selected exposure 

60 """ 

61 

62 def __init__(self, dataId, coordList): 

63 """Create exposure information that can be used to generate data references 

64 

65 The object has the following fields: 

66 - dataId: data ID of exposure (a dict) 

67 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

68 plus any others items that are desired 

69 """ 

70 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList) 

71 

72 

73class BaseSelectImagesTask(pipeBase.Task): 

74 """Base task for selecting images suitable for coaddition 

75 """ 

76 ConfigClass = pexConfig.Config 

77 _DefaultName = "selectImages" 

78 

79 @pipeBase.timeMethod 

80 def run(self, coordList): 

81 """Select images suitable for coaddition in a particular region 

82 

83 @param[in] coordList: list of coordinates defining region of interest; if None then select all images 

84 subclasses may add additional keyword arguments, as required 

85 

86 @return a pipeBase Struct containing: 

87 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo), 

88 which have at least the following fields: 

89 - dataId: data ID dictionary 

90 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

91 """ 

92 raise NotImplementedError() 

93 

94 def _runArgDictFromDataId(self, dataId): 

95 """Extract keyword arguments for run (other than coordList) from a data ID 

96 

97 @return keyword arguments for run (other than coordList), as a dict 

98 """ 

99 raise NotImplementedError() 

100 

101 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

102 """Run based on a data reference 

103 

104 This delegates to run() and _runArgDictFromDataId() to do the actual 

105 selection. In the event that the selectDataList is non-empty, this will 

106 be used to further restrict the selection, providing the user with 

107 additional control over the selection. 

108 

109 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass 

110 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky 

111 @param[in] makeDataRefList: if True, return dataRefList 

112 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection 

113 @return a pipeBase Struct containing: 

114 - exposureInfoList: a list of objects derived from ExposureInfo 

115 - dataRefList: a list of data references (None if makeDataRefList False) 

116 """ 

117 runArgDict = self._runArgDictFromDataId(dataRef.dataId) 

118 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList 

119 

120 if len(selectDataList) > 0 and len(exposureInfoList) > 0: 

121 # Restrict the exposure selection further 

122 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList) 

123 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys) 

124 inValues = set(inValues) 

125 newExposureInfoList = [] 

126 for info, ccdVal in zip(exposureInfoList, ccdValues): 

127 if ccdVal in inValues: 

128 newExposureInfoList.append(info) 

129 else: 

130 self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId) 

131 exposureInfoList = newExposureInfoList 

132 

133 if makeDataRefList: 

134 butler = dataRef.butlerSubset.butler 

135 dataRefList = [butler.dataRef(datasetType="calexp", 

136 dataId=expInfo.dataId, 

137 ) for expInfo in exposureInfoList] 

138 else: 

139 dataRefList = None 

140 

141 return pipeBase.Struct( 

142 dataRefList=dataRefList, 

143 exposureInfoList=exposureInfoList, 

144 ) 

145 

146 

147def _extractKeyValue(dataList, keys=None): 

148 """Extract the keys and values from a list of dataIds 

149 

150 The input dataList is a list of objects that have 'dataId' members. 

151 This allows it to be used for both a list of data references and a 

152 list of ExposureInfo 

153 """ 

154 assert len(dataList) > 0 

155 if keys is None: 

156 keys = sorted(dataList[0].dataId.keys()) 

157 keySet = set(keys) 

158 values = list() 

159 for data in dataList: 

160 thisKeys = set(data.dataId.keys()) 

161 if thisKeys != keySet: 

162 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys)) 

163 values.append(tuple(data.dataId[k] for k in keys)) 

164 return keys, values 

165 

166 

167class SelectStruct(pipeBase.Struct): 

168 """A container for data to be passed to the WcsSelectImagesTask""" 

169 

170 def __init__(self, dataRef, wcs, bbox): 

171 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox) 

172 

173 

174class WcsSelectImagesTask(BaseSelectImagesTask): 

175 """Select images using their Wcs 

176 

177 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define 

178 polygons on the celestial sphere, and test the polygon of the 

179 patch for overlap with the polygon of the image. 

180 

181 We use "convexHull" instead of generating a ConvexPolygon 

182 directly because the standard for the inputs to ConvexPolygon 

183 are pretty high and we don't want to be responsible for reaching them. 

184 """ 

185 

186 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

187 """Select images in the selectDataList that overlap the patch 

188 

189 This method is the old entry point for the Gen2 commandline tasks and drivers 

190 Will be deprecated in v22. 

191 

192 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

193 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

194 @param makeDataRefList: Construct a list of data references? 

195 @param selectDataList: List of SelectStruct, to consider for selection 

196 """ 

197 dataRefList = [] 

198 exposureInfoList = [] 

199 

200 patchVertices = [coord.getVector() for coord in coordList] 

201 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices) 

202 

203 for data in selectDataList: 

204 dataRef = data.dataRef 

205 imageWcs = data.wcs 

206 imageBox = data.bbox 

207 

208 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None) 

209 if imageCorners: 

210 dataRefList.append(dataRef) 

211 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners)) 

212 

213 return pipeBase.Struct( 

214 dataRefList=dataRefList if makeDataRefList else None, 

215 exposureInfoList=exposureInfoList, 

216 ) 

217 

218 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs): 

219 """Return indices of provided lists that meet the selection criteria 

220 

221 Parameters: 

222 ----------- 

223 wcsList : `list` of `lsst.afw.geom.SkyWcs` 

224 specifying the WCS's of the input ccds to be selected 

225 bboxList : `list` of `lsst.geom.Box2I` 

226 specifying the bounding boxes of the input ccds to be selected 

227 coordList : `list` of `lsst.geom.SpherePoint` 

228 ICRS coordinates specifying boundary of the patch. 

229 

230 Returns: 

231 -------- 

232 result: `list` of `int` 

233 of indices of selected ccds 

234 """ 

235 if dataIds is None: 

236 dataIds = [None] * len(wcsList) 

237 patchVertices = [coord.getVector() for coord in coordList] 

238 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices) 

239 result = [] 

240 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)): 

241 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId) 

242 if imageCorners: 

243 result.append(i) 

244 return result 

245 

246 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None): 

247 "Return corners or None if bad" 

248 try: 

249 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()] 

250 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e: 

251 # Protecting ourselves from awful Wcs solutions in input images 

252 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e) 

253 return 

254 

255 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners]) 

256 if imagePoly is None: 

257 self.log.debug("Unable to create polygon from image %s: deselecting", dataId) 

258 return 

259 

260 if patchPoly.intersects(imagePoly): 

261 # "intersects" also covers "contains" or "is contained by" 

262 self.log.info("Selecting calexp %s", dataId) 

263 return imageCorners 

264 

265 

266def sigmaMad(array): 

267 "Return median absolute deviation scaled to normally distributed data" 

268 return 1.4826*np.median(np.abs(array - np.median(array))) 

269 

270 

271class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections, 

272 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

273 defaultTemplates={"coaddName": "deep"}): 

274 pass 

275 

276 

277class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig, 

278 pipelineConnections=PsfWcsSelectImagesConnections): 

279 maxEllipResidual = pexConfig.Field( 

280 doc="Maximum median ellipticity residual", 

281 dtype=float, 

282 default=0.007, 

283 optional=True, 

284 ) 

285 maxSizeScatter = pexConfig.Field( 

286 doc="Maximum scatter in the size residuals", 

287 dtype=float, 

288 optional=True, 

289 ) 

290 maxScaledSizeScatter = pexConfig.Field( 

291 doc="Maximum scatter in the size residuals, scaled by the median size", 

292 dtype=float, 

293 default=0.009, 

294 optional=True, 

295 ) 

296 starSelection = pexConfig.Field( 

297 doc="select star with this field", 

298 dtype=str, 

299 default='calib_psf_used', 

300 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

301 'will be removed after v24.') 

302 ) 

303 starShape = pexConfig.Field( 

304 doc="name of star shape", 

305 dtype=str, 

306 default='base_SdssShape', 

307 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

308 'will be removed after v24.') 

309 ) 

310 psfShape = pexConfig.Field( 

311 doc="name of psf shape", 

312 dtype=str, 

313 default='base_SdssShape_psf', 

314 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

315 'will be removed after v24.') 

316 ) 

317 

318 

319class PsfWcsSelectImagesTask(WcsSelectImagesTask): 

320 """Select images using their Wcs and cuts on the PSF properties 

321 

322 The PSF quality criteria are based on the size and ellipticity residuals from the 

323 adaptive second moments of the star and the PSF. 

324 

325 The criteria are: 

326 - the median of the ellipticty residuals 

327 - the robust scatter of the size residuals (using the median absolute deviation) 

328 - the robust scatter of the size residuals scaled by the square of 

329 the median size 

330 """ 

331 

332 ConfigClass = PsfWcsSelectImagesConfig 

333 _DefaultName = "PsfWcsSelectImages" 

334 

335 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

336 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera. 

337 

338 This method is the old entry point for the Gen2 commandline tasks and drivers 

339 Will be deprecated in v22. 

340 

341 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

342 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

343 @param makeDataRefList: Construct a list of data references? 

344 @param selectDataList: List of SelectStruct, to consider for selection 

345 """ 

346 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList, 

347 selectDataList) 

348 

349 dataRefList = [] 

350 exposureInfoList = [] 

351 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

352 butler = dataRef.butlerSubset.butler 

353 srcCatalog = butler.get('src', dataRef.dataId) 

354 valid = self.isValidGen2(srcCatalog, dataRef.dataId) 

355 if valid is False: 

356 continue 

357 

358 dataRefList.append(dataRef) 

359 exposureInfoList.append(exposureInfo) 

360 

361 return pipeBase.Struct( 

362 dataRefList=dataRefList, 

363 exposureInfoList=exposureInfoList, 

364 ) 

365 

366 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs): 

367 """Return indices of provided lists that meet the selection criteria 

368 

369 Parameters: 

370 ----------- 

371 wcsList : `list` of `lsst.afw.geom.SkyWcs` 

372 specifying the WCS's of the input ccds to be selected 

373 bboxList : `list` of `lsst.geom.Box2I` 

374 specifying the bounding boxes of the input ccds to be selected 

375 coordList : `list` of `lsst.geom.SpherePoint` 

376 ICRS coordinates specifying boundary of the patch. 

377 visitSummary : `list` of `lsst.afw.table.ExposureCatalog` 

378 containing the PSF shape information for the input ccds to be selected 

379 

380 Returns: 

381 -------- 

382 goodPsf: `list` of `int` 

383 of indices of selected ccds 

384 """ 

385 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList, 

386 coordList=coordList, dataIds=dataIds) 

387 

388 goodPsf = [] 

389 

390 for i, dataId in enumerate(dataIds): 

391 if i not in goodWcs: 

392 continue 

393 if self.isValid(visitSummary, dataId['detector']): 

394 goodPsf.append(i) 

395 

396 return goodPsf 

397 

398 def isValid(self, visitSummary, detectorId): 

399 """Should this ccd be selected based on its PSF shape information. 

400 

401 Parameters 

402 ---------- 

403 visitSummary : `lsst.afw.table.ExposureCatalog` 

404 detectorId : `int` 

405 Detector identifier. 

406 

407 Returns 

408 ------- 

409 valid : `bool` 

410 True if selected. 

411 """ 

412 row = visitSummary.find(detectorId) 

413 if row is None: 

414 # This is not listed, so it must be bad. 

415 self.log.warning("Removing visit %d detector %d because summary stats not available.", 

416 row["visit"], detectorId) 

417 return False 

418 

419 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.) 

420 scatterSize = row["psfStarDeltaSizeScatter"] 

421 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"] 

422 

423 valid = True 

424 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: 

425 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f", 

426 row["visit"], detectorId, medianE, self.config.maxEllipResidual) 

427 valid = False 

428 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: 

429 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f", 

430 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter) 

431 valid = False 

432 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: 

433 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f", 

434 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter) 

435 valid = False 

436 

437 return valid 

438 

439 def isValidGen2(self, srcCatalog, dataId=None): 

440 """Should this ccd be selected based on its PSF shape information. 

441 

442 This routine is only used in Gen2 processing, and can be 

443 removed when Gen2 is retired. 

444 

445 Parameters 

446 ---------- 

447 srcCatalog : `lsst.afw.table.SourceCatalog` 

448 dataId : `dict` of dataId keys, optional. 

449 Used only for logging. Defaults to None. 

450 

451 Returns 

452 ------- 

453 valid : `bool` 

454 True if selected. 

455 """ 

456 mask = srcCatalog[self.config.starSelection] 

457 

458 starXX = srcCatalog[self.config.starShape+'_xx'][mask] 

459 starYY = srcCatalog[self.config.starShape+'_yy'][mask] 

460 starXY = srcCatalog[self.config.starShape+'_xy'][mask] 

461 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask] 

462 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask] 

463 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask] 

464 

465 starSize = np.power(starXX*starYY - starXY**2, 0.25) 

466 starE1 = (starXX - starYY)/(starXX + starYY) 

467 starE2 = 2*starXY/(starXX + starYY) 

468 medianSize = np.median(starSize) 

469 

470 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25) 

471 psfE1 = (psfXX - psfYY)/(psfXX + psfYY) 

472 psfE2 = 2*psfXY/(psfXX + psfYY) 

473 

474 medianE1 = np.abs(np.median(starE1 - psfE1)) 

475 medianE2 = np.abs(np.median(starE2 - psfE2)) 

476 medianE = np.sqrt(medianE1**2 + medianE2**2) 

477 

478 scatterSize = sigmaMad(starSize - psfSize) 

479 scaledScatterSize = scatterSize/medianSize**2 

480 

481 valid = True 

482 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: 

483 self.log.info("Removing visit %s because median e residual too large: %f vs %f", 

484 dataId, medianE, self.config.maxEllipResidual) 

485 valid = False 

486 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: 

487 self.log.info("Removing visit %s because size scatter is too large: %f vs %f", 

488 dataId, scatterSize, self.config.maxSizeScatter) 

489 valid = False 

490 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: 

491 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f", 

492 dataId, scaledScatterSize, self.config.maxScaledSizeScatter) 

493 valid = False 

494 

495 return valid 

496 

497 

498class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass): 

499 """Base configuration for BestSeeingSelectImagesTask. 

500 """ 

501 nImagesMax = pexConfig.RangeField( 

502 dtype=int, 

503 doc="Maximum number of images to select", 

504 default=5, 

505 min=0) 

506 maxPsfFwhm = pexConfig.Field( 

507 dtype=float, 

508 doc="Maximum PSF FWHM (in arcseconds) to select", 

509 default=1.5, 

510 optional=True) 

511 minPsfFwhm = pexConfig.Field( 

512 dtype=float, 

513 doc="Minimum PSF FWHM (in arcseconds) to select", 

514 default=0., 

515 optional=True) 

516 

517 

518class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask): 

519 """Select up to a maximum number of the best-seeing images using their Wcs. 

520 """ 

521 ConfigClass = BestSeeingWcsSelectImageConfig 

522 

523 def runDataRef(self, dataRef, coordList, makeDataRefList=True, 

524 selectDataList=None): 

525 """Select the best-seeing images in the selectDataList that overlap the patch. 

526 

527 This method is the old entry point for the Gen2 commandline tasks and drivers 

528 Will be deprecated in v22. 

529 

530 Parameters 

531 ---------- 

532 dataRef : `lsst.daf.persistence.ButlerDataRef` 

533 Data reference for coadd/tempExp (with tract, patch) 

534 coordList : `list` of `lsst.geom.SpherePoint` 

535 List of ICRS sky coordinates specifying boundary of patch 

536 makeDataRefList : `boolean`, optional 

537 Construct a list of data references? 

538 selectDataList : `list` of `SelectStruct` 

539 List of SelectStruct, to consider for selection 

540 

541 Returns 

542 ------- 

543 result : `lsst.pipe.base.Struct` 

544 Result struct with components: 

545 - ``exposureList``: the selected exposures 

546 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`). 

547 - ``dataRefList``: the optional data references corresponding to 

548 each element of ``exposureList`` 

549 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`). 

550 """ 

551 psfSizes = [] 

552 dataRefList = [] 

553 exposureInfoList = [] 

554 

555 if selectDataList is None: 

556 selectDataList = [] 

557 

558 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList) 

559 

560 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

561 cal = dataRef.get("calexp", immediate=True) 

562 

563 # if min/max PSF values are defined, remove images out of bounds 

564 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds() 

565 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds 

566 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.)) 

567 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm: 

568 continue 

569 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm: 

570 continue 

571 psfSizes.append(sizeFwhm) 

572 dataRefList.append(dataRef) 

573 exposureInfoList.append(exposureInfo) 

574 

575 if len(psfSizes) > self.config.nImagesMax: 

576 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax] 

577 filteredDataRefList = [dataRefList[i] for i in sortedIndices] 

578 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices] 

579 self.log.info("%d images selected with FWHM range of %f--%f arcseconds", 

580 len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]]) 

581 

582 else: 

583 if len(psfSizes) == 0: 

584 self.log.warning("0 images selected.") 

585 else: 

586 self.log.debug("%d images selected with FWHM range of %d--%d arcseconds", 

587 len(psfSizes), psfSizes[0], psfSizes[-1]) 

588 filteredDataRefList = dataRefList 

589 filteredExposureInfoList = exposureInfoList 

590 

591 return pipeBase.Struct( 

592 dataRefList=filteredDataRefList if makeDataRefList else None, 

593 exposureInfoList=filteredExposureInfoList, 

594 ) 

595 

596 

597class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections, 

598 dimensions=("tract", "patch", "skymap", "band", "instrument"), 

599 defaultTemplates={"coaddName": "goodSeeing"}): 

600 skyMap = pipeBase.connectionTypes.Input( 

601 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

602 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

603 storageClass="SkyMap", 

604 dimensions=("skymap",), 

605 ) 

606 visitSummaries = pipeBase.connectionTypes.Input( 

607 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask", 

608 name="visitSummary", 

609 storageClass="ExposureCatalog", 

610 dimensions=("instrument", "visit",), 

611 multiple=True, 

612 deferLoad=True 

613 ) 

614 goodVisits = pipeBase.connectionTypes.Output( 

615 doc="Selected visits to be coadded.", 

616 name="{coaddName}Visits", 

617 storageClass="StructuredDataDict", 

618 dimensions=("instrument", "tract", "patch", "skymap", "band"), 

619 ) 

620 

621 

622class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig, 

623 pipelineConnections=BestSeeingSelectVisitsConnections): 

624 nVisitsMax = pexConfig.RangeField( 

625 dtype=int, 

626 doc="Maximum number of visits to select", 

627 default=12, 

628 min=0 

629 ) 

630 maxPsfFwhm = pexConfig.Field( 

631 dtype=float, 

632 doc="Maximum PSF FWHM (in arcseconds) to select", 

633 default=1.5, 

634 optional=True 

635 ) 

636 minPsfFwhm = pexConfig.Field( 

637 dtype=float, 

638 doc="Minimum PSF FWHM (in arcseconds) to select", 

639 default=0., 

640 optional=True 

641 ) 

642 doConfirmOverlap = pexConfig.Field( 

643 dtype=bool, 

644 doc="Do remove visits that do not actually overlap the patch?", 

645 default=True, 

646 ) 

647 minMJD = pexConfig.Field( 

648 dtype=float, 

649 doc="Minimum visit MJD to select", 

650 default=None, 

651 optional=True 

652 ) 

653 maxMJD = pexConfig.Field( 

654 dtype=float, 

655 doc="Maximum visit MJD to select", 

656 default=None, 

657 optional=True 

658 ) 

659 

660 

661class BestSeeingSelectVisitsTask(pipeBase.PipelineTask): 

662 """Select up to a maximum number of the best-seeing visits 

663 

664 Don't exceed the FWHM range specified by configs min(max)PsfFwhm. 

665 This Task is a port of the Gen2 image-selector used in the AP pipeline: 

666 BestSeeingSelectImagesTask. This Task selects full visits based on the 

667 average PSF of the entire visit. 

668 """ 

669 ConfigClass = BestSeeingSelectVisitsConfig 

670 _DefaultName = 'bestSeeingSelectVisits' 

671 

672 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

673 inputs = butlerQC.get(inputRefs) 

674 quantumDataId = butlerQC.quantum.dataId 

675 outputs = self.run(**inputs, dataId=quantumDataId) 

676 butlerQC.put(outputs, outputRefs) 

677 

678 def run(self, visitSummaries, skyMap, dataId): 

679 """Run task 

680 

681 Parameters: 

682 ----------- 

683 visitSummary : `list` 

684 List of `lsst.pipe.base.connections.DeferredDatasetRef` of 

685 visitSummary tables of type `lsst.afw.table.ExposureCatalog` 

686 skyMap : `lsst.skyMap.SkyMap` 

687 SkyMap for checking visits overlap patch 

688 dataId : `dict` of dataId keys 

689 For retrieving patch info for checking visits overlap patch 

690 

691 Returns 

692 ------- 

693 result : `lsst.pipe.base.Struct` 

694 Result struct with components: 

695 

696 - `goodVisits`: `dict` with selected visit ids as keys, 

697 so that it can be be saved as a StructuredDataDict. 

698 StructuredDataList's are currently limited. 

699 """ 

700 

701 if self.config.doConfirmOverlap: 

702 patchPolygon = self.makePatchPolygon(skyMap, dataId) 

703 

704 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries] 

705 fwhmSizes = [] 

706 visits = [] 

707 for visit, visitSummary in zip(inputVisits, visitSummaries): 

708 # read in one-by-one and only once. There may be hundreds 

709 visitSummary = visitSummary.get() 

710 

711 # mjd is guaranteed to be the same for every detector in the visitSummary. 

712 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD) 

713 

714 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds() 

715 for vs in visitSummary] 

716 # psfSigma is PSF model determinant radius at chip center in pixels 

717 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary]) 

718 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.)) 

719 

720 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm: 

721 continue 

722 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm: 

723 continue 

724 if self.config.minMJD and mjd < self.config.minMJD: 

725 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD) 

726 continue 

727 if self.config.maxMJD and mjd > self.config.maxMJD: 

728 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD) 

729 continue 

730 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon): 

731 continue 

732 

733 fwhmSizes.append(fwhm) 

734 visits.append(visit) 

735 

736 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))] 

737 output = sortedVisits[:self.config.nVisitsMax] 

738 self.log.info("%d images selected with FWHM range of %d--%d arcseconds", 

739 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])]) 

740 

741 # In order to store as a StructuredDataDict, convert list to dict 

742 goodVisits = {key: True for key in output} 

743 return pipeBase.Struct(goodVisits=goodVisits) 

744 

745 def makePatchPolygon(self, skyMap, dataId): 

746 """Return True if sky polygon overlaps visit 

747 

748 Parameters: 

749 ----------- 

750 skyMap : `lsst.afw.table.ExposureCatalog` 

751 Exposure catalog with per-detector geometry 

752 dataId : `dict` of dataId keys 

753 For retrieving patch info 

754 

755 Returns: 

756 -------- 

757 result :` lsst.sphgeom.ConvexPolygon.convexHull` 

758 Polygon of patch's outer bbox 

759 """ 

760 wcs = skyMap[dataId['tract']].getWcs() 

761 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox() 

762 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners()) 

763 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners]) 

764 return result 

765 

766 def doesIntersectPolygon(self, visitSummary, polygon): 

767 """Return True if sky polygon overlaps visit 

768 

769 Parameters: 

770 ----------- 

771 visitSummary : `lsst.afw.table.ExposureCatalog` 

772 Exposure catalog with per-detector geometry 

773 polygon :` lsst.sphgeom.ConvexPolygon.convexHull` 

774 Polygon to check overlap 

775 

776 Returns: 

777 -------- 

778 doesIntersect: `bool` 

779 Does the visit overlap the polygon 

780 """ 

781 doesIntersect = False 

782 for detectorSummary in visitSummary: 

783 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in 

784 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])] 

785 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners) 

786 if detectorPolygon.intersects(polygon): 

787 doesIntersect = True 

788 break 

789 return doesIntersect 

790 

791 

792class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig, 

793 pipelineConnections=BestSeeingSelectVisitsConnections): 

794 qMin = pexConfig.RangeField( 

795 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, " 

796 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. " 

797 "This config should be changed from zero only for exploratory diffIm testing.", 

798 dtype=float, 

799 default=0, 

800 min=0, 

801 max=1, 

802 ) 

803 qMax = pexConfig.RangeField( 

804 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, " 

805 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.", 

806 dtype=float, 

807 default=0.33, 

808 min=0, 

809 max=1, 

810 ) 

811 nVisitsMin = pexConfig.Field( 

812 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits " 

813 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.", 

814 dtype=int, 

815 default=6, 

816 ) 

817 doConfirmOverlap = pexConfig.Field( 

818 dtype=bool, 

819 doc="Do remove visits that do not actually overlap the patch?", 

820 default=True, 

821 ) 

822 minMJD = pexConfig.Field( 

823 dtype=float, 

824 doc="Minimum visit MJD to select", 

825 default=None, 

826 optional=True 

827 ) 

828 maxMJD = pexConfig.Field( 

829 dtype=float, 

830 doc="Maximum visit MJD to select", 

831 default=None, 

832 optional=True 

833 ) 

834 

835 

836class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask): 

837 """Select a quantile of the best-seeing visits 

838 

839 Selects the best (for example, third) full visits based on the average 

840 PSF width in the entire visit. It can also be used for difference imaging 

841 experiments that require templates with the worst seeing visits. 

842 For example, selecting the worst third can be acheived by 

843 changing the config parameters qMin to 0.66 and qMax to 1. 

844 """ 

845 ConfigClass = BestSeeingQuantileSelectVisitsConfig 

846 _DefaultName = 'bestSeeingQuantileSelectVisits' 

847 

848 @utils.inheritDoc(BestSeeingSelectVisitsTask) 

849 def run(self, visitSummaries, skyMap, dataId): 

850 if self.config.doConfirmOverlap: 

851 patchPolygon = self.makePatchPolygon(skyMap, dataId) 

852 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]) 

853 radius = np.empty(len(visits)) 

854 intersects = np.full(len(visits), True) 

855 for i, visitSummary in enumerate(visitSummaries): 

856 # read in one-by-one and only once. There may be hundreds 

857 visitSummary = visitSummary.get() 

858 # psfSigma is PSF model determinant radius at chip center in pixels 

859 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary]) 

860 radius[i] = psfSigma 

861 if self.config.doConfirmOverlap: 

862 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon) 

863 if self.config.minMJD or self.config.maxMJD: 

864 # mjd is guaranteed to be the same for every detector in the visitSummary. 

865 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD) 

866 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True 

867 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True 

868 intersects[i] = intersects[i] and aboveMin and belowMax 

869 

870 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))] 

871 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))), 

872 max(0, len(visits[intersects]) - self.config.nVisitsMin)) 

873 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin) 

874 

875 # In order to store as a StructuredDataDict, convert list to dict 

876 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]} 

877 return pipeBase.Struct(goodVisits=goodVisits)