Coverage for python/lsst/pipe/tasks/selectImages.py: 18%

329 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 21:10 +0000

1# 

2# LSST Data Management System 

3# Copyright 2008, 2009, 2010 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import numpy as np 

23import lsst.sphgeom 

24import lsst.utils as utils 

25import lsst.pex.config as pexConfig 

26import lsst.pex.exceptions as pexExceptions 

27import lsst.geom as geom 

28import lsst.pipe.base as pipeBase 

29from lsst.skymap import BaseSkyMap 

30from lsst.daf.base import DateTime 

31 

32__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask", 

33 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask", 

34 "BestSeeingQuantileSelectVisitsTask"] 

35 

36 

37class DatabaseSelectImagesConfig(pexConfig.Config): 

38 """Base configuration for subclasses of BaseSelectImagesTask that use a database""" 

39 host = pexConfig.Field( 

40 doc="Database server host name", 

41 dtype=str, 

42 ) 

43 port = pexConfig.Field( 

44 doc="Database server port", 

45 dtype=int, 

46 ) 

47 database = pexConfig.Field( 

48 doc="Name of database", 

49 dtype=str, 

50 ) 

51 maxExposures = pexConfig.Field( 

52 doc="maximum exposures to select; intended for debugging; ignored if None", 

53 dtype=int, 

54 optional=True, 

55 ) 

56 

57 

58class BaseExposureInfo(pipeBase.Struct): 

59 """Data about a selected exposure 

60 """ 

61 

62 def __init__(self, dataId, coordList): 

63 """Create exposure information that can be used to generate data references 

64 

65 The object has the following fields: 

66 - dataId: data ID of exposure (a dict) 

67 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

68 plus any others items that are desired 

69 """ 

70 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList) 

71 

72 

73class BaseSelectImagesTask(pipeBase.Task): 

74 """Base task for selecting images suitable for coaddition 

75 """ 

76 ConfigClass = pexConfig.Config 

77 _DefaultName = "selectImages" 

78 

79 @pipeBase.timeMethod 

80 def run(self, coordList): 

81 """Select images suitable for coaddition in a particular region 

82 

83 @param[in] coordList: list of coordinates defining region of interest; if None then select all images 

84 subclasses may add additional keyword arguments, as required 

85 

86 @return a pipeBase Struct containing: 

87 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo), 

88 which have at least the following fields: 

89 - dataId: data ID dictionary 

90 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint) 

91 """ 

92 raise NotImplementedError() 

93 

94 def _runArgDictFromDataId(self, dataId): 

95 """Extract keyword arguments for run (other than coordList) from a data ID 

96 

97 @return keyword arguments for run (other than coordList), as a dict 

98 """ 

99 raise NotImplementedError() 

100 

101 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

102 """Run based on a data reference 

103 

104 This delegates to run() and _runArgDictFromDataId() to do the actual 

105 selection. In the event that the selectDataList is non-empty, this will 

106 be used to further restrict the selection, providing the user with 

107 additional control over the selection. 

108 

109 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass 

110 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky 

111 @param[in] makeDataRefList: if True, return dataRefList 

112 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection 

113 @return a pipeBase Struct containing: 

114 - exposureInfoList: a list of objects derived from ExposureInfo 

115 - dataRefList: a list of data references (None if makeDataRefList False) 

116 """ 

117 runArgDict = self._runArgDictFromDataId(dataRef.dataId) 

118 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList 

119 

120 if len(selectDataList) > 0 and len(exposureInfoList) > 0: 

121 # Restrict the exposure selection further 

122 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList) 

123 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys) 

124 inValues = set(inValues) 

125 newExposureInfoList = [] 

126 for info, ccdVal in zip(exposureInfoList, ccdValues): 

127 if ccdVal in inValues: 

128 newExposureInfoList.append(info) 

129 else: 

130 self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId) 

131 exposureInfoList = newExposureInfoList 

132 

133 if makeDataRefList: 

134 butler = dataRef.butlerSubset.butler 

135 dataRefList = [butler.dataRef(datasetType="calexp", 

136 dataId=expInfo.dataId, 

137 ) for expInfo in exposureInfoList] 

138 else: 

139 dataRefList = None 

140 

141 return pipeBase.Struct( 

142 dataRefList=dataRefList, 

143 exposureInfoList=exposureInfoList, 

144 ) 

145 

146 

147def _extractKeyValue(dataList, keys=None): 

148 """Extract the keys and values from a list of dataIds 

149 

150 The input dataList is a list of objects that have 'dataId' members. 

151 This allows it to be used for both a list of data references and a 

152 list of ExposureInfo 

153 """ 

154 assert len(dataList) > 0 

155 if keys is None: 

156 keys = sorted(dataList[0].dataId.keys()) 

157 keySet = set(keys) 

158 values = list() 

159 for data in dataList: 

160 thisKeys = set(data.dataId.keys()) 

161 if thisKeys != keySet: 

162 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys)) 

163 values.append(tuple(data.dataId[k] for k in keys)) 

164 return keys, values 

165 

166 

167class SelectStruct(pipeBase.Struct): 

168 """A container for data to be passed to the WcsSelectImagesTask""" 

169 

170 def __init__(self, dataRef, wcs, bbox): 

171 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox) 

172 

173 

174class WcsSelectImagesTask(BaseSelectImagesTask): 

175 """Select images using their Wcs 

176 

177 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define 

178 polygons on the celestial sphere, and test the polygon of the 

179 patch for overlap with the polygon of the image. 

180 

181 We use "convexHull" instead of generating a ConvexPolygon 

182 directly because the standard for the inputs to ConvexPolygon 

183 are pretty high and we don't want to be responsible for reaching them. 

184 """ 

185 

186 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

187 """Select images in the selectDataList that overlap the patch 

188 

189 This method is the old entry point for the Gen2 commandline tasks and drivers 

190 Will be deprecated in v22. 

191 

192 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

193 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

194 @param makeDataRefList: Construct a list of data references? 

195 @param selectDataList: List of SelectStruct, to consider for selection 

196 """ 

197 dataRefList = [] 

198 exposureInfoList = [] 

199 

200 patchVertices = [coord.getVector() for coord in coordList] 

201 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices) 

202 

203 for data in selectDataList: 

204 dataRef = data.dataRef 

205 imageWcs = data.wcs 

206 imageBox = data.bbox 

207 

208 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None) 

209 if imageCorners: 

210 dataRefList.append(dataRef) 

211 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners)) 

212 

213 return pipeBase.Struct( 

214 dataRefList=dataRefList if makeDataRefList else None, 

215 exposureInfoList=exposureInfoList, 

216 ) 

217 

218 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs): 

219 """Return indices of provided lists that meet the selection criteria 

220 

221 Parameters: 

222 ----------- 

223 wcsList : `list` of `lsst.afw.geom.SkyWcs` 

224 specifying the WCS's of the input ccds to be selected 

225 bboxList : `list` of `lsst.geom.Box2I` 

226 specifying the bounding boxes of the input ccds to be selected 

227 coordList : `list` of `lsst.geom.SpherePoint` 

228 ICRS coordinates specifying boundary of the patch. 

229 

230 Returns: 

231 -------- 

232 result: `list` of `int` 

233 of indices of selected ccds 

234 """ 

235 if dataIds is None: 

236 dataIds = [None] * len(wcsList) 

237 patchVertices = [coord.getVector() for coord in coordList] 

238 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices) 

239 result = [] 

240 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)): 

241 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId) 

242 if imageCorners: 

243 result.append(i) 

244 return result 

245 

246 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None): 

247 "Return corners or None if bad" 

248 try: 

249 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()] 

250 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e: 

251 # Protecting ourselves from awful Wcs solutions in input images 

252 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e) 

253 return 

254 

255 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners]) 

256 if imagePoly is None: 

257 self.log.debug("Unable to create polygon from image %s: deselecting", dataId) 

258 return 

259 

260 if patchPoly.intersects(imagePoly): 

261 # "intersects" also covers "contains" or "is contained by" 

262 self.log.info("Selecting calexp %s", dataId) 

263 return imageCorners 

264 

265 

266def sigmaMad(array): 

267 "Return median absolute deviation scaled to normally distributed data" 

268 return 1.4826*np.median(np.abs(array - np.median(array))) 

269 

270 

271class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections, 

272 dimensions=("tract", "patch", "skymap", "instrument", "visit"), 

273 defaultTemplates={"coaddName": "deep"}): 

274 pass 

275 

276 

277class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig, 

278 pipelineConnections=PsfWcsSelectImagesConnections): 

279 maxEllipResidual = pexConfig.Field( 

280 doc="Maximum median ellipticity residual", 

281 dtype=float, 

282 default=0.007, 

283 optional=True, 

284 ) 

285 maxSizeScatter = pexConfig.Field( 

286 doc="Maximum scatter in the size residuals", 

287 dtype=float, 

288 optional=True, 

289 ) 

290 maxScaledSizeScatter = pexConfig.Field( 

291 doc="Maximum scatter in the size residuals, scaled by the median size", 

292 dtype=float, 

293 default=0.009, 

294 optional=True, 

295 ) 

296 starSelection = pexConfig.Field( 

297 doc="select star with this field", 

298 dtype=str, 

299 default='calib_psf_used', 

300 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

301 'will be removed after v24.') 

302 ) 

303 starShape = pexConfig.Field( 

304 doc="name of star shape", 

305 dtype=str, 

306 default='base_SdssShape', 

307 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

308 'will be removed after v24.') 

309 ) 

310 psfShape = pexConfig.Field( 

311 doc="name of psf shape", 

312 dtype=str, 

313 default='base_SdssShape_psf', 

314 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and ' 

315 'will be removed after v24.') 

316 ) 

317 doLegacyStarSelectionComputation = pexConfig.Field( 

318 doc="Perform the legacy star selection computations (for backwards compatibility)", 

319 dtype=bool, 

320 default=False, 

321 deprecated=("This field is here for backwards compatibility and will be " 

322 "removed after v24.") 

323 ) 

324 

325 

326class PsfWcsSelectImagesTask(WcsSelectImagesTask): 

327 """Select images using their Wcs and cuts on the PSF properties 

328 

329 The PSF quality criteria are based on the size and ellipticity residuals from the 

330 adaptive second moments of the star and the PSF. 

331 

332 The criteria are: 

333 - the median of the ellipticty residuals 

334 - the robust scatter of the size residuals (using the median absolute deviation) 

335 - the robust scatter of the size residuals scaled by the square of 

336 the median size 

337 """ 

338 

339 ConfigClass = PsfWcsSelectImagesConfig 

340 _DefaultName = "PsfWcsSelectImages" 

341 

342 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]): 

343 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera. 

344 

345 This method is the old entry point for the Gen2 commandline tasks and drivers 

346 Will be deprecated in v22. 

347 

348 @param dataRef: Data reference for coadd/tempExp (with tract, patch) 

349 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch 

350 @param makeDataRefList: Construct a list of data references? 

351 @param selectDataList: List of SelectStruct, to consider for selection 

352 """ 

353 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList, 

354 selectDataList) 

355 

356 dataRefList = [] 

357 exposureInfoList = [] 

358 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

359 butler = dataRef.butlerSubset.butler 

360 srcCatalog = butler.get('src', dataRef.dataId) 

361 valid = self.isValidLegacy(srcCatalog, dataRef.dataId) 

362 if valid is False: 

363 continue 

364 

365 dataRefList.append(dataRef) 

366 exposureInfoList.append(exposureInfo) 

367 

368 return pipeBase.Struct( 

369 dataRefList=dataRefList, 

370 exposureInfoList=exposureInfoList, 

371 ) 

372 

373 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, srcList=None, **kwargs): 

374 """Return indices of provided lists that meet the selection criteria 

375 

376 Parameters: 

377 ----------- 

378 wcsList : `list` of `lsst.afw.geom.SkyWcs` 

379 specifying the WCS's of the input ccds to be selected 

380 bboxList : `list` of `lsst.geom.Box2I` 

381 specifying the bounding boxes of the input ccds to be selected 

382 coordList : `list` of `lsst.geom.SpherePoint` 

383 ICRS coordinates specifying boundary of the patch. 

384 visitSummary : `list` of `lsst.afw.table.ExposureCatalog` 

385 containing the PSF shape information for the input ccds to be selected. 

386 srcList : `list` of `lsst.afw.table.SourceCatalog`, optional 

387 containing the PSF shape information for the input ccds to be selected. 

388 This is only used if ``config.doLegacyStarSelectionComputation`` is 

389 True. 

390 

391 Returns: 

392 -------- 

393 goodPsf: `list` of `int` 

394 of indices of selected ccds 

395 """ 

396 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList, 

397 coordList=coordList, dataIds=dataIds) 

398 

399 goodPsf = [] 

400 

401 if not self.config.doLegacyStarSelectionComputation: 

402 # Check for old inputs, and give a helpful error message if so. 

403 if 'nPsfStar' not in visitSummary[0].schema.getNames(): 

404 raise RuntimeError("Old calexps detected. " 

405 "Please set config.doLegacyStarSelectionComputation=True for " 

406 "backwards compatibility.") 

407 

408 for i, dataId in enumerate(dataIds): 

409 if i not in goodWcs: 

410 continue 

411 if self.isValid(visitSummary, dataId["detector"]): 

412 goodPsf.append(i) 

413 else: 

414 if dataIds is None: 

415 dataIds = [None] * len(srcList) 

416 for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)): 

417 if i not in goodWcs: 

418 continue 

419 if self.isValidLegacy(srcCatalog, dataId): 

420 goodPsf.append(i) 

421 

422 return goodPsf 

423 

424 def isValid(self, visitSummary, detectorId): 

425 """Should this ccd be selected based on its PSF shape information. 

426 

427 Parameters 

428 ---------- 

429 visitSummary : `lsst.afw.table.ExposureCatalog` 

430 detectorId : `int` 

431 Detector identifier. 

432 

433 Returns 

434 ------- 

435 valid : `bool` 

436 True if selected. 

437 """ 

438 row = visitSummary.find(detectorId) 

439 if row is None: 

440 # This is not listed, so it must be bad. 

441 self.log.warning("Removing detector %d because summary stats not available.", detectorId) 

442 return False 

443 

444 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.) 

445 scatterSize = row["psfStarDeltaSizeScatter"] 

446 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"] 

447 

448 valid = True 

449 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: 

450 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f", 

451 row["visit"], detectorId, medianE, self.config.maxEllipResidual) 

452 valid = False 

453 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: 

454 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f", 

455 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter) 

456 valid = False 

457 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: 

458 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f", 

459 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter) 

460 valid = False 

461 

462 return valid 

463 

464 def isValidLegacy(self, srcCatalog, dataId=None): 

465 """Should this ccd be selected based on its PSF shape information. 

466 

467 This routine is only used in legacy processing (gen2 and 

468 backwards compatible old calexps) and should be removed after v24. 

469 

470 Parameters 

471 ---------- 

472 srcCatalog : `lsst.afw.table.SourceCatalog` 

473 dataId : `dict` of dataId keys, optional. 

474 Used only for logging. Defaults to None. 

475 

476 Returns 

477 ------- 

478 valid : `bool` 

479 True if selected. 

480 """ 

481 mask = srcCatalog[self.config.starSelection] 

482 

483 starXX = srcCatalog[self.config.starShape+'_xx'][mask] 

484 starYY = srcCatalog[self.config.starShape+'_yy'][mask] 

485 starXY = srcCatalog[self.config.starShape+'_xy'][mask] 

486 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask] 

487 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask] 

488 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask] 

489 

490 starSize = np.power(starXX*starYY - starXY**2, 0.25) 

491 starE1 = (starXX - starYY)/(starXX + starYY) 

492 starE2 = 2*starXY/(starXX + starYY) 

493 medianSize = np.median(starSize) 

494 

495 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25) 

496 psfE1 = (psfXX - psfYY)/(psfXX + psfYY) 

497 psfE2 = 2*psfXY/(psfXX + psfYY) 

498 

499 medianE1 = np.abs(np.median(starE1 - psfE1)) 

500 medianE2 = np.abs(np.median(starE2 - psfE2)) 

501 medianE = np.sqrt(medianE1**2 + medianE2**2) 

502 

503 scatterSize = sigmaMad(starSize - psfSize) 

504 scaledScatterSize = scatterSize/medianSize**2 

505 

506 valid = True 

507 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual: 

508 self.log.info("Removing visit %s because median e residual too large: %f vs %f", 

509 dataId, medianE, self.config.maxEllipResidual) 

510 valid = False 

511 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter: 

512 self.log.info("Removing visit %s because size scatter is too large: %f vs %f", 

513 dataId, scatterSize, self.config.maxSizeScatter) 

514 valid = False 

515 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter: 

516 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f", 

517 dataId, scaledScatterSize, self.config.maxScaledSizeScatter) 

518 valid = False 

519 

520 return valid 

521 

522 

523class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass): 

524 """Base configuration for BestSeeingSelectImagesTask. 

525 """ 

526 nImagesMax = pexConfig.RangeField( 

527 dtype=int, 

528 doc="Maximum number of images to select", 

529 default=5, 

530 min=0) 

531 maxPsfFwhm = pexConfig.Field( 

532 dtype=float, 

533 doc="Maximum PSF FWHM (in arcseconds) to select", 

534 default=1.5, 

535 optional=True) 

536 minPsfFwhm = pexConfig.Field( 

537 dtype=float, 

538 doc="Minimum PSF FWHM (in arcseconds) to select", 

539 default=0., 

540 optional=True) 

541 

542 

543class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask): 

544 """Select up to a maximum number of the best-seeing images using their Wcs. 

545 """ 

546 ConfigClass = BestSeeingWcsSelectImageConfig 

547 

548 def runDataRef(self, dataRef, coordList, makeDataRefList=True, 

549 selectDataList=None): 

550 """Select the best-seeing images in the selectDataList that overlap the patch. 

551 

552 This method is the old entry point for the Gen2 commandline tasks and drivers 

553 Will be deprecated in v22. 

554 

555 Parameters 

556 ---------- 

557 dataRef : `lsst.daf.persistence.ButlerDataRef` 

558 Data reference for coadd/tempExp (with tract, patch) 

559 coordList : `list` of `lsst.geom.SpherePoint` 

560 List of ICRS sky coordinates specifying boundary of patch 

561 makeDataRefList : `boolean`, optional 

562 Construct a list of data references? 

563 selectDataList : `list` of `SelectStruct` 

564 List of SelectStruct, to consider for selection 

565 

566 Returns 

567 ------- 

568 result : `lsst.pipe.base.Struct` 

569 Result struct with components: 

570 - ``exposureList``: the selected exposures 

571 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`). 

572 - ``dataRefList``: the optional data references corresponding to 

573 each element of ``exposureList`` 

574 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`). 

575 """ 

576 psfSizes = [] 

577 dataRefList = [] 

578 exposureInfoList = [] 

579 

580 if selectDataList is None: 

581 selectDataList = [] 

582 

583 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList) 

584 

585 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList): 

586 cal = dataRef.get("calexp", immediate=True) 

587 

588 # if min/max PSF values are defined, remove images out of bounds 

589 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds() 

590 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds 

591 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.)) 

592 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm: 

593 continue 

594 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm: 

595 continue 

596 psfSizes.append(sizeFwhm) 

597 dataRefList.append(dataRef) 

598 exposureInfoList.append(exposureInfo) 

599 

600 if len(psfSizes) > self.config.nImagesMax: 

601 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax] 

602 filteredDataRefList = [dataRefList[i] for i in sortedIndices] 

603 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices] 

604 self.log.info("%d images selected with FWHM range of %f--%f arcseconds", 

605 len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]]) 

606 

607 else: 

608 if len(psfSizes) == 0: 

609 self.log.warning("0 images selected.") 

610 else: 

611 self.log.debug("%d images selected with FWHM range of %d--%d arcseconds", 

612 len(psfSizes), psfSizes[0], psfSizes[-1]) 

613 filteredDataRefList = dataRefList 

614 filteredExposureInfoList = exposureInfoList 

615 

616 return pipeBase.Struct( 

617 dataRefList=filteredDataRefList if makeDataRefList else None, 

618 exposureInfoList=filteredExposureInfoList, 

619 ) 

620 

621 

622class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections, 

623 dimensions=("tract", "patch", "skymap", "band", "instrument"), 

624 defaultTemplates={"coaddName": "goodSeeing"}): 

625 skyMap = pipeBase.connectionTypes.Input( 

626 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures", 

627 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME, 

628 storageClass="SkyMap", 

629 dimensions=("skymap",), 

630 ) 

631 visitSummaries = pipeBase.connectionTypes.Input( 

632 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask", 

633 name="visitSummary", 

634 storageClass="ExposureCatalog", 

635 dimensions=("instrument", "visit",), 

636 multiple=True, 

637 deferLoad=True 

638 ) 

639 goodVisits = pipeBase.connectionTypes.Output( 

640 doc="Selected visits to be coadded.", 

641 name="{coaddName}Visits", 

642 storageClass="StructuredDataDict", 

643 dimensions=("instrument", "tract", "patch", "skymap", "band"), 

644 ) 

645 

646 

647class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig, 

648 pipelineConnections=BestSeeingSelectVisitsConnections): 

649 nVisitsMax = pexConfig.RangeField( 

650 dtype=int, 

651 doc="Maximum number of visits to select", 

652 default=12, 

653 min=0 

654 ) 

655 maxPsfFwhm = pexConfig.Field( 

656 dtype=float, 

657 doc="Maximum PSF FWHM (in arcseconds) to select", 

658 default=1.5, 

659 optional=True 

660 ) 

661 minPsfFwhm = pexConfig.Field( 

662 dtype=float, 

663 doc="Minimum PSF FWHM (in arcseconds) to select", 

664 default=0., 

665 optional=True 

666 ) 

667 doConfirmOverlap = pexConfig.Field( 

668 dtype=bool, 

669 doc="Do remove visits that do not actually overlap the patch?", 

670 default=True, 

671 ) 

672 minMJD = pexConfig.Field( 

673 dtype=float, 

674 doc="Minimum visit MJD to select", 

675 default=None, 

676 optional=True 

677 ) 

678 maxMJD = pexConfig.Field( 

679 dtype=float, 

680 doc="Maximum visit MJD to select", 

681 default=None, 

682 optional=True 

683 ) 

684 

685 

686class BestSeeingSelectVisitsTask(pipeBase.PipelineTask): 

687 """Select up to a maximum number of the best-seeing visits 

688 

689 Don't exceed the FWHM range specified by configs min(max)PsfFwhm. 

690 This Task is a port of the Gen2 image-selector used in the AP pipeline: 

691 BestSeeingSelectImagesTask. This Task selects full visits based on the 

692 average PSF of the entire visit. 

693 """ 

694 ConfigClass = BestSeeingSelectVisitsConfig 

695 _DefaultName = 'bestSeeingSelectVisits' 

696 

697 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

698 inputs = butlerQC.get(inputRefs) 

699 quantumDataId = butlerQC.quantum.dataId 

700 outputs = self.run(**inputs, dataId=quantumDataId) 

701 butlerQC.put(outputs, outputRefs) 

702 

703 def run(self, visitSummaries, skyMap, dataId): 

704 """Run task 

705 

706 Parameters: 

707 ----------- 

708 visitSummary : `list` 

709 List of `lsst.pipe.base.connections.DeferredDatasetRef` of 

710 visitSummary tables of type `lsst.afw.table.ExposureCatalog` 

711 skyMap : `lsst.skyMap.SkyMap` 

712 SkyMap for checking visits overlap patch 

713 dataId : `dict` of dataId keys 

714 For retrieving patch info for checking visits overlap patch 

715 

716 Returns 

717 ------- 

718 result : `lsst.pipe.base.Struct` 

719 Result struct with components: 

720 

721 - `goodVisits`: `dict` with selected visit ids as keys, 

722 so that it can be be saved as a StructuredDataDict. 

723 StructuredDataList's are currently limited. 

724 """ 

725 

726 if self.config.doConfirmOverlap: 

727 patchPolygon = self.makePatchPolygon(skyMap, dataId) 

728 

729 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries] 

730 fwhmSizes = [] 

731 visits = [] 

732 for visit, visitSummary in zip(inputVisits, visitSummaries): 

733 # read in one-by-one and only once. There may be hundreds 

734 visitSummary = visitSummary.get() 

735 

736 # mjd is guaranteed to be the same for every detector in the visitSummary. 

737 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD) 

738 

739 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds() 

740 for vs in visitSummary] 

741 # psfSigma is PSF model determinant radius at chip center in pixels 

742 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary]) 

743 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.)) 

744 

745 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm: 

746 continue 

747 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm: 

748 continue 

749 if self.config.minMJD and mjd < self.config.minMJD: 

750 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD) 

751 continue 

752 if self.config.maxMJD and mjd > self.config.maxMJD: 

753 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD) 

754 continue 

755 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon): 

756 continue 

757 

758 fwhmSizes.append(fwhm) 

759 visits.append(visit) 

760 

761 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))] 

762 output = sortedVisits[:self.config.nVisitsMax] 

763 self.log.info("%d images selected with FWHM range of %d--%d arcseconds", 

764 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])]) 

765 

766 # In order to store as a StructuredDataDict, convert list to dict 

767 goodVisits = {key: True for key in output} 

768 return pipeBase.Struct(goodVisits=goodVisits) 

769 

770 def makePatchPolygon(self, skyMap, dataId): 

771 """Return True if sky polygon overlaps visit 

772 

773 Parameters: 

774 ----------- 

775 skyMap : `lsst.afw.table.ExposureCatalog` 

776 Exposure catalog with per-detector geometry 

777 dataId : `dict` of dataId keys 

778 For retrieving patch info 

779 

780 Returns: 

781 -------- 

782 result :` lsst.sphgeom.ConvexPolygon.convexHull` 

783 Polygon of patch's outer bbox 

784 """ 

785 wcs = skyMap[dataId['tract']].getWcs() 

786 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox() 

787 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners()) 

788 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners]) 

789 return result 

790 

791 def doesIntersectPolygon(self, visitSummary, polygon): 

792 """Return True if sky polygon overlaps visit 

793 

794 Parameters: 

795 ----------- 

796 visitSummary : `lsst.afw.table.ExposureCatalog` 

797 Exposure catalog with per-detector geometry 

798 polygon :` lsst.sphgeom.ConvexPolygon.convexHull` 

799 Polygon to check overlap 

800 

801 Returns: 

802 -------- 

803 doesIntersect: `bool` 

804 Does the visit overlap the polygon 

805 """ 

806 doesIntersect = False 

807 for detectorSummary in visitSummary: 

808 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in 

809 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])] 

810 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners) 

811 if detectorPolygon.intersects(polygon): 

812 doesIntersect = True 

813 break 

814 return doesIntersect 

815 

816 

817class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig, 

818 pipelineConnections=BestSeeingSelectVisitsConnections): 

819 qMin = pexConfig.RangeField( 

820 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, " 

821 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. " 

822 "This config should be changed from zero only for exploratory diffIm testing.", 

823 dtype=float, 

824 default=0, 

825 min=0, 

826 max=1, 

827 ) 

828 qMax = pexConfig.RangeField( 

829 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, " 

830 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.", 

831 dtype=float, 

832 default=0.33, 

833 min=0, 

834 max=1, 

835 ) 

836 nVisitsMin = pexConfig.Field( 

837 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits " 

838 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.", 

839 dtype=int, 

840 default=6, 

841 ) 

842 doConfirmOverlap = pexConfig.Field( 

843 dtype=bool, 

844 doc="Do remove visits that do not actually overlap the patch?", 

845 default=True, 

846 ) 

847 minMJD = pexConfig.Field( 

848 dtype=float, 

849 doc="Minimum visit MJD to select", 

850 default=None, 

851 optional=True 

852 ) 

853 maxMJD = pexConfig.Field( 

854 dtype=float, 

855 doc="Maximum visit MJD to select", 

856 default=None, 

857 optional=True 

858 ) 

859 

860 

861class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask): 

862 """Select a quantile of the best-seeing visits 

863 

864 Selects the best (for example, third) full visits based on the average 

865 PSF width in the entire visit. It can also be used for difference imaging 

866 experiments that require templates with the worst seeing visits. 

867 For example, selecting the worst third can be acheived by 

868 changing the config parameters qMin to 0.66 and qMax to 1. 

869 """ 

870 ConfigClass = BestSeeingQuantileSelectVisitsConfig 

871 _DefaultName = 'bestSeeingQuantileSelectVisits' 

872 

873 @utils.inheritDoc(BestSeeingSelectVisitsTask) 

874 def run(self, visitSummaries, skyMap, dataId): 

875 if self.config.doConfirmOverlap: 

876 patchPolygon = self.makePatchPolygon(skyMap, dataId) 

877 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]) 

878 radius = np.empty(len(visits)) 

879 intersects = np.full(len(visits), True) 

880 for i, visitSummary in enumerate(visitSummaries): 

881 # read in one-by-one and only once. There may be hundreds 

882 visitSummary = visitSummary.get() 

883 # psfSigma is PSF model determinant radius at chip center in pixels 

884 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary]) 

885 radius[i] = psfSigma 

886 if self.config.doConfirmOverlap: 

887 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon) 

888 if self.config.minMJD or self.config.maxMJD: 

889 # mjd is guaranteed to be the same for every detector in the visitSummary. 

890 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD) 

891 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True 

892 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True 

893 intersects[i] = intersects[i] and aboveMin and belowMax 

894 

895 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))] 

896 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))), 

897 max(0, len(visits[intersects]) - self.config.nVisitsMin)) 

898 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin) 

899 

900 # In order to store as a StructuredDataDict, convert list to dict 

901 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]} 

902 return pipeBase.Struct(goodVisits=goodVisits)