lsst.pipe.tasks g97f4eaa8b5+110e61d94c
selectImages.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.utils as utils
25import lsst.pex.config as pexConfig
26import lsst.pex.exceptions as pexExceptions
27import lsst.geom as geom
28import lsst.pipe.base as pipeBase
29from lsst.skymap import BaseSkyMap
30from lsst.daf.base import DateTime
31from lsst.utils.timer import timeMethod
32
33__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
34 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask",
35 "BestSeeingQuantileSelectVisitsTask"]
36
37
38class DatabaseSelectImagesConfig(pexConfig.Config):
39 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
40 host = pexConfig.Field(
41 doc="Database server host name",
42 dtype=str,
43 )
44 port = pexConfig.Field(
45 doc="Database server port",
46 dtype=int,
47 )
48 database = pexConfig.Field(
49 doc="Name of database",
50 dtype=str,
51 )
52 maxExposures = pexConfig.Field(
53 doc="maximum exposures to select; intended for debugging; ignored if None",
54 dtype=int,
55 optional=True,
56 )
57
58
59class BaseExposureInfo(pipeBase.Struct):
60 """Data about a selected exposure
61 """
62
63 def __init__(self, dataId, coordList):
64 """Create exposure information that can be used to generate data references
65
66 The object has the following fields:
67 - dataId: data ID of exposure (a dict)
68 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
69 plus any others items that are desired
70 """
71 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
72
73
74class BaseSelectImagesTask(pipeBase.Task):
75 """Base task for selecting images suitable for coaddition
76 """
77 ConfigClass = pexConfig.Config
78 _DefaultName = "selectImages"
79
80 @timeMethod
81 def run(self, coordList):
82 """Select images suitable for coaddition in a particular region
83
84 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
85 subclasses may add additional keyword arguments, as required
86
87 @return a pipeBase Struct containing:
88 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
89 which have at least the following fields:
90 - dataId: data ID dictionary
91 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
92 """
93 raise NotImplementedError()
94
95 def _runArgDictFromDataId(self, dataId):
96 """Extract keyword arguments for run (other than coordList) from a data ID
97
98 @return keyword arguments for run (other than coordList), as a dict
99 """
100 raise NotImplementedError()
101
102 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
103 """Run based on a data reference
104
105 This delegates to run() and _runArgDictFromDataId() to do the actual
106 selection. In the event that the selectDataList is non-empty, this will
107 be used to further restrict the selection, providing the user with
108 additional control over the selection.
109
110 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
111 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
112 @param[in] makeDataRefList: if True, return dataRefList
113 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
114 @return a pipeBase Struct containing:
115 - exposureInfoList: a list of objects derived from ExposureInfo
116 - dataRefList: a list of data references (None if makeDataRefList False)
117 """
118 runArgDict = self._runArgDictFromDataId_runArgDictFromDataId(dataRef.dataId)
119 exposureInfoList = self.runrun(coordList, **runArgDict).exposureInfoList
120
121 if len(selectDataList) > 0 and len(exposureInfoList) > 0:
122 # Restrict the exposure selection further
123 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
124 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
125 inValues = set(inValues)
126 newExposureInfoList = []
127 for info, ccdVal in zip(exposureInfoList, ccdValues):
128 if ccdVal in inValues:
129 newExposureInfoList.append(info)
130 else:
131 self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId)
132 exposureInfoList = newExposureInfoList
133
134 if makeDataRefList:
135 butler = dataRef.butlerSubset.butler
136 dataRefList = [butler.dataRef(datasetType="calexp",
137 dataId=expInfo.dataId,
138 ) for expInfo in exposureInfoList]
139 else:
140 dataRefList = None
141
142 return pipeBase.Struct(
143 dataRefList=dataRefList,
144 exposureInfoList=exposureInfoList,
145 )
146
147
148def _extractKeyValue(dataList, keys=None):
149 """Extract the keys and values from a list of dataIds
150
151 The input dataList is a list of objects that have 'dataId' members.
152 This allows it to be used for both a list of data references and a
153 list of ExposureInfo
154 """
155 assert len(dataList) > 0
156 if keys is None:
157 keys = sorted(dataList[0].dataId.keys())
158 keySet = set(keys)
159 values = list()
160 for data in dataList:
161 thisKeys = set(data.dataId.keys())
162 if thisKeys != keySet:
163 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
164 values.append(tuple(data.dataId[k] for k in keys))
165 return keys, values
166
167
168class SelectStruct(pipeBase.Struct):
169 """A container for data to be passed to the WcsSelectImagesTask"""
170
171 def __init__(self, dataRef, wcs, bbox):
172 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
173
174
176 """Select images using their Wcs
177
178 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
179 polygons on the celestial sphere, and test the polygon of the
180 patch for overlap with the polygon of the image.
181
182 We use "convexHull" instead of generating a ConvexPolygon
183 directly because the standard for the inputs to ConvexPolygon
184 are pretty high and we don't want to be responsible for reaching them.
185 """
186
187 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
188 """Select images in the selectDataList that overlap the patch
189
190 This method is the old entry point for the Gen2 commandline tasks and drivers
191 Will be deprecated in v22.
192
193 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
194 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
195 @param makeDataRefList: Construct a list of data references?
196 @param selectDataList: List of SelectStruct, to consider for selection
197 """
198 dataRefList = []
199 exposureInfoList = []
200
201 patchVertices = [coord.getVector() for coord in coordList]
202 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
203
204 for data in selectDataList:
205 dataRef = data.dataRef
206 imageWcs = data.wcs
207 imageBox = data.bbox
208
209 imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
210 if imageCorners:
211 dataRefList.append(dataRef)
212 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
213
214 return pipeBase.Struct(
215 dataRefList=dataRefList if makeDataRefList else None,
216 exposureInfoList=exposureInfoList,
217 )
218
219 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
220 """Return indices of provided lists that meet the selection criteria
221
222 Parameters:
223 -----------
224 wcsList : `list` of `lsst.afw.geom.SkyWcs`
225 specifying the WCS's of the input ccds to be selected
226 bboxList : `list` of `lsst.geom.Box2I`
227 specifying the bounding boxes of the input ccds to be selected
228 coordList : `list` of `lsst.geom.SpherePoint`
229 ICRS coordinates specifying boundary of the patch.
230
231 Returns:
232 --------
233 result: `list` of `int`
234 of indices of selected ccds
235 """
236 if dataIds is None:
237 dataIds = [None] * len(wcsList)
238 patchVertices = [coord.getVector() for coord in coordList]
239 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
240 result = []
241 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
242 imageCorners = self.getValidImageCornersgetValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
243 if imageCorners:
244 result.append(i)
245 return result
246
247 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
248 "Return corners or None if bad"
249 try:
250 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
251 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
252 # Protecting ourselves from awful Wcs solutions in input images
253 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
254 return
255
256 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
257 if imagePoly is None:
258 self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
259 return
260
261 if patchPoly.intersects(imagePoly):
262 # "intersects" also covers "contains" or "is contained by"
263 self.log.info("Selecting calexp %s", dataId)
264 return imageCorners
265
266
267def sigmaMad(array):
268 "Return median absolute deviation scaled to normally distributed data"
269 return 1.4826*np.median(np.abs(array - np.median(array)))
270
271
272class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
273 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
274 defaultTemplates={"coaddName": "deep"}):
275 pass
276
277
278class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
279 pipelineConnections=PsfWcsSelectImagesConnections):
280 maxEllipResidual = pexConfig.Field(
281 doc="Maximum median ellipticity residual",
282 dtype=float,
283 default=0.007,
284 optional=True,
285 )
286 maxSizeScatter = pexConfig.Field(
287 doc="Maximum scatter in the size residuals",
288 dtype=float,
289 optional=True,
290 )
291 maxScaledSizeScatter = pexConfig.Field(
292 doc="Maximum scatter in the size residuals, scaled by the median size",
293 dtype=float,
294 default=0.009,
295 optional=True,
296 )
297 starSelection = pexConfig.Field(
298 doc="select star with this field",
299 dtype=str,
300 default='calib_psf_used',
301 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
302 'will be removed after v24.')
303 )
304 starShape = pexConfig.Field(
305 doc="name of star shape",
306 dtype=str,
307 default='base_SdssShape',
308 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
309 'will be removed after v24.')
310 )
311 psfShape = pexConfig.Field(
312 doc="name of psf shape",
313 dtype=str,
314 default='base_SdssShape_psf',
315 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
316 'will be removed after v24.')
317 )
318 doLegacyStarSelectionComputation = pexConfig.Field(
319 doc="Perform the legacy star selection computations (for backwards compatibility)",
320 dtype=bool,
321 default=False,
322 deprecated=("This field is here for backwards compatibility and will be "
323 "removed after v24.")
324 )
325
326
327class PsfWcsSelectImagesTask(WcsSelectImagesTask):
328 """Select images using their Wcs and cuts on the PSF properties
329
330 The PSF quality criteria are based on the size and ellipticity residuals from the
331 adaptive second moments of the star and the PSF.
332
333 The criteria are:
334 - the median of the ellipticty residuals
335 - the robust scatter of the size residuals (using the median absolute deviation)
336 - the robust scatter of the size residuals scaled by the square of
337 the median size
338 """
339
340 ConfigClass = PsfWcsSelectImagesConfig
341 _DefaultName = "PsfWcsSelectImages"
342
343 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
344 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
345
346 This method is the old entry point for the Gen2 commandline tasks and drivers
347 Will be deprecated in v22.
348
349 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
350 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
351 @param makeDataRefList: Construct a list of data references?
352 @param selectDataList: List of SelectStruct, to consider for selection
353 """
354 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
355 selectDataList)
356
357 dataRefList = []
358 exposureInfoList = []
359 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
360 butler = dataRef.butlerSubset.butler
361 srcCatalog = butler.get('src', dataRef.dataId)
362 valid = self.isValidLegacy(srcCatalog, dataRef.dataId)
363 if valid is False:
364 continue
365
366 dataRefList.append(dataRef)
367 exposureInfoList.append(exposureInfo)
368
369 return pipeBase.Struct(
370 dataRefList=dataRefList,
371 exposureInfoList=exposureInfoList,
372 )
373
374 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, srcList=None, **kwargs):
375 """Return indices of provided lists that meet the selection criteria
376
377 Parameters:
378 -----------
379 wcsList : `list` of `lsst.afw.geom.SkyWcs`
380 specifying the WCS's of the input ccds to be selected
381 bboxList : `list` of `lsst.geom.Box2I`
382 specifying the bounding boxes of the input ccds to be selected
383 coordList : `list` of `lsst.geom.SpherePoint`
384 ICRS coordinates specifying boundary of the patch.
385 visitSummary : `list` of `lsst.afw.table.ExposureCatalog`
386 containing the PSF shape information for the input ccds to be selected.
387 srcList : `list` of `lsst.afw.table.SourceCatalog`, optional
388 containing the PSF shape information for the input ccds to be selected.
389 This is only used if ``config.doLegacyStarSelectionComputation`` is
390 True.
391
392 Returns:
393 --------
394 goodPsf: `list` of `int`
395 of indices of selected ccds
396 """
397 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
398 coordList=coordList, dataIds=dataIds)
399
400 goodPsf = []
401
402 if not self.config.doLegacyStarSelectionComputation:
403 # Check for old inputs, and give a helpful error message if so.
404 if 'nPsfStar' not in visitSummary[0].schema.getNames():
405 raise RuntimeError("Old calexps detected. "
406 "Please set config.doLegacyStarSelectionComputation=True for "
407 "backwards compatibility.")
408
409 for i, dataId in enumerate(dataIds):
410 if i not in goodWcs:
411 continue
412 if self.isValid(visitSummary, dataId["detector"]):
413 goodPsf.append(i)
414 else:
415 if dataIds is None:
416 dataIds = [None] * len(srcList)
417 for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)):
418 if i not in goodWcs:
419 continue
420 if self.isValidLegacy(srcCatalog, dataId):
421 goodPsf.append(i)
422
423 return goodPsf
424
425 def isValid(self, visitSummary, detectorId):
426 """Should this ccd be selected based on its PSF shape information.
427
428 Parameters
429 ----------
430 visitSummary : `lsst.afw.table.ExposureCatalog`
431 detectorId : `int`
432 Detector identifier.
433
434 Returns
435 -------
436 valid : `bool`
437 True if selected.
438 """
439 row = visitSummary.find(detectorId)
440 if row is None:
441 # This is not listed, so it must be bad.
442 self.log.warning("Removing visit %d detector %d because summary stats not available.",
443 row["visit"], detectorId)
444 return False
445
446 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.)
447 scatterSize = row["psfStarDeltaSizeScatter"]
448 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"]
449
450 valid = True
451 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
452 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f",
453 row["visit"], detectorId, medianE, self.config.maxEllipResidual)
454 valid = False
455 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
456 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f",
457 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter)
458 valid = False
459 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
460 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
461 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
462 valid = False
463
464 return valid
465
466 def isValidLegacy(self, srcCatalog, dataId=None):
467 """Should this ccd be selected based on its PSF shape information.
468
469 This routine is only used in legacy processing (gen2 and
470 backwards compatible old calexps) and should be removed after v24.
471
472 Parameters
473 ----------
474 srcCatalog : `lsst.afw.table.SourceCatalog`
475 dataId : `dict` of dataId keys, optional.
476 Used only for logging. Defaults to None.
477
478 Returns
479 -------
480 valid : `bool`
481 True if selected.
482 """
483 mask = srcCatalog[self.config.starSelection]
484
485 starXX = srcCatalog[self.config.starShape+'_xx'][mask]
486 starYY = srcCatalog[self.config.starShape+'_yy'][mask]
487 starXY = srcCatalog[self.config.starShape+'_xy'][mask]
488 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
489 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
490 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
491
492 starSize = np.power(starXX*starYY - starXY**2, 0.25)
493 starE1 = (starXX - starYY)/(starXX + starYY)
494 starE2 = 2*starXY/(starXX + starYY)
495 medianSize = np.median(starSize)
496
497 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
498 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
499 psfE2 = 2*psfXY/(psfXX + psfYY)
500
501 medianE1 = np.abs(np.median(starE1 - psfE1))
502 medianE2 = np.abs(np.median(starE2 - psfE2))
503 medianE = np.sqrt(medianE1**2 + medianE2**2)
504
505 scatterSize = sigmaMad(starSize - psfSize)
506 scaledScatterSize = scatterSize/medianSize**2
507
508 valid = True
509 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
510 self.log.info("Removing visit %s because median e residual too large: %f vs %f",
511 dataId, medianE, self.config.maxEllipResidual)
512 valid = False
513 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
514 self.log.info("Removing visit %s because size scatter is too large: %f vs %f",
515 dataId, scatterSize, self.config.maxSizeScatter)
516 valid = False
517 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
518 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f",
519 dataId, scaledScatterSize, self.config.maxScaledSizeScatter)
520 valid = False
521
522 return valid
523
524
525class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
526 """Base configuration for BestSeeingSelectImagesTask.
527 """
528 nImagesMax = pexConfig.RangeField(
529 dtype=int,
530 doc="Maximum number of images to select",
531 default=5,
532 min=0)
533 maxPsfFwhm = pexConfig.Field(
534 dtype=float,
535 doc="Maximum PSF FWHM (in arcseconds) to select",
536 default=1.5,
537 optional=True)
538 minPsfFwhm = pexConfig.Field(
539 dtype=float,
540 doc="Minimum PSF FWHM (in arcseconds) to select",
541 default=0.,
542 optional=True)
543
544
545class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
546 """Select up to a maximum number of the best-seeing images using their Wcs.
547 """
548 ConfigClass = BestSeeingWcsSelectImageConfig
549
550 def runDataRef(self, dataRef, coordList, makeDataRefList=True,
551 selectDataList=None):
552 """Select the best-seeing images in the selectDataList that overlap the patch.
553
554 This method is the old entry point for the Gen2 commandline tasks and drivers
555 Will be deprecated in v22.
556
557 Parameters
558 ----------
559 dataRef : `lsst.daf.persistence.ButlerDataRef`
560 Data reference for coadd/tempExp (with tract, patch)
561 coordList : `list` of `lsst.geom.SpherePoint`
562 List of ICRS sky coordinates specifying boundary of patch
563 makeDataRefList : `boolean`, optional
564 Construct a list of data references?
565 selectDataList : `list` of `SelectStruct`
566 List of SelectStruct, to consider for selection
567
568 Returns
569 -------
570 result : `lsst.pipe.base.Struct`
571 Result struct with components:
572 - ``exposureList``: the selected exposures
574 - ``dataRefList``: the optional data references corresponding to
575 each element of ``exposureList``
576 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
577 """
578 psfSizes = []
579 dataRefList = []
580 exposureInfoList = []
581
582 if selectDataList is None:
583 selectDataList = []
584
585 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
586
587 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
588 cal = dataRef.get("calexp", immediate=True)
589
590 # if min/max PSF values are defined, remove images out of bounds
591 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
592 # Just need a rough estimate; average positions are fine
593 psfAvgPos = cal.getPsf().getAveragePosition()
594 psfSize = cal.getPsf().computeShape(psfAvgPos).getDeterminantRadius()*pixToArcseconds
595 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
596 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
597 continue
598 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
599 continue
600 psfSizes.append(sizeFwhm)
601 dataRefList.append(dataRef)
602 exposureInfoList.append(exposureInfo)
603
604 if len(psfSizes) > self.config.nImagesMax:
605 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
606 filteredDataRefList = [dataRefList[i] for i in sortedIndices]
607 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
608 self.log.info("%d images selected with FWHM range of %f--%f arcseconds",
609 len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]])
610
611 else:
612 if len(psfSizes) == 0:
613 self.log.warning("0 images selected.")
614 else:
615 self.log.debug("%d images selected with FWHM range of %d--%d arcseconds",
616 len(psfSizes), psfSizes[0], psfSizes[-1])
617 filteredDataRefList = dataRefList
618 filteredExposureInfoList = exposureInfoList
619
620 return pipeBase.Struct(
621 dataRefList=filteredDataRefList if makeDataRefList else None,
622 exposureInfoList=filteredExposureInfoList,
623 )
624
625
626class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
627 dimensions=("tract", "patch", "skymap", "band", "instrument"),
628 defaultTemplates={"coaddName": "goodSeeing"}):
629 skyMap = pipeBase.connectionTypes.Input(
630 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
631 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
632 storageClass="SkyMap",
633 dimensions=("skymap",),
634 )
635 visitSummaries = pipeBase.connectionTypes.Input(
636 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
637 name="visitSummary",
638 storageClass="ExposureCatalog",
639 dimensions=("instrument", "visit",),
640 multiple=True,
641 deferLoad=True
642 )
643 goodVisits = pipeBase.connectionTypes.Output(
644 doc="Selected visits to be coadded.",
645 name="{coaddName}Visits",
646 storageClass="StructuredDataDict",
647 dimensions=("instrument", "tract", "patch", "skymap", "band"),
648 )
649
650
651class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
652 pipelineConnections=BestSeeingSelectVisitsConnections):
653 nVisitsMax = pexConfig.RangeField(
654 dtype=int,
655 doc="Maximum number of visits to select",
656 default=12,
657 min=0
658 )
659 maxPsfFwhm = pexConfig.Field(
660 dtype=float,
661 doc="Maximum PSF FWHM (in arcseconds) to select",
662 default=1.5,
663 optional=True
664 )
665 minPsfFwhm = pexConfig.Field(
666 dtype=float,
667 doc="Minimum PSF FWHM (in arcseconds) to select",
668 default=0.,
669 optional=True
670 )
671 doConfirmOverlap = pexConfig.Field(
672 dtype=bool,
673 doc="Do remove visits that do not actually overlap the patch?",
674 default=True,
675 )
676 minMJD = pexConfig.Field(
677 dtype=float,
678 doc="Minimum visit MJD to select",
679 default=None,
680 optional=True
681 )
682 maxMJD = pexConfig.Field(
683 dtype=float,
684 doc="Maximum visit MJD to select",
685 default=None,
686 optional=True
687 )
688
689
690class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
691 """Select up to a maximum number of the best-seeing visits
692
693 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
694 This Task is a port of the Gen2 image-selector used in the AP pipeline:
695 BestSeeingSelectImagesTask. This Task selects full visits based on the
696 average PSF of the entire visit.
697 """
698 ConfigClass = BestSeeingSelectVisitsConfig
699 _DefaultName = 'bestSeeingSelectVisits'
700
701 def runQuantum(self, butlerQC, inputRefs, outputRefs):
702 inputs = butlerQC.get(inputRefs)
703 quantumDataId = butlerQC.quantum.dataId
704 outputs = self.run(**inputs, dataId=quantumDataId)
705 butlerQC.put(outputs, outputRefs)
706
707 def run(self, visitSummaries, skyMap, dataId):
708 """Run task
709
710 Parameters:
711 -----------
712 visitSummary : `list`
713 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
714 visitSummary tables of type `lsst.afw.table.ExposureCatalog`
715 skyMap : `lsst.skyMap.SkyMap`
716 SkyMap for checking visits overlap patch
717 dataId : `dict` of dataId keys
718 For retrieving patch info for checking visits overlap patch
719
720 Returns
721 -------
722 result : `lsst.pipe.base.Struct`
723 Result struct with components:
724
725 - `goodVisits`: `dict` with selected visit ids as keys,
726 so that it can be be saved as a StructuredDataDict.
727 StructuredDataList's are currently limited.
728 """
729
730 if self.config.doConfirmOverlap:
731 patchPolygon = self.makePatchPolygon(skyMap, dataId)
732
733 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
734 fwhmSizes = []
735 visits = []
736 for visit, visitSummary in zip(inputVisits, visitSummaries):
737 # read in one-by-one and only once. There may be hundreds
738 visitSummary = visitSummary.get()
739
740 # mjd is guaranteed to be the same for every detector in the visitSummary.
741 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
742
743 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
744 for vs in visitSummary]
745 # psfSigma is PSF model determinant radius at chip center in pixels
746 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
747 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
748
749 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
750 continue
751 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
752 continue
753 if self.config.minMJD and mjd < self.config.minMJD:
754 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
755 continue
756 if self.config.maxMJD and mjd > self.config.maxMJD:
757 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
758 continue
759 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
760 continue
761
762 fwhmSizes.append(fwhm)
763 visits.append(visit)
764
765 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
766 output = sortedVisits[:self.config.nVisitsMax]
767 self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
768 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
769
770 # In order to store as a StructuredDataDict, convert list to dict
771 goodVisits = {key: True for key in output}
772 return pipeBase.Struct(goodVisits=goodVisits)
773
774 def makePatchPolygon(self, skyMap, dataId):
775 """Return True if sky polygon overlaps visit
776
777 Parameters:
778 -----------
780 Exposure catalog with per-detector geometry
781 dataId : `dict` of dataId keys
782 For retrieving patch info
783
784 Returns:
785 --------
786 result :` lsst.sphgeom.ConvexPolygon.convexHull`
787 Polygon of patch's outer bbox
788 """
789 wcs = skyMap[dataId['tract']].getWcs()
790 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
791 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
792 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
793 return result
794
795 def doesIntersectPolygon(self, visitSummary, polygon):
796 """Return True if sky polygon overlaps visit
797
798 Parameters:
799 -----------
800 visitSummary : `lsst.afw.table.ExposureCatalog`
801 Exposure catalog with per-detector geometry
802 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
803 Polygon to check overlap
804
805 Returns:
806 --------
807 doesIntersect: `bool`
808 Does the visit overlap the polygon
809 """
810 doesIntersect = False
811 for detectorSummary in visitSummary:
812 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
813 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
814 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
815 if detectorPolygon.intersects(polygon):
816 doesIntersect = True
817 break
818 return doesIntersect
819
820
821class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
822 pipelineConnections=BestSeeingSelectVisitsConnections):
823 qMin = pexConfig.RangeField(
824 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
825 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
826 "This config should be changed from zero only for exploratory diffIm testing.",
827 dtype=float,
828 default=0,
829 min=0,
830 max=1,
831 )
832 qMax = pexConfig.RangeField(
833 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
834 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
835 dtype=float,
836 default=0.33,
837 min=0,
838 max=1,
839 )
840 nVisitsMin = pexConfig.Field(
841 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
842 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
843 dtype=int,
844 default=6,
845 )
846 doConfirmOverlap = pexConfig.Field(
847 dtype=bool,
848 doc="Do remove visits that do not actually overlap the patch?",
849 default=True,
850 )
851 minMJD = pexConfig.Field(
852 dtype=float,
853 doc="Minimum visit MJD to select",
854 default=None,
855 optional=True
856 )
857 maxMJD = pexConfig.Field(
858 dtype=float,
859 doc="Maximum visit MJD to select",
860 default=None,
861 optional=True
862 )
863
864
865class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
866 """Select a quantile of the best-seeing visits
867
868 Selects the best (for example, third) full visits based on the average
869 PSF width in the entire visit. It can also be used for difference imaging
870 experiments that require templates with the worst seeing visits.
871 For example, selecting the worst third can be acheived by
872 changing the config parameters qMin to 0.66 and qMax to 1.
873 """
874 ConfigClass = BestSeeingQuantileSelectVisitsConfig
875 _DefaultName = 'bestSeeingQuantileSelectVisits'
876
877 @utils.inheritDoc(BestSeeingSelectVisitsTask)
878 def run(self, visitSummaries, skyMap, dataId):
879 if self.config.doConfirmOverlap:
880 patchPolygon = self.makePatchPolygon(skyMap, dataId)
881 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
882 radius = np.empty(len(visits))
883 intersects = np.full(len(visits), True)
884 for i, visitSummary in enumerate(visitSummaries):
885 # read in one-by-one and only once. There may be hundreds
886 visitSummary = visitSummary.get()
887 # psfSigma is PSF model determinant radius at chip center in pixels
888 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
889 radius[i] = psfSigma
890 if self.config.doConfirmOverlap:
891 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
892 if self.config.minMJD or self.config.maxMJD:
893 # mjd is guaranteed to be the same for every detector in the visitSummary.
894 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
895 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
896 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
897 intersects[i] = intersects[i] and aboveMin and belowMax
898
899 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
900 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
901 max(0, len(visits[intersects]) - self.config.nVisitsMin))
902 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
903
904 # In order to store as a StructuredDataDict, convert list to dict
905 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
906 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
Definition: selectImages.py:63
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)