Coverage for python/lsst/pipe/tasks/selectImages.py: 23%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.utils as utils
25import lsst.pex.config as pexConfig
26import lsst.pex.exceptions as pexExceptions
27import lsst.geom as geom
28import lsst.pipe.base as pipeBase
29from lsst.skymap import BaseSkyMap
30from lsst.daf.base import DateTime
32__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
33 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask",
34 "BestSeeingQuantileSelectVisitsTask"]
37class DatabaseSelectImagesConfig(pexConfig.Config):
38 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
39 host = pexConfig.Field(
40 doc="Database server host name",
41 dtype=str,
42 )
43 port = pexConfig.Field(
44 doc="Database server port",
45 dtype=int,
46 )
47 database = pexConfig.Field(
48 doc="Name of database",
49 dtype=str,
50 )
51 maxExposures = pexConfig.Field(
52 doc="maximum exposures to select; intended for debugging; ignored if None",
53 dtype=int,
54 optional=True,
55 )
58class BaseExposureInfo(pipeBase.Struct):
59 """Data about a selected exposure
60 """
62 def __init__(self, dataId, coordList):
63 """Create exposure information that can be used to generate data references
65 The object has the following fields:
66 - dataId: data ID of exposure (a dict)
67 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
68 plus any others items that are desired
69 """
70 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
73class BaseSelectImagesTask(pipeBase.Task):
74 """Base task for selecting images suitable for coaddition
75 """
76 ConfigClass = pexConfig.Config
77 _DefaultName = "selectImages"
79 @pipeBase.timeMethod
80 def run(self, coordList):
81 """Select images suitable for coaddition in a particular region
83 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
84 subclasses may add additional keyword arguments, as required
86 @return a pipeBase Struct containing:
87 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
88 which have at least the following fields:
89 - dataId: data ID dictionary
90 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
91 """
92 raise NotImplementedError()
94 def _runArgDictFromDataId(self, dataId):
95 """Extract keyword arguments for run (other than coordList) from a data ID
97 @return keyword arguments for run (other than coordList), as a dict
98 """
99 raise NotImplementedError()
101 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
102 """Run based on a data reference
104 This delegates to run() and _runArgDictFromDataId() to do the actual
105 selection. In the event that the selectDataList is non-empty, this will
106 be used to further restrict the selection, providing the user with
107 additional control over the selection.
109 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
110 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
111 @param[in] makeDataRefList: if True, return dataRefList
112 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
113 @return a pipeBase Struct containing:
114 - exposureInfoList: a list of objects derived from ExposureInfo
115 - dataRefList: a list of data references (None if makeDataRefList False)
116 """
117 runArgDict = self._runArgDictFromDataId(dataRef.dataId)
118 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList
120 if len(selectDataList) > 0 and len(exposureInfoList) > 0:
121 # Restrict the exposure selection further
122 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
123 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
124 inValues = set(inValues)
125 newExposureInfoList = []
126 for info, ccdVal in zip(exposureInfoList, ccdValues):
127 if ccdVal in inValues:
128 newExposureInfoList.append(info)
129 else:
130 self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId)
131 exposureInfoList = newExposureInfoList
133 if makeDataRefList:
134 butler = dataRef.butlerSubset.butler
135 dataRefList = [butler.dataRef(datasetType="calexp",
136 dataId=expInfo.dataId,
137 ) for expInfo in exposureInfoList]
138 else:
139 dataRefList = None
141 return pipeBase.Struct(
142 dataRefList=dataRefList,
143 exposureInfoList=exposureInfoList,
144 )
147def _extractKeyValue(dataList, keys=None):
148 """Extract the keys and values from a list of dataIds
150 The input dataList is a list of objects that have 'dataId' members.
151 This allows it to be used for both a list of data references and a
152 list of ExposureInfo
153 """
154 assert len(dataList) > 0
155 if keys is None:
156 keys = sorted(dataList[0].dataId.keys())
157 keySet = set(keys)
158 values = list()
159 for data in dataList:
160 thisKeys = set(data.dataId.keys())
161 if thisKeys != keySet:
162 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
163 values.append(tuple(data.dataId[k] for k in keys))
164 return keys, values
167class SelectStruct(pipeBase.Struct):
168 """A container for data to be passed to the WcsSelectImagesTask"""
170 def __init__(self, dataRef, wcs, bbox):
171 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
174class WcsSelectImagesTask(BaseSelectImagesTask):
175 """Select images using their Wcs
177 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
178 polygons on the celestial sphere, and test the polygon of the
179 patch for overlap with the polygon of the image.
181 We use "convexHull" instead of generating a ConvexPolygon
182 directly because the standard for the inputs to ConvexPolygon
183 are pretty high and we don't want to be responsible for reaching them.
184 """
186 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
187 """Select images in the selectDataList that overlap the patch
189 This method is the old entry point for the Gen2 commandline tasks and drivers
190 Will be deprecated in v22.
192 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
193 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
194 @param makeDataRefList: Construct a list of data references?
195 @param selectDataList: List of SelectStruct, to consider for selection
196 """
197 dataRefList = []
198 exposureInfoList = []
200 patchVertices = [coord.getVector() for coord in coordList]
201 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
203 for data in selectDataList:
204 dataRef = data.dataRef
205 imageWcs = data.wcs
206 imageBox = data.bbox
208 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
209 if imageCorners:
210 dataRefList.append(dataRef)
211 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
213 return pipeBase.Struct(
214 dataRefList=dataRefList if makeDataRefList else None,
215 exposureInfoList=exposureInfoList,
216 )
218 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
219 """Return indices of provided lists that meet the selection criteria
221 Parameters:
222 -----------
223 wcsList : `list` of `lsst.afw.geom.SkyWcs`
224 specifying the WCS's of the input ccds to be selected
225 bboxList : `list` of `lsst.geom.Box2I`
226 specifying the bounding boxes of the input ccds to be selected
227 coordList : `list` of `lsst.geom.SpherePoint`
228 ICRS coordinates specifying boundary of the patch.
230 Returns:
231 --------
232 result: `list` of `int`
233 of indices of selected ccds
234 """
235 if dataIds is None:
236 dataIds = [None] * len(wcsList)
237 patchVertices = [coord.getVector() for coord in coordList]
238 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
239 result = []
240 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
241 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
242 if imageCorners:
243 result.append(i)
244 return result
246 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
247 "Return corners or None if bad"
248 try:
249 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
250 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
251 # Protecting ourselves from awful Wcs solutions in input images
252 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
253 return
255 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
256 if imagePoly is None:
257 self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
258 return
260 if patchPoly.intersects(imagePoly):
261 # "intersects" also covers "contains" or "is contained by"
262 self.log.info("Selecting calexp %s", dataId)
263 return imageCorners
266def sigmaMad(array):
267 "Return median absolute deviation scaled to normally distributed data"
268 return 1.4826*np.median(np.abs(array - np.median(array)))
271class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
272 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
273 defaultTemplates={"coaddName": "deep"}):
274 pass
277class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
278 pipelineConnections=PsfWcsSelectImagesConnections):
279 maxEllipResidual = pexConfig.Field(
280 doc="Maximum median ellipticity residual",
281 dtype=float,
282 default=0.007,
283 optional=True,
284 )
285 maxSizeScatter = pexConfig.Field(
286 doc="Maximum scatter in the size residuals",
287 dtype=float,
288 optional=True,
289 )
290 maxScaledSizeScatter = pexConfig.Field(
291 doc="Maximum scatter in the size residuals, scaled by the median size",
292 dtype=float,
293 default=0.009,
294 optional=True,
295 )
296 starSelection = pexConfig.Field(
297 doc="select star with this field",
298 dtype=str,
299 default='calib_psf_used',
300 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
301 'will be removed after v24.')
302 )
303 starShape = pexConfig.Field(
304 doc="name of star shape",
305 dtype=str,
306 default='base_SdssShape',
307 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
308 'will be removed after v24.')
309 )
310 psfShape = pexConfig.Field(
311 doc="name of psf shape",
312 dtype=str,
313 default='base_SdssShape_psf',
314 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
315 'will be removed after v24.')
316 )
317 doLegacyStarSelectionComputation = pexConfig.Field(
318 doc="Perform the legacy star selection computations (for backwards compatibility)",
319 dtype=bool,
320 default=False,
321 deprecated=("This field is here for backwards compatibility and will be "
322 "removed after v24.")
323 )
326class PsfWcsSelectImagesTask(WcsSelectImagesTask):
327 """Select images using their Wcs and cuts on the PSF properties
329 The PSF quality criteria are based on the size and ellipticity residuals from the
330 adaptive second moments of the star and the PSF.
332 The criteria are:
333 - the median of the ellipticty residuals
334 - the robust scatter of the size residuals (using the median absolute deviation)
335 - the robust scatter of the size residuals scaled by the square of
336 the median size
337 """
339 ConfigClass = PsfWcsSelectImagesConfig
340 _DefaultName = "PsfWcsSelectImages"
342 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
343 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
345 This method is the old entry point for the Gen2 commandline tasks and drivers
346 Will be deprecated in v22.
348 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
349 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
350 @param makeDataRefList: Construct a list of data references?
351 @param selectDataList: List of SelectStruct, to consider for selection
352 """
353 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
354 selectDataList)
356 dataRefList = []
357 exposureInfoList = []
358 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
359 butler = dataRef.butlerSubset.butler
360 srcCatalog = butler.get('src', dataRef.dataId)
361 valid = self.isValidLegacy(srcCatalog, dataRef.dataId)
362 if valid is False:
363 continue
365 dataRefList.append(dataRef)
366 exposureInfoList.append(exposureInfo)
368 return pipeBase.Struct(
369 dataRefList=dataRefList,
370 exposureInfoList=exposureInfoList,
371 )
373 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, srcList=None, **kwargs):
374 """Return indices of provided lists that meet the selection criteria
376 Parameters:
377 -----------
378 wcsList : `list` of `lsst.afw.geom.SkyWcs`
379 specifying the WCS's of the input ccds to be selected
380 bboxList : `list` of `lsst.geom.Box2I`
381 specifying the bounding boxes of the input ccds to be selected
382 coordList : `list` of `lsst.geom.SpherePoint`
383 ICRS coordinates specifying boundary of the patch.
384 visitSummary : `list` of `lsst.afw.table.ExposureCatalog`
385 containing the PSF shape information for the input ccds to be selected.
386 srcList : `list` of `lsst.afw.table.SourceCatalog`, optional
387 containing the PSF shape information for the input ccds to be selected.
388 This is only used if ``config.doLegacyStarSelectionComputation`` is
389 True.
391 Returns:
392 --------
393 goodPsf: `list` of `int`
394 of indices of selected ccds
395 """
396 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
397 coordList=coordList, dataIds=dataIds)
399 goodPsf = []
401 if not self.config.doLegacyStarSelectionComputation:
402 # Check for old inputs, and give a helpful error message if so.
403 if 'nPsfStar' not in visitSummary[0].schema.getNames():
404 raise RuntimeError("Old calexps detected. "
405 "Please set config.doLegacyStarSelectionComputation=True for "
406 "backwards compatibility.")
408 for i, dataId in enumerate(dataIds):
409 if i not in goodWcs:
410 continue
411 if self.isValid(visitSummary, dataId["detector"]):
412 goodPsf.append(i)
413 else:
414 if dataIds is None:
415 dataIds = [None] * len(srcList)
416 for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)):
417 if i not in goodWcs:
418 continue
419 if self.isValidLegacy(srcCatalog, dataId):
420 goodPsf.append(i)
422 return goodPsf
424 def isValid(self, visitSummary, detectorId):
425 """Should this ccd be selected based on its PSF shape information.
427 Parameters
428 ----------
429 visitSummary : `lsst.afw.table.ExposureCatalog`
430 detectorId : `int`
431 Detector identifier.
433 Returns
434 -------
435 valid : `bool`
436 True if selected.
437 """
438 row = visitSummary.find(detectorId)
439 if row is None:
440 # This is not listed, so it must be bad.
441 self.log.warning("Removing visit %d detector %d because summary stats not available.",
442 row["visit"], detectorId)
443 return False
445 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.)
446 scatterSize = row["psfStarDeltaSizeScatter"]
447 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"]
449 valid = True
450 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
451 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f",
452 row["visit"], detectorId, medianE, self.config.maxEllipResidual)
453 valid = False
454 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
455 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f",
456 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter)
457 valid = False
458 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
459 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
460 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
461 valid = False
463 return valid
465 def isValidLegacy(self, srcCatalog, dataId=None):
466 """Should this ccd be selected based on its PSF shape information.
468 This routine is only used in legacy processing (gen2 and
469 backwards compatible old calexps) and should be removed after v24.
471 Parameters
472 ----------
473 srcCatalog : `lsst.afw.table.SourceCatalog`
474 dataId : `dict` of dataId keys, optional.
475 Used only for logging. Defaults to None.
477 Returns
478 -------
479 valid : `bool`
480 True if selected.
481 """
482 mask = srcCatalog[self.config.starSelection]
484 starXX = srcCatalog[self.config.starShape+'_xx'][mask]
485 starYY = srcCatalog[self.config.starShape+'_yy'][mask]
486 starXY = srcCatalog[self.config.starShape+'_xy'][mask]
487 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
488 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
489 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
491 starSize = np.power(starXX*starYY - starXY**2, 0.25)
492 starE1 = (starXX - starYY)/(starXX + starYY)
493 starE2 = 2*starXY/(starXX + starYY)
494 medianSize = np.median(starSize)
496 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
497 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
498 psfE2 = 2*psfXY/(psfXX + psfYY)
500 medianE1 = np.abs(np.median(starE1 - psfE1))
501 medianE2 = np.abs(np.median(starE2 - psfE2))
502 medianE = np.sqrt(medianE1**2 + medianE2**2)
504 scatterSize = sigmaMad(starSize - psfSize)
505 scaledScatterSize = scatterSize/medianSize**2
507 valid = True
508 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
509 self.log.info("Removing visit %s because median e residual too large: %f vs %f",
510 dataId, medianE, self.config.maxEllipResidual)
511 valid = False
512 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
513 self.log.info("Removing visit %s because size scatter is too large: %f vs %f",
514 dataId, scatterSize, self.config.maxSizeScatter)
515 valid = False
516 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
517 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f",
518 dataId, scaledScatterSize, self.config.maxScaledSizeScatter)
519 valid = False
521 return valid
524class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
525 """Base configuration for BestSeeingSelectImagesTask.
526 """
527 nImagesMax = pexConfig.RangeField(
528 dtype=int,
529 doc="Maximum number of images to select",
530 default=5,
531 min=0)
532 maxPsfFwhm = pexConfig.Field(
533 dtype=float,
534 doc="Maximum PSF FWHM (in arcseconds) to select",
535 default=1.5,
536 optional=True)
537 minPsfFwhm = pexConfig.Field(
538 dtype=float,
539 doc="Minimum PSF FWHM (in arcseconds) to select",
540 default=0.,
541 optional=True)
544class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
545 """Select up to a maximum number of the best-seeing images using their Wcs.
546 """
547 ConfigClass = BestSeeingWcsSelectImageConfig
549 def runDataRef(self, dataRef, coordList, makeDataRefList=True,
550 selectDataList=None):
551 """Select the best-seeing images in the selectDataList that overlap the patch.
553 This method is the old entry point for the Gen2 commandline tasks and drivers
554 Will be deprecated in v22.
556 Parameters
557 ----------
558 dataRef : `lsst.daf.persistence.ButlerDataRef`
559 Data reference for coadd/tempExp (with tract, patch)
560 coordList : `list` of `lsst.geom.SpherePoint`
561 List of ICRS sky coordinates specifying boundary of patch
562 makeDataRefList : `boolean`, optional
563 Construct a list of data references?
564 selectDataList : `list` of `SelectStruct`
565 List of SelectStruct, to consider for selection
567 Returns
568 -------
569 result : `lsst.pipe.base.Struct`
570 Result struct with components:
571 - ``exposureList``: the selected exposures
572 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`).
573 - ``dataRefList``: the optional data references corresponding to
574 each element of ``exposureList``
575 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
576 """
577 psfSizes = []
578 dataRefList = []
579 exposureInfoList = []
581 if selectDataList is None:
582 selectDataList = []
584 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
586 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
587 cal = dataRef.get("calexp", immediate=True)
589 # if min/max PSF values are defined, remove images out of bounds
590 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
591 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
592 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
593 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
594 continue
595 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
596 continue
597 psfSizes.append(sizeFwhm)
598 dataRefList.append(dataRef)
599 exposureInfoList.append(exposureInfo)
601 if len(psfSizes) > self.config.nImagesMax:
602 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
603 filteredDataRefList = [dataRefList[i] for i in sortedIndices]
604 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
605 self.log.info("%d images selected with FWHM range of %f--%f arcseconds",
606 len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]])
608 else:
609 if len(psfSizes) == 0:
610 self.log.warning("0 images selected.")
611 else:
612 self.log.debug("%d images selected with FWHM range of %d--%d arcseconds",
613 len(psfSizes), psfSizes[0], psfSizes[-1])
614 filteredDataRefList = dataRefList
615 filteredExposureInfoList = exposureInfoList
617 return pipeBase.Struct(
618 dataRefList=filteredDataRefList if makeDataRefList else None,
619 exposureInfoList=filteredExposureInfoList,
620 )
623class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
624 dimensions=("tract", "patch", "skymap", "band", "instrument"),
625 defaultTemplates={"coaddName": "goodSeeing"}):
626 skyMap = pipeBase.connectionTypes.Input(
627 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
628 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
629 storageClass="SkyMap",
630 dimensions=("skymap",),
631 )
632 visitSummaries = pipeBase.connectionTypes.Input(
633 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
634 name="visitSummary",
635 storageClass="ExposureCatalog",
636 dimensions=("instrument", "visit",),
637 multiple=True,
638 deferLoad=True
639 )
640 goodVisits = pipeBase.connectionTypes.Output(
641 doc="Selected visits to be coadded.",
642 name="{coaddName}Visits",
643 storageClass="StructuredDataDict",
644 dimensions=("instrument", "tract", "patch", "skymap", "band"),
645 )
648class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
649 pipelineConnections=BestSeeingSelectVisitsConnections):
650 nVisitsMax = pexConfig.RangeField(
651 dtype=int,
652 doc="Maximum number of visits to select",
653 default=12,
654 min=0
655 )
656 maxPsfFwhm = pexConfig.Field(
657 dtype=float,
658 doc="Maximum PSF FWHM (in arcseconds) to select",
659 default=1.5,
660 optional=True
661 )
662 minPsfFwhm = pexConfig.Field(
663 dtype=float,
664 doc="Minimum PSF FWHM (in arcseconds) to select",
665 default=0.,
666 optional=True
667 )
668 doConfirmOverlap = pexConfig.Field(
669 dtype=bool,
670 doc="Do remove visits that do not actually overlap the patch?",
671 default=True,
672 )
673 minMJD = pexConfig.Field(
674 dtype=float,
675 doc="Minimum visit MJD to select",
676 default=None,
677 optional=True
678 )
679 maxMJD = pexConfig.Field(
680 dtype=float,
681 doc="Maximum visit MJD to select",
682 default=None,
683 optional=True
684 )
687class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
688 """Select up to a maximum number of the best-seeing visits
690 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
691 This Task is a port of the Gen2 image-selector used in the AP pipeline:
692 BestSeeingSelectImagesTask. This Task selects full visits based on the
693 average PSF of the entire visit.
694 """
695 ConfigClass = BestSeeingSelectVisitsConfig
696 _DefaultName = 'bestSeeingSelectVisits'
698 def runQuantum(self, butlerQC, inputRefs, outputRefs):
699 inputs = butlerQC.get(inputRefs)
700 quantumDataId = butlerQC.quantum.dataId
701 outputs = self.run(**inputs, dataId=quantumDataId)
702 butlerQC.put(outputs, outputRefs)
704 def run(self, visitSummaries, skyMap, dataId):
705 """Run task
707 Parameters:
708 -----------
709 visitSummary : `list`
710 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
711 visitSummary tables of type `lsst.afw.table.ExposureCatalog`
712 skyMap : `lsst.skyMap.SkyMap`
713 SkyMap for checking visits overlap patch
714 dataId : `dict` of dataId keys
715 For retrieving patch info for checking visits overlap patch
717 Returns
718 -------
719 result : `lsst.pipe.base.Struct`
720 Result struct with components:
722 - `goodVisits`: `dict` with selected visit ids as keys,
723 so that it can be be saved as a StructuredDataDict.
724 StructuredDataList's are currently limited.
725 """
727 if self.config.doConfirmOverlap:
728 patchPolygon = self.makePatchPolygon(skyMap, dataId)
730 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
731 fwhmSizes = []
732 visits = []
733 for visit, visitSummary in zip(inputVisits, visitSummaries):
734 # read in one-by-one and only once. There may be hundreds
735 visitSummary = visitSummary.get()
737 # mjd is guaranteed to be the same for every detector in the visitSummary.
738 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
740 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
741 for vs in visitSummary]
742 # psfSigma is PSF model determinant radius at chip center in pixels
743 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
744 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
746 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
747 continue
748 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
749 continue
750 if self.config.minMJD and mjd < self.config.minMJD:
751 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
752 continue
753 if self.config.maxMJD and mjd > self.config.maxMJD:
754 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
755 continue
756 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
757 continue
759 fwhmSizes.append(fwhm)
760 visits.append(visit)
762 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
763 output = sortedVisits[:self.config.nVisitsMax]
764 self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
765 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
767 # In order to store as a StructuredDataDict, convert list to dict
768 goodVisits = {key: True for key in output}
769 return pipeBase.Struct(goodVisits=goodVisits)
771 def makePatchPolygon(self, skyMap, dataId):
772 """Return True if sky polygon overlaps visit
774 Parameters:
775 -----------
776 skyMap : `lsst.afw.table.ExposureCatalog`
777 Exposure catalog with per-detector geometry
778 dataId : `dict` of dataId keys
779 For retrieving patch info
781 Returns:
782 --------
783 result :` lsst.sphgeom.ConvexPolygon.convexHull`
784 Polygon of patch's outer bbox
785 """
786 wcs = skyMap[dataId['tract']].getWcs()
787 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
788 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
789 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
790 return result
792 def doesIntersectPolygon(self, visitSummary, polygon):
793 """Return True if sky polygon overlaps visit
795 Parameters:
796 -----------
797 visitSummary : `lsst.afw.table.ExposureCatalog`
798 Exposure catalog with per-detector geometry
799 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
800 Polygon to check overlap
802 Returns:
803 --------
804 doesIntersect: `bool`
805 Does the visit overlap the polygon
806 """
807 doesIntersect = False
808 for detectorSummary in visitSummary:
809 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
810 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
811 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
812 if detectorPolygon.intersects(polygon):
813 doesIntersect = True
814 break
815 return doesIntersect
818class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
819 pipelineConnections=BestSeeingSelectVisitsConnections):
820 qMin = pexConfig.RangeField(
821 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
822 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
823 "This config should be changed from zero only for exploratory diffIm testing.",
824 dtype=float,
825 default=0,
826 min=0,
827 max=1,
828 )
829 qMax = pexConfig.RangeField(
830 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
831 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
832 dtype=float,
833 default=0.33,
834 min=0,
835 max=1,
836 )
837 nVisitsMin = pexConfig.Field(
838 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
839 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
840 dtype=int,
841 default=6,
842 )
843 doConfirmOverlap = pexConfig.Field(
844 dtype=bool,
845 doc="Do remove visits that do not actually overlap the patch?",
846 default=True,
847 )
848 minMJD = pexConfig.Field(
849 dtype=float,
850 doc="Minimum visit MJD to select",
851 default=None,
852 optional=True
853 )
854 maxMJD = pexConfig.Field(
855 dtype=float,
856 doc="Maximum visit MJD to select",
857 default=None,
858 optional=True
859 )
862class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
863 """Select a quantile of the best-seeing visits
865 Selects the best (for example, third) full visits based on the average
866 PSF width in the entire visit. It can also be used for difference imaging
867 experiments that require templates with the worst seeing visits.
868 For example, selecting the worst third can be acheived by
869 changing the config parameters qMin to 0.66 and qMax to 1.
870 """
871 ConfigClass = BestSeeingQuantileSelectVisitsConfig
872 _DefaultName = 'bestSeeingQuantileSelectVisits'
874 @utils.inheritDoc(BestSeeingSelectVisitsTask)
875 def run(self, visitSummaries, skyMap, dataId):
876 if self.config.doConfirmOverlap:
877 patchPolygon = self.makePatchPolygon(skyMap, dataId)
878 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
879 radius = np.empty(len(visits))
880 intersects = np.full(len(visits), True)
881 for i, visitSummary in enumerate(visitSummaries):
882 # read in one-by-one and only once. There may be hundreds
883 visitSummary = visitSummary.get()
884 # psfSigma is PSF model determinant radius at chip center in pixels
885 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
886 radius[i] = psfSigma
887 if self.config.doConfirmOverlap:
888 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
889 if self.config.minMJD or self.config.maxMJD:
890 # mjd is guaranteed to be the same for every detector in the visitSummary.
891 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
892 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
893 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
894 intersects[i] = intersects[i] and aboveMin and belowMax
896 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
897 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
898 max(0, len(visits[intersects]) - self.config.nVisitsMin))
899 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
901 # In order to store as a StructuredDataDict, convert list to dict
902 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
903 return pipeBase.Struct(goodVisits=goodVisits)