Coverage for python/lsst/pipe/tasks/selectImages.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.pex.config as pexConfig
25import lsst.pex.exceptions as pexExceptions
26import lsst.geom as geom
27import lsst.pipe.base as pipeBase
29__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
30 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask"]
33class DatabaseSelectImagesConfig(pexConfig.Config):
34 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
35 host = pexConfig.Field(
36 doc="Database server host name",
37 dtype=str,
38 )
39 port = pexConfig.Field(
40 doc="Database server port",
41 dtype=int,
42 )
43 database = pexConfig.Field(
44 doc="Name of database",
45 dtype=str,
46 )
47 maxExposures = pexConfig.Field(
48 doc="maximum exposures to select; intended for debugging; ignored if None",
49 dtype=int,
50 optional=True,
51 )
54class BaseExposureInfo(pipeBase.Struct):
55 """Data about a selected exposure
56 """
58 def __init__(self, dataId, coordList):
59 """Create exposure information that can be used to generate data references
61 The object has the following fields:
62 - dataId: data ID of exposure (a dict)
63 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
64 plus any others items that are desired
65 """
66 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
69class BaseSelectImagesTask(pipeBase.Task):
70 """Base task for selecting images suitable for coaddition
71 """
72 ConfigClass = pexConfig.Config
73 _DefaultName = "selectImages"
75 @pipeBase.timeMethod
76 def run(self, coordList):
77 """Select images suitable for coaddition in a particular region
79 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
80 subclasses may add additional keyword arguments, as required
82 @return a pipeBase Struct containing:
83 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
84 which have at least the following fields:
85 - dataId: data ID dictionary
86 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
87 """
88 raise NotImplementedError()
90 def _runArgDictFromDataId(self, dataId):
91 """Extract keyword arguments for run (other than coordList) from a data ID
93 @return keyword arguments for run (other than coordList), as a dict
94 """
95 raise NotImplementedError()
97 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
98 """Run based on a data reference
100 This delegates to run() and _runArgDictFromDataId() to do the actual
101 selection. In the event that the selectDataList is non-empty, this will
102 be used to further restrict the selection, providing the user with
103 additional control over the selection.
105 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
106 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
107 @param[in] makeDataRefList: if True, return dataRefList
108 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
109 @return a pipeBase Struct containing:
110 - exposureInfoList: a list of objects derived from ExposureInfo
111 - dataRefList: a list of data references (None if makeDataRefList False)
112 """
113 runArgDict = self._runArgDictFromDataId(dataRef.dataId)
114 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList
116 if len(selectDataList) > 0 and len(exposureInfoList) > 0:
117 # Restrict the exposure selection further
118 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
119 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
120 inValues = set(inValues)
121 newExposureInfoList = []
122 for info, ccdVal in zip(exposureInfoList, ccdValues):
123 if ccdVal in inValues:
124 newExposureInfoList.append(info)
125 else:
126 self.log.info("De-selecting exposure %s: not in selectDataList" % info.dataId)
127 exposureInfoList = newExposureInfoList
129 if makeDataRefList:
130 butler = dataRef.butlerSubset.butler
131 dataRefList = [butler.dataRef(datasetType="calexp",
132 dataId=expInfo.dataId,
133 ) for expInfo in exposureInfoList]
134 else:
135 dataRefList = None
137 return pipeBase.Struct(
138 dataRefList=dataRefList,
139 exposureInfoList=exposureInfoList,
140 )
143def _extractKeyValue(dataList, keys=None):
144 """Extract the keys and values from a list of dataIds
146 The input dataList is a list of objects that have 'dataId' members.
147 This allows it to be used for both a list of data references and a
148 list of ExposureInfo
149 """
150 assert len(dataList) > 0
151 if keys is None:
152 keys = sorted(dataList[0].dataId.keys())
153 keySet = set(keys)
154 values = list()
155 for data in dataList:
156 thisKeys = set(data.dataId.keys())
157 if thisKeys != keySet:
158 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
159 values.append(tuple(data.dataId[k] for k in keys))
160 return keys, values
163class SelectStruct(pipeBase.Struct):
164 """A container for data to be passed to the WcsSelectImagesTask"""
166 def __init__(self, dataRef, wcs, bbox):
167 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
170class WcsSelectImagesTask(BaseSelectImagesTask):
171 """Select images using their Wcs
173 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
174 polygons on the celestial sphere, and test the polygon of the
175 patch for overlap with the polygon of the image.
177 We use "convexHull" instead of generating a ConvexPolygon
178 directly because the standard for the inputs to ConvexPolygon
179 are pretty high and we don't want to be responsible for reaching them.
180 """
182 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
183 """Select images in the selectDataList that overlap the patch
185 This method is the old entry point for the Gen2 commandline tasks and drivers
186 Will be deprecated in v22.
188 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
189 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
190 @param makeDataRefList: Construct a list of data references?
191 @param selectDataList: List of SelectStruct, to consider for selection
192 """
193 dataRefList = []
194 exposureInfoList = []
196 patchVertices = [coord.getVector() for coord in coordList]
197 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
199 for data in selectDataList:
200 dataRef = data.dataRef
201 imageWcs = data.wcs
202 imageBox = data.bbox
204 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
205 if imageCorners:
206 dataRefList.append(dataRef)
207 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
209 return pipeBase.Struct(
210 dataRefList=dataRefList if makeDataRefList else None,
211 exposureInfoList=exposureInfoList,
212 )
214 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
215 """Return indices of provided lists that meet the selection criteria
217 Parameters:
218 -----------
219 wcsList : `list` of `lsst.afw.geom.SkyWcs`
220 specifying the WCS's of the input ccds to be selected
221 bboxList : `list` of `lsst.geom.Box2I`
222 specifying the bounding boxes of the input ccds to be selected
223 coordList : `list` of `lsst.geom.SpherePoint`
224 ICRS coordinates specifying boundary of the patch.
226 Returns:
227 --------
228 result: `list` of `int`
229 of indices of selected ccds
230 """
231 if dataIds is None:
232 dataIds = [None] * len(wcsList)
233 patchVertices = [coord.getVector() for coord in coordList]
234 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
235 result = []
236 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
237 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
238 if imageCorners:
239 result.append(i)
240 return result
242 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
243 "Return corners or None if bad"
244 try:
245 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
246 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
247 # Protecting ourselves from awful Wcs solutions in input images
248 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
249 return
251 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
252 if imagePoly is None:
253 self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
254 return
256 if patchPoly.intersects(imagePoly):
257 # "intersects" also covers "contains" or "is contained by"
258 self.log.info("Selecting calexp %s" % dataId)
259 return imageCorners
262def sigmaMad(array):
263 "Return median absolute deviation scaled to normally distributed data"
264 return 1.4826*np.median(np.abs(array - np.median(array)))
267class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
268 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
269 defaultTemplates={"coaddName": "deep"}):
270 pass
273class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
274 pipelineConnections=PsfWcsSelectImagesConnections):
275 maxEllipResidual = pexConfig.Field(
276 doc="Maximum median ellipticity residual",
277 dtype=float,
278 default=0.007,
279 optional=True,
280 )
281 maxSizeScatter = pexConfig.Field(
282 doc="Maximum scatter in the size residuals",
283 dtype=float,
284 optional=True,
285 )
286 maxScaledSizeScatter = pexConfig.Field(
287 doc="Maximum scatter in the size residuals, scaled by the median size",
288 dtype=float,
289 default=0.009,
290 optional=True,
291 )
292 starSelection = pexConfig.Field(
293 doc="select star with this field",
294 dtype=str,
295 default='calib_psf_used'
296 )
297 starShape = pexConfig.Field(
298 doc="name of star shape",
299 dtype=str,
300 default='base_SdssShape'
301 )
302 psfShape = pexConfig.Field(
303 doc="name of psf shape",
304 dtype=str,
305 default='base_SdssShape_psf'
306 )
309class PsfWcsSelectImagesTask(WcsSelectImagesTask):
310 """Select images using their Wcs and cuts on the PSF properties
312 The PSF quality criteria are based on the size and ellipticity residuals from the
313 adaptive second moments of the star and the PSF.
315 The criteria are:
316 - the median of the ellipticty residuals
317 - the robust scatter of the size residuals (using the median absolute deviation)
318 - the robust scatter of the size residuals scaled by the square of
319 the median size
320 """
322 ConfigClass = PsfWcsSelectImagesConfig
323 _DefaultName = "PsfWcsSelectImages"
325 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
326 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
328 This method is the old entry point for the Gen2 commandline tasks and drivers
329 Will be deprecated in v22.
331 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
332 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
333 @param makeDataRefList: Construct a list of data references?
334 @param selectDataList: List of SelectStruct, to consider for selection
335 """
336 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
337 selectDataList)
339 dataRefList = []
340 exposureInfoList = []
341 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
342 butler = dataRef.butlerSubset.butler
343 srcCatalog = butler.get('src', dataRef.dataId)
344 valid = self.isValid(srcCatalog, dataRef.dataId)
345 if valid is False:
346 continue
348 dataRefList.append(dataRef)
349 exposureInfoList.append(exposureInfo)
351 return pipeBase.Struct(
352 dataRefList=dataRefList,
353 exposureInfoList=exposureInfoList,
354 )
356 def run(self, wcsList, bboxList, coordList, srcList, dataIds=None, **kwargs):
357 """Return indices of provided lists that meet the selection criteria
359 Parameters:
360 -----------
361 wcsList : `list` of `lsst.afw.geom.SkyWcs`
362 specifying the WCS's of the input ccds to be selected
363 bboxList : `list` of `lsst.geom.Box2I`
364 specifying the bounding boxes of the input ccds to be selected
365 coordList : `list` of `lsst.geom.SpherePoint`
366 ICRS coordinates specifying boundary of the patch.
367 srcList : `list` of `lsst.afw.table.SourceCatalog`
368 containing the PSF shape information for the input ccds to be selected
370 Returns:
371 --------
372 goodPsf: `list` of `int`
373 of indices of selected ccds
374 """
375 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
376 coordList=coordList, dataIds=dataIds)
378 goodPsf = []
379 if dataIds is None:
380 dataIds = [None] * len(srcList)
381 for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)):
382 if i not in goodWcs:
383 continue
384 if self.isValid(srcCatalog, dataId):
385 goodPsf.append(i)
387 return goodPsf
389 def isValid(self, srcCatalog, dataId=None):
390 """Should this ccd be selected based on its PSF shape information
392 Parameters
393 ----------
394 srcCatalog : `lsst.afw.table.SourceCatalog`
395 dataId : `dict` of dataId keys, optional.
396 Used only for logging. Defaults to None.
398 Returns
399 -------
400 valid : `bool`
401 True if selected.
402 """
403 mask = srcCatalog[self.config.starSelection]
405 starXX = srcCatalog[self.config.starShape+'_xx'][mask]
406 starYY = srcCatalog[self.config.starShape+'_yy'][mask]
407 starXY = srcCatalog[self.config.starShape+'_xy'][mask]
408 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
409 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
410 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
412 starSize = np.power(starXX*starYY - starXY**2, 0.25)
413 starE1 = (starXX - starYY)/(starXX + starYY)
414 starE2 = 2*starXY/(starXX + starYY)
415 medianSize = np.median(starSize)
417 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
418 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
419 psfE2 = 2*psfXY/(psfXX + psfYY)
421 medianE1 = np.abs(np.median(starE1 - psfE1))
422 medianE2 = np.abs(np.median(starE2 - psfE2))
423 medianE = np.sqrt(medianE1**2 + medianE2**2)
425 scatterSize = sigmaMad(starSize - psfSize)
426 scaledScatterSize = scatterSize/medianSize**2
428 valid = True
429 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
430 self.log.info("Removing visit %s because median e residual too large: %f vs %f" %
431 (dataId, medianE, self.config.maxEllipResidual))
432 valid = False
433 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
434 self.log.info("Removing visit %s because size scatter is too large: %f vs %f" %
435 (dataId, scatterSize, self.config.maxSizeScatter))
436 valid = False
437 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
438 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f" %
439 (dataId, scaledScatterSize, self.config.maxScaledSizeScatter))
440 valid = False
442 return valid
445class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
446 """Base configuration for BestSeeingSelectImagesTask.
447 """
448 nImagesMax = pexConfig.RangeField(
449 dtype=int,
450 doc="Maximum number of images to select",
451 default=5,
452 min=0)
453 maxPsfFwhm = pexConfig.Field(
454 dtype=float,
455 doc="Maximum PSF FWHM (in arcseconds) to select",
456 default=1.5,
457 optional=True)
458 minPsfFwhm = pexConfig.Field(
459 dtype=float,
460 doc="Minimum PSF FWHM (in arcseconds) to select",
461 default=0.,
462 optional=True)
465class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
466 """Select up to a maximum number of the best-seeing images using their Wcs.
467 """
468 ConfigClass = BestSeeingWcsSelectImageConfig
470 def runDataRef(self, dataRef, coordList, makeDataRefList=True,
471 selectDataList=None):
472 """Select the best-seeing images in the selectDataList that overlap the patch.
474 This method is the old entry point for the Gen2 commandline tasks and drivers
475 Will be deprecated in v22.
477 Parameters
478 ----------
479 dataRef : `lsst.daf.persistence.ButlerDataRef`
480 Data reference for coadd/tempExp (with tract, patch)
481 coordList : `list` of `lsst.geom.SpherePoint`
482 List of ICRS sky coordinates specifying boundary of patch
483 makeDataRefList : `boolean`, optional
484 Construct a list of data references?
485 selectDataList : `list` of `SelectStruct`
486 List of SelectStruct, to consider for selection
488 Returns
489 -------
490 result : `lsst.pipe.base.Struct`
491 Result struct with components:
492 - ``exposureList``: the selected exposures
493 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`).
494 - ``dataRefList``: the optional data references corresponding to
495 each element of ``exposureList``
496 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
497 """
498 psfSizes = []
499 dataRefList = []
500 exposureInfoList = []
502 if selectDataList is None:
503 selectDataList = []
505 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
507 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
508 cal = dataRef.get("calexp", immediate=True)
510 # if min/max PSF values are defined, remove images out of bounds
511 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
512 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
513 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
514 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
515 continue
516 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
517 continue
518 psfSizes.append(psfSize)
519 dataRefList.append(dataRef)
520 exposureInfoList.append(exposureInfo)
522 if len(psfSizes) > self.config.nImagesMax:
523 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
524 filteredDataRefList = [dataRefList[i] for i in sortedIndices]
525 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
526 self.log.info(f"{len(sortedIndices)} images selected with FWHM "
527 f"range of {psfSizes[sortedIndices[0]]}--{psfSizes[sortedIndices[-1]]} arcseconds")
529 else:
530 if len(psfSizes) == 0:
531 self.log.warn("0 images selected.")
532 else:
533 self.log.debug(f"{len(psfSizes)} images selected with FWHM range "
534 f"of {psfSizes[0]}--{psfSizes[-1]} arcseconds")
535 filteredDataRefList = dataRefList
536 filteredExposureInfoList = exposureInfoList
538 return pipeBase.Struct(
539 dataRefList=filteredDataRefList if makeDataRefList else None,
540 exposureInfoList=filteredExposureInfoList,
541 )
543 def run(self, wcsList, bboxList, coordList, psfList, dataIds, **kwargs):
544 """Return indices of good calexps from a list.
546 This task does not make sense for use with makeWarp where there quanta
547 are per-visit rather than per-patch. This task selectes the best ccds
548 of ONE VISIT that overlap the patch.
550 This includes some code duplication with runDataRef,
551 but runDataRef will be deprecated as of v22.
553 Parameters:
554 -----------
555 wcsList : `list` of `lsst.afw.geom.SkyWcs`
556 specifying the WCS's of the input ccds to be selected
557 bboxList : `list` of `lsst.geom.Box2I`
558 specifying the bounding boxes of the input ccds to be selected
559 coordList : `list` of `lsst.geom.SpherePoint`
560 ICRS coordinates specifying boundary of the patch.
561 psfList : `list` of `lsst.afw.detection.Psf`
562 specifying the PSF model of the input ccds to be selected
564 Returns:
565 --------
566 output: `list` of `int`
567 of indices of selected ccds sorted by seeing
568 """
569 goodWcs = super().run(wcsList=wcsList, bboxList=bboxList, coordList=coordList, dataIds=dataIds)
571 psfSizes = []
572 indices = []
573 for i, (wcs, psf) in enumerate(wcsList, psfList):
574 if i not in goodWcs:
575 continue
576 # if min/max PSF values are defined, remove images out of bounds
577 pixToArcseconds = wcs.getPixelScale().asArcseconds()
578 psfSize = psf.computeShape().getDeterminantRadius()*pixToArcseconds
579 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
580 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
581 continue
582 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
583 continue
584 psfSizes.append(psfSize)
585 indices.append(i)
587 sortedIndices = [ind for (_, ind) in sorted(zip(psfSizes, indices))]
588 output = sortedIndices[:self.config.nImagesMax]
589 self.log.info(f"{len(output)} images selected with FWHM "
590 f"range of {psfSizes[indices.index(output[0])]}"
591 f"--{psfSizes[indices.index(output[-1])]} arcseconds")
592 return output