Coverage for python/lsst/pipe/tasks/selectImages.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.pex.config as pexConfig
25import lsst.pex.exceptions as pexExceptions
26import lsst.geom as geom
27import lsst.pipe.base as pipeBase
29__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
30 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask"]
33class DatabaseSelectImagesConfig(pexConfig.Config):
34 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
35 host = pexConfig.Field(
36 doc="Database server host name",
37 dtype=str,
38 )
39 port = pexConfig.Field(
40 doc="Database server port",
41 dtype=int,
42 )
43 database = pexConfig.Field(
44 doc="Name of database",
45 dtype=str,
46 )
47 maxExposures = pexConfig.Field(
48 doc="maximum exposures to select; intended for debugging; ignored if None",
49 dtype=int,
50 optional=True,
51 )
54class BaseExposureInfo(pipeBase.Struct):
55 """Data about a selected exposure
56 """
58 def __init__(self, dataId, coordList):
59 """Create exposure information that can be used to generate data references
61 The object has the following fields:
62 - dataId: data ID of exposure (a dict)
63 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
64 plus any others items that are desired
65 """
66 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
69class BaseSelectImagesTask(pipeBase.Task):
70 """Base task for selecting images suitable for coaddition
71 """
72 ConfigClass = pexConfig.Config
73 _DefaultName = "selectImages"
75 @pipeBase.timeMethod
76 def run(self, coordList):
77 """Select images suitable for coaddition in a particular region
79 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
80 subclasses may add additional keyword arguments, as required
82 @return a pipeBase Struct containing:
83 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
84 which have at least the following fields:
85 - dataId: data ID dictionary
86 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
87 """
88 raise NotImplementedError()
90 def _runArgDictFromDataId(self, dataId):
91 """Extract keyword arguments for run (other than coordList) from a data ID
93 @return keyword arguments for run (other than coordList), as a dict
94 """
95 raise NotImplementedError()
97 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
98 """Run based on a data reference
100 This delegates to run() and _runArgDictFromDataId() to do the actual
101 selection. In the event that the selectDataList is non-empty, this will
102 be used to further restrict the selection, providing the user with
103 additional control over the selection.
105 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
106 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
107 @param[in] makeDataRefList: if True, return dataRefList
108 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
109 @return a pipeBase Struct containing:
110 - exposureInfoList: a list of objects derived from ExposureInfo
111 - dataRefList: a list of data references (None if makeDataRefList False)
112 """
113 runArgDict = self._runArgDictFromDataId(dataRef.dataId)
114 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList
116 if len(selectDataList) > 0 and len(exposureInfoList) > 0:
117 # Restrict the exposure selection further
118 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
119 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
120 inValues = set(inValues)
121 newExposureInfoList = []
122 for info, ccdVal in zip(exposureInfoList, ccdValues):
123 if ccdVal in inValues:
124 newExposureInfoList.append(info)
125 else:
126 self.log.info("De-selecting exposure %s: not in selectDataList" % info.dataId)
127 exposureInfoList = newExposureInfoList
129 if makeDataRefList:
130 butler = dataRef.butlerSubset.butler
131 dataRefList = [butler.dataRef(datasetType="calexp",
132 dataId=expInfo.dataId,
133 ) for expInfo in exposureInfoList]
134 else:
135 dataRefList = None
137 return pipeBase.Struct(
138 dataRefList=dataRefList,
139 exposureInfoList=exposureInfoList,
140 )
143def _extractKeyValue(dataList, keys=None):
144 """Extract the keys and values from a list of dataIds
146 The input dataList is a list of objects that have 'dataId' members.
147 This allows it to be used for both a list of data references and a
148 list of ExposureInfo
149 """
150 assert len(dataList) > 0
151 if keys is None:
152 keys = sorted(dataList[0].dataId.keys())
153 keySet = set(keys)
154 values = list()
155 for data in dataList:
156 thisKeys = set(data.dataId.keys())
157 if thisKeys != keySet:
158 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
159 values.append(tuple(data.dataId[k] for k in keys))
160 return keys, values
163class SelectStruct(pipeBase.Struct):
164 """A container for data to be passed to the WcsSelectImagesTask"""
166 def __init__(self, dataRef, wcs, bbox):
167 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
170class WcsSelectImagesTask(BaseSelectImagesTask):
171 """Select images using their Wcs"""
173 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
174 """Select images in the selectDataList that overlap the patch
176 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
177 polygons on the celestial sphere, and test the polygon of the
178 patch for overlap with the polygon of the image.
180 We use "convexHull" instead of generating a ConvexPolygon
181 directly because the standard for the inputs to ConvexPolygon
182 are pretty high and we don't want to be responsible for reaching them.
184 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
185 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
186 @param makeDataRefList: Construct a list of data references?
187 @param selectDataList: List of SelectStruct, to consider for selection
188 """
189 dataRefList = []
190 exposureInfoList = []
192 patchVertices = [coord.getVector() for coord in coordList]
193 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
195 for data in selectDataList:
196 dataRef = data.dataRef
197 imageWcs = data.wcs
198 imageBox = data.bbox
200 try:
201 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
202 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
203 # Protecting ourselves from awful Wcs solutions in input images
204 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataRef.dataId, e)
205 continue
207 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
208 if imagePoly is None:
209 self.log.debug("Unable to create polygon from image %s: deselecting", dataRef.dataId)
210 continue
211 if patchPoly.intersects(imagePoly): # "intersects" also covers "contains" or "is contained by"
212 self.log.info("Selecting calexp %s" % dataRef.dataId)
213 dataRefList.append(dataRef)
214 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
216 return pipeBase.Struct(
217 dataRefList=dataRefList if makeDataRefList else None,
218 exposureInfoList=exposureInfoList,
219 )
222class PsfWcsSelectImagesConfig(pexConfig.Config):
223 maxEllipResidual = pexConfig.Field(
224 doc="Maximum median ellipticity residual",
225 dtype=float,
226 default=0.007,
227 optional=True,
228 )
229 maxSizeScatter = pexConfig.Field(
230 doc="Maximum scatter in the size residuals",
231 dtype=float,
232 optional=True,
233 )
234 maxScaledSizeScatter = pexConfig.Field(
235 doc="Maximum scatter in the size residuals, scaled by the median size",
236 dtype=float,
237 default=0.009,
238 optional=True,
239 )
240 starSelection = pexConfig.Field(
241 doc="select star with this field",
242 dtype=str,
243 default='calib_psf_used'
244 )
245 starShape = pexConfig.Field(
246 doc="name of star shape",
247 dtype=str,
248 default='base_SdssShape'
249 )
250 psfShape = pexConfig.Field(
251 doc="name of psf shape",
252 dtype=str,
253 default='base_SdssShape_psf'
254 )
257def sigmaMad(array):
258 "Return median absolute deviation scaled to normally distributed data"
259 return 1.4826*np.median(np.abs(array - np.median(array)))
262class PsfWcsSelectImagesTask(WcsSelectImagesTask):
263 """Select images using their Wcs and cuts on the PSF properties"""
265 ConfigClass = PsfWcsSelectImagesConfig
266 _DefaultName = "PsfWcsSelectImages"
268 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
269 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
271 The PSF quality criteria are based on the size and ellipticity residuals from the
272 adaptive second moments of the star and the PSF.
274 The criteria are:
275 - the median of the ellipticty residuals
276 - the robust scatter of the size residuals (using the median absolute deviation)
277 - the robust scatter of the size residuals scaled by the square of
278 the median size
280 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
281 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
282 @param makeDataRefList: Construct a list of data references?
283 @param selectDataList: List of SelectStruct, to consider for selection
284 """
285 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
286 selectDataList)
288 dataRefList = []
289 exposureInfoList = []
290 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
291 butler = dataRef.butlerSubset.butler
292 srcCatalog = butler.get('src', dataRef.dataId)
293 mask = srcCatalog[self.config.starSelection]
295 starXX = srcCatalog[self.config.starShape+'_xx'][mask]
296 starYY = srcCatalog[self.config.starShape+'_yy'][mask]
297 starXY = srcCatalog[self.config.starShape+'_xy'][mask]
298 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
299 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
300 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
302 starSize = np.power(starXX*starYY - starXY**2, 0.25)
303 starE1 = (starXX - starYY)/(starXX + starYY)
304 starE2 = 2*starXY/(starXX + starYY)
305 medianSize = np.median(starSize)
307 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
308 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
309 psfE2 = 2*psfXY/(psfXX + psfYY)
311 medianE1 = np.abs(np.median(starE1 - psfE1))
312 medianE2 = np.abs(np.median(starE2 - psfE2))
313 medianE = np.sqrt(medianE1**2 + medianE2**2)
315 scatterSize = sigmaMad(starSize - psfSize)
316 scaledScatterSize = scatterSize/medianSize**2
318 valid = True
319 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
320 self.log.info("Removing visit %s because median e residual too large: %f vs %f" %
321 (dataRef.dataId, medianE, self.config.maxEllipResidual))
322 valid = False
323 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
324 self.log.info("Removing visit %s because size scatter is too large: %f vs %f" %
325 (dataRef.dataId, scatterSize, self.config.maxSizeScatter))
326 valid = False
327 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
328 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f" %
329 (dataRef.dataId, scaledScatterSize, self.config.maxScaledSizeScatter))
330 valid = False
332 if valid is False:
333 continue
335 dataRefList.append(dataRef)
336 exposureInfoList.append(exposureInfo)
338 return pipeBase.Struct(
339 dataRefList=dataRefList,
340 exposureInfoList=exposureInfoList,
341 )
344class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
345 """Base configuration for BestSeeingSelectImagesTask.
346 """
347 nImagesMax = pexConfig.Field(
348 dtype=int,
349 doc="Maximum number of images to select",
350 default=5)
351 maxPsfFwhm = pexConfig.Field(
352 dtype=float,
353 doc="Maximum PSF FWHM (in arcseconds) to select",
354 default=1.5,
355 optional=True)
356 minPsfFwhm = pexConfig.Field(
357 dtype=float,
358 doc="Minimum PSF FWHM (in arcseconds) to select",
359 default=0.,
360 optional=True)
363class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
364 """Select up to a maximum number of the best-seeing images using their Wcs.
365 """
366 ConfigClass = BestSeeingWcsSelectImageConfig
368 def runDataRef(self, dataRef, coordList, makeDataRefList=True,
369 selectDataList=None):
370 """Select the best-seeing images in the selectDataList that overlap the patch.
372 Parameters
373 ----------
374 dataRef : `lsst.daf.persistence.ButlerDataRef`
375 Data reference for coadd/tempExp (with tract, patch)
376 coordList : `list` of `lsst.geom.SpherePoint`
377 List of ICRS sky coordinates specifying boundary of patch
378 makeDataRefList : `boolean`, optional
379 Construct a list of data references?
380 selectDataList : `list` of `SelectStruct`
381 List of SelectStruct, to consider for selection
383 Returns
384 -------
385 result : `lsst.pipe.base.Struct`
386 Result struct with components:
387 - ``exposureList``: the selected exposures
388 (`list` of `lsst.pipe.tasks.selectImages.BaseExposureInfo`).
389 - ``dataRefList``: the optional data references corresponding to
390 each element of ``exposureList``
391 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
392 """
393 if self.config.nImagesMax <= 0:
394 raise RuntimeError(f"nImagesMax must be greater than zero: {self.config.nImagesMax}")
396 psfSizes = []
397 dataRefList = []
398 exposureInfoList = []
400 if selectDataList is None:
401 selectDataList = []
403 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
405 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
406 cal = dataRef.get("calexp", immediate=True)
408 # if min/max PSF values are defined, remove images out of bounds
409 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
410 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
411 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
412 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
413 continue
414 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
415 continue
416 psfSizes.append(psfSize)
417 dataRefList.append(dataRef)
418 exposureInfoList.append(exposureInfo)
420 if len(psfSizes) > self.config.nImagesMax:
421 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
422 filteredDataRefList = [dataRefList[i] for i in sortedIndices]
423 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
424 self.log.info(f"{len(sortedIndices)} images selected with FWHM "
425 f"range of {psfSizes[sortedIndices[0]]}--{psfSizes[sortedIndices[-1]]} arcseconds")
427 else:
428 if len(psfSizes) == 0:
429 self.log.warn(f"0 images selected.")
430 else:
431 self.log.debug(f"{len(psfSizes)} images selected with FWHM range "
432 f"of {psfSizes[0]}--{psfSizes[-1]} arcseconds")
433 filteredDataRefList = dataRefList
434 filteredExposureInfoList = exposureInfoList
436 return pipeBase.Struct(
437 dataRefList=filteredDataRefList if makeDataRefList else None,
438 exposureInfoList=filteredExposureInfoList,
439 )