lsst.pipe.tasks g6a99470703+51f2fbd04f
Loading...
Searching...
No Matches
selectImages.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.utils as utils
25import lsst.pex.config as pexConfig
26import lsst.pex.exceptions as pexExceptions
27import lsst.geom as geom
28import lsst.pipe.base as pipeBase
29from lsst.skymap import BaseSkyMap
30from lsst.daf.base import DateTime
31from lsst.utils.timer import timeMethod
32
33__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
34 "DatabaseSelectImagesConfig", "BestSeeingSelectVisitsTask",
35 "BestSeeingQuantileSelectVisitsTask"]
36
37
38class DatabaseSelectImagesConfig(pexConfig.Config):
39 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
40 host = pexConfig.Field(
41 doc="Database server host name",
42 dtype=str,
43 )
44 port = pexConfig.Field(
45 doc="Database server port",
46 dtype=int,
47 )
48 database = pexConfig.Field(
49 doc="Name of database",
50 dtype=str,
51 )
52 maxExposures = pexConfig.Field(
53 doc="maximum exposures to select; intended for debugging; ignored if None",
54 dtype=int,
55 optional=True,
56 )
57
58
59class BaseExposureInfo(pipeBase.Struct):
60 """Data about a selected exposure
61 """
62
63 def __init__(self, dataId, coordList):
64 """Create exposure information that can be used to generate data references
65
66 The object has the following fields:
67 - dataId: data ID of exposure (a dict)
68 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
69 plus any others items that are desired
70 """
71 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
72
73
74class BaseSelectImagesTask(pipeBase.Task):
75 """Base task for selecting images suitable for coaddition
76 """
77 ConfigClass = pexConfig.Config
78 _DefaultName = "selectImages"
79
80 @timeMethod
81 def run(self, coordList):
82 """Select images suitable for coaddition in a particular region
83
84 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
85 subclasses may add additional keyword arguments, as required
86
87 @return a pipeBase Struct containing:
88 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
89 which have at least the following fields:
90 - dataId: data ID dictionary
91 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
92 """
93 raise NotImplementedError()
94
95
96def _extractKeyValue(dataList, keys=None):
97 """Extract the keys and values from a list of dataIds
98
99 The input dataList is a list of objects that have 'dataId' members.
100 This allows it to be used for both a list of data references and a
101 list of ExposureInfo
102 """
103 assert len(dataList) > 0
104 if keys is None:
105 keys = sorted(dataList[0].dataId.keys())
106 keySet = set(keys)
107 values = list()
108 for data in dataList:
109 thisKeys = set(data.dataId.keys())
110 if thisKeys != keySet:
111 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
112 values.append(tuple(data.dataId[k] for k in keys))
113 return keys, values
114
115
116class SelectStruct(pipeBase.Struct):
117 """A container for data to be passed to the WcsSelectImagesTask"""
118
119 def __init__(self, dataRef, wcs, bbox):
120 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
121
122
124 """Select images using their Wcs
125
126 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
127 polygons on the celestial sphere, and test the polygon of the
128 patch for overlap with the polygon of the image.
129
130 We use "convexHull" instead of generating a ConvexPolygon
131 directly because the standard for the inputs to ConvexPolygon
132 are pretty high and we don't want to be responsible for reaching them.
133 """
134
135 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
136 """Return indices of provided lists that meet the selection criteria
137
138 Parameters:
139 -----------
140 wcsList : `list` of `lsst.afw.geom.SkyWcs`
141 specifying the WCS's of the input ccds to be selected
142 bboxList : `list` of `lsst.geom.Box2I`
143 specifying the bounding boxes of the input ccds to be selected
144 coordList : `list` of `lsst.geom.SpherePoint`
145 ICRS coordinates specifying boundary of the patch.
146
147 Returns:
148 --------
149 result: `list` of `int`
150 of indices of selected ccds
151 """
152 if dataIds is None:
153 dataIds = [None] * len(wcsList)
154 patchVertices = [coord.getVector() for coord in coordList]
155 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
156 result = []
157 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
158 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
159 if imageCorners:
160 result.append(i)
161 return result
162
163 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
164 "Return corners or None if bad"
165 try:
166 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
167 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
168 # Protecting ourselves from awful Wcs solutions in input images
169 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
170 return
171
172 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
173 if imagePoly is None:
174 self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
175 return
176
177 if patchPoly.intersects(imagePoly):
178 # "intersects" also covers "contains" or "is contained by"
179 self.log.info("Selecting calexp %s", dataId)
180 return imageCorners
181
182
183class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
184 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
185 defaultTemplates={"coaddName": "deep"}):
186 pass
187
188
189class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
190 pipelineConnections=PsfWcsSelectImagesConnections):
191 maxEllipResidual = pexConfig.Field(
192 doc="Maximum median ellipticity residual",
193 dtype=float,
194 default=0.007,
195 optional=True,
196 )
197 maxSizeScatter = pexConfig.Field(
198 doc="Maximum scatter in the size residuals",
199 dtype=float,
200 optional=True,
201 )
202 maxScaledSizeScatter = pexConfig.Field(
203 doc="Maximum scatter in the size residuals, scaled by the median size",
204 dtype=float,
205 default=0.009,
206 optional=True,
207 )
208
209
210class PsfWcsSelectImagesTask(WcsSelectImagesTask):
211 """Select images using their Wcs and cuts on the PSF properties
212
213 The PSF quality criteria are based on the size and ellipticity residuals from the
214 adaptive second moments of the star and the PSF.
215
216 The criteria are:
217 - the median of the ellipticty residuals
218 - the robust scatter of the size residuals (using the median absolute deviation)
219 - the robust scatter of the size residuals scaled by the square of
220 the median size
221 """
222
223 ConfigClass = PsfWcsSelectImagesConfig
224 _DefaultName = "PsfWcsSelectImages"
225
226 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
227 """Return indices of provided lists that meet the selection criteria
228
229 Parameters:
230 -----------
231 wcsList : `list` of `lsst.afw.geom.SkyWcs`
232 specifying the WCS's of the input ccds to be selected
233 bboxList : `list` of `lsst.geom.Box2I`
234 specifying the bounding boxes of the input ccds to be selected
235 coordList : `list` of `lsst.geom.SpherePoint`
236 ICRS coordinates specifying boundary of the patch.
237 visitSummary : `list` of `lsst.afw.table.ExposureCatalog`
238 containing the PSF shape information for the input ccds to be selected.
239
240 Returns:
241 --------
242 goodPsf: `list` of `int`
243 of indices of selected ccds
244 """
245 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
246 coordList=coordList, dataIds=dataIds)
247
248 goodPsf = []
249
250 for i, dataId in enumerate(dataIds):
251 if i not in goodWcs:
252 continue
253 if self.isValid(visitSummary, dataId["detector"]):
254 goodPsf.append(i)
255
256 return goodPsf
257
258 def isValid(self, visitSummary, detectorId):
259 """Should this ccd be selected based on its PSF shape information.
260
261 Parameters
262 ----------
263 visitSummary : `lsst.afw.table.ExposureCatalog`
264 detectorId : `int`
265 Detector identifier.
266
267 Returns
268 -------
269 valid : `bool`
270 True if selected.
271 """
272 row = visitSummary.find(detectorId)
273 if row is None:
274 # This is not listed, so it must be bad.
275 self.log.warning("Removing detector %d because summary stats not available.", detectorId)
276 return False
277
278 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.)
279 scatterSize = row["psfStarDeltaSizeScatter"]
280 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"]
281
282 valid = True
283 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
284 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f",
285 row["visit"], detectorId, medianE, self.config.maxEllipResidual)
286 valid = False
287 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
288 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f",
289 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter)
290 valid = False
291 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
292 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
293 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
294 valid = False
295
296 return valid
297
298
299class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
300 dimensions=("tract", "patch", "skymap", "band", "instrument"),
301 defaultTemplates={"coaddName": "goodSeeing"}):
302 skyMap = pipeBase.connectionTypes.Input(
303 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
304 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
305 storageClass="SkyMap",
306 dimensions=("skymap",),
307 )
308 visitSummaries = pipeBase.connectionTypes.Input(
309 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
310 name="visitSummary",
311 storageClass="ExposureCatalog",
312 dimensions=("instrument", "visit",),
313 multiple=True,
314 deferLoad=True
315 )
316 goodVisits = pipeBase.connectionTypes.Output(
317 doc="Selected visits to be coadded.",
318 name="{coaddName}Visits",
319 storageClass="StructuredDataDict",
320 dimensions=("instrument", "tract", "patch", "skymap", "band"),
321 )
322
323
324class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
325 pipelineConnections=BestSeeingSelectVisitsConnections):
326 nVisitsMax = pexConfig.RangeField(
327 dtype=int,
328 doc="Maximum number of visits to select",
329 default=12,
330 min=0
331 )
332 maxPsfFwhm = pexConfig.Field(
333 dtype=float,
334 doc="Maximum PSF FWHM (in arcseconds) to select",
335 default=1.5,
336 optional=True
337 )
338 minPsfFwhm = pexConfig.Field(
339 dtype=float,
340 doc="Minimum PSF FWHM (in arcseconds) to select",
341 default=0.,
342 optional=True
343 )
344 doConfirmOverlap = pexConfig.Field(
345 dtype=bool,
346 doc="Do remove visits that do not actually overlap the patch?",
347 default=True,
348 )
349 minMJD = pexConfig.Field(
350 dtype=float,
351 doc="Minimum visit MJD to select",
352 default=None,
353 optional=True
354 )
355 maxMJD = pexConfig.Field(
356 dtype=float,
357 doc="Maximum visit MJD to select",
358 default=None,
359 optional=True
360 )
361
362
363class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
364 """Select up to a maximum number of the best-seeing visits
365
366 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
367 This Task is a port of the Gen2 image-selector used in the AP pipeline:
368 BestSeeingSelectImagesTask. This Task selects full visits based on the
369 average PSF of the entire visit.
370 """
371 ConfigClass = BestSeeingSelectVisitsConfig
372 _DefaultName = 'bestSeeingSelectVisits'
373
374 def runQuantum(self, butlerQC, inputRefs, outputRefs):
375 inputs = butlerQC.get(inputRefs)
376 quantumDataId = butlerQC.quantum.dataId
377 outputs = self.run(**inputs, dataId=quantumDataId)
378 butlerQC.put(outputs, outputRefs)
379
380 def run(self, visitSummaries, skyMap, dataId):
381 """Run task
382
383 Parameters:
384 -----------
385 visitSummary : `list`
386 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
387 visitSummary tables of type `lsst.afw.table.ExposureCatalog`
388 skyMap : `lsst.skyMap.SkyMap`
389 SkyMap for checking visits overlap patch
390 dataId : `dict` of dataId keys
391 For retrieving patch info for checking visits overlap patch
392
393 Returns
394 -------
395 result : `lsst.pipe.base.Struct`
396 Result struct with components:
397
398 - `goodVisits`: `dict` with selected visit ids as keys,
399 so that it can be be saved as a StructuredDataDict.
400 StructuredDataList's are currently limited.
401 """
402
403 if self.config.doConfirmOverlap:
404 patchPolygon = self.makePatchPolygon(skyMap, dataId)
405
406 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
407 fwhmSizes = []
408 visits = []
409 for visit, visitSummary in zip(inputVisits, visitSummaries):
410 # read in one-by-one and only once. There may be hundreds
411 visitSummary = visitSummary.get()
412
413 # mjd is guaranteed to be the same for every detector in the visitSummary.
414 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
415
416 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
417 for vs in visitSummary]
418 # psfSigma is PSF model determinant radius at chip center in pixels
419 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
420 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
421
422 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
423 continue
424 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
425 continue
426 if self.config.minMJD and mjd < self.config.minMJD:
427 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
428 continue
429 if self.config.maxMJD and mjd > self.config.maxMJD:
430 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
431 continue
432 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
433 continue
434
435 fwhmSizes.append(fwhm)
436 visits.append(visit)
437
438 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
439 output = sortedVisits[:self.config.nVisitsMax]
440 self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
441 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
442
443 # In order to store as a StructuredDataDict, convert list to dict
444 goodVisits = {key: True for key in output}
445 return pipeBase.Struct(goodVisits=goodVisits)
446
447 def makePatchPolygon(self, skyMap, dataId):
448 """Return True if sky polygon overlaps visit
449
450 Parameters:
451 -----------
453 Exposure catalog with per-detector geometry
454 dataId : `dict` of dataId keys
455 For retrieving patch info
456
457 Returns:
458 --------
459 result :` lsst.sphgeom.ConvexPolygon.convexHull`
460 Polygon of patch's outer bbox
461 """
462 wcs = skyMap[dataId['tract']].getWcs()
463 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
464 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
465 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
466 return result
467
468 def doesIntersectPolygon(self, visitSummary, polygon):
469 """Return True if sky polygon overlaps visit
470
471 Parameters:
472 -----------
473 visitSummary : `lsst.afw.table.ExposureCatalog`
474 Exposure catalog with per-detector geometry
475 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
476 Polygon to check overlap
477
478 Returns:
479 --------
480 doesIntersect: `bool`
481 Does the visit overlap the polygon
482 """
483 doesIntersect = False
484 for detectorSummary in visitSummary:
485 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
486 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
487 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
488 if detectorPolygon.intersects(polygon):
489 doesIntersect = True
490 break
491 return doesIntersect
492
493
494class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
495 pipelineConnections=BestSeeingSelectVisitsConnections):
496 qMin = pexConfig.RangeField(
497 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
498 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
499 "This config should be changed from zero only for exploratory diffIm testing.",
500 dtype=float,
501 default=0,
502 min=0,
503 max=1,
504 )
505 qMax = pexConfig.RangeField(
506 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
507 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
508 dtype=float,
509 default=0.33,
510 min=0,
511 max=1,
512 )
513 nVisitsMin = pexConfig.Field(
514 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
515 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
516 dtype=int,
517 default=6,
518 )
519 doConfirmOverlap = pexConfig.Field(
520 dtype=bool,
521 doc="Do remove visits that do not actually overlap the patch?",
522 default=True,
523 )
524 minMJD = pexConfig.Field(
525 dtype=float,
526 doc="Minimum visit MJD to select",
527 default=None,
528 optional=True
529 )
530 maxMJD = pexConfig.Field(
531 dtype=float,
532 doc="Maximum visit MJD to select",
533 default=None,
534 optional=True
535 )
536
537
538class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
539 """Select a quantile of the best-seeing visits
540
541 Selects the best (for example, third) full visits based on the average
542 PSF width in the entire visit. It can also be used for difference imaging
543 experiments that require templates with the worst seeing visits.
544 For example, selecting the worst third can be acheived by
545 changing the config parameters qMin to 0.66 and qMax to 1.
546 """
547 ConfigClass = BestSeeingQuantileSelectVisitsConfig
548 _DefaultName = 'bestSeeingQuantileSelectVisits'
549
550 @utils.inheritDoc(BestSeeingSelectVisitsTask)
551 def run(self, visitSummaries, skyMap, dataId):
552 if self.config.doConfirmOverlap:
553 patchPolygon = self.makePatchPolygon(skyMap, dataId)
554 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
555 radius = np.empty(len(visits))
556 intersects = np.full(len(visits), True)
557 for i, visitSummary in enumerate(visitSummaries):
558 # read in one-by-one and only once. There may be hundreds
559 visitSummary = visitSummary.get()
560 # psfSigma is PSF model determinant radius at chip center in pixels
561 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
562 radius[i] = psfSigma
563 if self.config.doConfirmOverlap:
564 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
565 if self.config.minMJD or self.config.maxMJD:
566 # mjd is guaranteed to be the same for every detector in the visitSummary.
567 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
568 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
569 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
570 intersects[i] = intersects[i] and aboveMin and belowMax
571
572 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
573 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
574 max(0, len(visits[intersects]) - self.config.nVisitsMin))
575 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
576
577 # In order to store as a StructuredDataDict, convert list to dict
578 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
579 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
Definition: selectImages.py:63
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)