lsst.pipe.tasks g4a6547c0d5+de68eba77a
Loading...
Searching...
No Matches
selectImages.py
Go to the documentation of this file.
2# LSST Data Management System
3# Copyright 2008, 2009, 2010 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import numpy as np
23import lsst.sphgeom
24import lsst.utils as utils
25import lsst.pex.config as pexConfig
26import lsst.pex.exceptions as pexExceptions
27import lsst.geom as geom
28import lsst.pipe.base as pipeBase
29from lsst.skymap import BaseSkyMap
30from lsst.daf.base import DateTime
31
32__all__ = ["BaseSelectImagesTask", "BaseExposureInfo", "WcsSelectImagesTask", "PsfWcsSelectImagesTask",
33 "DatabaseSelectImagesConfig", "BestSeeingWcsSelectImagesTask", "BestSeeingSelectVisitsTask",
34 "BestSeeingQuantileSelectVisitsTask"]
35
36
37class DatabaseSelectImagesConfig(pexConfig.Config):
38 """Base configuration for subclasses of BaseSelectImagesTask that use a database"""
39 host = pexConfig.Field(
40 doc="Database server host name",
41 dtype=str,
42 )
43 port = pexConfig.Field(
44 doc="Database server port",
45 dtype=int,
46 )
47 database = pexConfig.Field(
48 doc="Name of database",
49 dtype=str,
50 )
51 maxExposures = pexConfig.Field(
52 doc="maximum exposures to select; intended for debugging; ignored if None",
53 dtype=int,
54 optional=True,
55 )
56
57
58class BaseExposureInfo(pipeBase.Struct):
59 """Data about a selected exposure
60 """
61
62 def __init__(self, dataId, coordList):
63 """Create exposure information that can be used to generate data references
64
65 The object has the following fields:
66 - dataId: data ID of exposure (a dict)
67 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
68 plus any others items that are desired
69 """
70 super(BaseExposureInfo, self).__init__(dataId=dataId, coordList=coordList)
71
72
73class BaseSelectImagesTask(pipeBase.Task):
74 """Base task for selecting images suitable for coaddition
75 """
76 ConfigClass = pexConfig.Config
77 _DefaultName = "selectImages"
78
79 @pipeBase.timeMethod
80 def run(self, coordList):
81 """Select images suitable for coaddition in a particular region
82
83 @param[in] coordList: list of coordinates defining region of interest; if None then select all images
84 subclasses may add additional keyword arguments, as required
85
86 @return a pipeBase Struct containing:
87 - exposureInfoList: a list of exposure information objects (subclasses of BaseExposureInfo),
88 which have at least the following fields:
89 - dataId: data ID dictionary
90 - coordList: ICRS coordinates of the corners of the exposure (list of lsst.geom.SpherePoint)
91 """
92 raise NotImplementedError()
93
94 def _runArgDictFromDataId(self, dataId):
95 """Extract keyword arguments for run (other than coordList) from a data ID
96
97 @return keyword arguments for run (other than coordList), as a dict
98 """
99 raise NotImplementedError()
100
101 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
102 """Run based on a data reference
103
104 This delegates to run() and _runArgDictFromDataId() to do the actual
105 selection. In the event that the selectDataList is non-empty, this will
106 be used to further restrict the selection, providing the user with
107 additional control over the selection.
108
109 @param[in] dataRef: data reference; must contain any extra keys needed by the subclass
110 @param[in] coordList: list of coordinates defining region of interest; if None, search the whole sky
111 @param[in] makeDataRefList: if True, return dataRefList
112 @param[in] selectDataList: List of SelectStruct with dataRefs to consider for selection
113 @return a pipeBase Struct containing:
114 - exposureInfoList: a list of objects derived from ExposureInfo
115 - dataRefList: a list of data references (None if makeDataRefList False)
116 """
117 runArgDict = self._runArgDictFromDataId(dataRef.dataId)
118 exposureInfoList = self.run(coordList, **runArgDict).exposureInfoList
119
120 if len(selectDataList) > 0 and len(exposureInfoList) > 0:
121 # Restrict the exposure selection further
122 ccdKeys, ccdValues = _extractKeyValue(exposureInfoList)
123 inKeys, inValues = _extractKeyValue([s.dataRef for s in selectDataList], keys=ccdKeys)
124 inValues = set(inValues)
125 newExposureInfoList = []
126 for info, ccdVal in zip(exposureInfoList, ccdValues):
127 if ccdVal in inValues:
128 newExposureInfoList.append(info)
129 else:
130 self.log.info("De-selecting exposure %s: not in selectDataList", info.dataId)
131 exposureInfoList = newExposureInfoList
132
133 if makeDataRefList:
134 butler = dataRef.butlerSubset.butler
135 dataRefList = [butler.dataRef(datasetType="calexp",
136 dataId=expInfo.dataId,
137 ) for expInfo in exposureInfoList]
138 else:
139 dataRefList = None
140
141 return pipeBase.Struct(
142 dataRefList=dataRefList,
143 exposureInfoList=exposureInfoList,
144 )
145
146
147def _extractKeyValue(dataList, keys=None):
148 """Extract the keys and values from a list of dataIds
149
150 The input dataList is a list of objects that have 'dataId' members.
151 This allows it to be used for both a list of data references and a
152 list of ExposureInfo
153 """
154 assert len(dataList) > 0
155 if keys is None:
156 keys = sorted(dataList[0].dataId.keys())
157 keySet = set(keys)
158 values = list()
159 for data in dataList:
160 thisKeys = set(data.dataId.keys())
161 if thisKeys != keySet:
162 raise RuntimeError("DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
163 values.append(tuple(data.dataId[k] for k in keys))
164 return keys, values
165
166
167class SelectStruct(pipeBase.Struct):
168 """A container for data to be passed to the WcsSelectImagesTask"""
169
170 def __init__(self, dataRef, wcs, bbox):
171 super(SelectStruct, self).__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
172
173
175 """Select images using their Wcs
176
177 We use the "convexHull" method of lsst.sphgeom.ConvexPolygon to define
178 polygons on the celestial sphere, and test the polygon of the
179 patch for overlap with the polygon of the image.
180
181 We use "convexHull" instead of generating a ConvexPolygon
182 directly because the standard for the inputs to ConvexPolygon
183 are pretty high and we don't want to be responsible for reaching them.
184 """
185
186 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
187 """Select images in the selectDataList that overlap the patch
188
189 This method is the old entry point for the Gen2 commandline tasks and drivers
190 Will be deprecated in v22.
191
192 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
193 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
194 @param makeDataRefList: Construct a list of data references?
195 @param selectDataList: List of SelectStruct, to consider for selection
196 """
197 dataRefList = []
198 exposureInfoList = []
199
200 patchVertices = [coord.getVector() for coord in coordList]
201 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
202
203 for data in selectDataList:
204 dataRef = data.dataRef
205 imageWcs = data.wcs
206 imageBox = data.bbox
207
208 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId=None)
209 if imageCorners:
210 dataRefList.append(dataRef)
211 exposureInfoList.append(BaseExposureInfo(dataRef.dataId, imageCorners))
212
213 return pipeBase.Struct(
214 dataRefList=dataRefList if makeDataRefList else None,
215 exposureInfoList=exposureInfoList,
216 )
217
218 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
219 """Return indices of provided lists that meet the selection criteria
220
221 Parameters:
222 -----------
223 wcsList : `list` of `lsst.afw.geom.SkyWcs`
224 specifying the WCS's of the input ccds to be selected
225 bboxList : `list` of `lsst.geom.Box2I`
226 specifying the bounding boxes of the input ccds to be selected
227 coordList : `list` of `lsst.geom.SpherePoint`
228 ICRS coordinates specifying boundary of the patch.
229
230 Returns:
231 --------
232 result: `list` of `int`
233 of indices of selected ccds
234 """
235 if dataIds is None:
236 dataIds = [None] * len(wcsList)
237 patchVertices = [coord.getVector() for coord in coordList]
238 patchPoly = lsst.sphgeom.ConvexPolygon.convexHull(patchVertices)
239 result = []
240 for i, (imageWcs, imageBox, dataId) in enumerate(zip(wcsList, bboxList, dataIds)):
241 imageCorners = self.getValidImageCorners(imageWcs, imageBox, patchPoly, dataId)
242 if imageCorners:
243 result.append(i)
244 return result
245
246 def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None):
247 "Return corners or None if bad"
248 try:
249 imageCorners = [imageWcs.pixelToSky(pix) for pix in geom.Box2D(imageBox).getCorners()]
250 except (pexExceptions.DomainError, pexExceptions.RuntimeError) as e:
251 # Protecting ourselves from awful Wcs solutions in input images
252 self.log.debug("WCS error in testing calexp %s (%s): deselecting", dataId, e)
253 return
254
255 imagePoly = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in imageCorners])
256 if imagePoly is None:
257 self.log.debug("Unable to create polygon from image %s: deselecting", dataId)
258 return
259
260 if patchPoly.intersects(imagePoly):
261 # "intersects" also covers "contains" or "is contained by"
262 self.log.info("Selecting calexp %s", dataId)
263 return imageCorners
264
265
266def sigmaMad(array):
267 "Return median absolute deviation scaled to normally distributed data"
268 return 1.4826*np.median(np.abs(array - np.median(array)))
269
270
271class PsfWcsSelectImagesConnections(pipeBase.PipelineTaskConnections,
272 dimensions=("tract", "patch", "skymap", "instrument", "visit"),
273 defaultTemplates={"coaddName": "deep"}):
274 pass
275
276
277class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
278 pipelineConnections=PsfWcsSelectImagesConnections):
279 maxEllipResidual = pexConfig.Field(
280 doc="Maximum median ellipticity residual",
281 dtype=float,
282 default=0.007,
283 optional=True,
284 )
285 maxSizeScatter = pexConfig.Field(
286 doc="Maximum scatter in the size residuals",
287 dtype=float,
288 optional=True,
289 )
290 maxScaledSizeScatter = pexConfig.Field(
291 doc="Maximum scatter in the size residuals, scaled by the median size",
292 dtype=float,
293 default=0.009,
294 optional=True,
295 )
296 starSelection = pexConfig.Field(
297 doc="select star with this field",
298 dtype=str,
299 default='calib_psf_used',
300 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
301 'will be removed after v24.')
302 )
303 starShape = pexConfig.Field(
304 doc="name of star shape",
305 dtype=str,
306 default='base_SdssShape',
307 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
308 'will be removed after v24.')
309 )
310 psfShape = pexConfig.Field(
311 doc="name of psf shape",
312 dtype=str,
313 default='base_SdssShape_psf',
314 deprecated=('This field has been moved to ComputeExposureSummaryStatsTask and '
315 'will be removed after v24.')
316 )
317 doLegacyStarSelectionComputation = pexConfig.Field(
318 doc="Perform the legacy star selection computations (for backwards compatibility)",
319 dtype=bool,
320 default=False,
321 deprecated=("This field is here for backwards compatibility and will be "
322 "removed after v24.")
323 )
324
325
326class PsfWcsSelectImagesTask(WcsSelectImagesTask):
327 """Select images using their Wcs and cuts on the PSF properties
328
329 The PSF quality criteria are based on the size and ellipticity residuals from the
330 adaptive second moments of the star and the PSF.
331
332 The criteria are:
333 - the median of the ellipticty residuals
334 - the robust scatter of the size residuals (using the median absolute deviation)
335 - the robust scatter of the size residuals scaled by the square of
336 the median size
337 """
338
339 ConfigClass = PsfWcsSelectImagesConfig
340 _DefaultName = "PsfWcsSelectImages"
341
342 def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[]):
343 """Select images in the selectDataList that overlap the patch and satisfy PSF quality critera.
344
345 This method is the old entry point for the Gen2 commandline tasks and drivers
346 Will be deprecated in v22.
347
348 @param dataRef: Data reference for coadd/tempExp (with tract, patch)
349 @param coordList: List of ICRS coordinates (lsst.geom.SpherePoint) specifying boundary of patch
350 @param makeDataRefList: Construct a list of data references?
351 @param selectDataList: List of SelectStruct, to consider for selection
352 """
353 result = super(PsfWcsSelectImagesTask, self).runDataRef(dataRef, coordList, makeDataRefList,
354 selectDataList)
355
356 dataRefList = []
357 exposureInfoList = []
358 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
359 butler = dataRef.butlerSubset.butler
360 srcCatalog = butler.get('src', dataRef.dataId)
361 valid = self.isValidLegacy(srcCatalog, dataRef.dataId)
362 if valid is False:
363 continue
364
365 dataRefList.append(dataRef)
366 exposureInfoList.append(exposureInfo)
367
368 return pipeBase.Struct(
369 dataRefList=dataRefList,
370 exposureInfoList=exposureInfoList,
371 )
372
373 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, srcList=None, **kwargs):
374 """Return indices of provided lists that meet the selection criteria
375
376 Parameters:
377 -----------
378 wcsList : `list` of `lsst.afw.geom.SkyWcs`
379 specifying the WCS's of the input ccds to be selected
380 bboxList : `list` of `lsst.geom.Box2I`
381 specifying the bounding boxes of the input ccds to be selected
382 coordList : `list` of `lsst.geom.SpherePoint`
383 ICRS coordinates specifying boundary of the patch.
384 visitSummary : `list` of `lsst.afw.table.ExposureCatalog`
385 containing the PSF shape information for the input ccds to be selected.
386 srcList : `list` of `lsst.afw.table.SourceCatalog`, optional
387 containing the PSF shape information for the input ccds to be selected.
388 This is only used if ``config.doLegacyStarSelectionComputation`` is
389 True.
390
391 Returns:
392 --------
393 goodPsf: `list` of `int`
394 of indices of selected ccds
395 """
396 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
397 coordList=coordList, dataIds=dataIds)
398
399 goodPsf = []
400
401 if not self.config.doLegacyStarSelectionComputation:
402 # Check for old inputs, and give a helpful error message if so.
403 if 'nPsfStar' not in visitSummary[0].schema.getNames():
404 raise RuntimeError("Old calexps detected. "
405 "Please set config.doLegacyStarSelectionComputation=True for "
406 "backwards compatibility.")
407
408 for i, dataId in enumerate(dataIds):
409 if i not in goodWcs:
410 continue
411 if self.isValid(visitSummary, dataId["detector"]):
412 goodPsf.append(i)
413 else:
414 if dataIds is None:
415 dataIds = [None] * len(srcList)
416 for i, (srcCatalog, dataId) in enumerate(zip(srcList, dataIds)):
417 if i not in goodWcs:
418 continue
419 if self.isValidLegacy(srcCatalog, dataId):
420 goodPsf.append(i)
421
422 return goodPsf
423
424 def isValid(self, visitSummary, detectorId):
425 """Should this ccd be selected based on its PSF shape information.
426
427 Parameters
428 ----------
429 visitSummary : `lsst.afw.table.ExposureCatalog`
430 detectorId : `int`
431 Detector identifier.
432
433 Returns
434 -------
435 valid : `bool`
436 True if selected.
437 """
438 row = visitSummary.find(detectorId)
439 if row is None:
440 # This is not listed, so it must be bad.
441 self.log.warning("Removing detector %d because summary stats not available.", detectorId)
442 return False
443
444 medianE = np.sqrt(row["psfStarDeltaE1Median"]**2. + row["psfStarDeltaE2Median"]**2.)
445 scatterSize = row["psfStarDeltaSizeScatter"]
446 scaledScatterSize = row["psfStarScaledDeltaSizeScatter"]
447
448 valid = True
449 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
450 self.log.info("Removing visit %d detector %d because median e residual too large: %f vs %f",
451 row["visit"], detectorId, medianE, self.config.maxEllipResidual)
452 valid = False
453 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
454 self.log.info("Removing visit %d detector %d because size scatter too large: %f vs %f",
455 row["visit"], detectorId, scatterSize, self.config.maxSizeScatter)
456 valid = False
457 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
458 self.log.info("Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
459 row["visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
460 valid = False
461
462 return valid
463
464 def isValidLegacy(self, srcCatalog, dataId=None):
465 """Should this ccd be selected based on its PSF shape information.
466
467 This routine is only used in legacy processing (gen2 and
468 backwards compatible old calexps) and should be removed after v24.
469
470 Parameters
471 ----------
472 srcCatalog : `lsst.afw.table.SourceCatalog`
473 dataId : `dict` of dataId keys, optional.
474 Used only for logging. Defaults to None.
475
476 Returns
477 -------
478 valid : `bool`
479 True if selected.
480 """
481 mask = srcCatalog[self.config.starSelection]
482
483 starXX = srcCatalog[self.config.starShape+'_xx'][mask]
484 starYY = srcCatalog[self.config.starShape+'_yy'][mask]
485 starXY = srcCatalog[self.config.starShape+'_xy'][mask]
486 psfXX = srcCatalog[self.config.psfShape+'_xx'][mask]
487 psfYY = srcCatalog[self.config.psfShape+'_yy'][mask]
488 psfXY = srcCatalog[self.config.psfShape+'_xy'][mask]
489
490 starSize = np.power(starXX*starYY - starXY**2, 0.25)
491 starE1 = (starXX - starYY)/(starXX + starYY)
492 starE2 = 2*starXY/(starXX + starYY)
493 medianSize = np.median(starSize)
494
495 psfSize = np.power(psfXX*psfYY - psfXY**2, 0.25)
496 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
497 psfE2 = 2*psfXY/(psfXX + psfYY)
498
499 medianE1 = np.abs(np.median(starE1 - psfE1))
500 medianE2 = np.abs(np.median(starE2 - psfE2))
501 medianE = np.sqrt(medianE1**2 + medianE2**2)
502
503 scatterSize = sigmaMad(starSize - psfSize)
504 scaledScatterSize = scatterSize/medianSize**2
505
506 valid = True
507 if self.config.maxEllipResidual and medianE > self.config.maxEllipResidual:
508 self.log.info("Removing visit %s because median e residual too large: %f vs %f",
509 dataId, medianE, self.config.maxEllipResidual)
510 valid = False
511 elif self.config.maxSizeScatter and scatterSize > self.config.maxSizeScatter:
512 self.log.info("Removing visit %s because size scatter is too large: %f vs %f",
513 dataId, scatterSize, self.config.maxSizeScatter)
514 valid = False
515 elif self.config.maxScaledSizeScatter and scaledScatterSize > self.config.maxScaledSizeScatter:
516 self.log.info("Removing visit %s because scaled size scatter is too large: %f vs %f",
517 dataId, scaledScatterSize, self.config.maxScaledSizeScatter)
518 valid = False
519
520 return valid
521
522
523class BestSeeingWcsSelectImageConfig(WcsSelectImagesTask.ConfigClass):
524 """Base configuration for BestSeeingSelectImagesTask.
525 """
526 nImagesMax = pexConfig.RangeField(
527 dtype=int,
528 doc="Maximum number of images to select",
529 default=5,
530 min=0)
531 maxPsfFwhm = pexConfig.Field(
532 dtype=float,
533 doc="Maximum PSF FWHM (in arcseconds) to select",
534 default=1.5,
535 optional=True)
536 minPsfFwhm = pexConfig.Field(
537 dtype=float,
538 doc="Minimum PSF FWHM (in arcseconds) to select",
539 default=0.,
540 optional=True)
541
542
543class BestSeeingWcsSelectImagesTask(WcsSelectImagesTask):
544 """Select up to a maximum number of the best-seeing images using their Wcs.
545 """
546 ConfigClass = BestSeeingWcsSelectImageConfig
547
548 def runDataRef(self, dataRef, coordList, makeDataRefList=True,
549 selectDataList=None):
550 """Select the best-seeing images in the selectDataList that overlap the patch.
551
552 This method is the old entry point for the Gen2 commandline tasks and drivers
553 Will be deprecated in v22.
554
555 Parameters
556 ----------
557 dataRef : `lsst.daf.persistence.ButlerDataRef`
558 Data reference for coadd/tempExp (with tract, patch)
559 coordList : `list` of `lsst.geom.SpherePoint`
560 List of ICRS sky coordinates specifying boundary of patch
561 makeDataRefList : `boolean`, optional
562 Construct a list of data references?
563 selectDataList : `list` of `SelectStruct`
564 List of SelectStruct, to consider for selection
565
566 Returns
567 -------
568 result : `lsst.pipe.base.Struct`
569 Result struct with components:
570 - ``exposureList``: the selected exposures
572 - ``dataRefList``: the optional data references corresponding to
573 each element of ``exposureList``
574 (`list` of `lsst.daf.persistence.ButlerDataRef`, or `None`).
575 """
576 psfSizes = []
577 dataRefList = []
578 exposureInfoList = []
579
580 if selectDataList is None:
581 selectDataList = []
582
583 result = super().runDataRef(dataRef, coordList, makeDataRefList=True, selectDataList=selectDataList)
584
585 for dataRef, exposureInfo in zip(result.dataRefList, result.exposureInfoList):
586 cal = dataRef.get("calexp", immediate=True)
587
588 # if min/max PSF values are defined, remove images out of bounds
589 pixToArcseconds = cal.getWcs().getPixelScale().asArcseconds()
590 psfSize = cal.getPsf().computeShape().getDeterminantRadius()*pixToArcseconds
591 sizeFwhm = psfSize * np.sqrt(8.*np.log(2.))
592 if self.config.maxPsfFwhm and sizeFwhm > self.config.maxPsfFwhm:
593 continue
594 if self.config.minPsfFwhm and sizeFwhm < self.config.minPsfFwhm:
595 continue
596 psfSizes.append(sizeFwhm)
597 dataRefList.append(dataRef)
598 exposureInfoList.append(exposureInfo)
599
600 if len(psfSizes) > self.config.nImagesMax:
601 sortedIndices = np.argsort(psfSizes)[:self.config.nImagesMax]
602 filteredDataRefList = [dataRefList[i] for i in sortedIndices]
603 filteredExposureInfoList = [exposureInfoList[i] for i in sortedIndices]
604 self.log.info("%d images selected with FWHM range of %f--%f arcseconds",
605 len(sortedIndices), psfSizes[sortedIndices[0]], psfSizes[sortedIndices[-1]])
606
607 else:
608 if len(psfSizes) == 0:
609 self.log.warning("0 images selected.")
610 else:
611 self.log.debug("%d images selected with FWHM range of %d--%d arcseconds",
612 len(psfSizes), psfSizes[0], psfSizes[-1])
613 filteredDataRefList = dataRefList
614 filteredExposureInfoList = exposureInfoList
615
616 return pipeBase.Struct(
617 dataRefList=filteredDataRefList if makeDataRefList else None,
618 exposureInfoList=filteredExposureInfoList,
619 )
620
621
622class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
623 dimensions=("tract", "patch", "skymap", "band", "instrument"),
624 defaultTemplates={"coaddName": "goodSeeing"}):
625 skyMap = pipeBase.connectionTypes.Input(
626 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
627 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
628 storageClass="SkyMap",
629 dimensions=("skymap",),
630 )
631 visitSummaries = pipeBase.connectionTypes.Input(
632 doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
633 name="visitSummary",
634 storageClass="ExposureCatalog",
635 dimensions=("instrument", "visit",),
636 multiple=True,
637 deferLoad=True
638 )
639 goodVisits = pipeBase.connectionTypes.Output(
640 doc="Selected visits to be coadded.",
641 name="{coaddName}Visits",
642 storageClass="StructuredDataDict",
643 dimensions=("instrument", "tract", "patch", "skymap", "band"),
644 )
645
646
647class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
648 pipelineConnections=BestSeeingSelectVisitsConnections):
649 nVisitsMax = pexConfig.RangeField(
650 dtype=int,
651 doc="Maximum number of visits to select",
652 default=12,
653 min=0
654 )
655 maxPsfFwhm = pexConfig.Field(
656 dtype=float,
657 doc="Maximum PSF FWHM (in arcseconds) to select",
658 default=1.5,
659 optional=True
660 )
661 minPsfFwhm = pexConfig.Field(
662 dtype=float,
663 doc="Minimum PSF FWHM (in arcseconds) to select",
664 default=0.,
665 optional=True
666 )
667 doConfirmOverlap = pexConfig.Field(
668 dtype=bool,
669 doc="Do remove visits that do not actually overlap the patch?",
670 default=True,
671 )
672 minMJD = pexConfig.Field(
673 dtype=float,
674 doc="Minimum visit MJD to select",
675 default=None,
676 optional=True
677 )
678 maxMJD = pexConfig.Field(
679 dtype=float,
680 doc="Maximum visit MJD to select",
681 default=None,
682 optional=True
683 )
684
685
686class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
687 """Select up to a maximum number of the best-seeing visits
688
689 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
690 This Task is a port of the Gen2 image-selector used in the AP pipeline:
691 BestSeeingSelectImagesTask. This Task selects full visits based on the
692 average PSF of the entire visit.
693 """
694 ConfigClass = BestSeeingSelectVisitsConfig
695 _DefaultName = 'bestSeeingSelectVisits'
696
697 def runQuantum(self, butlerQC, inputRefs, outputRefs):
698 inputs = butlerQC.get(inputRefs)
699 quantumDataId = butlerQC.quantum.dataId
700 outputs = self.run(**inputs, dataId=quantumDataId)
701 butlerQC.put(outputs, outputRefs)
702
703 def run(self, visitSummaries, skyMap, dataId):
704 """Run task
705
706 Parameters:
707 -----------
708 visitSummary : `list`
709 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
710 visitSummary tables of type `lsst.afw.table.ExposureCatalog`
711 skyMap : `lsst.skyMap.SkyMap`
712 SkyMap for checking visits overlap patch
713 dataId : `dict` of dataId keys
714 For retrieving patch info for checking visits overlap patch
715
716 Returns
717 -------
718 result : `lsst.pipe.base.Struct`
719 Result struct with components:
720
721 - `goodVisits`: `dict` with selected visit ids as keys,
722 so that it can be be saved as a StructuredDataDict.
723 StructuredDataList's are currently limited.
724 """
725
726 if self.config.doConfirmOverlap:
727 patchPolygon = self.makePatchPolygon(skyMap, dataId)
728
729 inputVisits = [visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries]
730 fwhmSizes = []
731 visits = []
732 for visit, visitSummary in zip(inputVisits, visitSummaries):
733 # read in one-by-one and only once. There may be hundreds
734 visitSummary = visitSummary.get()
735
736 # mjd is guaranteed to be the same for every detector in the visitSummary.
737 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
738
739 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
740 for vs in visitSummary]
741 # psfSigma is PSF model determinant radius at chip center in pixels
742 psfSigmas = np.array([vs['psfSigma'] for vs in visitSummary])
743 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
744
745 if self.config.maxPsfFwhm and fwhm > self.config.maxPsfFwhm:
746 continue
747 if self.config.minPsfFwhm and fwhm < self.config.minPsfFwhm:
748 continue
749 if self.config.minMJD and mjd < self.config.minMJD:
750 self.log.debug('MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
751 continue
752 if self.config.maxMJD and mjd > self.config.maxMJD:
753 self.log.debug('MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
754 continue
755 if self.config.doConfirmOverlap and not self.doesIntersectPolygon(visitSummary, patchPolygon):
756 continue
757
758 fwhmSizes.append(fwhm)
759 visits.append(visit)
760
761 sortedVisits = [ind for (_, ind) in sorted(zip(fwhmSizes, visits))]
762 output = sortedVisits[:self.config.nVisitsMax]
763 self.log.info("%d images selected with FWHM range of %d--%d arcseconds",
764 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
765
766 # In order to store as a StructuredDataDict, convert list to dict
767 goodVisits = {key: True for key in output}
768 return pipeBase.Struct(goodVisits=goodVisits)
769
770 def makePatchPolygon(self, skyMap, dataId):
771 """Return True if sky polygon overlaps visit
772
773 Parameters:
774 -----------
776 Exposure catalog with per-detector geometry
777 dataId : `dict` of dataId keys
778 For retrieving patch info
779
780 Returns:
781 --------
782 result :` lsst.sphgeom.ConvexPolygon.convexHull`
783 Polygon of patch's outer bbox
784 """
785 wcs = skyMap[dataId['tract']].getWcs()
786 bbox = skyMap[dataId['tract']][dataId['patch']].getOuterBBox()
787 sphCorners = wcs.pixelToSky(lsst.geom.Box2D(bbox).getCorners())
788 result = lsst.sphgeom.ConvexPolygon.convexHull([coord.getVector() for coord in sphCorners])
789 return result
790
791 def doesIntersectPolygon(self, visitSummary, polygon):
792 """Return True if sky polygon overlaps visit
793
794 Parameters:
795 -----------
796 visitSummary : `lsst.afw.table.ExposureCatalog`
797 Exposure catalog with per-detector geometry
798 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
799 Polygon to check overlap
800
801 Returns:
802 --------
803 doesIntersect: `bool`
804 Does the visit overlap the polygon
805 """
806 doesIntersect = False
807 for detectorSummary in visitSummary:
808 corners = [lsst.geom.SpherePoint(ra, decl, units=lsst.geom.degrees).getVector() for (ra, decl) in
809 zip(detectorSummary['raCorners'], detectorSummary['decCorners'])]
810 detectorPolygon = lsst.sphgeom.ConvexPolygon.convexHull(corners)
811 if detectorPolygon.intersects(polygon):
812 doesIntersect = True
813 break
814 return doesIntersect
815
816
817class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
818 pipelineConnections=BestSeeingSelectVisitsConnections):
819 qMin = pexConfig.RangeField(
820 doc="Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
821 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
822 "This config should be changed from zero only for exploratory diffIm testing.",
823 dtype=float,
824 default=0,
825 min=0,
826 max=1,
827 )
828 qMax = pexConfig.RangeField(
829 doc="Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
830 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
831 dtype=float,
832 default=0.33,
833 min=0,
834 max=1,
835 )
836 nVisitsMin = pexConfig.Field(
837 doc="At least this number of visits selected and supercedes quantile. For example, if 10 visits "
838 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
839 dtype=int,
840 default=6,
841 )
842 doConfirmOverlap = pexConfig.Field(
843 dtype=bool,
844 doc="Do remove visits that do not actually overlap the patch?",
845 default=True,
846 )
847 minMJD = pexConfig.Field(
848 dtype=float,
849 doc="Minimum visit MJD to select",
850 default=None,
851 optional=True
852 )
853 maxMJD = pexConfig.Field(
854 dtype=float,
855 doc="Maximum visit MJD to select",
856 default=None,
857 optional=True
858 )
859
860
861class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
862 """Select a quantile of the best-seeing visits
863
864 Selects the best (for example, third) full visits based on the average
865 PSF width in the entire visit. It can also be used for difference imaging
866 experiments that require templates with the worst seeing visits.
867 For example, selecting the worst third can be acheived by
868 changing the config parameters qMin to 0.66 and qMax to 1.
869 """
870 ConfigClass = BestSeeingQuantileSelectVisitsConfig
871 _DefaultName = 'bestSeeingQuantileSelectVisits'
872
873 @utils.inheritDoc(BestSeeingSelectVisitsTask)
874 def run(self, visitSummaries, skyMap, dataId):
875 if self.config.doConfirmOverlap:
876 patchPolygon = self.makePatchPolygon(skyMap, dataId)
877 visits = np.array([visitSummary.ref.dataId['visit'] for visitSummary in visitSummaries])
878 radius = np.empty(len(visits))
879 intersects = np.full(len(visits), True)
880 for i, visitSummary in enumerate(visitSummaries):
881 # read in one-by-one and only once. There may be hundreds
882 visitSummary = visitSummary.get()
883 # psfSigma is PSF model determinant radius at chip center in pixels
884 psfSigma = np.nanmedian([vs['psfSigma'] for vs in visitSummary])
885 radius[i] = psfSigma
886 if self.config.doConfirmOverlap:
887 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
888 if self.config.minMJD or self.config.maxMJD:
889 # mjd is guaranteed to be the same for every detector in the visitSummary.
890 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
891 aboveMin = mjd > self.config.minMJD if self.config.minMJD else True
892 belowMax = mjd < self.config.maxMJD if self.config.maxMJD else True
893 intersects[i] = intersects[i] and aboveMin and belowMax
894
895 sortedVisits = [v for rad, v in sorted(zip(radius[intersects], visits[intersects]))]
896 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
897 max(0, len(visits[intersects]) - self.config.nVisitsMin))
898 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
899
900 # In order to store as a StructuredDataDict, convert list to dict
901 goodVisits = {int(visit): True for visit in sortedVisits[lowerBound:upperBound]}
902 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
Definition: selectImages.py:62
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
def runDataRef(self, dataRef, coordList, makeDataRefList=True, selectDataList=[])
def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)