22__all__ = [
"BaseSelectImagesTask",
"BaseExposureInfo",
"WcsSelectImagesTask",
"PsfWcsSelectImagesTask",
23 "DatabaseSelectImagesConfig",
"BestSeeingSelectVisitsTask",
24 "BestSeeingQuantileSelectVisitsTask"]
28import lsst.utils
as utils
35from lsst.utils.timer
import timeMethod
39 """Base configuration for subclasses of BaseSelectImagesTask that use a
43 host = pexConfig.Field(
44 doc="Database server host name",
47 port = pexConfig.Field(
48 doc=
"Database server port",
51 database = pexConfig.Field(
52 doc=
"Name of database",
55 maxExposures = pexConfig.Field(
56 doc=
"maximum exposures to select; intended for debugging; ignored if None",
63 """Data about a selected exposure.
68 Data ID keys of exposure.
69 coordList : `list` [`lsst.afw.geom.SpherePoint`]
70 ICRS coordinates of the corners of the exposure
71 plus any others items that are desired.
75 super(BaseExposureInfo, self).
__init__(dataId=dataId, coordList=coordList)
79 """Base task for selecting images suitable for coaddition.
82 ConfigClass = pexConfig.Config
83 _DefaultName = "selectImages"
86 def run(self, coordList):
87 """Select images suitable for coaddition in a particular region.
92 List of coordinates defining region of interest;
if `
None`, then
93 select all images subclasses may add additional keyword arguments,
98 result : `pipeBase.Struct`
99 Results
as a struct
with attributes:
102 A list of exposure information objects (subclasses of
103 BaseExposureInfo), which have at least the following fields:
104 - dataId: Data ID dictionary (`dict`).
105 - coordList: ICRS coordinates of the corners of the exposure.
108 raise NotImplementedError()
111def _extractKeyValue(dataList, keys=None):
112 """Extract the keys and values from a list of dataIds.
114 The input dataList is a list of objects that have
'dataId' members.
115 This allows it to be used
for both a list of data references
and a
116 list of ExposureInfo.
131 Raised
if DataId keys are inconsistent.
133 assert len(dataList) > 0
135 keys = sorted(dataList[0].dataId.keys())
138 for data
in dataList:
139 thisKeys = set(data.dataId.keys())
140 if thisKeys != keySet:
141 raise RuntimeError(
"DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
142 values.append(tuple(data.dataId[k]
for k
in keys))
147 """A container for data to be passed to the WcsSelectImagesTask.
154 Coordinate system definition (wcs).
155 bbox : `lsst.geom.box.Box2I`
156 Integer bounding box for image.
160 super(SelectStruct, self).
__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
164 """Select images using their Wcs.
167 polygons on the celestial sphere,
and test the polygon of the
168 patch
for overlap
with the polygon of the image.
170 We use
"convexHull" instead of generating a ConvexPolygon
171 directly because the standard
for the inputs to ConvexPolygon
172 are pretty high
and we don
't want to be responsible for reaching them.
175 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
176 """Return indices of provided lists that meet the selection criteria.
181 Specifying the WCS's of the input ccds to be selected.
183 Specifying the bounding boxes of the input ccds to be selected.
185 ICRS coordinates specifying boundary of the patch.
186 dataIds : iterable [`lsst.daf.butler.dataId`] or `
None`, optional
187 An iterable object of dataIds which point to reference catalogs.
189 Additional keyword arguments.
193 result : `list` [`int`]
194 The indices of selected ccds.
197 dataIds = [
None] * len(wcsList)
198 patchVertices = [coord.getVector()
for coord
in coordList]
201 for i, (imageWcs, imageBox, dataId)
in enumerate(zip(wcsList, bboxList, dataIds)):
203 self.log.info(
"De-selecting exposure %s: Exposure has no WCS.", dataId)
211 """Return corners or `None` if bad.
217 patchPoly : `Unknown`
221 imageCorners = [imageWcs.pixelToSky(pix)
for pix
in geom.Box2D(imageBox).getCorners()]
222 except (pexExceptions.DomainError, pexExceptions.RuntimeError)
as e:
224 self.log.debug(
"WCS error in testing calexp %s (%s): deselecting", dataId, e)
228 if imagePoly
is None:
229 self.log.debug(
"Unable to create polygon from image %s: deselecting", dataId)
232 if patchPoly.intersects(imagePoly):
234 self.log.info(
"Selecting calexp %s", dataId)
241 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
242 defaultTemplates={
"coaddName":
"deep"}):
246class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
247 pipelineConnections=PsfWcsSelectImagesConnections):
248 maxEllipResidual = pexConfig.Field(
249 doc=
"Maximum median ellipticity residual",
254 maxSizeScatter = pexConfig.Field(
255 doc=
"Maximum scatter in the size residuals",
259 maxScaledSizeScatter = pexConfig.Field(
260 doc=
"Maximum scatter in the size residuals, scaled by the median size",
265 maxPsfTraceRadiusDelta = pexConfig.Field(
266 doc=
"Maximum delta (max - min) of model PSF trace radius values evaluated on a grid on "
267 "the unmasked detector pixels (pixel).",
275 """Select images using their Wcs and cuts on the PSF properties.
277 The PSF quality criteria are based on the size and ellipticity
278 residuals
from the adaptive second moments of the star
and the PSF.
281 - the median of the ellipticty residuals.
282 - the robust scatter of the size residuals (using the median absolute
284 - the robust scatter of the size residuals scaled by the square of
288 ConfigClass = PsfWcsSelectImagesConfig
289 _DefaultName = "PsfWcsSelectImages"
291 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
292 """Return indices of provided lists that meet the selection criteria.
297 Specifying the WCS's of the input ccds to be selected.
299 Specifying the bounding boxes of the input ccds to be selected.
301 ICRS coordinates specifying boundary of the patch.
303 containing the PSF shape information for the input ccds to be
305 dataIds : iterable [`lsst.daf.butler.dataId`]
or `
None`, optional
306 An iterable object of dataIds which point to reference catalogs.
308 Additional keyword arguments.
312 goodPsf : `list` [`int`]
313 The indices of selected ccds.
315 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
316 coordList=coordList, dataIds=dataIds)
320 for i, dataId
in enumerate(dataIds):
323 if self.isValid(visitSummary, dataId[
"detector"]):
328 def isValid(self, visitSummary, detectorId):
329 """Should this ccd be selected based on its PSF shape information.
334 Exposure catalog with per-detector summary information.
343 row = visitSummary.find(detectorId)
346 self.log.warning(
"Removing detector %d because summary stats not available.", detectorId)
349 medianE = np.sqrt(row[
"psfStarDeltaE1Median"]**2. + row[
"psfStarDeltaE2Median"]**2.)
350 scatterSize = row[
"psfStarDeltaSizeScatter"]
351 scaledScatterSize = row[
"psfStarScaledDeltaSizeScatter"]
352 psfTraceRadiusDelta = row[
"psfTraceRadiusDelta"]
355 if self.config.maxEllipResidual
and medianE > self.config.maxEllipResidual:
356 self.log.info(
"Removing visit %d detector %d because median e residual too large: %f vs %f",
357 row[
"visit"], detectorId, medianE, self.config.maxEllipResidual)
359 elif self.config.maxSizeScatter
and scatterSize > self.config.maxSizeScatter:
360 self.log.info(
"Removing visit %d detector %d because size scatter too large: %f vs %f",
361 row[
"visit"], detectorId, scatterSize, self.config.maxSizeScatter)
363 elif self.config.maxScaledSizeScatter
and scaledScatterSize > self.config.maxScaledSizeScatter:
364 self.log.info(
"Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
365 row[
"visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
368 self.config.maxPsfTraceRadiusDelta
370 psfTraceRadiusDelta > self.config.maxPsfTraceRadiusDelta
371 or ~np.isfinite(psfTraceRadiusDelta)
375 "Removing visit %d detector %d because max-min delta of model PSF trace radius values "
376 "across the unmasked detector pixels is not finite or too large: %.3f vs %.3f (pixels)",
377 row[
"visit"], detectorId, psfTraceRadiusDelta, self.config.maxPsfTraceRadiusDelta
384class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
385 dimensions=(
"tract",
"patch",
"skymap",
"band",
"instrument"),
386 defaultTemplates={
"coaddName":
"goodSeeing"}):
387 skyMap = pipeBase.connectionTypes.Input(
388 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
389 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
390 storageClass=
"SkyMap",
391 dimensions=(
"skymap",),
393 visitSummaries = pipeBase.connectionTypes.Input(
394 doc=
"Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
396 storageClass=
"ExposureCatalog",
397 dimensions=(
"instrument",
"visit",),
401 goodVisits = pipeBase.connectionTypes.Output(
402 doc=
"Selected visits to be coadded.",
403 name=
"{coaddName}Visits",
404 storageClass=
"StructuredDataDict",
405 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band"),
409class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
410 pipelineConnections=BestSeeingSelectVisitsConnections):
411 nVisitsMax = pexConfig.RangeField(
413 doc=
"Maximum number of visits to select",
417 maxPsfFwhm = pexConfig.Field(
419 doc=
"Maximum PSF FWHM (in arcseconds) to select",
423 minPsfFwhm = pexConfig.Field(
425 doc=
"Minimum PSF FWHM (in arcseconds) to select",
429 doConfirmOverlap = pexConfig.Field(
431 doc=
"Do remove visits that do not actually overlap the patch?",
434 minMJD = pexConfig.Field(
436 doc=
"Minimum visit MJD to select",
440 maxMJD = pexConfig.Field(
442 doc=
"Maximum visit MJD to select",
448class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
449 """Select up to a maximum number of the best-seeing visits.
451 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
452 This Task is a port of the Gen2 image-selector used
in the AP pipeline:
453 BestSeeingSelectImagesTask. This Task selects full visits based on the
454 average PSF of the entire visit.
457 ConfigClass = BestSeeingSelectVisitsConfig
458 _DefaultName = 'bestSeeingSelectVisits'
460 def runQuantum(self, butlerQC, inputRefs, outputRefs):
461 inputs = butlerQC.get(inputRefs)
462 quantumDataId = butlerQC.quantum.dataId
463 outputs = self.run(**inputs, dataId=quantumDataId)
464 butlerQC.put(outputs, outputRefs)
466 def run(self, visitSummaries, skyMap, dataId):
471 visitSummary : `list` [`lsst.pipe.base.connections.DeferredDatasetRef`]
472 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
474 skyMap : `lsst.skyMap.SkyMap`
475 SkyMap for checking visits overlap patch.
476 dataId : `dict` of dataId keys
477 For retrieving patch info
for checking visits overlap patch.
481 result : `lsst.pipe.base.Struct`
482 Results
as a struct
with attributes:
485 A `dict`
with selected visit ids
as keys,
486 so that it can be be saved
as a StructuredDataDict.
487 StructuredDataList
's are currently limited.
489 if self.config.doConfirmOverlap:
490 patchPolygon = self.makePatchPolygon(skyMap, dataId)
492 inputVisits = [visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries]
495 for visit, visitSummary
in zip(inputVisits, visitSummaries):
497 visitSummary = visitSummary.get()
501 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
503 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
504 for vs
in visitSummary
if vs.getWcs()]
506 psfSigmas = np.array([vs[
'psfSigma']
for vs
in visitSummary
if vs.getWcs()])
507 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
509 if self.config.maxPsfFwhm
and fwhm > self.config.maxPsfFwhm:
511 if self.config.minPsfFwhm
and fwhm < self.config.minPsfFwhm:
513 if self.config.minMJD
and mjd < self.config.minMJD:
514 self.log.debug(
'MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
516 if self.config.maxMJD
and mjd > self.config.maxMJD:
517 self.log.debug(
'MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
519 if self.config.doConfirmOverlap
and not self.doesIntersectPolygon(visitSummary, patchPolygon):
522 fwhmSizes.append(fwhm)
525 sortedVisits = [ind
for (_, ind)
in sorted(zip(fwhmSizes, visits))]
526 output = sortedVisits[:self.config.nVisitsMax]
527 self.log.info(
"%d images selected with FWHM range of %d--%d arcseconds",
528 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
531 goodVisits = {key:
True for key
in output}
532 return pipeBase.Struct(goodVisits=goodVisits)
534 def makePatchPolygon(self, skyMap, dataId):
535 """Return True if sky polygon overlaps visit.
540 Exposure catalog with per-detector geometry.
541 dataId : `dict` of dataId keys
542 For retrieving patch info.
546 result : `lsst.sphgeom.ConvexPolygon.convexHull`
547 Polygon of patch
's outer bbox.
549 wcs = skyMap[dataId['tract']].getWcs()
550 bbox = skyMap[dataId[
'tract']][dataId[
'patch']].getOuterBBox()
555 def doesIntersectPolygon(self, visitSummary, polygon):
556 """Return True if sky polygon overlaps visit.
561 Exposure catalog with per-detector geometry.
562 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
563 Polygon to check overlap.
567 doesIntersect : `bool`
568 True if the visit overlaps the polygon.
570 doesIntersect = False
571 for detectorSummary
in visitSummary:
572 if (np.all(np.isfinite(detectorSummary[
'raCorners']))
573 and np.all(np.isfinite(detectorSummary[
'decCorners']))):
576 zip(detectorSummary[
'raCorners'], detectorSummary[
'decCorners'])]
578 if detectorPolygon.intersects(polygon):
584class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
585 pipelineConnections=BestSeeingSelectVisitsConnections):
586 qMin = pexConfig.RangeField(
587 doc=
"Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
588 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
589 "This config should be changed from zero only for exploratory diffIm testing.",
595 qMax = pexConfig.RangeField(
596 doc=
"Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
597 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
603 nVisitsMin = pexConfig.Field(
604 doc=
"At least this number of visits selected and supercedes quantile. For example, if 10 visits "
605 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
609 doConfirmOverlap = pexConfig.Field(
611 doc=
"Do remove visits that do not actually overlap the patch?",
614 minMJD = pexConfig.Field(
616 doc=
"Minimum visit MJD to select",
620 maxMJD = pexConfig.Field(
622 doc=
"Maximum visit MJD to select",
628class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
629 """Select a quantile of the best-seeing visits.
631 Selects the best (for example, third) full visits based on the average
632 PSF width
in the entire visit. It can also be used
for difference imaging
633 experiments that require templates
with the worst seeing visits.
634 For example, selecting the worst third can be acheived by
635 changing the config parameters qMin to 0.66
and qMax to 1.
637 ConfigClass = BestSeeingQuantileSelectVisitsConfig
638 _DefaultName = 'bestSeeingQuantileSelectVisits'
640 @utils.inheritDoc(BestSeeingSelectVisitsTask)
641 def run(self, visitSummaries, skyMap, dataId):
642 if self.config.doConfirmOverlap:
643 patchPolygon = self.makePatchPolygon(skyMap, dataId)
644 visits = np.array([visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries])
645 radius = np.empty(len(visits))
646 intersects = np.full(len(visits),
True)
647 for i, visitSummary
in enumerate(visitSummaries):
649 visitSummary = visitSummary.get()
651 psfSigma = np.nanmedian([vs[
'psfSigma']
for vs
in visitSummary])
653 if self.config.doConfirmOverlap:
654 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
655 if self.config.minMJD
or self.config.maxMJD:
658 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
659 aboveMin = mjd > self.config.minMJD
if self.config.minMJD
else True
660 belowMax = mjd < self.config.maxMJD
if self.config.maxMJD
else True
661 intersects[i] = intersects[i]
and aboveMin
and belowMax
663 sortedVisits = [v
for rad, v
in sorted(zip(radius[intersects], visits[intersects]))]
664 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
665 max(0, len(visits[intersects]) - self.config.nVisitsMin))
666 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
669 goodVisits = {int(visit):
True for visit
in sortedVisits[lowerBound:upperBound]}
670 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)