22__all__ = [
"BaseSelectImagesTask",
"BaseExposureInfo",
"WcsSelectImagesTask",
"PsfWcsSelectImagesTask",
23 "DatabaseSelectImagesConfig",
"BestSeeingSelectVisitsTask",
24 "BestSeeingQuantileSelectVisitsTask"]
28import lsst.utils
as utils
35from lsst.utils.timer
import timeMethod
39 """Base configuration for subclasses of BaseSelectImagesTask that use a database."""
41 host = pexConfig.Field(
42 doc=
"Database server host name",
45 port = pexConfig.Field(
46 doc=
"Database server port",
49 database = pexConfig.Field(
50 doc=
"Name of database",
53 maxExposures = pexConfig.Field(
54 doc=
"maximum exposures to select; intended for debugging; ignored if None",
61 """Data about a selected exposure.
65 dataId : `dict` of dataId keys
67 coordList : `list` of `lsst.afw.geom.SpherePoint`
68 ICRS coordinates of the corners of the exposure
69 plus any others items that are desired.
73 super(BaseExposureInfo, self).
__init__(dataId=dataId, coordList=coordList)
77 """Base task for selecting images suitable for coaddition.
80 ConfigClass = pexConfig.Config
81 _DefaultName = "selectImages"
84 def run(self, coordList):
85 """Select images suitable for coaddition in a particular region.
90 List of coordinates defining region of interest;
if None then select all images
91 subclasses may add additional keyword arguments,
as required.
95 result : `pipeBase.Struct`
96 Results
as a struct
with attributes:
99 A list of exposure information objects (subclasses of BaseExposureInfo),
100 which have at least the following fields:
101 - dataId: Data ID dictionary (`dict`).
102 - coordList: ICRS coordinates of the corners of the exposure.
105 raise NotImplementedError()
108def _extractKeyValue(dataList, keys=None):
109 """Extract the keys and values from a list of dataIds.
111 The input dataList is a list of objects that have
'dataId' members.
112 This allows it to be used
for both a list of data references
and a
113 list of ExposureInfo.
128 Raised
if DataId keys are inconsistent.
130 assert len(dataList) > 0
132 keys = sorted(dataList[0].dataId.keys())
135 for data
in dataList:
136 thisKeys = set(data.dataId.keys())
137 if thisKeys != keySet:
138 raise RuntimeError(
"DataId keys inconsistent: %s vs %s" % (keySet, thisKeys))
139 values.append(tuple(data.dataId[k]
for k
in keys))
144 """A container for data to be passed to the WcsSelectImagesTask.
151 Coordinate system definition (wcs).
152 bbox : `lsst.geom.box.Box2I`
153 Integer bounding box for image.
157 super(SelectStruct, self).
__init__(dataRef=dataRef, wcs=wcs, bbox=bbox)
161 """Select images using their Wcs.
164 polygons on the celestial sphere,
and test the polygon of the
165 patch
for overlap
with the polygon of the image.
167 We use
"convexHull" instead of generating a ConvexPolygon
168 directly because the standard
for the inputs to ConvexPolygon
169 are pretty high
and we don
't want to be responsible for reaching them.
172 def run(self, wcsList, bboxList, coordList, dataIds=None, **kwargs):
173 """Return indices of provided lists that meet the selection criteria.
178 Specifying the WCS's of the input ccds to be selected.
180 Specifying the bounding boxes of the input ccds to be selected.
182 ICRS coordinates specifying boundary of the patch.
183 dataIds : iterable of `lsst.daf.butler.dataId` or `
None`, optional
184 An iterable object of dataIds which point to reference catalogs.
186 Additional keyword arguments.
190 result : `list` of `int`
191 The indices of selected ccds.
194 dataIds = [
None] * len(wcsList)
195 patchVertices = [coord.getVector()
for coord
in coordList]
198 for i, (imageWcs, imageBox, dataId)
in enumerate(zip(wcsList, bboxList, dataIds)):
205 """Return corners or `None` if bad.
211 patchPoly : `Unknown`
215 imageCorners = [imageWcs.pixelToSky(pix)
for pix
in geom.Box2D(imageBox).getCorners()]
216 except (pexExceptions.DomainError, pexExceptions.RuntimeError)
as e:
218 self.log.debug(
"WCS error in testing calexp %s (%s): deselecting", dataId, e)
222 if imagePoly
is None:
223 self.log.debug(
"Unable to create polygon from image %s: deselecting", dataId)
226 if patchPoly.intersects(imagePoly):
228 self.log.info(
"Selecting calexp %s", dataId)
233 dimensions=(
"tract",
"patch",
"skymap",
"instrument",
"visit"),
234 defaultTemplates={
"coaddName":
"deep"}):
238class PsfWcsSelectImagesConfig(pipeBase.PipelineTaskConfig,
239 pipelineConnections=PsfWcsSelectImagesConnections):
240 maxEllipResidual = pexConfig.Field(
241 doc=
"Maximum median ellipticity residual",
246 maxSizeScatter = pexConfig.Field(
247 doc=
"Maximum scatter in the size residuals",
251 maxScaledSizeScatter = pexConfig.Field(
252 doc=
"Maximum scatter in the size residuals, scaled by the median size",
260 """Select images using their Wcs and cuts on the PSF properties.
262 The PSF quality criteria are based on the size and ellipticity residuals
from the
263 adaptive second moments of the star
and the PSF.
266 - the median of the ellipticty residuals.
267 - the robust scatter of the size residuals (using the median absolute deviation).
268 - the robust scatter of the size residuals scaled by the square of
272 ConfigClass = PsfWcsSelectImagesConfig
273 _DefaultName = "PsfWcsSelectImages"
275 def run(self, wcsList, bboxList, coordList, visitSummary, dataIds=None, **kwargs):
276 """Return indices of provided lists that meet the selection criteria.
281 Specifying the WCS's of the input ccds to be selected.
283 Specifying the bounding boxes of the input ccds to be selected.
285 ICRS coordinates specifying boundary of the patch.
287 containing the PSF shape information for the input ccds to be selected.
288 dataIds : iterable of `lsst.daf.butler.dataId`
or `
None`, optional
289 An iterable object of dataIds which point to reference catalogs.
291 Additional keyword arguments.
295 goodPsf: `list` of `int`
296 The indices of selected ccds.
298 goodWcs = super(PsfWcsSelectImagesTask, self).run(wcsList=wcsList, bboxList=bboxList,
299 coordList=coordList, dataIds=dataIds)
303 for i, dataId
in enumerate(dataIds):
306 if self.isValid(visitSummary, dataId[
"detector"]):
311 def isValid(self, visitSummary, detectorId):
312 """Should this ccd be selected based on its PSF shape information.
317 Exposure catalog with per-detector summary information.
326 row = visitSummary.find(detectorId)
329 self.log.warning(
"Removing detector %d because summary stats not available.", detectorId)
332 medianE = np.sqrt(row[
"psfStarDeltaE1Median"]**2. + row[
"psfStarDeltaE2Median"]**2.)
333 scatterSize = row[
"psfStarDeltaSizeScatter"]
334 scaledScatterSize = row[
"psfStarScaledDeltaSizeScatter"]
337 if self.config.maxEllipResidual
and medianE > self.config.maxEllipResidual:
338 self.log.info(
"Removing visit %d detector %d because median e residual too large: %f vs %f",
339 row[
"visit"], detectorId, medianE, self.config.maxEllipResidual)
341 elif self.config.maxSizeScatter
and scatterSize > self.config.maxSizeScatter:
342 self.log.info(
"Removing visit %d detector %d because size scatter too large: %f vs %f",
343 row[
"visit"], detectorId, scatterSize, self.config.maxSizeScatter)
345 elif self.config.maxScaledSizeScatter
and scaledScatterSize > self.config.maxScaledSizeScatter:
346 self.log.info(
"Removing visit %d detector %d because scaled size scatter too large: %f vs %f",
347 row[
"visit"], detectorId, scaledScatterSize, self.config.maxScaledSizeScatter)
353class BestSeeingSelectVisitsConnections(pipeBase.PipelineTaskConnections,
354 dimensions=(
"tract",
"patch",
"skymap",
"band",
"instrument"),
355 defaultTemplates={
"coaddName":
"goodSeeing"}):
356 skyMap = pipeBase.connectionTypes.Input(
357 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
358 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
359 storageClass=
"SkyMap",
360 dimensions=(
"skymap",),
362 visitSummaries = pipeBase.connectionTypes.Input(
363 doc=
"Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
365 storageClass=
"ExposureCatalog",
366 dimensions=(
"instrument",
"visit",),
370 goodVisits = pipeBase.connectionTypes.Output(
371 doc=
"Selected visits to be coadded.",
372 name=
"{coaddName}Visits",
373 storageClass=
"StructuredDataDict",
374 dimensions=(
"instrument",
"tract",
"patch",
"skymap",
"band"),
378class BestSeeingSelectVisitsConfig(pipeBase.PipelineTaskConfig,
379 pipelineConnections=BestSeeingSelectVisitsConnections):
380 nVisitsMax = pexConfig.RangeField(
382 doc=
"Maximum number of visits to select",
386 maxPsfFwhm = pexConfig.Field(
388 doc=
"Maximum PSF FWHM (in arcseconds) to select",
392 minPsfFwhm = pexConfig.Field(
394 doc=
"Minimum PSF FWHM (in arcseconds) to select",
398 doConfirmOverlap = pexConfig.Field(
400 doc=
"Do remove visits that do not actually overlap the patch?",
403 minMJD = pexConfig.Field(
405 doc=
"Minimum visit MJD to select",
409 maxMJD = pexConfig.Field(
411 doc=
"Maximum visit MJD to select",
417class BestSeeingSelectVisitsTask(pipeBase.PipelineTask):
418 """Select up to a maximum number of the best-seeing visits.
420 Don't exceed the FWHM range specified by configs min(max)PsfFwhm.
421 This Task is a port of the Gen2 image-selector used
in the AP pipeline:
422 BestSeeingSelectImagesTask. This Task selects full visits based on the
423 average PSF of the entire visit.
426 ConfigClass = BestSeeingSelectVisitsConfig
427 _DefaultName = 'bestSeeingSelectVisits'
429 def runQuantum(self, butlerQC, inputRefs, outputRefs):
430 inputs = butlerQC.get(inputRefs)
431 quantumDataId = butlerQC.quantum.dataId
432 outputs = self.run(**inputs, dataId=quantumDataId)
433 butlerQC.put(outputs, outputRefs)
435 def run(self, visitSummaries, skyMap, dataId):
440 visitSummary : `list` of `lsst.pipe.base.connections.DeferredDatasetRef`
441 List of `lsst.pipe.base.connections.DeferredDatasetRef` of
443 skyMap : `lsst.skyMap.SkyMap`
444 SkyMap for checking visits overlap patch.
445 dataId : `dict` of dataId keys
446 For retrieving patch info
for checking visits overlap patch.
450 result : `lsst.pipe.base.Struct`
451 Results
as a struct
with attributes:
454 A `dict`
with selected visit ids
as keys,
455 so that it can be be saved
as a StructuredDataDict.
456 StructuredDataList
's are currently limited.
458 if self.config.doConfirmOverlap:
459 patchPolygon = self.makePatchPolygon(skyMap, dataId)
461 inputVisits = [visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries]
464 for visit, visitSummary
in zip(inputVisits, visitSummaries):
466 visitSummary = visitSummary.get()
469 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
471 pixToArcseconds = [vs.getWcs().getPixelScale(vs.getBBox().getCenter()).asArcseconds()
472 for vs
in visitSummary]
474 psfSigmas = np.array([vs[
'psfSigma']
for vs
in visitSummary])
475 fwhm = np.nanmean(psfSigmas * pixToArcseconds) * np.sqrt(8.*np.log(2.))
477 if self.config.maxPsfFwhm
and fwhm > self.config.maxPsfFwhm:
479 if self.config.minPsfFwhm
and fwhm < self.config.minPsfFwhm:
481 if self.config.minMJD
and mjd < self.config.minMJD:
482 self.log.debug(
'MJD %f earlier than %.2f; rejecting', mjd, self.config.minMJD)
484 if self.config.maxMJD
and mjd > self.config.maxMJD:
485 self.log.debug(
'MJD %f later than %.2f; rejecting', mjd, self.config.maxMJD)
487 if self.config.doConfirmOverlap
and not self.doesIntersectPolygon(visitSummary, patchPolygon):
490 fwhmSizes.append(fwhm)
493 sortedVisits = [ind
for (_, ind)
in sorted(zip(fwhmSizes, visits))]
494 output = sortedVisits[:self.config.nVisitsMax]
495 self.log.info(
"%d images selected with FWHM range of %d--%d arcseconds",
496 len(output), fwhmSizes[visits.index(output[0])], fwhmSizes[visits.index(output[-1])])
499 goodVisits = {key:
True for key
in output}
500 return pipeBase.Struct(goodVisits=goodVisits)
502 def makePatchPolygon(self, skyMap, dataId):
503 """Return True if sky polygon overlaps visit.
508 Exposure catalog with per-detector geometry.
509 dataId : `dict` of dataId keys
510 For retrieving patch info.
514 result : `lsst.sphgeom.ConvexPolygon.convexHull`
515 Polygon of patch
's outer bbox.
517 wcs = skyMap[dataId['tract']].getWcs()
518 bbox = skyMap[dataId[
'tract']][dataId[
'patch']].getOuterBBox()
523 def doesIntersectPolygon(self, visitSummary, polygon):
524 """Return True if sky polygon overlaps visit.
529 Exposure catalog with per-detector geometry.
530 polygon :` lsst.sphgeom.ConvexPolygon.convexHull`
531 Polygon to check overlap.
535 doesIntersect : `bool`
536 True if the visit overlaps the polygon.
538 doesIntersect = False
539 for detectorSummary
in visitSummary:
541 zip(detectorSummary[
'raCorners'], detectorSummary[
'decCorners'])]
543 if detectorPolygon.intersects(polygon):
549class BestSeeingQuantileSelectVisitsConfig(pipeBase.PipelineTaskConfig,
550 pipelineConnections=BestSeeingSelectVisitsConnections):
551 qMin = pexConfig.RangeField(
552 doc=
"Lower bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
553 "and select those in the interquantile range (qMin, qMax). Set qMin to 0 for Best Seeing. "
554 "This config should be changed from zero only for exploratory diffIm testing.",
560 qMax = pexConfig.RangeField(
561 doc=
"Upper bound of quantile range to select. Sorts visits by seeing from narrow to wide, "
562 "and select those in the interquantile range (qMin, qMax). Set qMax to 1 for Worst Seeing.",
568 nVisitsMin = pexConfig.Field(
569 doc=
"At least this number of visits selected and supercedes quantile. For example, if 10 visits "
570 "cover this patch, qMin=0.33, and nVisitsMin=5, the best 5 visits will be selected.",
574 doConfirmOverlap = pexConfig.Field(
576 doc=
"Do remove visits that do not actually overlap the patch?",
579 minMJD = pexConfig.Field(
581 doc=
"Minimum visit MJD to select",
585 maxMJD = pexConfig.Field(
587 doc=
"Maximum visit MJD to select",
593class BestSeeingQuantileSelectVisitsTask(BestSeeingSelectVisitsTask):
594 """Select a quantile of the best-seeing visits.
596 Selects the best (for example, third) full visits based on the average
597 PSF width
in the entire visit. It can also be used
for difference imaging
598 experiments that require templates
with the worst seeing visits.
599 For example, selecting the worst third can be acheived by
600 changing the config parameters qMin to 0.66
and qMax to 1.
602 ConfigClass = BestSeeingQuantileSelectVisitsConfig
603 _DefaultName = 'bestSeeingQuantileSelectVisits'
605 @utils.inheritDoc(BestSeeingSelectVisitsTask)
606 def run(self, visitSummaries, skyMap, dataId):
607 if self.config.doConfirmOverlap:
608 patchPolygon = self.makePatchPolygon(skyMap, dataId)
609 visits = np.array([visitSummary.ref.dataId[
'visit']
for visitSummary
in visitSummaries])
610 radius = np.empty(len(visits))
611 intersects = np.full(len(visits),
True)
612 for i, visitSummary
in enumerate(visitSummaries):
614 visitSummary = visitSummary.get()
616 psfSigma = np.nanmedian([vs[
'psfSigma']
for vs
in visitSummary])
618 if self.config.doConfirmOverlap:
619 intersects[i] = self.doesIntersectPolygon(visitSummary, patchPolygon)
620 if self.config.minMJD
or self.config.maxMJD:
622 mjd = visitSummary[0].getVisitInfo().getDate().get(system=DateTime.MJD)
623 aboveMin = mjd > self.config.minMJD
if self.config.minMJD
else True
624 belowMax = mjd < self.config.maxMJD
if self.config.maxMJD
else True
625 intersects[i] = intersects[i]
and aboveMin
and belowMax
627 sortedVisits = [v
for rad, v
in sorted(zip(radius[intersects], visits[intersects]))]
628 lowerBound = min(int(np.round(self.config.qMin*len(visits[intersects]))),
629 max(0, len(visits[intersects]) - self.config.nVisitsMin))
630 upperBound = max(int(np.round(self.config.qMax*len(visits[intersects]))), self.config.nVisitsMin)
633 goodVisits = {int(visit):
True for visit
in sortedVisits[lowerBound:upperBound]}
634 return pipeBase.Struct(goodVisits=goodVisits)
def __init__(self, dataId, coordList)
def __init__(self, dataRef, wcs, bbox)
def getValidImageCorners(self, imageWcs, imageBox, patchPoly, dataId=None)
static ConvexPolygon convexHull(std::vector< UnitVector3d > const &points)