Coverage for python/lsst/pipe/tasks/healSparseMapping.py : 23%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2021 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22from collections import defaultdict
23import numbers
24import numpy as np
25import healpy as hp
26import healsparse as hsp
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.geom
31import lsst.afw.geom as afwGeom
32from lsst.daf.butler import Formatter
35__all__ = ["HealSparseInputMapTask", "HealSparseInputMapConfig",
36 "HealSparseMapFormatter"]
39class HealSparseMapFormatter(Formatter):
40 """Interface for reading and writing healsparse.HealSparseMap files
41 """
42 unsupportedParameters = frozenset()
43 supportedExtensions = frozenset({".hsp", ".fit", ".fits"})
44 extension = '.hsp'
46 def read(self, component=None):
47 # Docstring inherited from Formatter.read.
48 path = self.fileDescriptor.location.path
50 if component == 'coverage':
51 try:
52 data = hsp.HealSparseCoverage.read(path)
53 except (OSError, RuntimeError):
54 raise ValueError(f"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
56 return data
58 if self.fileDescriptor.parameters is None:
59 pixels = None
60 degrade_nside = None
61 else:
62 pixels = self.fileDescriptor.parameters.get('pixels', None)
63 degrade_nside = self.fileDescriptor.parameters.get('degrade_nside', None)
64 try:
65 data = hsp.HealSparseMap.read(path, pixels=pixels, degrade_nside=degrade_nside)
66 except (OSError, RuntimeError):
67 raise ValueError(f"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
69 return data
71 def write(self, inMemoryDataset):
72 # Docstring inherited from Formatter.write.
73 # Update the location with the formatter-preferred file extension
74 self.fileDescriptor.location.updateExtension(self.extension)
75 inMemoryDataset.write(self.fileDescriptor.location.path, clobber=True)
78def _is_power_of_two(value):
79 """Check that value is a power of two.
81 Parameters
82 ----------
83 value : `int`
84 Value to check.
86 Returns
87 -------
88 is_power_of_two : `bool`
89 True if value is a power of two; False otherwise, or
90 if value is not an integer.
91 """
92 if not isinstance(value, numbers.Integral): 92 ↛ 93line 92 didn't jump to line 93, because the condition on line 92 was never true
93 return False
95 # See https://stackoverflow.com/questions/57025836
96 # Every power of 2 has exactly 1 bit set to 1; subtracting
97 # 1 therefore flips every preceding bit. If you and that
98 # together with the original value it must be 0.
99 return (value & (value - 1) == 0) and value != 0
102class HealSparseInputMapConfig(pexConfig.Config):
103 """Configuration parameters for HealSparseInputMapTask"""
104 nside = pexConfig.Field(
105 doc="Mapping healpix nside. Must be power of 2.",
106 dtype=int,
107 default=32768,
108 check=_is_power_of_two,
109 )
110 nside_coverage = pexConfig.Field(
111 doc="HealSparse coverage map nside. Must be power of 2.",
112 dtype=int,
113 default=256,
114 check=_is_power_of_two,
115 )
116 bad_mask_min_coverage = pexConfig.Field(
117 doc=("Minimum area fraction of a map healpixel pixel that must be "
118 "covered by bad pixels to be removed from the input map. "
119 "This is approximate."),
120 dtype=float,
121 default=0.5,
122 )
125class HealSparseInputMapTask(pipeBase.Task):
126 """Task for making a HealSparse input map."""
128 ConfigClass = HealSparseInputMapConfig
129 _DefaultName = "healSparseInputMap"
131 def __init__(self, **kwargs):
132 pipeBase.Task.__init__(self, **kwargs)
134 self.ccd_input_map = None
136 def build_ccd_input_map(self, bbox, wcs, ccds):
137 """Build a map from ccd valid polygons or bounding boxes.
139 Parameters
140 ----------
141 bbox : `lsst.geom.Box2I`
142 Bounding box for region to build input map.
143 wcs : `lsst.afw.geom.SkyWcs`
144 WCS object for region to build input map.
145 ccds : `lsst.afw.table.ExposureCatalog`
146 Exposure catalog with ccd data from coadd inputs.
147 """
148 self.ccd_input_map = hsp.HealSparseMap.make_empty(nside_coverage=self.config.nside_coverage,
149 nside_sparse=self.config.nside,
150 dtype=hsp.WIDE_MASK,
151 wide_mask_maxbits=len(ccds))
152 self._wcs = wcs
153 self._bbox = bbox
154 self._ccds = ccds
156 pixel_scale = wcs.getPixelScale().asArcseconds()
157 hpix_area_arcsec2 = hp.nside2pixarea(self.config.nside, degrees=True)*(3600.**2.)
158 self._min_bad = self.config.bad_mask_min_coverage*hpix_area_arcsec2/(pixel_scale**2.)
160 metadata = {}
161 self._bits_per_visit_ccd = {}
162 self._bits_per_visit = defaultdict(list)
163 for bit, ccd in enumerate(ccds):
164 metadata[f'B{bit:04d}CCD'] = ccd['ccd']
165 metadata[f'B{bit:04d}VIS'] = ccd['visit']
166 metadata[f'B{bit:04d}WT'] = ccd['weight']
168 self._bits_per_visit_ccd[(ccd['visit'], ccd['ccd'])] = bit
169 self._bits_per_visit[ccd['visit']].append(bit)
171 ccd_poly = ccd.getValidPolygon()
172 if ccd_poly is None:
173 ccd_poly = afwGeom.Polygon(lsst.geom.Box2D(ccd.getBBox()))
174 # Detectors need to be rendered with their own wcs.
175 ccd_poly_radec = self._pixels_to_radec(ccd.getWcs(), ccd_poly.convexHull().getVertices())
177 # Create a ccd healsparse polygon
178 poly = hsp.Polygon(ra=ccd_poly_radec[: -1, 0],
179 dec=ccd_poly_radec[: -1, 1],
180 value=[bit])
181 self.ccd_input_map.set_bits_pix(poly.get_pixels(nside=self.ccd_input_map.nside_sparse),
182 [bit])
184 # Cut down to the overall bounding box with associated wcs.
185 bbox_afw_poly = afwGeom.Polygon(lsst.geom.Box2D(bbox))
186 bbox_poly_radec = self._pixels_to_radec(self._wcs,
187 bbox_afw_poly.convexHull().getVertices())
188 bbox_poly = hsp.Polygon(ra=bbox_poly_radec[: -1, 0], dec=bbox_poly_radec[: -1, 1],
189 value=np.arange(self.ccd_input_map.wide_mask_maxbits))
190 bbox_poly_map = bbox_poly.get_map_like(self.ccd_input_map)
191 self.ccd_input_map = hsp.and_intersection([self.ccd_input_map, bbox_poly_map])
192 self.ccd_input_map.metadata = metadata
194 # Create a temporary map to hold the count of bad pixels in each healpix pixel
195 self._ccd_input_pixels = self.ccd_input_map.valid_pixels
197 dtype = [(f'v{visit}', 'i4') for visit in self._bits_per_visit.keys()]
199 cov = self.config.nside_coverage
200 ns = self.config.nside
201 self._ccd_input_bad_count_map = hsp.HealSparseMap.make_empty(nside_coverage=cov,
202 nside_sparse=ns,
203 dtype=dtype,
204 primary=dtype[0][0])
205 # Don't set input bad map if there are no ccds which overlap the bbox.
206 if len(self._ccd_input_pixels) > 0:
207 self._ccd_input_bad_count_map[self._ccd_input_pixels] = np.zeros(1, dtype=dtype)
209 def mask_warp_bbox(self, bbox, visit, mask, bit_mask_value):
210 """Mask a subregion from a visit.
211 This must be run after build_ccd_input_map initializes
212 the overall map.
214 Parameters
215 ----------
216 bbox : `lsst.geom.Box2I`
217 Bounding box from region to mask.
218 visit : `int`
219 Visit number corresponding to warp with mask.
220 mask : `lsst.afw.image.MaskX`
221 Mask plane from warp exposure.
222 bit_mask_value : `int`
223 Bit mask to check for bad pixels.
225 Raises
226 ------
227 RuntimeError : Raised if build_ccd_input_map was not run first.
228 """
229 if self.ccd_input_map is None:
230 raise RuntimeError("Must run build_ccd_input_map before mask_warp_bbox")
232 # Find the bad pixels and convert to healpix
233 bad_pixels = np.where(mask.array & bit_mask_value)
234 if len(bad_pixels[0]) == 0:
235 # No bad pixels
236 return
238 # Bad pixels come from warps which use the overall wcs.
239 bad_ra, bad_dec = self._wcs.pixelToSkyArray(bad_pixels[1].astype(np.float64),
240 bad_pixels[0].astype(np.float64),
241 degrees=True)
242 bad_hpix = hp.ang2pix(self.config.nside, bad_ra, bad_dec,
243 lonlat=True, nest=True)
245 # Count the number of bad image pixels in each healpix pixel
246 min_bad_hpix = bad_hpix.min()
247 bad_hpix_count = np.zeros(bad_hpix.max() - min_bad_hpix + 1, dtype=np.int32)
248 np.add.at(bad_hpix_count, bad_hpix - min_bad_hpix, 1)
250 # Add these to the accumulator map.
251 # We need to make sure that the "primary" array has valid values for
252 # this pixel to be registered in the accumulator map.
253 pix_to_add, = np.where(bad_hpix_count > 0)
254 count_map_arr = self._ccd_input_bad_count_map[min_bad_hpix + pix_to_add]
255 primary = self._ccd_input_bad_count_map.primary
256 count_map_arr[primary] = np.clip(count_map_arr[primary], 0, None)
258 count_map_arr[f'v{visit}'] = np.clip(count_map_arr[f'v{visit}'], 0, None)
259 count_map_arr[f'v{visit}'] += bad_hpix_count[pix_to_add]
261 self._ccd_input_bad_count_map[min_bad_hpix + pix_to_add] = count_map_arr
263 def finalize_ccd_input_map_mask(self):
264 """Use accumulated mask information to finalize the masking of
265 ccd_input_map.
267 Raises
268 ------
269 RuntimeError : Raised if build_ccd_input_map was not run first.
270 """
271 if self.ccd_input_map is None:
272 raise RuntimeError("Must run build_ccd_input_map before finalize_ccd_input_map_mask.")
274 count_map_arr = self._ccd_input_bad_count_map[self._ccd_input_pixels]
275 for visit in self._bits_per_visit:
276 to_mask, = np.where(count_map_arr[f'v{visit}'] > self._min_bad)
277 if to_mask.size == 0:
278 continue
279 self.ccd_input_map.clear_bits_pix(self._ccd_input_pixels[to_mask],
280 self._bits_per_visit[visit])
282 # Clear memory
283 self._ccd_input_bad_count_map = None
285 def _pixels_to_radec(self, wcs, pixels):
286 """Convert pixels to ra/dec positions using a wcs.
288 Parameters
289 ----------
290 wcs : `lsst.afw.geom.SkyWcs`
291 WCS object.
292 pixels : `list` [`lsst.geom.Point2D`]
293 List of pixels to convert.
295 Returns
296 -------
297 radec : `numpy.ndarray`
298 Nx2 array of ra/dec positions associated with pixels.
299 """
300 sph_pts = wcs.pixelToSky(pixels)
301 return np.array([(sph.getRa().asDegrees(), sph.getDec().asDegrees())
302 for sph in sph_pts])