Coverage for python/lsst/pipe/tasks/healSparseMapping.py: 17%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#
2# LSST Data Management System
3# Copyright 2008-2021 AURA/LSST.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22from collections import defaultdict
23import numbers
24import numpy as np
25import healpy as hp
26import healsparse as hsp
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.geom
31import lsst.afw.geom as afwGeom
32from lsst.daf.butler import Formatter
33from lsst.skymap import BaseSkyMap
34from .healSparseMappingProperties import (BasePropertyMap, BasePropertyMapConfig,
35 PropertyMapMap, compute_approx_psf_size_and_shape)
38__all__ = ["HealSparseInputMapTask", "HealSparseInputMapConfig",
39 "HealSparseMapFormatter", "HealSparsePropertyMapConnections",
40 "HealSparsePropertyMapConfig", "HealSparsePropertyMapTask",
41 "ConsolidateHealSparsePropertyMapConnections",
42 "ConsolidateHealSparsePropertyMapConfig",
43 "ConsolidateHealSparsePropertyMapTask"]
46class HealSparseMapFormatter(Formatter):
47 """Interface for reading and writing healsparse.HealSparseMap files."""
48 unsupportedParameters = frozenset()
49 supportedExtensions = frozenset({".hsp", ".fit", ".fits"})
50 extension = '.hsp'
52 def read(self, component=None):
53 # Docstring inherited from Formatter.read.
54 path = self.fileDescriptor.location.path
56 if component == 'coverage':
57 try:
58 data = hsp.HealSparseCoverage.read(path)
59 except (OSError, RuntimeError):
60 raise ValueError(f"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
62 return data
64 if self.fileDescriptor.parameters is None:
65 pixels = None
66 degrade_nside = None
67 else:
68 pixels = self.fileDescriptor.parameters.get('pixels', None)
69 degrade_nside = self.fileDescriptor.parameters.get('degrade_nside', None)
70 try:
71 data = hsp.HealSparseMap.read(path, pixels=pixels, degrade_nside=degrade_nside)
72 except (OSError, RuntimeError):
73 raise ValueError(f"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
75 return data
77 def write(self, inMemoryDataset):
78 # Docstring inherited from Formatter.write.
79 # Update the location with the formatter-preferred file extension
80 self.fileDescriptor.location.updateExtension(self.extension)
81 inMemoryDataset.write(self.fileDescriptor.location.path, clobber=True)
84def _is_power_of_two(value):
85 """Check that value is a power of two.
87 Parameters
88 ----------
89 value : `int`
90 Value to check.
92 Returns
93 -------
94 is_power_of_two : `bool`
95 True if value is a power of two; False otherwise, or
96 if value is not an integer.
97 """
98 if not isinstance(value, numbers.Integral):
99 return False
101 # See https://stackoverflow.com/questions/57025836
102 # Every power of 2 has exactly 1 bit set to 1; subtracting
103 # 1 therefore flips every preceding bit. If you and that
104 # together with the original value it must be 0.
105 return (value & (value - 1) == 0) and value != 0
108class HealSparseInputMapConfig(pexConfig.Config):
109 """Configuration parameters for HealSparseInputMapTask"""
110 nside = pexConfig.Field(
111 doc="Mapping healpix nside. Must be power of 2.",
112 dtype=int,
113 default=32768,
114 check=_is_power_of_two,
115 )
116 nside_coverage = pexConfig.Field(
117 doc="HealSparse coverage map nside. Must be power of 2.",
118 dtype=int,
119 default=256,
120 check=_is_power_of_two,
121 )
122 bad_mask_min_coverage = pexConfig.Field(
123 doc=("Minimum area fraction of a map healpixel pixel that must be "
124 "covered by bad pixels to be removed from the input map. "
125 "This is approximate."),
126 dtype=float,
127 default=0.5,
128 )
131class HealSparseInputMapTask(pipeBase.Task):
132 """Task for making a HealSparse input map."""
134 ConfigClass = HealSparseInputMapConfig
135 _DefaultName = "healSparseInputMap"
137 def __init__(self, **kwargs):
138 pipeBase.Task.__init__(self, **kwargs)
140 self.ccd_input_map = None
142 def build_ccd_input_map(self, bbox, wcs, ccds):
143 """Build a map from ccd valid polygons or bounding boxes.
145 Parameters
146 ----------
147 bbox : `lsst.geom.Box2I`
148 Bounding box for region to build input map.
149 wcs : `lsst.afw.geom.SkyWcs`
150 WCS object for region to build input map.
151 ccds : `lsst.afw.table.ExposureCatalog`
152 Exposure catalog with ccd data from coadd inputs.
153 """
154 self.ccd_input_map = hsp.HealSparseMap.make_empty(nside_coverage=self.config.nside_coverage,
155 nside_sparse=self.config.nside,
156 dtype=hsp.WIDE_MASK,
157 wide_mask_maxbits=len(ccds))
158 self._wcs = wcs
159 self._bbox = bbox
160 self._ccds = ccds
162 pixel_scale = wcs.getPixelScale().asArcseconds()
163 hpix_area_arcsec2 = hp.nside2pixarea(self.config.nside, degrees=True)*(3600.**2.)
164 self._min_bad = self.config.bad_mask_min_coverage*hpix_area_arcsec2/(pixel_scale**2.)
166 metadata = {}
167 self._bits_per_visit_ccd = {}
168 self._bits_per_visit = defaultdict(list)
169 for bit, ccd_row in enumerate(ccds):
170 metadata[f"B{bit:04d}CCD"] = ccd_row["ccd"]
171 metadata[f"B{bit:04d}VIS"] = ccd_row["visit"]
172 metadata[f"B{bit:04d}WT"] = ccd_row["weight"]
174 self._bits_per_visit_ccd[(ccd_row["visit"], ccd_row["ccd"])] = bit
175 self._bits_per_visit[ccd_row["visit"]].append(bit)
177 ccd_poly = ccd_row.getValidPolygon()
178 if ccd_poly is None:
179 ccd_poly = afwGeom.Polygon(lsst.geom.Box2D(ccd_row.getBBox()))
180 # Detectors need to be rendered with their own wcs.
181 ccd_poly_radec = self._pixels_to_radec(ccd_row.getWcs(), ccd_poly.convexHull().getVertices())
183 # Create a ccd healsparse polygon
184 poly = hsp.Polygon(ra=ccd_poly_radec[: -1, 0],
185 dec=ccd_poly_radec[: -1, 1],
186 value=[bit])
187 self.ccd_input_map.set_bits_pix(poly.get_pixels(nside=self.ccd_input_map.nside_sparse),
188 [bit])
190 # Cut down to the overall bounding box with associated wcs.
191 bbox_afw_poly = afwGeom.Polygon(lsst.geom.Box2D(bbox))
192 bbox_poly_radec = self._pixels_to_radec(self._wcs,
193 bbox_afw_poly.convexHull().getVertices())
194 bbox_poly = hsp.Polygon(ra=bbox_poly_radec[: -1, 0], dec=bbox_poly_radec[: -1, 1],
195 value=np.arange(self.ccd_input_map.wide_mask_maxbits))
196 bbox_poly_map = bbox_poly.get_map_like(self.ccd_input_map)
197 self.ccd_input_map = hsp.and_intersection([self.ccd_input_map, bbox_poly_map])
198 self.ccd_input_map.metadata = metadata
200 # Create a temporary map to hold the count of bad pixels in each healpix pixel
201 self._ccd_input_pixels = self.ccd_input_map.valid_pixels
203 dtype = [(f"v{visit}", "i4") for visit in self._bits_per_visit.keys()]
205 cov = self.config.nside_coverage
206 ns = self.config.nside
207 self._ccd_input_bad_count_map = hsp.HealSparseMap.make_empty(nside_coverage=cov,
208 nside_sparse=ns,
209 dtype=dtype,
210 primary=dtype[0][0])
211 # Don't set input bad map if there are no ccds which overlap the bbox.
212 if len(self._ccd_input_pixels) > 0:
213 self._ccd_input_bad_count_map[self._ccd_input_pixels] = np.zeros(1, dtype=dtype)
215 def mask_warp_bbox(self, bbox, visit, mask, bit_mask_value):
216 """Mask a subregion from a visit.
217 This must be run after build_ccd_input_map initializes
218 the overall map.
220 Parameters
221 ----------
222 bbox : `lsst.geom.Box2I`
223 Bounding box from region to mask.
224 visit : `int`
225 Visit number corresponding to warp with mask.
226 mask : `lsst.afw.image.MaskX`
227 Mask plane from warp exposure.
228 bit_mask_value : `int`
229 Bit mask to check for bad pixels.
231 Raises
232 ------
233 RuntimeError : Raised if build_ccd_input_map was not run first.
234 """
235 if self.ccd_input_map is None:
236 raise RuntimeError("Must run build_ccd_input_map before mask_warp_bbox")
238 # Find the bad pixels and convert to healpix
239 bad_pixels = np.where(mask.array & bit_mask_value)
240 if len(bad_pixels[0]) == 0:
241 # No bad pixels
242 return
244 # Bad pixels come from warps which use the overall wcs.
245 bad_ra, bad_dec = self._wcs.pixelToSkyArray(bad_pixels[1].astype(np.float64),
246 bad_pixels[0].astype(np.float64),
247 degrees=True)
248 bad_hpix = hp.ang2pix(self.config.nside, bad_ra, bad_dec,
249 lonlat=True, nest=True)
251 # Count the number of bad image pixels in each healpix pixel
252 min_bad_hpix = bad_hpix.min()
253 bad_hpix_count = np.zeros(bad_hpix.max() - min_bad_hpix + 1, dtype=np.int32)
254 np.add.at(bad_hpix_count, bad_hpix - min_bad_hpix, 1)
256 # Add these to the accumulator map.
257 # We need to make sure that the "primary" array has valid values for
258 # this pixel to be registered in the accumulator map.
259 pix_to_add, = np.where(bad_hpix_count > 0)
260 count_map_arr = self._ccd_input_bad_count_map[min_bad_hpix + pix_to_add]
261 primary = self._ccd_input_bad_count_map.primary
262 count_map_arr[primary] = np.clip(count_map_arr[primary], 0, None)
264 count_map_arr[f"v{visit}"] = np.clip(count_map_arr[f"v{visit}"], 0, None)
265 count_map_arr[f"v{visit}"] += bad_hpix_count[pix_to_add]
267 self._ccd_input_bad_count_map[min_bad_hpix + pix_to_add] = count_map_arr
269 def finalize_ccd_input_map_mask(self):
270 """Use accumulated mask information to finalize the masking of
271 ccd_input_map.
273 Raises
274 ------
275 RuntimeError : Raised if build_ccd_input_map was not run first.
276 """
277 if self.ccd_input_map is None:
278 raise RuntimeError("Must run build_ccd_input_map before finalize_ccd_input_map_mask.")
280 count_map_arr = self._ccd_input_bad_count_map[self._ccd_input_pixels]
281 for visit in self._bits_per_visit:
282 to_mask, = np.where(count_map_arr[f"v{visit}"] > self._min_bad)
283 if to_mask.size == 0:
284 continue
285 self.ccd_input_map.clear_bits_pix(self._ccd_input_pixels[to_mask],
286 self._bits_per_visit[visit])
288 # Clear memory
289 self._ccd_input_bad_count_map = None
291 def _pixels_to_radec(self, wcs, pixels):
292 """Convert pixels to ra/dec positions using a wcs.
294 Parameters
295 ----------
296 wcs : `lsst.afw.geom.SkyWcs`
297 WCS object.
298 pixels : `list` [`lsst.geom.Point2D`]
299 List of pixels to convert.
301 Returns
302 -------
303 radec : `numpy.ndarray`
304 Nx2 array of ra/dec positions associated with pixels.
305 """
306 sph_pts = wcs.pixelToSky(pixels)
307 return np.array([(sph.getRa().asDegrees(), sph.getDec().asDegrees())
308 for sph in sph_pts])
311class HealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections,
312 dimensions=("tract", "band", "skymap",),
313 defaultTemplates={"coaddName": "deep"}):
314 input_maps = pipeBase.connectionTypes.Input(
315 doc="Healsparse bit-wise coadd input maps",
316 name="{coaddName}Coadd_inputMap",
317 storageClass="HealSparseMap",
318 dimensions=("tract", "patch", "skymap", "band"),
319 multiple=True,
320 deferLoad=True,
321 )
322 coadd_exposures = pipeBase.connectionTypes.Input(
323 doc="Coadded exposures associated with input_maps",
324 name="{coaddName}Coadd",
325 storageClass="ExposureF",
326 dimensions=("tract", "patch", "skymap", "band"),
327 multiple=True,
328 deferLoad=True,
329 )
330 visit_summaries = pipeBase.connectionTypes.Input(
331 doc="Visit summary tables with aggregated statistics",
332 name="visitSummary",
333 storageClass="ExposureCatalog",
334 dimensions=("instrument", "visit"),
335 multiple=True,
336 deferLoad=True,
337 )
338 sky_map = pipeBase.connectionTypes.Input(
339 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
340 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
341 storageClass="SkyMap",
342 dimensions=("skymap",),
343 )
345 # Create output connections for all possible maps defined in the
346 # registry. The vars() trick used here allows us to set class attributes
347 # programatically. Taken from
348 # https://stackoverflow.com/questions/2519807/
349 # setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
350 for name in BasePropertyMap.registry:
351 vars()[f"{name}_map_min"] = pipeBase.connectionTypes.Output(
352 doc=f"Minimum-value map of {name}",
353 name=f"{{coaddName}}Coadd_{name}_map_min",
354 storageClass="HealSparseMap",
355 dimensions=("tract", "skymap", "band"),
356 )
357 vars()[f"{name}_map_max"] = pipeBase.connectionTypes.Output(
358 doc=f"Maximum-value map of {name}",
359 name=f"{{coaddName}}Coadd_{name}_map_max",
360 storageClass="HealSparseMap",
361 dimensions=("tract", "skymap", "band"),
362 )
363 vars()[f"{name}_map_mean"] = pipeBase.connectionTypes.Output(
364 doc=f"Mean-value map of {name}",
365 name=f"{{coaddName}}Coadd_{name}_map_mean",
366 storageClass="HealSparseMap",
367 dimensions=("tract", "skymap", "band"),
368 )
369 vars()[f"{name}_map_weighted_mean"] = pipeBase.connectionTypes.Output(
370 doc=f"Weighted mean-value map of {name}",
371 name=f"{{coaddName}}Coadd_{name}_map_weighted_mean",
372 storageClass="HealSparseMap",
373 dimensions=("tract", "skymap", "band"),
374 )
375 vars()[f"{name}_map_sum"] = pipeBase.connectionTypes.Output(
376 doc=f"Sum-value map of {name}",
377 name=f"{{coaddName}}Coadd_{name}_map_sum",
378 storageClass="HealSparseMap",
379 dimensions=("tract", "skymap", "band"),
380 )
382 def __init__(self, *, config=None):
383 super().__init__(config=config)
385 # Not all possible maps in the registry will be configured to run.
386 # Here we remove the unused connections.
387 for name in BasePropertyMap.registry:
388 if name not in config.property_maps:
389 prop_config = BasePropertyMapConfig()
390 prop_config.do_min = False
391 prop_config.do_max = False
392 prop_config.do_mean = False
393 prop_config.do_weighted_mean = False
394 prop_config.do_sum = False
395 else:
396 prop_config = config.property_maps[name]
398 if not prop_config.do_min:
399 self.outputs.remove(f"{name}_map_min")
400 if not prop_config.do_max:
401 self.outputs.remove(f"{name}_map_max")
402 if not prop_config.do_mean:
403 self.outputs.remove(f"{name}_map_mean")
404 if not prop_config.do_weighted_mean:
405 self.outputs.remove(f"{name}_map_weighted_mean")
406 if not prop_config.do_sum:
407 self.outputs.remove(f"{name}_map_sum")
410class HealSparsePropertyMapConfig(pipeBase.PipelineTaskConfig,
411 pipelineConnections=HealSparsePropertyMapConnections):
412 """Configuration parameters for HealSparsePropertyMapTask"""
413 property_maps = BasePropertyMap.registry.makeField(
414 multi=True,
415 default=["exposure_time",
416 "psf_size",
417 "psf_e1",
418 "psf_e2",
419 "psf_maglim",
420 "sky_noise",
421 "sky_background",
422 "dcr_dra",
423 "dcr_ddec",
424 "dcr_e1",
425 "dcr_e2"],
426 doc="Property map computation objects",
427 )
429 def setDefaults(self):
430 self.property_maps["exposure_time"].do_sum = True
431 self.property_maps["psf_size"].do_weighted_mean = True
432 self.property_maps["psf_e1"].do_weighted_mean = True
433 self.property_maps["psf_e2"].do_weighted_mean = True
434 self.property_maps["psf_maglim"].do_weighted_mean = True
435 self.property_maps["sky_noise"].do_weighted_mean = True
436 self.property_maps["sky_background"].do_weighted_mean = True
437 self.property_maps["dcr_dra"].do_weighted_mean = True
438 self.property_maps["dcr_ddec"].do_weighted_mean = True
439 self.property_maps["dcr_e1"].do_weighted_mean = True
440 self.property_maps["dcr_e2"].do_weighted_mean = True
443class HealSparsePropertyMapTask(pipeBase.PipelineTask):
444 """Task to compute Healsparse property maps.
446 This task will compute individual property maps (per tract, per
447 map type, per band). These maps cover the full coadd tract, and
448 are not truncated to the inner tract region.
449 """
450 ConfigClass = HealSparsePropertyMapConfig
451 _DefaultName = "healSparsePropertyMapTask"
453 def __init__(self, **kwargs):
454 super().__init__(**kwargs)
455 self.property_maps = PropertyMapMap()
456 for name, config, PropertyMapClass in self.config.property_maps.apply():
457 self.property_maps[name] = PropertyMapClass(config, name)
459 @pipeBase.timeMethod
460 def runQuantum(self, butlerQC, inputRefs, outputRefs):
461 inputs = butlerQC.get(inputRefs)
463 sky_map = inputs.pop("sky_map")
465 tract = butlerQC.quantum.dataId["tract"]
466 band = butlerQC.quantum.dataId["band"]
468 input_map_dict = {ref.dataId["patch"]: ref for ref in inputs["input_maps"]}
469 coadd_dict = {ref.dataId["patch"]: ref for ref in inputs["coadd_exposures"]}
471 visit_summary_dict = {ref.dataId["visit"]: ref.get()
472 for ref in inputs["visit_summaries"]}
474 self.run(sky_map, tract, band, coadd_dict, input_map_dict, visit_summary_dict)
476 # Write the outputs
477 for name, property_map in self.property_maps.items():
478 if property_map.config.do_min:
479 butlerQC.put(property_map.min_map,
480 getattr(outputRefs, f"{name}_map_min"))
481 if property_map.config.do_max:
482 butlerQC.put(property_map.max_map,
483 getattr(outputRefs, f"{name}_map_max"))
484 if property_map.config.do_mean:
485 butlerQC.put(property_map.mean_map,
486 getattr(outputRefs, f"{name}_map_mean"))
487 if property_map.config.do_weighted_mean:
488 butlerQC.put(property_map.weighted_mean_map,
489 getattr(outputRefs, f"{name}_map_weighted_mean"))
490 if property_map.config.do_sum:
491 butlerQC.put(property_map.sum_map,
492 getattr(outputRefs, f"{name}_map_sum"))
494 def run(self, sky_map, tract, band, coadd_dict, input_map_dict, visit_summary_dict):
495 """Run the healsparse property task.
497 Parameters
498 ----------
499 sky_map : Sky map object
500 tract : `int`
501 Tract number.
502 band : `str`
503 Band name for logging.
504 coadd_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
505 Dictionary of coadd exposure references. Keys are patch numbers.
506 input_map_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
507 Dictionary of input map references. Keys are patch numbers.
508 visit_summary_dict : `dict` [`int`: `lsst.afw.table.ExposureCatalog`]
509 Dictionary of visit summary tables. Keys are visit numbers.
511 Raises
512 ------
513 RepeatableQuantumError
514 If visit_summary_dict is missing any visits or detectors found in an
515 input map. This leads to an inconsistency between what is in the coadd
516 (via the input map) and the visit summary tables which contain data
517 to compute the maps.
518 """
519 tract_info = sky_map[tract]
521 tract_maps_initialized = False
523 for patch in input_map_dict.keys():
524 self.log.info("Making maps for band %s, tract %d, patch %d.",
525 band, tract, patch)
527 patch_info = tract_info[patch]
529 input_map = input_map_dict[patch].get()
530 coadd_photo_calib = coadd_dict[patch].get(component="photoCalib")
531 coadd_inputs = coadd_dict[patch].get(component="coaddInputs")
533 coadd_zeropoint = 2.5*np.log10(coadd_photo_calib.getInstFluxAtZeroMagnitude())
535 # Crop input_map to the inner polygon of the patch
536 poly_vertices = patch_info.getInnerSkyPolygon(tract_info.getWcs()).getVertices()
537 patch_radec = self._vertices_to_radec(poly_vertices)
538 patch_poly = hsp.Polygon(ra=patch_radec[:, 0], dec=patch_radec[:, 1],
539 value=np.arange(input_map.wide_mask_maxbits))
540 patch_poly_map = patch_poly.get_map_like(input_map)
541 input_map = hsp.and_intersection([input_map, patch_poly_map])
543 if not tract_maps_initialized:
544 # We use the first input map nside information to initialize
545 # the tract maps
546 nside_coverage = self._compute_nside_coverage_tract(tract_info)
547 nside = input_map.nside_sparse
549 do_compute_approx_psf = False
550 # Initialize the tract maps
551 for property_map in self.property_maps:
552 property_map.initialize_tract_maps(nside_coverage, nside)
553 if property_map.requires_psf:
554 do_compute_approx_psf = True
556 tract_maps_initialized = True
558 valid_pixels, vpix_ra, vpix_dec = input_map.valid_pixels_pos(return_pixels=True)
560 # Check if there are no valid pixels for the inner (unique) patch region
561 if valid_pixels.size == 0:
562 continue
564 # Initialize the value accumulators
565 for property_map in self.property_maps:
566 property_map.initialize_values(valid_pixels.size)
567 property_map.zeropoint = coadd_zeropoint
569 # Initialize the weight and counter accumulators
570 total_weights = np.zeros(valid_pixels.size)
571 total_inputs = np.zeros(valid_pixels.size, dtype=np.int32)
573 for bit, ccd_row in enumerate(coadd_inputs.ccds):
574 # Which pixels in the map are used by this visit/detector
575 inmap, = np.where(input_map.check_bits_pix(valid_pixels, [bit]))
577 # Check if there are any valid pixels in the map from this deteector.
578 if inmap.size == 0:
579 continue
581 # visit, detector_id, weight = input_dict[bit]
582 visit = ccd_row["visit"]
583 detector_id = ccd_row["ccd"]
584 weight = ccd_row["weight"]
586 x, y = ccd_row.getWcs().skyToPixelArray(vpix_ra[inmap], vpix_dec[inmap], degrees=True)
587 scalings = self._compute_calib_scale(ccd_row, x, y)
589 if do_compute_approx_psf:
590 psf_array = compute_approx_psf_size_and_shape(ccd_row, vpix_ra[inmap], vpix_dec[inmap])
591 else:
592 psf_array = None
594 total_weights[inmap] += weight
595 total_inputs[inmap] += 1
597 # Retrieve the correct visitSummary row
598 if visit not in visit_summary_dict:
599 msg = f"Visit {visit} not found in visit_summaries."
600 raise pipeBase.RepeatableQuantumError(msg)
601 row = visit_summary_dict[visit].find(detector_id)
602 if row is None:
603 msg = f"Visit {visit} / detector_id {detector_id} not found in visit_summaries."
604 raise pipeBase.RepeatableQuantumError(msg)
606 # Accumulate the values
607 for property_map in self.property_maps:
608 property_map.accumulate_values(inmap,
609 vpix_ra[inmap],
610 vpix_dec[inmap],
611 weight,
612 scalings,
613 row,
614 psf_array=psf_array)
616 # Finalize the mean values and set the tract maps
617 for property_map in self.property_maps:
618 property_map.finalize_mean_values(total_weights, total_inputs)
619 property_map.set_map_values(valid_pixels)
621 def _compute_calib_scale(self, ccd_row, x, y):
622 """Compute calibration scaling values.
624 Parameters
625 ----------
626 ccd_row : `lsst.afw.table.ExposureRecord`
627 Exposure metadata for a given detector exposure.
628 x : `np.ndarray`
629 Array of x positions.
630 y : `np.ndarray`
631 Array of y positions.
633 Returns
634 -------
635 calib_scale : `np.ndarray`
636 Array of calibration scale values.
637 """
638 photo_calib = ccd_row.getPhotoCalib()
639 bf = photo_calib.computeScaledCalibration()
640 if bf.getBBox() == ccd_row.getBBox():
641 # Track variable calibration over the detector
642 calib_scale = photo_calib.getCalibrationMean()*bf.evaluate(x, y)
643 else:
644 # Spatially constant calibration
645 calib_scale = photo_calib.getCalibrationMean()
647 return calib_scale
649 def _vertices_to_radec(self, vertices):
650 """Convert polygon vertices to ra/dec.
652 Parameters
653 ----------
654 vertices : `list` [ `lsst.sphgeom.UnitVector3d` ]
655 Vertices for bounding polygon.
657 Returns
658 -------
659 radec : `numpy.ndarray`
660 Nx2 array of ra/dec positions (in degrees) associated with vertices.
661 """
662 lonlats = [lsst.sphgeom.LonLat(x) for x in vertices]
663 radec = np.array([(x.getLon().asDegrees(), x.getLat().asDegrees()) for
664 x in lonlats])
665 return radec
667 def _compute_nside_coverage_tract(self, tract_info):
668 """Compute the optimal coverage nside for a tract.
670 Parameters
671 ----------
672 tract_info : `lsst.skymap.tractInfo.ExplicitTractInfo`
673 Tract information object.
675 Returns
676 -------
677 nside_coverage : `int`
678 Optimal coverage nside for a tract map.
679 """
680 num_patches = tract_info.getNumPatches()
682 # Compute approximate patch area
683 patch_info = tract_info.getPatchInfo(0)
684 vertices = patch_info.getInnerSkyPolygon(tract_info.getWcs()).getVertices()
685 radec = self._vertices_to_radec(vertices)
686 delta_ra = np.max(radec[:, 0]) - np.min(radec[:, 0])
687 delta_dec = np.max(radec[:, 1]) - np.min(radec[:, 1])
688 patch_area = delta_ra*delta_dec*np.cos(np.deg2rad(np.mean(radec[:, 1])))
690 tract_area = num_patches[0]*num_patches[1]*patch_area
691 # Start with a fairly low nside and increase until we find the approximate area.
692 nside_coverage_tract = 32
693 while hp.nside2pixarea(nside_coverage_tract, degrees=True) > tract_area:
694 nside_coverage_tract = 2*nside_coverage_tract
695 # Step back one, but don't go bigger pixels than nside=32
696 nside_coverage_tract = int(np.clip(nside_coverage_tract/2, 32, None))
698 return nside_coverage_tract
701class ConsolidateHealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections,
702 dimensions=("band", "skymap",),
703 defaultTemplates={"coaddName": "deep"}):
704 sky_map = pipeBase.connectionTypes.Input(
705 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
706 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
707 storageClass="SkyMap",
708 dimensions=("skymap",),
709 )
711 # Create output connections for all possible maps defined in the
712 # registry. The vars() trick used here allows us to set class attributes
713 # programatically. Taken from
714 # https://stackoverflow.com/questions/2519807/
715 # setting-a-class-attribute-with-a-given-name-in-python-while-defining-the-class
716 for name in BasePropertyMap.registry:
717 vars()[f"{name}_map_min"] = pipeBase.connectionTypes.Input(
718 doc=f"Minimum-value map of {name}",
719 name=f"{{coaddName}}Coadd_{name}_map_min",
720 storageClass="HealSparseMap",
721 dimensions=("tract", "skymap", "band"),
722 multiple=True,
723 deferLoad=True,
724 )
725 vars()[f"{name}_consolidated_map_min"] = pipeBase.connectionTypes.Output(
726 doc=f"Minumum-value map of {name}",
727 name=f"{{coaddName}}Coadd_{name}_consolidated_map_min",
728 storageClass="HealSparseMap",
729 dimensions=("skymap", "band"),
730 )
731 vars()[f"{name}_map_max"] = pipeBase.connectionTypes.Input(
732 doc=f"Maximum-value map of {name}",
733 name=f"{{coaddName}}Coadd_{name}_map_max",
734 storageClass="HealSparseMap",
735 dimensions=("tract", "skymap", "band"),
736 multiple=True,
737 deferLoad=True,
738 )
739 vars()[f"{name}_consolidated_map_max"] = pipeBase.connectionTypes.Output(
740 doc=f"Minumum-value map of {name}",
741 name=f"{{coaddName}}Coadd_{name}_consolidated_map_max",
742 storageClass="HealSparseMap",
743 dimensions=("skymap", "band"),
744 )
745 vars()[f"{name}_map_mean"] = pipeBase.connectionTypes.Input(
746 doc=f"Mean-value map of {name}",
747 name=f"{{coaddName}}Coadd_{name}_map_mean",
748 storageClass="HealSparseMap",
749 dimensions=("tract", "skymap", "band"),
750 multiple=True,
751 deferLoad=True,
752 )
753 vars()[f"{name}_consolidated_map_mean"] = pipeBase.connectionTypes.Output(
754 doc=f"Minumum-value map of {name}",
755 name=f"{{coaddName}}Coadd_{name}_consolidated_map_mean",
756 storageClass="HealSparseMap",
757 dimensions=("skymap", "band"),
758 )
759 vars()[f"{name}_map_weighted_mean"] = pipeBase.connectionTypes.Input(
760 doc=f"Weighted mean-value map of {name}",
761 name=f"{{coaddName}}Coadd_{name}_map_weighted_mean",
762 storageClass="HealSparseMap",
763 dimensions=("tract", "skymap", "band"),
764 multiple=True,
765 deferLoad=True,
766 )
767 vars()[f"{name}_consolidated_map_weighted_mean"] = pipeBase.connectionTypes.Output(
768 doc=f"Minumum-value map of {name}",
769 name=f"{{coaddName}}Coadd_{name}_consolidated_map_weighted_mean",
770 storageClass="HealSparseMap",
771 dimensions=("skymap", "band"),
772 )
773 vars()[f"{name}_map_sum"] = pipeBase.connectionTypes.Input(
774 doc=f"Sum-value map of {name}",
775 name=f"{{coaddName}}Coadd_{name}_map_sum",
776 storageClass="HealSparseMap",
777 dimensions=("tract", "skymap", "band"),
778 multiple=True,
779 deferLoad=True,
780 )
781 vars()[f"{name}_consolidated_map_sum"] = pipeBase.connectionTypes.Output(
782 doc=f"Minumum-value map of {name}",
783 name=f"{{coaddName}}Coadd_{name}_consolidated_map_sum",
784 storageClass="HealSparseMap",
785 dimensions=("skymap", "band"),
786 )
788 def __init__(self, *, config=None):
789 super().__init__(config=config)
791 # Not all possible maps in the registry will be configured to run.
792 # Here we remove the unused connections.
793 for name in BasePropertyMap.registry:
794 if name not in config.property_maps:
795 prop_config = BasePropertyMapConfig()
796 prop_config.do_min = False
797 prop_config.do_max = False
798 prop_config.do_mean = False
799 prop_config.do_weighted_mean = False
800 prop_config.do_sum = False
801 else:
802 prop_config = config.property_maps[name]
804 if not prop_config.do_min:
805 self.inputs.remove(f"{name}_map_min")
806 self.outputs.remove(f"{name}_consolidated_map_min")
807 if not prop_config.do_max:
808 self.inputs.remove(f"{name}_map_max")
809 self.outputs.remove(f"{name}_consolidated_map_max")
810 if not prop_config.do_mean:
811 self.inputs.remove(f"{name}_map_mean")
812 self.outputs.remove(f"{name}_consolidated_map_mean")
813 if not prop_config.do_weighted_mean:
814 self.inputs.remove(f"{name}_map_weighted_mean")
815 self.outputs.remove(f"{name}_consolidated_map_weighted_mean")
816 if not prop_config.do_sum:
817 self.inputs.remove(f"{name}_map_sum")
818 self.outputs.remove(f"{name}_consolidated_map_sum")
821class ConsolidateHealSparsePropertyMapConfig(pipeBase.PipelineTaskConfig,
822 pipelineConnections=ConsolidateHealSparsePropertyMapConnections):
823 """Configuration parameters for ConsolidateHealSparsePropertyMapTask"""
824 property_maps = BasePropertyMap.registry.makeField(
825 multi=True,
826 default=["exposure_time",
827 "psf_size",
828 "psf_e1",
829 "psf_e2",
830 "psf_maglim",
831 "sky_noise",
832 "sky_background",
833 "dcr_dra",
834 "dcr_ddec",
835 "dcr_e1",
836 "dcr_e2"],
837 doc="Property map computation objects",
838 )
840 def setDefaults(self):
841 self.property_maps["exposure_time"].do_sum = True
842 self.property_maps["psf_size"].do_weighted_mean = True
843 self.property_maps["psf_e1"].do_weighted_mean = True
844 self.property_maps["psf_e2"].do_weighted_mean = True
845 self.property_maps["psf_maglim"].do_weighted_mean = True
846 self.property_maps["sky_noise"].do_weighted_mean = True
847 self.property_maps["sky_background"].do_weighted_mean = True
848 self.property_maps["dcr_dra"].do_weighted_mean = True
849 self.property_maps["dcr_ddec"].do_weighted_mean = True
850 self.property_maps["dcr_e1"].do_weighted_mean = True
851 self.property_maps["dcr_e2"].do_weighted_mean = True
854class ConsolidateHealSparsePropertyMapTask(pipeBase.PipelineTask):
855 """Task to consolidate HealSparse property maps.
857 This task will take all the individual tract-based maps (per map type,
858 per band) and consolidate them into one survey-wide map (per map type,
859 per band). Each tract map is truncated to its inner region before
860 consolidation.
861 """
862 ConfigClass = ConsolidateHealSparsePropertyMapConfig
863 _DefaultName = "consolidateHealSparsePropertyMapTask"
865 def __init__(self, **kwargs):
866 super().__init__(**kwargs)
867 self.property_maps = PropertyMapMap()
868 for name, config, PropertyMapClass in self.config.property_maps.apply():
869 self.property_maps[name] = PropertyMapClass(config, name)
871 @pipeBase.timeMethod
872 def runQuantum(self, butlerQC, inputRefs, outputRefs):
873 inputs = butlerQC.get(inputRefs)
875 sky_map = inputs.pop("sky_map")
877 # These need to be consolidated one at a time to conserve memory.
878 for name in self.config.property_maps.names:
879 for type_ in ['min', 'max', 'mean', 'weighted_mean', 'sum']:
880 map_type = f"{name}_map_{type_}"
881 if map_type in inputs:
882 input_refs = {ref.dataId['tract']: ref
883 for ref in inputs[map_type]}
884 consolidated_map = self.consolidate_map(sky_map, input_refs)
885 butlerQC.put(consolidated_map,
886 getattr(outputRefs, f"{name}_consolidated_map_{type_}"))
888 def consolidate_map(self, sky_map, input_refs):
889 """Consolidate the healsparse property maps.
891 Parameters
892 ----------
893 sky_map : Sky map object
894 input_refs : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
895 Dictionary of tract_id mapping to dataref.
897 Returns
898 -------
899 consolidated_map : `healsparse.HealSparseMap`
900 Consolidated HealSparse map.
901 """
902 # First, we read in the coverage maps to know how much memory
903 # to allocate
904 cov_mask = None
905 for tract_id in input_refs:
906 cov = input_refs[tract_id].get(component='coverage')
907 if cov_mask is None:
908 cov_mask = cov.coverage_mask
909 else:
910 cov_mask |= cov.coverage_mask
912 cov_pix, = np.where(cov_mask)
914 # Now read in each tract map and build the consolidated map.
915 consolidated_map = None
916 for tract_id in input_refs:
917 input_map = input_refs[tract_id].get()
918 if consolidated_map is None:
919 dtype = input_map.dtype
920 sentinel = input_map._sentinel
921 nside_coverage = input_map.nside_coverage
922 nside_sparse = input_map.nside_sparse
923 consolidated_map = hsp.HealSparseMap.make_empty(nside_coverage,
924 nside_sparse,
925 dtype,
926 sentinel=sentinel)
927 consolidated_map._reserve_cov_pix(cov_pix)
929 # Only use pixels that are properly inside the tract.
930 vpix, ra, dec = input_map.valid_pixels_pos(return_pixels=True)
931 vpix_tract_ids = sky_map.findTractIdArray(ra, dec, degrees=True)
933 in_tract = (vpix_tract_ids == tract_id)
935 consolidated_map[vpix[in_tract]] = input_map[vpix[in_tract]]
937 return consolidated_map