22 from collections
import defaultdict
26 import healsparse
as hsp
32 from lsst.daf.butler
import Formatter
34 from .healSparseMappingProperties
import (BasePropertyMap, BasePropertyMapConfig,
35 PropertyMapMap, compute_approx_psf_size_and_shape)
38 __all__ = [
"HealSparseInputMapTask",
"HealSparseInputMapConfig",
39 "HealSparseMapFormatter",
"HealSparsePropertyMapConnections",
40 "HealSparsePropertyMapConfig",
"HealSparsePropertyMapTask"]
44 """Interface for reading and writing healsparse.HealSparseMap files."""
45 unsupportedParameters = frozenset()
46 supportedExtensions = frozenset({
".hsp",
".fit",
".fits"})
49 def read(self, component=None):
51 path = self.fileDescriptor.location.path
53 if component ==
'coverage':
55 data = hsp.HealSparseCoverage.read(path)
56 except (OSError, RuntimeError):
57 raise ValueError(f
"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
61 if self.fileDescriptor.parameters
is None:
65 pixels = self.fileDescriptor.parameters.get(
'pixels',
None)
66 degrade_nside = self.fileDescriptor.parameters.get(
'degrade_nside',
None)
68 data = hsp.HealSparseMap.read(path, pixels=pixels, degrade_nside=degrade_nside)
69 except (OSError, RuntimeError):
70 raise ValueError(f
"Unable to read healsparse map with URI {self.fileDescriptor.location.uri}")
74 def write(self, inMemoryDataset):
77 self.fileDescriptor.location.updateExtension(self.
extensionextension)
78 inMemoryDataset.write(self.fileDescriptor.location.path, clobber=
True)
81 def _is_power_of_two(value):
82 """Check that value is a power of two.
91 is_power_of_two : `bool`
92 True if value is a power of two; False otherwise, or
93 if value is not an integer.
95 if not isinstance(value, numbers.Integral):
102 return (value & (value - 1) == 0)
and value != 0
106 """Configuration parameters for HealSparseInputMapTask"""
107 nside = pexConfig.Field(
108 doc=
"Mapping healpix nside. Must be power of 2.",
111 check=_is_power_of_two,
113 nside_coverage = pexConfig.Field(
114 doc=
"HealSparse coverage map nside. Must be power of 2.",
117 check=_is_power_of_two,
119 bad_mask_min_coverage = pexConfig.Field(
120 doc=(
"Minimum area fraction of a map healpixel pixel that must be "
121 "covered by bad pixels to be removed from the input map. "
122 "This is approximate."),
129 """Task for making a HealSparse input map."""
131 ConfigClass = HealSparseInputMapConfig
132 _DefaultName =
"healSparseInputMap"
135 pipeBase.Task.__init__(self, **kwargs)
140 """Build a map from ccd valid polygons or bounding boxes.
144 bbox : `lsst.geom.Box2I`
145 Bounding box for region to build input map.
146 wcs : `lsst.afw.geom.SkyWcs`
147 WCS object for region to build input map.
148 ccds : `lsst.afw.table.ExposureCatalog`
149 Exposure catalog with ccd data from coadd inputs.
151 self.
ccd_input_mapccd_input_map = hsp.HealSparseMap.make_empty(nside_coverage=self.config.nside_coverage,
152 nside_sparse=self.config.nside,
154 wide_mask_maxbits=len(ccds))
156 self.
_bbox_bbox = bbox
157 self.
_ccds_ccds = ccds
159 pixel_scale = wcs.getPixelScale().asArcseconds()
160 hpix_area_arcsec2 = hp.nside2pixarea(self.config.nside, degrees=
True)*(3600.**2.)
161 self.
_min_bad_min_bad = self.config.bad_mask_min_coverage*hpix_area_arcsec2/(pixel_scale**2.)
166 for bit, ccd_row
in enumerate(ccds):
167 metadata[f
"B{bit:04d}CCD"] = ccd_row[
"ccd"]
168 metadata[f
"B{bit:04d}VIS"] = ccd_row[
"visit"]
169 metadata[f
"B{bit:04d}WT"] = ccd_row[
"weight"]
174 ccd_poly = ccd_row.getValidPolygon()
178 ccd_poly_radec = self.
_pixels_to_radec_pixels_to_radec(ccd_row.getWcs(), ccd_poly.convexHull().getVertices())
181 poly = hsp.Polygon(ra=ccd_poly_radec[: -1, 0],
182 dec=ccd_poly_radec[: -1, 1],
190 bbox_afw_poly.convexHull().getVertices())
191 bbox_poly = hsp.Polygon(ra=bbox_poly_radec[: -1, 0], dec=bbox_poly_radec[: -1, 1],
192 value=np.arange(self.
ccd_input_mapccd_input_map.wide_mask_maxbits))
193 bbox_poly_map = bbox_poly.get_map_like(self.
ccd_input_mapccd_input_map)
200 dtype = [(f
"v{visit}",
"i4")
for visit
in self.
_bits_per_visit_bits_per_visit.keys()]
202 cov = self.config.nside_coverage
203 ns = self.config.nside
213 """Mask a subregion from a visit.
214 This must be run after build_ccd_input_map initializes
219 bbox : `lsst.geom.Box2I`
220 Bounding box from region to mask.
222 Visit number corresponding to warp with mask.
223 mask : `lsst.afw.image.MaskX`
224 Mask plane from warp exposure.
225 bit_mask_value : `int`
226 Bit mask to check for bad pixels.
230 RuntimeError : Raised if build_ccd_input_map was not run first.
233 raise RuntimeError(
"Must run build_ccd_input_map before mask_warp_bbox")
236 bad_pixels = np.where(mask.array & bit_mask_value)
237 if len(bad_pixels[0]) == 0:
242 bad_ra, bad_dec = self.
_wcs_wcs.pixelToSkyArray(bad_pixels[1].astype(np.float64),
243 bad_pixels[0].astype(np.float64),
245 bad_hpix = hp.ang2pix(self.config.nside, bad_ra, bad_dec,
246 lonlat=
True, nest=
True)
249 min_bad_hpix = bad_hpix.min()
250 bad_hpix_count = np.zeros(bad_hpix.max() - min_bad_hpix + 1, dtype=np.int32)
251 np.add.at(bad_hpix_count, bad_hpix - min_bad_hpix, 1)
256 pix_to_add, = np.where(bad_hpix_count > 0)
259 count_map_arr[primary] = np.clip(count_map_arr[primary], 0,
None)
261 count_map_arr[f
"v{visit}"] = np.clip(count_map_arr[f
"v{visit}"], 0,
None)
262 count_map_arr[f
"v{visit}"] += bad_hpix_count[pix_to_add]
267 """Use accumulated mask information to finalize the masking of
272 RuntimeError : Raised if build_ccd_input_map was not run first.
275 raise RuntimeError(
"Must run build_ccd_input_map before finalize_ccd_input_map_mask.")
279 to_mask, = np.where(count_map_arr[f
"v{visit}"] > self.
_min_bad_min_bad)
280 if to_mask.size == 0:
288 def _pixels_to_radec(self, wcs, pixels):
289 """Convert pixels to ra/dec positions using a wcs.
293 wcs : `lsst.afw.geom.SkyWcs`
295 pixels : `list` [`lsst.geom.Point2D`]
296 List of pixels to convert.
300 radec : `numpy.ndarray`
301 Nx2 array of ra/dec positions associated with pixels.
303 sph_pts = wcs.pixelToSky(pixels)
304 return np.array([(sph.getRa().asDegrees(), sph.getDec().asDegrees())
309 dimensions=(
"tract",
"band",
"skymap",),
310 defaultTemplates={
"coaddName":
"deep"}):
311 input_maps = pipeBase.connectionTypes.Input(
312 doc=
"Healsparse bit-wise coadd input maps",
313 name=
"{coaddName}Coadd_inputMap",
314 storageClass=
"HealSparseMap",
315 dimensions=(
"tract",
"patch",
"skymap",
"band"),
319 coadd_exposures = pipeBase.connectionTypes.Input(
320 doc=
"Coadded exposures associated with input_maps",
321 name=
"{coaddName}Coadd",
322 storageClass=
"ExposureF",
323 dimensions=(
"tract",
"patch",
"skymap",
"band"),
327 visit_summaries = pipeBase.connectionTypes.Input(
328 doc=
"Visit summary tables with aggregated statistics",
330 storageClass=
"ExposureCatalog",
331 dimensions=(
"instrument",
"visit"),
335 sky_map = pipeBase.connectionTypes.Input(
336 doc=
"Input definition of geometry/bbox and projection/wcs for coadded exposures",
337 name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
338 storageClass=
"SkyMap",
339 dimensions=(
"skymap",),
344 for name
in BasePropertyMap.registry:
345 vars()[f
"{name}_map_min"] = pipeBase.connectionTypes.Output(
346 doc=f
"Minimum-value map of {name}",
347 name=f
"{{coaddName}}Coadd_{name}_map_min",
348 storageClass=
"HealSparseMap",
349 dimensions=(
"tract",
"skymap",
"band"),
351 vars()[f
"{name}_map_max"] = pipeBase.connectionTypes.Output(
352 doc=f
"Maximum-value map of {name}",
353 name=f
"{{coaddName}}Coadd_{name}_map_max",
354 storageClass=
"HealSparseMap",
355 dimensions=(
"tract",
"skymap",
"band"),
357 vars()[f
"{name}_map_mean"] = pipeBase.connectionTypes.Output(
358 doc=f
"Mean-value map of {name}",
359 name=f
"{{coaddName}}Coadd_{name}_map_mean",
360 storageClass=
"HealSparseMap",
361 dimensions=(
"tract",
"skymap",
"band"),
363 vars()[f
"{name}_map_weighted_mean"] = pipeBase.connectionTypes.Output(
364 doc=f
"Weighted mean-value map of {name}",
365 name=f
"{{coaddName}}Coadd_{name}_map_weighted_mean",
366 storageClass=
"HealSparseMap",
367 dimensions=(
"tract",
"skymap",
"band"),
369 vars()[f
"{name}_map_sum"] = pipeBase.connectionTypes.Output(
370 doc=f
"Sum-value map of {name}",
371 name=f
"{{coaddName}}Coadd_{name}_map_sum",
372 storageClass=
"HealSparseMap",
373 dimensions=(
"tract",
"skymap",
"band"),
376 def __init__(self, *, config=None):
377 super().__init__(config=config)
380 for name
in BasePropertyMap.registry:
381 if name
not in config.property_maps:
383 prop_config.do_min =
False
384 prop_config.do_max =
False
385 prop_config.do_mean =
False
386 prop_config.do_weighted_mean =
False
387 prop_config.do_sum =
False
389 prop_config = config.property_maps[name]
391 if not prop_config.do_min:
392 self.outputs.remove(f
"{name}_map_min")
393 if not prop_config.do_max:
394 self.outputs.remove(f
"{name}_map_max")
395 if not prop_config.do_mean:
396 self.outputs.remove(f
"{name}_map_mean")
397 if not prop_config.do_weighted_mean:
398 self.outputs.remove(f
"{name}_map_weighted_mean")
399 if not prop_config.do_sum:
400 self.outputs.remove(f
"{name}_map_sum")
403 class HealSparsePropertyMapConfig(pipeBase.PipelineTaskConfig,
404 pipelineConnections=HealSparsePropertyMapConnections):
405 """Configuration parameters for HealSparsePropertyMapTask"""
406 property_maps = BasePropertyMap.registry.makeField(
408 default=[
"exposure_time",
419 doc=
"Property map computation objects",
422 def setDefaults(self):
423 self.property_maps[
"exposure_time"].do_sum =
True
424 self.property_maps[
"psf_size"].do_weighted_mean =
True
425 self.property_maps[
"psf_e1"].do_weighted_mean =
True
426 self.property_maps[
"psf_e2"].do_weighted_mean =
True
427 self.property_maps[
"psf_maglim"].do_weighted_mean =
True
428 self.property_maps[
"sky_noise"].do_weighted_mean =
True
429 self.property_maps[
"sky_background"].do_weighted_mean =
True
430 self.property_maps[
"dcr_dra"].do_weighted_mean =
True
431 self.property_maps[
"dcr_ddec"].do_weighted_mean =
True
432 self.property_maps[
"dcr_e1"].do_weighted_mean =
True
433 self.property_maps[
"dcr_e2"].do_weighted_mean =
True
436 class HealSparsePropertyMapTask(pipeBase.PipelineTask):
437 """Task to compute Healsparse property maps."""
438 ConfigClass = HealSparsePropertyMapConfig
439 _DefaultName =
"healSparsePropertyMapTask"
441 def __init__(self, **kwargs):
442 super().__init__(**kwargs)
444 for name, config, PropertyMapClass
in self.config.property_maps.apply():
445 self.property_maps[name] = PropertyMapClass(config, name)
448 def runQuantum(self, butlerQC, inputRefs, outputRefs):
449 inputs = butlerQC.get(inputRefs)
451 sky_map = inputs.pop(
"sky_map")
453 tract = butlerQC.quantum.dataId[
"tract"]
454 band = butlerQC.quantum.dataId[
"band"]
456 input_map_dict = {ref.dataId[
"patch"]: ref
for ref
in inputs[
"input_maps"]}
457 coadd_dict = {ref.dataId[
"patch"]: ref
for ref
in inputs[
"coadd_exposures"]}
459 visit_summary_dict = {ref.dataId[
"visit"]: ref.get()
460 for ref
in inputs[
"visit_summaries"]}
462 self.run(sky_map, tract, band, coadd_dict, input_map_dict, visit_summary_dict)
465 for name, property_map
in self.property_maps.items():
466 if property_map.config.do_min:
467 butlerQC.put(property_map.min_map,
468 getattr(outputRefs, f
"{name}_map_min"))
469 if property_map.config.do_max:
470 butlerQC.put(property_map.max_map,
471 getattr(outputRefs, f
"{name}_map_max"))
472 if property_map.config.do_mean:
473 butlerQC.put(property_map.mean_map,
474 getattr(outputRefs, f
"{name}_map_mean"))
475 if property_map.config.do_weighted_mean:
476 butlerQC.put(property_map.weighted_mean_map,
477 getattr(outputRefs, f
"{name}_map_weighted_mean"))
478 if property_map.config.do_sum:
479 butlerQC.put(property_map.sum_map,
480 getattr(outputRefs, f
"{name}_map_sum"))
482 def run(self, sky_map, tract, band, coadd_dict, input_map_dict, visit_summary_dict):
483 """Run the healsparse property task.
487 sky_map : Sky map object
491 Band name for logging.
492 coadd_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
493 Dictionary of coadd exposure references. Keys are patch numbers.
494 input_map_dict : `dict` [`int`: `lsst.daf.butler.DeferredDatasetHandle`]
495 Dictionary of input map references. Keys are patch numbers.
496 visit_summary_dict : `dict` [`int`: `lsst.afw.table.ExposureCatalog`]
497 Dictionary of visit summary tables. Keys are visit numbers.
501 RepeatableQuantumError
502 If visit_summary_dict is missing any visits or detectors found in an
503 input map. This leads to an inconsistency between what is in the coadd
504 (via the input map) and the visit summary tables which contain data
507 tract_info = sky_map[tract]
509 tract_maps_initialized =
False
511 for patch
in input_map_dict.keys():
512 self.log.info(
"Making maps for band %s, tract %d, patch %d.",
515 patch_info = tract_info[patch]
517 input_map = input_map_dict[patch].get()
518 coadd_photo_calib = coadd_dict[patch].get(component=
"photoCalib")
519 coadd_inputs = coadd_dict[patch].get(component=
"coaddInputs")
521 coadd_zeropoint = 2.5*np.log10(coadd_photo_calib.getInstFluxAtZeroMagnitude())
524 poly_vertices = patch_info.getInnerSkyPolygon(tract_info.getWcs()).getVertices()
525 patch_radec = self._vertices_to_radec(poly_vertices)
526 patch_poly = hsp.Polygon(ra=patch_radec[:, 0], dec=patch_radec[:, 1],
527 value=np.arange(input_map.wide_mask_maxbits))
528 patch_poly_map = patch_poly.get_map_like(input_map)
529 input_map = hsp.and_intersection([input_map, patch_poly_map])
531 if not tract_maps_initialized:
534 nside_coverage = self._compute_nside_coverage_tract(tract_info)
535 nside = input_map.nside_sparse
537 do_compute_approx_psf =
False
539 for property_map
in self.property_maps:
540 property_map.initialize_tract_maps(nside_coverage, nside)
541 if property_map.requires_psf:
542 do_compute_approx_psf =
True
544 tract_maps_initialized =
True
546 valid_pixels, vpix_ra, vpix_dec = input_map.valid_pixels_pos(return_pixels=
True)
549 if valid_pixels.size == 0:
553 for property_map
in self.property_maps:
554 property_map.initialize_values(valid_pixels.size)
555 property_map.zeropoint = coadd_zeropoint
558 total_weights = np.zeros(valid_pixels.size)
559 total_inputs = np.zeros(valid_pixels.size, dtype=np.int32)
561 for bit, ccd_row
in enumerate(coadd_inputs.ccds):
563 inmap, = np.where(input_map.check_bits_pix(valid_pixels, [bit]))
570 visit = ccd_row[
"visit"]
571 detector_id = ccd_row[
"ccd"]
572 weight = ccd_row[
"weight"]
574 x, y = ccd_row.getWcs().skyToPixelArray(vpix_ra[inmap], vpix_dec[inmap], degrees=
True)
575 scalings = self._compute_calib_scale(ccd_row, x, y)
577 if do_compute_approx_psf:
582 total_weights[inmap] += weight
583 total_inputs[inmap] += 1
586 if visit
not in visit_summary_dict:
587 msg = f
"Visit {visit} not found in visit_summaries."
588 raise pipeBase.RepeatableQuantumError(msg)
589 row = visit_summary_dict[visit].find(detector_id)
591 msg = f
"Visit {visit} / detector_id {detector_id} not found in visit_summaries."
592 raise pipeBase.RepeatableQuantumError(msg)
595 for property_map
in self.property_maps:
596 property_map.accumulate_values(inmap,
605 for property_map
in self.property_maps:
606 property_map.finalize_mean_values(total_weights, total_inputs)
607 property_map.set_map_values(valid_pixels)
609 def _compute_calib_scale(self, ccd_row, x, y):
610 """Compute calibration scaling values.
614 ccd_row : `lsst.afw.table.ExposureRecord`
615 Exposure metadata for a given detector exposure.
617 Array of x positions.
619 Array of y positions.
623 calib_scale : `np.ndarray`
624 Array of calibration scale values.
626 photo_calib = ccd_row.getPhotoCalib()
627 bf = photo_calib.computeScaledCalibration()
628 if bf.getBBox() == ccd_row.getBBox():
630 calib_scale = photo_calib.getCalibrationMean()*bf.evaluate(x, y)
633 calib_scale = photo_calib.getCalibrationMean()
637 def _vertices_to_radec(self, vertices):
638 """Convert polygon vertices to ra/dec.
642 vertices : `list` [ `lsst.sphgeom.UnitVector3d` ]
643 Vertices for bounding polygon.
647 radec : `numpy.ndarray`
648 Nx2 array of ra/dec positions (in degrees) associated with vertices.
651 radec = np.array([(x.getLon().asDegrees(), x.getLat().asDegrees())
for
655 def _compute_nside_coverage_tract(self, tract_info):
656 """Compute the optimal coverage nside for a tract.
660 tract_info : `lsst.skymap.tractInfo.ExplicitTractInfo`
661 Tract information object.
665 nside_coverage : `int`
666 Optimal coverage nside for a tract map.
668 num_patches = tract_info.getNumPatches()
671 patch_info = tract_info.getPatchInfo(0)
672 vertices = patch_info.getInnerSkyPolygon(tract_info.getWcs()).getVertices()
673 radec = self._vertices_to_radec(vertices)
674 delta_ra = np.max(radec[:, 0]) - np.min(radec[:, 0])
675 delta_dec = np.max(radec[:, 1]) - np.min(radec[:, 1])
676 patch_area = delta_ra*delta_dec*np.cos(np.deg2rad(np.mean(radec[:, 1])))
678 tract_area = num_patches[0]*num_patches[1]*patch_area
680 nside_coverage_tract = 32
681 while hp.nside2pixarea(nside_coverage_tract, degrees=
True) > tract_area:
682 nside_coverage_tract = 2*nside_coverage_tract
684 nside_coverage_tract = int(np.clip(nside_coverage_tract/2, 32,
None))
686 return nside_coverage_tract
def compute_approx_psf_size_and_shape(ccd_row, ra, dec, nx=20, ny=20, orderx=2, ordery=2)