22"""Tasks for making and manipulating HIPS images."""
24__all__ = [
"HighResolutionHipsTask",
"HighResolutionHipsConfig",
"HighResolutionHipsConnections",
25 "GenerateHipsTask",
"GenerateHipsConfig",
"GenerateColorHipsTask",
"GenerateColorHipsConfig"]
27from collections
import defaultdict
35from datetime
import datetime
37import healsparse
as hsp
38from astropy.io
import fits
39from astropy.visualization.lupton_rgb
import AsinhMapping
43from lsst.utils.timer
import timeMethod
44from lsst.daf.butler
import Butler, DatasetRef, Quantum, SkyPixDimension, UnresolvedRefWarning
52from lsst.resources
import ResourcePath
56 dimensions=(
"healpix9",
"band"),
57 defaultTemplates={
"coaddName":
"deep"}):
58 coadd_exposure_handles = pipeBase.connectionTypes.Input(
59 doc=
"Coadded exposures to convert to HIPS format.",
60 name=
"{coaddName}Coadd_calexp",
61 storageClass=
"ExposureF",
62 dimensions=(
"tract",
"patch",
"skymap",
"band"),
66 hips_exposures = pipeBase.connectionTypes.Output(
67 doc=
"HiPS-compatible HPX image.",
68 name=
"{coaddName}Coadd_hpx",
69 storageClass=
"ExposureF",
70 dimensions=(
"healpix11",
"band"),
74 def __init__(self, *, config=None):
75 super().__init__(config=config)
78 for dim
in self.dimensions:
80 if quantum_order
is not None:
81 raise ValueError(
"Must not specify more than one quantum healpix dimension.")
82 quantum_order = int(dim.split(
"healpix")[1])
83 if quantum_order
is None:
84 raise ValueError(
"Must specify a healpix dimension in quantum dimensions.")
86 if quantum_order > config.hips_order:
87 raise ValueError(
"Quantum healpix dimension order must not be greater than hips_order")
90 for dim
in self.hips_exposures.dimensions:
93 raise ValueError(
"Must not specify more than one healpix dimension.")
94 order = int(dim.split(
"healpix")[1])
96 raise ValueError(
"Must specify a healpix dimension in hips_exposure dimensions.")
98 if order != config.hips_order:
99 raise ValueError(
"healpix dimension order must match config.hips_order.")
102class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig,
103 pipelineConnections=HighResolutionHipsConnections):
104 """Configuration parameters for HighResolutionHipsTask.
108 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to
109 2**hips_order. Each cell
is 'shift_order' orders deeper than the HEALPix
110 cell,
with 2**shift_order x 2**shift_order sub-pixels on a side, which
111 defines the target resolution of the HiPS image. The IVOA recommends
112 shift_order=9,
for 2**9=512 pixels on a side.
115 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf
116 shows the relationship between hips_order, number of tiles (full
117 sky coverage), cell size,
and sub-pixel size/image resolution (
with
118 the default shift_order=9):
119 +------------+-----------------+--------------+------------------+
120 | hips_order | Number of Tiles | Cell Size | Image Resolution |
121 +============+=================+==============+==================+
122 | 0 | 12 | 58.63 deg | 6.871 arcmin |
123 | 1 | 48 | 29.32 deg | 3.435 arcmin |
124 | 2 | 192 | 14.66 deg | 1.718 arcmin |
125 | 3 | 768 | 7.329 deg | 51.53 arcsec |
126 | 4 | 3072 | 3.665 deg | 25.77 arcsec |
127 | 5 | 12288 | 1.832 deg | 12.88 arcsec |
128 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec |
129 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec |
130 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec |
131 | 9 | 3145728 | 6.871 arcmin | 805.2mas |
132 | 10 | 12582912 | 3.435 arcmin | 402.6mas |
133 | 11 | 50331648 | 1.718 arcmin | 201.3mas |
134 | 12 | 201326592 | 51.53 arcsec | 100.6mas |
135 | 13 | 805306368 | 25.77 arcsec | 50.32mas |
136 +------------+-----------------+--------------+------------------+
138 hips_order = pexConfig.Field(
139 doc="HIPS image order.",
143 shift_order = pexConfig.Field(
144 doc=
"HIPS shift order (such that each tile is 2**shift_order pixels on a side)",
148 warp = pexConfig.ConfigField(
149 dtype=afwMath.Warper.ConfigClass,
150 doc=
"Warper configuration",
153 def setDefaults(self):
154 self.warp.warpingKernelName =
"lanczos5"
157class HipsTaskNameDescriptor:
158 """Descriptor used create a DefaultName that matches the order of
159 the defined dimensions in the connections
class.
164 The prefix of the Default name, to which the order will be
167 def __init__(self, prefix):
169 self._defaultName = f
"{prefix}{{}}"
172 def __get__(self, obj, klass=None):
175 "HipsTaskDescriptor was used in an unexpected context"
177 if self._order
is None:
178 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions
179 for dim
in klassDimensions:
180 if (match := re.match(
r"^healpix(\d*)$", dim))
is not None:
181 self._order = int(match.group(1))
185 "Could not find healpix dimension in connections class"
187 return self._defaultName.format(self._order)
190class HighResolutionHipsTask(pipeBase.PipelineTask):
191 """Task for making high resolution HiPS images."""
192 ConfigClass = HighResolutionHipsConfig
193 _DefaultName = HipsTaskNameDescriptor(
"highResolutionHips")
195 def __init__(self, **kwargs):
196 super().__init__(**kwargs)
197 self.warper = afwMath.Warper.fromConfig(self.config.warp)
200 def runQuantum(self, butlerQC, inputRefs, outputRefs):
201 inputs = butlerQC.get(inputRefs)
203 healpix_dim = f
"healpix{self.config.hips_order}"
205 pixels = [hips_exposure.dataId[healpix_dim]
206 for hips_exposure
in outputRefs.hips_exposures]
208 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs[
"coadd_exposure_handles"])
210 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]:
211 hips_exposure_ref
for hips_exposure_ref
in outputRefs.hips_exposures}
212 for pixel, hips_exposure
in outputs.hips_exposures.items():
213 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel])
215 def run(self, pixels, coadd_exposure_handles):
216 """Run the HighResolutionHipsTask.
220 pixels : `Iterable` [ `int` ]
221 Iterable of healpix pixels (nest ordering) to warp to.
222 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
223 Handles for the coadd exposures.
227 outputs : `lsst.pipe.base.Struct`
228 ``hips_exposures``
is a dict
with pixel (key)
and hips_exposure (value)
230 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order)
232 npix = 2**self.config.shift_order
242 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order)
243 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx)
244 exp_hpx_dict[pixel] = exp_hpx
245 warp_dict[pixel] = []
250 for handle
in coadd_exposure_handles:
251 coadd_exp = handle.get()
255 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx)
257 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs())
258 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
263 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict())
264 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter())
265 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib())
267 if warped.getBBox().getArea() == 0
or not np.any(np.isfinite(warped.image.array)):
270 "No overlap between output HPX %d and input exposure %s",
276 exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
277 warp_dict[pixel].append(exp.maskedImage)
281 stats_flags = afwMath.stringToStatisticsProperty(
"MEAN")
282 stats_ctrl = afwMath.StatisticsControl()
283 stats_ctrl.setNanSafe(
True)
284 stats_ctrl.setWeighted(
True)
285 stats_ctrl.setCalcErrorFromInputVariance(
True)
291 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
293 if not warp_dict[pixel]:
295 self.log.debug(
"No data in HPX pixel %d", pixel)
298 exp_hpx_dict.pop(pixel)
301 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack(
305 [1.0]*len(warp_dict[pixel]),
310 return pipeBase.Struct(hips_exposures=exp_hpx_dict)
313 def build_quantum_graph_cli(cls, argv):
314 """A command-line interface entry point to `build_quantum_graph`.
315 This method provides the implementation for the
316 ``build-high-resolution-hips-qg`` script.
320 argv : `Sequence` [ `str` ]
321 Command-line arguments (e.g. ``sys.argv[1:]``).
323 parser = cls._make_cli_parser()
325 args = parser.parse_args(argv)
327 if args.subparser_name
is None:
331 pipeline = pipeBase.Pipeline.from_uri(args.pipeline)
332 expanded_pipeline = list(pipeline.toExpandedPipeline())
334 if len(expanded_pipeline) != 1:
335 raise RuntimeError(f
"Pipeline file {args.pipeline} may only contain one task.")
337 (task_def,) = expanded_pipeline
339 butler = Butler(args.butler_config, collections=args.input)
341 if args.subparser_name ==
"segment":
343 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order)
344 dataset = task_def.connections.coadd_exposure_handles.name
345 data_ids = set(butler.registry.queryDataIds(
"tract", datasets=dataset).expanded())
347 for data_id
in data_ids:
348 region = data_id.region
349 pixel_range = hpix_pixelization.envelope(region)
350 for r
in pixel_range.ranges():
351 region_pixels.extend(range(r[0], r[1]))
352 indices = np.unique(region_pixels)
354 print(f
"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:")
355 for pixel
in indices:
358 elif args.subparser_name ==
"build":
361 build_ranges =
RangeSet(sorted(args.pixels))
363 qg = cls.build_quantum_graph(
366 args.hpix_build_order,
369 collections=args.input
371 qg.saveUri(args.save_qgraph)
374 def _make_cli_parser(cls):
375 """Make the command-line parser.
379 parser : `argparse.ArgumentParser`
381 parser = argparse.ArgumentParser(
383 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets."
386 subparsers = parser.add_subparsers(help=
"sub-command help", dest=
"subparser_name")
388 parser_segment = subparsers.add_parser(
"segment",
389 help=
"Determine survey segments for workflow.")
390 parser_build = subparsers.add_parser(
"build",
391 help=
"Build quantum graph for HighResolutionHipsTask")
393 for sub
in [parser_segment, parser_build]:
399 help=
"Path to data repository or butler configuration.",
406 help=
"Pipeline file, limited to one task.",
414 help=
"Input collection(s) to search for coadd exposures.",
419 "--hpix_build_order",
422 help=
"HEALPix order to segment sky for building quantum graph files.",
429 help=
"Data ID expression used when querying for input coadd datasets.",
432 parser_build.add_argument(
436 help=
"Output filename for QuantumGraph.",
439 parser_build.add_argument(
444 help=
"Pixels at --hpix_build_order to generate quantum graph.",
451 def build_quantum_graph(
460 """Generate a `QuantumGraph` for running just this task.
462 This is a temporary workaround
for incomplete butler query support
for
467 task_def : `lsst.pipe.base.TaskDef`
469 registry : `lsst.daf.butler.Registry`
470 Client
for the butler database. May be read-only.
471 constraint_order : `int`
472 HEALPix order used to contrain which quanta are generated, via
473 ``constraint_indices``. This should be a coarser grid (smaller
474 order) than the order used
for the task
's quantum and output data
475 IDs, and ideally something between the spatial scale of a patch
or
476 the data repository
's "common skypix" system (usually ``htm7``).
478 RangeSet which describes constraint pixels (HEALPix NEST, with order
479 constraint_order) to constrain generated quanta.
480 where : `str`, optional
481 A boolean `str` expression of the form accepted by
482 `Registry.queryDatasets` to constrain input datasets. This may
483 contain a constraint on tracts, patches,
or bands, but
not HEALPix
484 indices. Constraints on tracts
and patches should usually be
485 unnecessary, however - existing coadds that overlap the given
486 HEALpix indices will be selected without such a constraint,
and
487 providing one may reject some that should normally be included.
488 collections : `str`
or `Iterable` [ `str` ], optional
489 Collection
or collections to search
for input datasets,
in order.
490 If
not provided, ``registry.defaults.collections`` will be
493 config = task_def.config
495 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry)
498 (input_dataset_type,) = dataset_types.inputs
503 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name]
504 incidental_output_dataset_types = dataset_types.outputs.copy()
505 incidental_output_dataset_types.remove(output_dataset_type)
506 (hpx_output_dimension,) = (d
for d
in output_dataset_type.dimensions
507 if isinstance(d, SkyPixDimension))
509 constraint_hpx_pixelization = registry.dimensions[f
"healpix{constraint_order}"].pixelization
510 common_skypix_name = registry.dimensions.commonSkyPix.name
511 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization
514 task_dimensions = registry.dimensions.extract(task_def.connections.dimensions)
515 (hpx_dimension,) = (d
for d
in task_dimensions
if d.name !=
"band")
516 hpx_pixelization = hpx_dimension.pixelization
518 if hpx_pixelization.level < constraint_order:
519 raise ValueError(f
"Quantum order {hpx_pixelization.level} must be < {constraint_order}")
520 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order))
525 for begin, end
in constraint_ranges:
526 for hpx_index
in range(begin, end):
527 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index)
528 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region)
532 for simp
in range(1, 10):
533 if len(common_skypix_ranges) < 100:
535 common_skypix_ranges.simplify(simp)
542 for n, (begin, end)
in enumerate(common_skypix_ranges):
545 where_terms.append(f
"{common_skypix_name} = cpx{n}")
546 bind[f
"cpx{n}"] = begin
548 where_terms.append(f
"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)")
549 bind[f
"cpx{n}a"] = begin
550 bind[f
"cpx{n}b"] = stop
552 where =
" OR ".join(where_terms)
554 where = f
"({where}) AND ({' OR '.join(where_terms)})"
558 input_refs = registry.queryDatasets(
562 collections=collections,
565 inputs_by_patch = defaultdict(set)
566 patch_dimensions = registry.dimensions.extract([
"patch"])
567 for input_ref
in input_refs:
568 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref)
569 if not inputs_by_patch:
570 message_body =
"\n".join(input_refs.explain_no_results())
571 raise RuntimeError(f
"No inputs found:\n{message_body}")
576 inputs_by_hpx = defaultdict(set)
577 for patch_data_id, input_refs_for_patch
in inputs_by_patch.items():
578 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region)
579 for begin, end
in patch_hpx_ranges & hpx_ranges:
580 for hpx_index
in range(begin, end):
581 inputs_by_hpx[hpx_index].update(input_refs_for_patch)
584 for hpx_index, input_refs_for_hpx_index
in inputs_by_hpx.items():
586 input_refs_by_band = defaultdict(list)
587 for input_ref
in input_refs_for_hpx_index:
588 input_refs_by_band[input_ref.dataId[
"band"]].append(input_ref)
590 for band, input_refs_for_band
in input_refs_by_band.items():
591 data_id = registry.expandDataId({hpx_dimension: hpx_index,
"band": band})
593 hpx_pixel_ranges =
RangeSet(hpx_index)
594 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level))
596 for begin, end
in hpx_output_ranges:
597 for hpx_output_index
in range(begin, end):
598 output_data_ids.append(
599 registry.expandDataId({hpx_output_dimension: hpx_output_index,
"band": band})
601 with warnings.catch_warnings():
602 warnings.simplefilter(
"ignore", category=UnresolvedRefWarning)
603 outputs = {dt: [DatasetRef(dt, data_id)]
for dt
in incidental_output_dataset_types}
604 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id)
605 for data_id
in output_data_ids]
608 taskName=task_def.taskName,
609 taskClass=task_def.taskClass,
612 inputs={input_dataset_type: input_refs_for_band},
618 raise RuntimeError(
"Given constraints yielded empty quantum graph.")
620 return pipeBase.QuantumGraph(quanta={task_def: quanta})
623class HipsPropertiesSpectralTerm(pexConfig.Config):
624 lambda_min = pexConfig.Field(
625 doc=
"Minimum wavelength (nm)",
628 lambda_max = pexConfig.Field(
629 doc=
"Maximum wavelength (nm)",
634class HipsPropertiesConfig(pexConfig.Config):
635 """Configuration parameters for writing a HiPS properties file."""
636 creator_did_template = pexConfig.Field(
637 doc=(
"Unique identifier of the HiPS - Format: IVOID. "
638 "Use ``{band}`` to substitute the band name."),
642 obs_collection = pexConfig.Field(
643 doc=
"Short name of original data set - Format: one word",
647 obs_description_template = pexConfig.Field(
648 doc=(
"Data set description - Format: free text, longer free text "
649 "description of the dataset. Use ``{band}`` to substitute "
653 prov_progenitor = pexConfig.ListField(
654 doc=
"Provenance of the original data - Format: free text",
658 obs_title_template = pexConfig.Field(
659 doc=(
"Data set title format: free text, but should be short. "
660 "Use ``{band}`` to substitute the band name."),
664 spectral_ranges = pexConfig.ConfigDictField(
665 doc=(
"Mapping from band to lambda_min, lamba_max (nm). May be approximate."),
667 itemtype=HipsPropertiesSpectralTerm,
670 initial_ra = pexConfig.Field(
671 doc=
"Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.",
675 initial_dec = pexConfig.Field(
676 doc=
"Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.",
680 initial_fov = pexConfig.Field(
681 doc=
"Initial field-of-view (deg). If not set will use ~1 healpix tile.",
685 obs_ack = pexConfig.Field(
686 doc=
"Observation acknowledgements (free text).",
690 t_min = pexConfig.Field(
691 doc=
"Time (MJD) of earliest observation included in HiPS",
695 t_max = pexConfig.Field(
696 doc=
"Time (MJD) of latest observation included in HiPS",
704 if self.obs_collection
is not None:
705 if re.search(
r"\s", self.obs_collection):
706 raise ValueError(
"obs_collection cannot contain any space characters.")
708 def setDefaults(self):
711 u_term = HipsPropertiesSpectralTerm()
712 u_term.lambda_min = 330.
713 u_term.lambda_max = 400.
714 self.spectral_ranges[
"u"] = u_term
715 g_term = HipsPropertiesSpectralTerm()
716 g_term.lambda_min = 402.
717 g_term.lambda_max = 552.
718 self.spectral_ranges[
"g"] = g_term
719 r_term = HipsPropertiesSpectralTerm()
720 r_term.lambda_min = 552.
721 r_term.lambda_max = 691.
722 self.spectral_ranges[
"r"] = r_term
723 i_term = HipsPropertiesSpectralTerm()
724 i_term.lambda_min = 691.
725 i_term.lambda_max = 818.
726 self.spectral_ranges[
"i"] = i_term
727 z_term = HipsPropertiesSpectralTerm()
728 z_term.lambda_min = 818.
729 z_term.lambda_max = 922.
730 self.spectral_ranges[
"z"] = z_term
731 y_term = HipsPropertiesSpectralTerm()
732 y_term.lambda_min = 970.
733 y_term.lambda_max = 1060.
734 self.spectral_ranges[
"y"] = y_term
737class GenerateHipsConnections(pipeBase.PipelineTaskConnections,
738 dimensions=(
"instrument",
"band"),
739 defaultTemplates={
"coaddName":
"deep"}):
740 hips_exposure_handles = pipeBase.connectionTypes.Input(
741 doc=
"HiPS-compatible HPX images.",
742 name=
"{coaddName}Coadd_hpx",
743 storageClass=
"ExposureF",
744 dimensions=(
"healpix11",
"band"),
750class GenerateHipsConfig(pipeBase.PipelineTaskConfig,
751 pipelineConnections=GenerateHipsConnections):
752 """Configuration parameters for GenerateHipsTask."""
757 hips_base_uri = pexConfig.Field(
758 doc=
"URI to HiPS base for output.",
762 min_order = pexConfig.Field(
763 doc=
"Minimum healpix order for HiPS tree.",
767 properties = pexConfig.ConfigField(
768 dtype=HipsPropertiesConfig,
769 doc=
"Configuration for properties file.",
771 allsky_tilesize = pexConfig.Field(
773 doc=
"Allsky.png tile size. Must be power of 2.",
776 png_gray_asinh_minimum = pexConfig.Field(
777 doc=
"AsinhMapping intensity to be mapped to black for grayscale png scaling.",
781 png_gray_asinh_stretch = pexConfig.Field(
782 doc=
"AsinhMapping linear stretch for grayscale png scaling.",
786 png_gray_asinh_softening = pexConfig.Field(
787 doc=
"AsinhMapping softening parameter (Q) for grayscale png scaling.",
793class GenerateHipsTask(pipeBase.PipelineTask):
794 """Task for making a HiPS tree with FITS and grayscale PNGs."""
795 ConfigClass = GenerateHipsConfig
796 _DefaultName =
"generateHips"
800 def runQuantum(self, butlerQC, inputRefs, outputRefs):
801 inputs = butlerQC.get(inputRefs)
803 dims = inputRefs.hips_exposure_handles[0].dataId.names
807 order = int(dim.split(
"healpix")[1])
811 raise RuntimeError(
"Could not determine healpix order for input exposures.")
813 hips_exposure_handle_dict = {
814 (hips_exposure_handle.dataId[healpix_dim],
815 hips_exposure_handle.dataId[
"band"]): hips_exposure_handle
816 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]
819 data_bands = {hips_exposure_handle.dataId[
"band"]
820 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]}
821 bands = self._check_data_bands(data_bands)
826 hips_exposure_handle_dict=hips_exposure_handle_dict,
827 do_color=self.color_task,
830 def _check_data_bands(self, data_bands):
831 """Check that the data has only a single band.
835 data_bands : `set` [`str`]
836 Bands from the input data.
840 bands : `list` [`str`]
841 List of single band to process.
845 RuntimeError
if there
is not exactly one band.
847 if len(data_bands) != 1:
848 raise RuntimeError(
"GenerateHipsTask can only use data from a single band.")
850 return list(data_bands)
853 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False):
854 """Run the GenerateHipsTask.
858 bands : `list [ `str` ]
859 List of bands to be processed (or single band).
861 HEALPix order of the maximum (native) HPX exposures.
862 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`}
863 Dict of handles
for the HiPS high-resolution exposures.
864 Key
is (pixel number, ``band``).
865 do_color : `bool`, optional
866 Do color pngs instead of per-band grayscale.
868 min_order = self.config.min_order
871 png_grayscale_mapping = AsinhMapping(
872 self.config.png_gray_asinh_minimum,
873 self.config.png_gray_asinh_stretch,
874 Q=self.config.png_gray_asinh_softening,
877 png_color_mapping = AsinhMapping(
878 self.config.png_color_asinh_minimum,
879 self.config.png_color_asinh_stretch,
880 Q=self.config.png_color_asinh_softening,
883 bcb = self.config.blue_channel_band
884 gcb = self.config.green_channel_band
885 rcb = self.config.red_channel_band
886 colorstr = f
"{bcb}{gcb}{rcb}"
889 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=
True)
893 pixels = np.unique(np.array([pixel
for pixel, _
in hips_exposure_handle_dict.keys()]))
896 pixels = np.append(pixels, [0])
900 pixels_shifted[max_order] = pixels
901 for order
in range(max_order - 1, min_order - 1, -1):
902 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2)
905 for order
in range(min_order, max_order + 1):
906 pixels_shifted[order][-1] = -1
909 exp0 = list(hips_exposure_handle_dict.values())[0].get()
910 bbox = exp0.getBBox()
911 npix = bbox.getWidth()
912 shift_order = int(np.round(np.log2(npix)))
919 for order
in range(min_order, max_order + 1):
920 exp = exp0.Factory(bbox=bbox)
921 exp.image.array[:, :] = np.nan
922 exposures[(band, order)] = exp
925 for pixel_counter, pixel
in enumerate(pixels[:-1]):
926 self.log.debug(
"Working on high resolution pixel %d", pixel)
932 if (pixel, band)
in hips_exposure_handle_dict:
933 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get()
939 for order
in range(max_order, min_order - 1, -1):
940 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]:
948 self._write_hips_image(
949 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
951 pixels_shifted[order][pixel_counter],
952 exposures[(band, order)].image,
953 png_grayscale_mapping,
954 shift_order=shift_order,
958 self._write_hips_color_png(
959 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
961 pixels_shifted[order][pixel_counter],
962 exposures[(self.config.red_channel_band, order)].image,
963 exposures[(self.config.green_channel_band, order)].image,
964 exposures[(self.config.blue_channel_band, order)].image,
968 log_level = self.log.INFO
if order == (max_order - 3)
else self.log.DEBUG
971 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)",
974 pixels_shifted[order][pixel_counter],
980 if order == min_order:
982 exposures[(band, order)].image.array[:, :] = np.nan
987 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2)
988 with warnings.catch_warnings():
989 warnings.simplefilter(
"ignore")
990 binned_image_arr = np.nanmean(arr, axis=(1, 3))
994 sub_index = (pixels_shifted[order][pixel_counter]
995 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2))
998 exp = exposures[(band, order - 1)]
1002 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr
1003 elif sub_index == 1:
1004 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr
1005 elif sub_index == 2:
1006 exp.image.array[npix//2:, npix//2:] = binned_image_arr
1007 elif sub_index == 3:
1008 exp.image.array[0: npix//2, npix//2:] = binned_image_arr
1011 raise ValueError(
"Illegal pixel sub index")
1014 if order < max_order:
1015 exposures[(band, order)].image.array[:, :] = np.nan
1020 band_pixels = np.array([pixel
1021 for pixel, band_
in hips_exposure_handle_dict.keys()
1023 band_pixels = np.sort(band_pixels)
1025 self._write_properties_and_moc(
1026 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1034 self._write_allsky_file(
1035 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1039 self._write_properties_and_moc(
1040 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1048 self._write_allsky_file(
1049 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1053 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9):
1054 """Write a HiPS image.
1058 hips_base_path : `lsst.resources.ResourcePath`
1059 Resource path to the base of the HiPS directory tree.
1061 HEALPix order of the HiPS image to write.
1063 HEALPix pixel of the HiPS image.
1066 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1067 Mapping to convert image to scaled png.
1068 shift_order : `int`, optional
1076 dir_number = self._get_dir_number(pixel)
1077 hips_dir = hips_base_path.join(
1085 wcs = makeHpxWcs(order, pixel, shift_order=shift_order)
1087 uri = hips_dir.join(f
"Npix{pixel}.fits")
1089 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1090 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata())
1092 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1096 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8)
1097 vals[~np.isfinite(image.array) | (image.array < 0)] = 0
1098 im = Image.fromarray(vals[::-1, :],
"L")
1100 uri = hips_dir.join(f
"Npix{pixel}.png")
1102 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1103 im.save(temporary_uri.ospath)
1105 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1107 def _write_hips_color_png(
1117 """Write a color png HiPS image.
1121 hips_base_path : `lsst.resources.ResourcePath`
1122 Resource path to the base of the HiPS directory tree.
1124 HEALPix order of the HiPS image to write.
1126 HEALPix pixel of the HiPS image.
1128 Input for red channel of output png.
1130 Input
for green channel of output png.
1132 Input
for blue channel of output png.
1133 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1134 Mapping to convert image to scaled png.
1141 dir_number = self._get_dir_number(pixel)
1142 hips_dir = hips_base_path.join(
1151 arr_red = image_red.array.copy()
1152 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0]
1153 arr_green = image_green.array.copy()
1154 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1]
1155 arr_blue = image_blue.array.copy()
1156 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2]
1158 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue)
1160 im = Image.fromarray(image_array[::-1, :, :], mode=
"RGB")
1162 uri = hips_dir.join(f
"Npix{pixel}.png")
1164 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1165 im.save(temporary_uri.ospath)
1167 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1169 def _write_properties_and_moc(
1179 """Write HiPS properties file and MOC.
1183 hips_base_path : : `lsst.resources.ResourcePath`
1184 Resource path to the base of the HiPS directory tree.
1186 Maximum HEALPix order.
1187 pixels : `np.ndarray` (N,)
1188 Array of pixels used.
1190 Sample HPX exposure used for generating HiPS tiles.
1196 Is band multiband / color?
1198 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels)
1200 initial_ra = self.config.properties.initial_ra
1201 initial_dec = self.config.properties.initial_dec
1202 initial_fov = self.config.properties.initial_fov
1204 if initial_ra
is None or initial_dec
is None or initial_fov
is None:
1207 temp_pixels = pixels.copy()
1208 if temp_pixels.size % 2 == 0:
1209 temp_pixels = np.append(temp_pixels, [temp_pixels[0]])
1210 medpix = int(np.median(temp_pixels))
1211 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix)
1212 _initial_fov = hpg.nside_to_resolution(2**max_order, units=
'arcminutes')/60.
1214 if initial_ra
is None or initial_dec
is None:
1215 initial_ra = _initial_ra
1216 initial_dec = _initial_dec
1217 if initial_fov
is None:
1218 initial_fov = _initial_fov
1220 self._write_hips_properties_file(
1222 self.config.properties,
1235 self._write_hips_moc_file(
1241 def _write_hips_properties_file(
1255 """Write HiPS properties file.
1259 hips_base_path : `lsst.resources.ResourcePath`
1260 ResourcePath at top of HiPS tree. File will be written
1261 to this path as ``properties``.
1262 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig`
1263 Configuration
for properties values.
1265 Name of band(s)
for HiPS tree.
1267 Is multiband / color?
1269 Sample HPX exposure used
for generating HiPS tiles.
1271 Maximum HEALPix order.
1275 Coverage area
in square degrees.
1276 initial_ra : `float`
1277 Initial HiPS RA position (degrees).
1278 initial_dec : `float`
1279 Initial HiPS Dec position (degrees).
1280 initial_fov : `float`
1281 Initial HiPS display size (degrees).
1287 def _write_property(fh, name, value):
1288 """Write a property name/value to a file handle.
1292 fh : file handle (blah)
1301 if re.search(
r"\s", name):
1302 raise ValueError(f
"``{name}`` cannot contain any space characters.")
1304 raise ValueError(f
"``{name}`` cannot contain an ``=``")
1306 fh.write(f
"{name:25}= {value}\n")
1308 if exposure.image.array.dtype == np.dtype(
"float32"):
1310 elif exposure.image.array.dtype == np.dtype(
"float64"):
1312 elif exposure.image.array.dtype == np.dtype(
"int32"):
1315 date_iso8601 = datetime.utcnow().isoformat(timespec=
"seconds") +
"Z"
1316 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units=
'degrees')
1318 uri = hips_base_path.join(
"properties")
1319 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1320 with open(temporary_uri.ospath,
"w")
as fh:
1324 properties_config.creator_did_template.format(band=band),
1326 if properties_config.obs_collection
is not None:
1327 _write_property(fh,
"obs_collection", properties_config.obs_collection)
1331 properties_config.obs_title_template.format(band=band),
1333 if properties_config.obs_description_template
is not None:
1337 properties_config.obs_description_template.format(band=band),
1339 if len(properties_config.prov_progenitor) > 0:
1340 for prov_progenitor
in properties_config.prov_progenitor:
1341 _write_property(fh,
"prov_progenitor", prov_progenitor)
1342 if properties_config.obs_ack
is not None:
1343 _write_property(fh,
"obs_ack", properties_config.obs_ack)
1344 _write_property(fh,
"obs_regime",
"Optical")
1345 _write_property(fh,
"data_pixel_bitpix",
str(bitpix))
1346 _write_property(fh,
"dataproduct_type",
"image")
1347 _write_property(fh,
"moc_sky_fraction",
str(area/41253.))
1348 _write_property(fh,
"data_ucd",
"phot.flux")
1349 _write_property(fh,
"hips_creation_date", date_iso8601)
1350 _write_property(fh,
"hips_builder",
"lsst.pipe.tasks.hips.GenerateHipsTask")
1351 _write_property(fh,
"hips_creator",
"Vera C. Rubin Observatory")
1352 _write_property(fh,
"hips_version",
"1.4")
1353 _write_property(fh,
"hips_release_date", date_iso8601)
1354 _write_property(fh,
"hips_frame",
"equatorial")
1355 _write_property(fh,
"hips_order",
str(max_order))
1356 _write_property(fh,
"hips_tile_width",
str(exposure.getBBox().getWidth()))
1357 _write_property(fh,
"hips_status",
"private master clonableOnce")
1359 _write_property(fh,
"hips_tile_format",
"png")
1360 _write_property(fh,
"dataproduct_subtype",
"color")
1362 _write_property(fh,
"hips_tile_format",
"png fits")
1363 _write_property(fh,
"hips_pixel_bitpix",
str(bitpix))
1364 _write_property(fh,
"hips_pixel_scale",
str(pixel_scale))
1365 _write_property(fh,
"hips_initial_ra",
str(initial_ra))
1366 _write_property(fh,
"hips_initial_dec",
str(initial_dec))
1367 _write_property(fh,
"hips_initial_fov",
str(initial_fov))
1369 if self.config.blue_channel_band
in properties_config.spectral_ranges:
1370 em_min = properties_config.spectral_ranges[
1371 self.config.blue_channel_band
1374 self.log.warning(
"blue band %s not in self.config.spectral_ranges.", band)
1376 if self.config.red_channel_band
in properties_config.spectral_ranges:
1377 em_max = properties_config.spectral_ranges[
1378 self.config.red_channel_band
1381 self.log.warning(
"red band %s not in self.config.spectral_ranges.", band)
1384 if band
in properties_config.spectral_ranges:
1385 em_min = properties_config.spectral_ranges[band].lambda_min/1e9
1386 em_max = properties_config.spectral_ranges[band].lambda_max/1e9
1388 self.log.warning(
"band %s not in self.config.spectral_ranges.", band)
1391 _write_property(fh,
"em_min",
str(em_min))
1392 _write_property(fh,
"em_max",
str(em_max))
1393 if properties_config.t_min
is not None:
1394 _write_property(fh,
"t_min", properties_config.t_min)
1395 if properties_config.t_max
is not None:
1396 _write_property(fh,
"t_max", properties_config.t_max)
1398 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1400 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1):
1401 """Write HiPS MOC file.
1405 hips_base_path : `lsst.resources.ResourcePath`
1406 ResourcePath to top of HiPS tree. File will be written as
1407 to this path
as ``Moc.fits``.
1409 Maximum HEALPix order.
1410 pixels : `np.ndarray`
1411 Array of pixels covered.
1412 min_uniq_order : `int`, optional
1413 Minimum HEALPix order
for looking
for fully covered pixels.
1421 uniq = 4*(4**max_order) + pixels
1424 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32)
1425 hspmap[pixels] = 1.0
1428 for uniq_order
in range(max_order - 1, min_uniq_order - 1, -1):
1429 hspmap = hspmap.degrade(2**uniq_order, reduction=
"sum")
1430 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order))
1432 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero()
1433 if covered.size == 0:
1437 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered]
1440 uniq = np.unique(uniq)
1443 tbl = np.zeros(uniq.size, dtype=[(
"UNIQ",
"i8")])
1446 order = np.log2(tbl[
"UNIQ"]//4).astype(np.int32)//2
1447 moc_order = np.max(order)
1449 hdu = fits.BinTableHDU(tbl)
1450 hdu.header[
"PIXTYPE"] =
"HEALPIX"
1451 hdu.header[
"ORDERING"] =
"NUNIQ"
1452 hdu.header[
"COORDSYS"] =
"C"
1453 hdu.header[
"MOCORDER"] = moc_order
1454 hdu.header[
"MOCTOOL"] =
"lsst.pipe.tasks.hips.GenerateHipsTask"
1456 uri = hips_base_path.join(
"Moc.fits")
1458 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1459 hdu.writeto(temporary_uri.ospath)
1461 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1463 def _write_allsky_file(self, hips_base_path, allsky_order):
1464 """Write an Allsky.png file.
1468 hips_base_path : `lsst.resources.ResourcePath`
1469 Resource path to the base of the HiPS directory tree.
1470 allsky_order : `int`
1471 HEALPix order of the minimum order to make allsky file.
1473 tile_size = self.config.allsky_tilesize
1474 n_tiles_per_side = int(np.sqrt(hpg.nside_to_npixel(hpg.order_to_nside(allsky_order))))
1478 allsky_order_uri = hips_base_path.join(f
"Norder{allsky_order}", forceDirectory=
True)
1479 pixel_regex = re.compile(
r"Npix([0-9]+)\.png$")
1481 ResourcePath.findFileResources(
1482 candidates=[allsky_order_uri],
1483 file_filter=pixel_regex,
1487 for png_uri
in png_uris:
1488 matches = re.match(pixel_regex, png_uri.basename())
1489 pix_num = int(matches.group(1))
1490 tile_image = Image.open(io.BytesIO(png_uri.read()))
1491 row = math.floor(pix_num//n_tiles_per_side)
1492 column = pix_num % n_tiles_per_side
1493 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size)
1494 tile_image_shrunk = tile_image.resize((tile_size, tile_size))
1496 if allsky_image
is None:
1497 allsky_image = Image.new(
1499 (n_tiles_per_side*tile_size, n_tiles_per_side*tile_size),
1501 allsky_image.paste(tile_image_shrunk, box)
1503 uri = allsky_order_uri.join(
"Allsky.png")
1505 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1506 allsky_image.save(temporary_uri.ospath)
1508 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1510 def _get_dir_number(self, pixel):
1511 """Compute the directory number from a pixel.
1516 HEALPix pixel number.
1521 HiPS directory number.
1523 return (pixel//10000)*10000
1526class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections,
1527 dimensions=(
"instrument", ),
1528 defaultTemplates={
"coaddName":
"deep"}):
1529 hips_exposure_handles = pipeBase.connectionTypes.Input(
1530 doc=
"HiPS-compatible HPX images.",
1531 name=
"{coaddName}Coadd_hpx",
1532 storageClass=
"ExposureF",
1533 dimensions=(
"healpix11",
"band"),
1539class GenerateColorHipsConfig(GenerateHipsConfig,
1540 pipelineConnections=GenerateColorHipsConnections):
1541 """Configuration parameters for GenerateColorHipsTask."""
1542 blue_channel_band = pexConfig.Field(
1543 doc=
"Band to use for blue channel of color pngs.",
1547 green_channel_band = pexConfig.Field(
1548 doc=
"Band to use for green channel of color pngs.",
1552 red_channel_band = pexConfig.Field(
1553 doc=
"Band to use for red channel of color pngs.",
1557 png_color_asinh_minimum = pexConfig.Field(
1558 doc=
"AsinhMapping intensity to be mapped to black for color png scaling.",
1562 png_color_asinh_stretch = pexConfig.Field(
1563 doc=
"AsinhMapping linear stretch for color png scaling.",
1567 png_color_asinh_softening = pexConfig.Field(
1568 doc=
"AsinhMapping softening parameter (Q) for color png scaling.",
1574class GenerateColorHipsTask(GenerateHipsTask):
1575 """Task for making a HiPS tree with color pngs."""
1576 ConfigClass = GenerateColorHipsConfig
1577 _DefaultName =
"generateColorHips"
1580 def _check_data_bands(self, data_bands):
1581 """Check the data for configured bands.
1583 Warn if any color bands are missing data.
1587 data_bands : `set` [`str`]
1588 Bands
from the input data.
1592 bands : `list` [`str`]
1593 List of bands
in bgr color order.
1595 if len(data_bands) == 0:
1596 raise RuntimeError(
"GenerateColorHipsTask must have data from at least one band.")
1598 if self.config.blue_channel_band
not in data_bands:
1600 "Color png blue_channel_band %s not in dataset.",
1601 self.config.blue_channel_band
1603 if self.config.green_channel_band
not in data_bands:
1605 "Color png green_channel_band %s not in dataset.",
1606 self.config.green_channel_band
1608 if self.config.red_channel_band
not in data_bands:
1610 "Color png red_channel_band %s not in dataset.",
1611 self.config.red_channel_band
1615 self.config.blue_channel_band,
1616 self.config.green_channel_band,
1617 self.config.red_channel_band,