22"""Tasks for making and manipulating HIPS images."""
24__all__ = [
"HighResolutionHipsTask",
"HighResolutionHipsConfig",
"HighResolutionHipsConnections",
25 "GenerateHipsTask",
"GenerateHipsConfig",
"GenerateColorHipsTask",
"GenerateColorHipsConfig"]
27from collections
import defaultdict
35from datetime
import datetime
37import healsparse
as hsp
38from astropy.io
import fits
39from astropy.visualization.lupton_rgb
import AsinhMapping
43from lsst.utils.timer
import timeMethod
44from lsst.daf.butler
import Butler, DataCoordinate, DatasetRef, Quantum
46import lsst.pipe.base
as pipeBase
52from lsst.resources
import ResourcePath
54from .healSparseMapping
import _is_power_of_two
58 dimensions=(
"healpix9",
"band"),
59 defaultTemplates={
"coaddName":
"deep"}):
60 coadd_exposure_handles = pipeBase.connectionTypes.Input(
61 doc=
"Coadded exposures to convert to HIPS format.",
62 name=
"{coaddName}Coadd_calexp",
63 storageClass=
"ExposureF",
64 dimensions=(
"tract",
"patch",
"skymap",
"band"),
68 hips_exposures = pipeBase.connectionTypes.Output(
69 doc=
"HiPS-compatible HPX image.",
70 name=
"{coaddName}Coadd_hpx",
71 storageClass=
"ExposureF",
72 dimensions=(
"healpix11",
"band"),
76 def __init__(self, *, config=None):
77 super().__init__(config=config)
80 for dim
in self.dimensions:
82 if quantum_order
is not None:
83 raise ValueError(
"Must not specify more than one quantum healpix dimension.")
84 quantum_order = int(dim.split(
"healpix")[1])
85 if quantum_order
is None:
86 raise ValueError(
"Must specify a healpix dimension in quantum dimensions.")
88 if quantum_order > config.hips_order:
89 raise ValueError(
"Quantum healpix dimension order must not be greater than hips_order")
92 for dim
in self.hips_exposures.dimensions:
95 raise ValueError(
"Must not specify more than one healpix dimension.")
96 order = int(dim.split(
"healpix")[1])
98 raise ValueError(
"Must specify a healpix dimension in hips_exposure dimensions.")
100 if order != config.hips_order:
101 raise ValueError(
"healpix dimension order must match config.hips_order.")
104class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=HighResolutionHipsConnections):
106 """Configuration parameters for HighResolutionHipsTask.
110 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to
111 2**hips_order. Each cell is 'shift_order' orders deeper than the HEALPix
112 cell, with 2**shift_order x 2**shift_order sub-pixels on a side, which
113 defines the target resolution of the HiPS image. The IVOA recommends
114 shift_order=9, for 2**9=512 pixels on a side.
117 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf
118 shows the relationship between hips_order, number of tiles (full
119 sky coverage), cell size, and sub-pixel size/image resolution (with
120 the default shift_order=9):
121 +------------+-----------------+--------------+------------------+
122 | hips_order | Number of Tiles | Cell Size | Image Resolution |
123 +============+=================+==============+==================+
124 | 0 | 12 | 58.63 deg | 6.871 arcmin |
125 | 1 | 48 | 29.32 deg | 3.435 arcmin |
126 | 2 | 192 | 14.66 deg | 1.718 arcmin |
127 | 3 | 768 | 7.329 deg | 51.53 arcsec |
128 | 4 | 3072 | 3.665 deg | 25.77 arcsec |
129 | 5 | 12288 | 1.832 deg | 12.88 arcsec |
130 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec |
131 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec |
132 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec |
133 | 9 | 3145728 | 6.871 arcmin | 805.2mas |
134 | 10 | 12582912 | 3.435 arcmin | 402.6mas |
135 | 11 | 50331648 | 1.718 arcmin | 201.3mas |
136 | 12 | 201326592 | 51.53 arcsec | 100.6mas |
137 | 13 | 805306368 | 25.77 arcsec | 50.32mas |
138 +------------+-----------------+--------------+------------------+
140 hips_order = pexConfig.Field(
141 doc=
"HIPS image order.",
145 shift_order = pexConfig.Field(
146 doc=
"HIPS shift order (such that each tile is 2**shift_order pixels on a side)",
150 warp = pexConfig.ConfigField(
151 dtype=afwMath.Warper.ConfigClass,
152 doc=
"Warper configuration",
155 def setDefaults(self):
156 self.warp.warpingKernelName =
"lanczos5"
159class HipsTaskNameDescriptor:
160 """Descriptor used create a DefaultName that matches the order of
161 the defined dimensions in the connections class.
166 The prefix of the Default name, to which the order will be
169 def __init__(self, prefix):
171 self._defaultName = f
"{prefix}{{}}"
174 def __get__(self, obj, klass=None):
177 "HipsTaskDescriptor was used in an unexpected context"
179 if self._order
is None:
180 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions
181 for dim
in klassDimensions:
182 if (match := re.match(
r"^healpix(\d*)$", dim))
is not None:
183 self._order = int(match.group(1))
187 "Could not find healpix dimension in connections class"
189 return self._defaultName.format(self._order)
192class HighResolutionHipsTask(pipeBase.PipelineTask):
193 """Task for making high resolution HiPS images."""
194 ConfigClass = HighResolutionHipsConfig
195 _DefaultName = HipsTaskNameDescriptor(
"highResolutionHips")
197 def __init__(self, **kwargs):
198 super().__init__(**kwargs)
199 self.warper = afwMath.Warper.fromConfig(self.config.warp)
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
203 inputs = butlerQC.get(inputRefs)
205 healpix_dim = f
"healpix{self.config.hips_order}"
207 pixels = [hips_exposure.dataId[healpix_dim]
208 for hips_exposure
in outputRefs.hips_exposures]
210 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs[
"coadd_exposure_handles"])
212 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]:
213 hips_exposure_ref
for hips_exposure_ref
in outputRefs.hips_exposures}
214 for pixel, hips_exposure
in outputs.hips_exposures.items():
215 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel])
217 def run(self, pixels, coadd_exposure_handles):
218 """Run the HighResolutionHipsTask.
222 pixels : `Iterable` [ `int` ]
223 Iterable of healpix pixels (nest ordering) to warp to.
224 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
225 Handles for the coadd exposures.
229 outputs : `lsst.pipe.base.Struct`
230 ``hips_exposures`` is a dict with pixel (key) and hips_exposure (value)
232 self.log.info(
"Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order)
234 npix = 2**self.config.shift_order
244 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order)
245 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx)
246 exp_hpx_dict[pixel] = exp_hpx
247 warp_dict[pixel] = []
252 for handle
in coadd_exposure_handles:
253 coadd_exp = handle.get()
257 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx)
259 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs())
260 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
265 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict())
266 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter())
267 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib())
269 if warped.getBBox().getArea() == 0
or not np.any(np.isfinite(warped.image.array)):
272 "No overlap between output HPX %d and input exposure %s",
278 exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
279 warp_dict[pixel].append(exp.maskedImage)
283 stats_flags = afwMath.stringToStatisticsProperty(
"MEAN")
284 stats_ctrl = afwMath.StatisticsControl()
285 stats_ctrl.setNanSafe(
True)
286 stats_ctrl.setWeighted(
True)
287 stats_ctrl.setCalcErrorFromInputVariance(
True)
293 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
295 if not warp_dict[pixel]:
297 self.log.debug(
"No data in HPX pixel %d", pixel)
300 exp_hpx_dict.pop(pixel)
303 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack(
307 [1.0]*len(warp_dict[pixel]),
312 return pipeBase.Struct(hips_exposures=exp_hpx_dict)
315 def build_quantum_graph_cli(cls, argv):
316 """A command-line interface entry point to `build_quantum_graph`.
317 This method provides the implementation for the
318 ``build-high-resolution-hips-qg`` script.
322 argv : `Sequence` [ `str` ]
323 Command-line arguments (e.g. ``sys.argv[1:]``).
325 parser = cls._make_cli_parser()
327 args = parser.parse_args(argv)
329 if args.subparser_name
is None:
333 pipeline = pipeBase.Pipeline.from_uri(args.pipeline)
334 expanded_pipeline = list(pipeline.toExpandedPipeline())
336 if len(expanded_pipeline) != 1:
337 raise RuntimeError(f
"Pipeline file {args.pipeline} may only contain one task.")
339 (task_def,) = expanded_pipeline
341 butler = Butler(args.butler_config, collections=args.input)
343 if args.subparser_name ==
"segment":
345 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order)
346 dataset = task_def.connections.coadd_exposure_handles.name
347 data_ids = set(butler.registry.queryDataIds(
"tract", datasets=dataset).expanded())
349 for data_id
in data_ids:
350 region = data_id.region
351 pixel_range = hpix_pixelization.envelope(region)
352 for r
in pixel_range.ranges():
353 region_pixels.extend(range(r[0], r[1]))
354 indices = np.unique(region_pixels)
356 print(f
"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:")
357 for pixel
in indices:
360 elif args.subparser_name ==
"build":
364 if args.output_run
is None:
365 if args.output
is None:
366 raise ValueError(
"At least one of --output or --output-run options is required.")
367 args.output_run =
"{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp())
369 build_ranges =
RangeSet(sorted(args.pixels))
374 "butler_argument": args.butler_config,
375 "output": args.output,
376 "output_run": args.output_run,
377 "data_query": args.where,
378 "time": f
"{datetime.now()}",
381 qg = cls.build_quantum_graph(
384 args.hpix_build_order,
387 collections=args.input,
390 qg.saveUri(args.save_qgraph)
393 def _make_cli_parser(cls):
394 """Make the command-line parser.
398 parser : `argparse.ArgumentParser`
400 parser = argparse.ArgumentParser(
402 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets."
405 subparsers = parser.add_subparsers(help=
"sub-command help", dest=
"subparser_name")
407 parser_segment = subparsers.add_parser(
"segment",
408 help=
"Determine survey segments for workflow.")
409 parser_build = subparsers.add_parser(
"build",
410 help=
"Build quantum graph for HighResolutionHipsTask")
412 for sub
in [parser_segment, parser_build]:
418 help=
"Path to data repository or butler configuration.",
425 help=
"Pipeline file, limited to one task.",
433 help=
"Input collection(s) to search for coadd exposures.",
438 "--hpix_build_order",
441 help=
"HEALPix order to segment sky for building quantum graph files.",
448 help=
"Data ID expression used when querying for input coadd datasets.",
451 parser_build.add_argument(
455 "Name of the output CHAINED collection. If this options is specified and "
456 "--output-run is not, then a new RUN collection will be created by appending "
457 "a timestamp to the value of this option."
462 parser_build.add_argument(
466 "Output RUN collection to write resulting images. If not provided "
467 "then --output must be provided and a new RUN collection will be created "
468 "by appending a timestamp to the value passed with --output."
473 parser_build.add_argument(
477 help=
"Output filename for QuantumGraph.",
480 parser_build.add_argument(
485 help=
"Pixels at --hpix_build_order to generate quantum graph.",
492 def build_quantum_graph(
502 """Generate a `QuantumGraph` for running just this task.
504 This is a temporary workaround for incomplete butler query support for
509 task_def : `lsst.pipe.base.TaskDef`
511 registry : `lsst.daf.butler.Registry`
512 Client for the butler database. May be read-only.
513 constraint_order : `int`
514 HEALPix order used to contrain which quanta are generated, via
515 ``constraint_indices``. This should be a coarser grid (smaller
516 order) than the order used for the task's quantum and output data
517 IDs, and ideally something between the spatial scale of a patch or
518 the data repository's "common skypix" system (usually ``htm7``).
519 constraint_ranges : `lsst.sphgeom.RangeSet`
520 RangeSet which describes constraint pixels (HEALPix NEST, with order
521 constraint_order) to constrain generated quanta.
522 where : `str`, optional
523 A boolean `str` expression of the form accepted by
524 `Registry.queryDatasets` to constrain input datasets. This may
525 contain a constraint on tracts, patches, or bands, but not HEALPix
526 indices. Constraints on tracts and patches should usually be
527 unnecessary, however - existing coadds that overlap the given
528 HEALpix indices will be selected without such a constraint, and
529 providing one may reject some that should normally be included.
530 collections : `str` or `Iterable` [ `str` ], optional
531 Collection or collections to search for input datasets, in order.
532 If not provided, ``registry.defaults.collections`` will be
534 metadata : `dict` [ `str`, `Any` ]
535 Graph metadata. It is required to contain "output_run" key with the
536 name of the output RUN collection.
538 config = task_def.config
540 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry)
543 (input_dataset_type,) = dataset_types.inputs
548 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name]
549 incidental_output_dataset_types = dataset_types.outputs.copy()
550 incidental_output_dataset_types.remove(output_dataset_type)
551 (hpx_output_dimension,) = (
552 registry.dimensions.skypix_dimensions[d]
for d
in output_dataset_type.dimensions.skypix.names
555 constraint_hpx_pixelization = registry.dimensions[f
"healpix{constraint_order}"].pixelization
556 common_skypix_name = registry.dimensions.commonSkyPix.name
557 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization
560 task_dimensions = registry.dimensions.conform(task_def.connections.dimensions)
562 registry.dimensions.skypix_dimensions[d]
for d
in task_dimensions.names
if d !=
"band"
564 hpx_pixelization = hpx_dimension.pixelization
566 if hpx_pixelization.level < constraint_order:
567 raise ValueError(f
"Quantum order {hpx_pixelization.level} must be < {constraint_order}")
568 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order))
573 for begin, end
in constraint_ranges:
574 for hpx_index
in range(begin, end):
575 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index)
576 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region)
580 for simp
in range(1, 10):
581 if len(common_skypix_ranges) < 100:
583 common_skypix_ranges.simplify(simp)
590 for n, (begin, end)
in enumerate(common_skypix_ranges):
593 where_terms.append(f
"{common_skypix_name} = cpx{n}")
594 bind[f
"cpx{n}"] = begin
596 where_terms.append(f
"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)")
597 bind[f
"cpx{n}a"] = begin
598 bind[f
"cpx{n}b"] = stop
600 where =
" OR ".join(where_terms)
602 where = f
"({where}) AND ({' OR '.join(where_terms)})"
606 input_refs = registry.queryDatasets(
610 collections=collections,
613 inputs_by_patch = defaultdict(set)
614 patch_dimensions = registry.dimensions.conform([
"patch"])
615 for input_ref
in input_refs:
616 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref)
617 if not inputs_by_patch:
618 message_body =
"\n".join(input_refs.explain_no_results())
619 raise RuntimeError(f
"No inputs found:\n{message_body}")
624 inputs_by_hpx = defaultdict(set)
625 for patch_data_id, input_refs_for_patch
in inputs_by_patch.items():
626 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region)
627 for begin, end
in patch_hpx_ranges & hpx_ranges:
628 for hpx_index
in range(begin, end):
629 inputs_by_hpx[hpx_index].update(input_refs_for_patch)
632 output_run = metadata[
"output_run"]
633 for hpx_index, input_refs_for_hpx_index
in inputs_by_hpx.items():
635 input_refs_by_band = defaultdict(list)
636 for input_ref
in input_refs_for_hpx_index:
637 input_refs_by_band[input_ref.dataId[
"band"]].append(input_ref)
639 for band, input_refs_for_band
in input_refs_by_band.items():
640 data_id = registry.expandDataId({hpx_dimension: hpx_index,
"band": band})
642 hpx_pixel_ranges =
RangeSet(hpx_index)
643 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level))
645 for begin, end
in hpx_output_ranges:
646 for hpx_output_index
in range(begin, end):
647 output_data_ids.append(
648 registry.expandDataId({hpx_output_dimension: hpx_output_index,
"band": band})
651 dt: [DatasetRef(dt, data_id, run=output_run)]
for dt
in incidental_output_dataset_types
653 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id, run=output_run)
654 for data_id
in output_data_ids]
657 taskName=task_def.taskName,
658 taskClass=task_def.taskClass,
661 inputs={input_dataset_type: input_refs_for_band},
667 raise RuntimeError(
"Given constraints yielded empty quantum graph.")
670 empty_data_id = DataCoordinate.make_empty(registry.dimensions)
672 global_init_outputs = []
673 if config_dataset_type := dataset_types.initOutputs.get(task_def.configDatasetName):
674 init_outputs[task_def] = [DatasetRef(config_dataset_type, empty_data_id, run=output_run)]
675 packages_dataset_name = pipeBase.PipelineDatasetTypes.packagesDatasetName
676 if packages_dataset_type := dataset_types.initOutputs.get(packages_dataset_name):
677 global_init_outputs.append(DatasetRef(packages_dataset_type, empty_data_id, run=output_run))
679 return pipeBase.QuantumGraph(
680 quanta={task_def: quanta},
681 initOutputs=init_outputs,
682 globalInitOutputs=global_init_outputs,
687class HipsPropertiesSpectralTerm(pexConfig.Config):
688 lambda_min = pexConfig.Field(
689 doc=
"Minimum wavelength (nm)",
692 lambda_max = pexConfig.Field(
693 doc=
"Maximum wavelength (nm)",
698class HipsPropertiesConfig(pexConfig.Config):
699 """Configuration parameters for writing a HiPS properties file."""
700 creator_did_template = pexConfig.Field(
701 doc=(
"Unique identifier of the HiPS - Format: IVOID. "
702 "Use ``{band}`` to substitute the band name."),
706 obs_collection = pexConfig.Field(
707 doc=
"Short name of original data set - Format: one word",
711 obs_description_template = pexConfig.Field(
712 doc=(
"Data set description - Format: free text, longer free text "
713 "description of the dataset. Use ``{band}`` to substitute "
717 prov_progenitor = pexConfig.ListField(
718 doc=
"Provenance of the original data - Format: free text",
722 obs_title_template = pexConfig.Field(
723 doc=(
"Data set title format: free text, but should be short. "
724 "Use ``{band}`` to substitute the band name."),
728 spectral_ranges = pexConfig.ConfigDictField(
729 doc=(
"Mapping from band to lambda_min, lamba_max (nm). May be approximate."),
731 itemtype=HipsPropertiesSpectralTerm,
734 initial_ra = pexConfig.Field(
735 doc=
"Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.",
739 initial_dec = pexConfig.Field(
740 doc=
"Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.",
744 initial_fov = pexConfig.Field(
745 doc=
"Initial field-of-view (deg). If not set will use ~1 healpix tile.",
749 obs_ack = pexConfig.Field(
750 doc=
"Observation acknowledgements (free text).",
754 t_min = pexConfig.Field(
755 doc=
"Time (MJD) of earliest observation included in HiPS",
759 t_max = pexConfig.Field(
760 doc=
"Time (MJD) of latest observation included in HiPS",
768 if self.obs_collection
is not None:
769 if re.search(
r"\s", self.obs_collection):
770 raise ValueError(
"obs_collection cannot contain any space characters.")
772 def setDefaults(self):
775 u_term = HipsPropertiesSpectralTerm()
776 u_term.lambda_min = 330.
777 u_term.lambda_max = 400.
778 self.spectral_ranges[
"u"] = u_term
779 g_term = HipsPropertiesSpectralTerm()
780 g_term.lambda_min = 402.
781 g_term.lambda_max = 552.
782 self.spectral_ranges[
"g"] = g_term
783 r_term = HipsPropertiesSpectralTerm()
784 r_term.lambda_min = 552.
785 r_term.lambda_max = 691.
786 self.spectral_ranges[
"r"] = r_term
787 i_term = HipsPropertiesSpectralTerm()
788 i_term.lambda_min = 691.
789 i_term.lambda_max = 818.
790 self.spectral_ranges[
"i"] = i_term
791 z_term = HipsPropertiesSpectralTerm()
792 z_term.lambda_min = 818.
793 z_term.lambda_max = 922.
794 self.spectral_ranges[
"z"] = z_term
795 y_term = HipsPropertiesSpectralTerm()
796 y_term.lambda_min = 970.
797 y_term.lambda_max = 1060.
798 self.spectral_ranges[
"y"] = y_term
801class GenerateHipsConnections(pipeBase.PipelineTaskConnections,
802 dimensions=(
"instrument",
"band"),
803 defaultTemplates={
"coaddName":
"deep"}):
804 hips_exposure_handles = pipeBase.connectionTypes.Input(
805 doc=
"HiPS-compatible HPX images.",
806 name=
"{coaddName}Coadd_hpx",
807 storageClass=
"ExposureF",
808 dimensions=(
"healpix11",
"band"),
814class GenerateHipsConfig(pipeBase.PipelineTaskConfig,
815 pipelineConnections=GenerateHipsConnections):
816 """Configuration parameters for GenerateHipsTask."""
821 hips_base_uri = pexConfig.Field(
822 doc=
"URI to HiPS base for output.",
826 min_order = pexConfig.Field(
827 doc=
"Minimum healpix order for HiPS tree.",
831 properties = pexConfig.ConfigField(
832 dtype=HipsPropertiesConfig,
833 doc=
"Configuration for properties file.",
835 allsky_tilesize = pexConfig.Field(
837 doc=
"Allsky tile size; must be power of 2. HiPS standard recommends 64x64 tiles.",
839 check=_is_power_of_two,
841 png_gray_asinh_minimum = pexConfig.Field(
842 doc=
"AsinhMapping intensity to be mapped to black for grayscale png scaling.",
846 png_gray_asinh_stretch = pexConfig.Field(
847 doc=
"AsinhMapping linear stretch for grayscale png scaling.",
851 png_gray_asinh_softening = pexConfig.Field(
852 doc=
"AsinhMapping softening parameter (Q) for grayscale png scaling.",
858class GenerateHipsTask(pipeBase.PipelineTask):
859 """Task for making a HiPS tree with FITS and grayscale PNGs."""
860 ConfigClass = GenerateHipsConfig
861 _DefaultName =
"generateHips"
865 def runQuantum(self, butlerQC, inputRefs, outputRefs):
866 inputs = butlerQC.get(inputRefs)
868 dims = inputRefs.hips_exposure_handles[0].dataId.dimensions.names
872 order = int(dim.split(
"healpix")[1])
876 raise RuntimeError(
"Could not determine healpix order for input exposures.")
878 hips_exposure_handle_dict = {
879 (hips_exposure_handle.dataId[healpix_dim],
880 hips_exposure_handle.dataId[
"band"]): hips_exposure_handle
881 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]
884 data_bands = {hips_exposure_handle.dataId[
"band"]
885 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]}
886 bands = self._check_data_bands(data_bands)
891 hips_exposure_handle_dict=hips_exposure_handle_dict,
892 do_color=self.color_task,
895 def _check_data_bands(self, data_bands):
896 """Check that the data has only a single band.
900 data_bands : `set` [`str`]
901 Bands from the input data.
905 bands : `list` [`str`]
906 List of single band to process.
910 RuntimeError if there is not exactly one band.
912 if len(data_bands) != 1:
913 raise RuntimeError(
"GenerateHipsTask can only use data from a single band.")
915 return list(data_bands)
918 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False):
919 """Run the GenerateHipsTask.
923 bands : `list [ `str` ]
924 List of bands to be processed (or single band).
926 HEALPix order of the maximum (native) HPX exposures.
927 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`}
928 Dict of handles for the HiPS high-resolution exposures.
929 Key is (pixel number, ``band``).
930 do_color : `bool`, optional
931 Do color pngs instead of per-band grayscale.
933 min_order = self.config.min_order
936 png_grayscale_mapping = AsinhMapping(
937 self.config.png_gray_asinh_minimum,
938 self.config.png_gray_asinh_stretch,
939 Q=self.config.png_gray_asinh_softening,
942 png_color_mapping = AsinhMapping(
943 self.config.png_color_asinh_minimum,
944 self.config.png_color_asinh_stretch,
945 Q=self.config.png_color_asinh_softening,
948 bcb = self.config.blue_channel_band
949 gcb = self.config.green_channel_band
950 rcb = self.config.red_channel_band
951 colorstr = f
"{bcb}{gcb}{rcb}"
954 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=
True)
958 pixels = np.unique(np.array([pixel
for pixel, _
in hips_exposure_handle_dict.keys()]))
961 pixels = np.append(pixels, [0])
965 pixels_shifted[max_order] = pixels
966 for order
in range(max_order - 1, min_order - 1, -1):
967 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2)
970 for order
in range(min_order, max_order + 1):
971 pixels_shifted[order][-1] = -1
974 exp0 = list(hips_exposure_handle_dict.values())[0].get()
975 bbox = exp0.getBBox()
976 npix = bbox.getWidth()
977 shift_order = int(np.round(np.log2(npix)))
984 for order
in range(min_order, max_order + 1):
985 exp = exp0.Factory(bbox=bbox)
986 exp.image.array[:, :] = np.nan
987 exposures[(band, order)] = exp
990 for pixel_counter, pixel
in enumerate(pixels[:-1]):
991 self.log.debug(
"Working on high resolution pixel %d", pixel)
997 if (pixel, band)
in hips_exposure_handle_dict:
998 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get()
1004 for order
in range(max_order, min_order - 1, -1):
1005 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]:
1013 self._write_hips_image(
1014 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1016 pixels_shifted[order][pixel_counter],
1017 exposures[(band, order)].image,
1018 png_grayscale_mapping,
1019 shift_order=shift_order,
1023 self._write_hips_color_png(
1024 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1026 pixels_shifted[order][pixel_counter],
1027 exposures[(self.config.red_channel_band, order)].image,
1028 exposures[(self.config.green_channel_band, order)].image,
1029 exposures[(self.config.blue_channel_band, order)].image,
1033 log_level = self.log.INFO
if order == (max_order - 3)
else self.log.DEBUG
1036 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)",
1039 pixels_shifted[order][pixel_counter],
1045 if order == min_order:
1047 exposures[(band, order)].image.array[:, :] = np.nan
1052 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2)
1053 with warnings.catch_warnings():
1054 warnings.simplefilter(
"ignore")
1055 binned_image_arr = np.nanmean(arr, axis=(1, 3))
1059 sub_index = (pixels_shifted[order][pixel_counter]
1060 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2))
1063 exp = exposures[(band, order - 1)]
1067 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr
1068 elif sub_index == 1:
1069 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr
1070 elif sub_index == 2:
1071 exp.image.array[npix//2:, npix//2:] = binned_image_arr
1072 elif sub_index == 3:
1073 exp.image.array[0: npix//2, npix//2:] = binned_image_arr
1076 raise ValueError(
"Illegal pixel sub index")
1079 if order < max_order:
1080 exposures[(band, order)].image.array[:, :] = np.nan
1085 band_pixels = np.array([pixel
1086 for pixel, band_
in hips_exposure_handle_dict.keys()
1088 band_pixels = np.sort(band_pixels)
1090 self._write_properties_and_moc(
1091 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1099 self._write_allsky_file(
1100 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1104 self._write_properties_and_moc(
1105 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1113 self._write_allsky_file(
1114 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1118 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9):
1119 """Write a HiPS image.
1123 hips_base_path : `lsst.resources.ResourcePath`
1124 Resource path to the base of the HiPS directory tree.
1126 HEALPix order of the HiPS image to write.
1128 HEALPix pixel of the HiPS image.
1129 image : `lsst.afw.image.Image`
1131 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1132 Mapping to convert image to scaled png.
1133 shift_order : `int`, optional
1141 dir_number = self._get_dir_number(pixel)
1142 hips_dir = hips_base_path.join(
1150 wcs = makeHpxWcs(order, pixel, shift_order=shift_order)
1152 uri = hips_dir.join(f
"Npix{pixel}.fits")
1154 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1155 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata())
1157 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1161 with np.errstate(invalid=
"ignore"):
1162 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8)
1164 vals[~np.isfinite(image.array) | (image.array < 0)] = 0
1165 im = Image.fromarray(vals[::-1, :],
"L")
1167 uri = hips_dir.join(f
"Npix{pixel}.png")
1169 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1170 im.save(temporary_uri.ospath)
1172 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1174 def _write_hips_color_png(
1184 """Write a color png HiPS image.
1188 hips_base_path : `lsst.resources.ResourcePath`
1189 Resource path to the base of the HiPS directory tree.
1191 HEALPix order of the HiPS image to write.
1193 HEALPix pixel of the HiPS image.
1194 image_red : `lsst.afw.image.Image`
1195 Input for red channel of output png.
1196 image_green : `lsst.afw.image.Image`
1197 Input for green channel of output png.
1198 image_blue : `lsst.afw.image.Image`
1199 Input for blue channel of output png.
1200 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1201 Mapping to convert image to scaled png.
1208 dir_number = self._get_dir_number(pixel)
1209 hips_dir = hips_base_path.join(
1218 arr_red = image_red.array.copy()
1219 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0]
1220 arr_green = image_green.array.copy()
1221 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1]
1222 arr_blue = image_blue.array.copy()
1223 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2]
1225 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue)
1227 im = Image.fromarray(image_array[::-1, :, :], mode=
"RGB")
1229 uri = hips_dir.join(f
"Npix{pixel}.png")
1231 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1232 im.save(temporary_uri.ospath)
1234 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1236 def _write_properties_and_moc(
1246 """Write HiPS properties file and MOC.
1250 hips_base_path : : `lsst.resources.ResourcePath`
1251 Resource path to the base of the HiPS directory tree.
1253 Maximum HEALPix order.
1254 pixels : `np.ndarray` (N,)
1255 Array of pixels used.
1256 exposure : `lsst.afw.image.Exposure`
1257 Sample HPX exposure used for generating HiPS tiles.
1263 Is band multiband / color?
1265 area = hpg.nside_to_pixel_area(2**max_order, degrees=
True)*len(pixels)
1267 initial_ra = self.config.properties.initial_ra
1268 initial_dec = self.config.properties.initial_dec
1269 initial_fov = self.config.properties.initial_fov
1271 if initial_ra
is None or initial_dec
is None or initial_fov
is None:
1274 temp_pixels = pixels.copy()
1275 if temp_pixels.size % 2 == 0:
1276 temp_pixels = np.append(temp_pixels, [temp_pixels[0]])
1277 medpix = int(np.median(temp_pixels))
1278 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix)
1279 _initial_fov = hpg.nside_to_resolution(2**max_order, units=
'arcminutes')/60.
1281 if initial_ra
is None or initial_dec
is None:
1282 initial_ra = _initial_ra
1283 initial_dec = _initial_dec
1284 if initial_fov
is None:
1285 initial_fov = _initial_fov
1287 self._write_hips_properties_file(
1289 self.config.properties,
1302 self._write_hips_moc_file(
1308 def _write_hips_properties_file(
1322 """Write HiPS properties file.
1326 hips_base_path : `lsst.resources.ResourcePath`
1327 ResourcePath at top of HiPS tree. File will be written
1328 to this path as ``properties``.
1329 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig`
1330 Configuration for properties values.
1332 Name of band(s) for HiPS tree.
1334 Is multiband / color?
1335 exposure : `lsst.afw.image.Exposure`
1336 Sample HPX exposure used for generating HiPS tiles.
1338 Maximum HEALPix order.
1342 Coverage area in square degrees.
1343 initial_ra : `float`
1344 Initial HiPS RA position (degrees).
1345 initial_dec : `float`
1346 Initial HiPS Dec position (degrees).
1347 initial_fov : `float`
1348 Initial HiPS display size (degrees).
1354 def _write_property(fh, name, value):
1355 """Write a property name/value to a file handle.
1359 fh : file handle (blah)
1368 if re.search(
r"\s", name):
1369 raise ValueError(f
"``{name}`` cannot contain any space characters.")
1371 raise ValueError(f
"``{name}`` cannot contain an ``=``")
1373 fh.write(f
"{name:25}= {value}\n")
1375 if exposure.image.array.dtype == np.dtype(
"float32"):
1377 elif exposure.image.array.dtype == np.dtype(
"float64"):
1379 elif exposure.image.array.dtype == np.dtype(
"int32"):
1382 date_iso8601 = datetime.utcnow().isoformat(timespec=
"seconds") +
"Z"
1383 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units=
'degrees')
1385 uri = hips_base_path.join(
"properties")
1386 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1387 with open(temporary_uri.ospath,
"w")
as fh:
1391 properties_config.creator_did_template.format(band=band),
1393 if properties_config.obs_collection
is not None:
1394 _write_property(fh,
"obs_collection", properties_config.obs_collection)
1398 properties_config.obs_title_template.format(band=band),
1400 if properties_config.obs_description_template
is not None:
1404 properties_config.obs_description_template.format(band=band),
1406 if len(properties_config.prov_progenitor) > 0:
1407 for prov_progenitor
in properties_config.prov_progenitor:
1408 _write_property(fh,
"prov_progenitor", prov_progenitor)
1409 if properties_config.obs_ack
is not None:
1410 _write_property(fh,
"obs_ack", properties_config.obs_ack)
1411 _write_property(fh,
"obs_regime",
"Optical")
1412 _write_property(fh,
"data_pixel_bitpix", str(bitpix))
1413 _write_property(fh,
"dataproduct_type",
"image")
1414 _write_property(fh,
"moc_sky_fraction", str(area/41253.))
1415 _write_property(fh,
"data_ucd",
"phot.flux")
1416 _write_property(fh,
"hips_creation_date", date_iso8601)
1417 _write_property(fh,
"hips_builder",
"lsst.pipe.tasks.hips.GenerateHipsTask")
1418 _write_property(fh,
"hips_creator",
"Vera C. Rubin Observatory")
1419 _write_property(fh,
"hips_version",
"1.4")
1420 _write_property(fh,
"hips_release_date", date_iso8601)
1421 _write_property(fh,
"hips_frame",
"equatorial")
1422 _write_property(fh,
"hips_order", str(max_order))
1423 _write_property(fh,
"hips_tile_width", str(exposure.getBBox().getWidth()))
1424 _write_property(fh,
"hips_status",
"private master clonableOnce")
1426 _write_property(fh,
"hips_tile_format",
"png")
1427 _write_property(fh,
"dataproduct_subtype",
"color")
1429 _write_property(fh,
"hips_tile_format",
"png fits")
1430 _write_property(fh,
"hips_pixel_bitpix", str(bitpix))
1431 _write_property(fh,
"hips_pixel_scale", str(pixel_scale))
1432 _write_property(fh,
"hips_initial_ra", str(initial_ra))
1433 _write_property(fh,
"hips_initial_dec", str(initial_dec))
1434 _write_property(fh,
"hips_initial_fov", str(initial_fov))
1436 if self.config.blue_channel_band
in properties_config.spectral_ranges:
1437 em_min = properties_config.spectral_ranges[
1438 self.config.blue_channel_band
1441 self.log.warning(
"blue band %s not in self.config.spectral_ranges.", band)
1443 if self.config.red_channel_band
in properties_config.spectral_ranges:
1444 em_max = properties_config.spectral_ranges[
1445 self.config.red_channel_band
1448 self.log.warning(
"red band %s not in self.config.spectral_ranges.", band)
1451 if band
in properties_config.spectral_ranges:
1452 em_min = properties_config.spectral_ranges[band].lambda_min/1e9
1453 em_max = properties_config.spectral_ranges[band].lambda_max/1e9
1455 self.log.warning(
"band %s not in self.config.spectral_ranges.", band)
1458 _write_property(fh,
"em_min", str(em_min))
1459 _write_property(fh,
"em_max", str(em_max))
1460 if properties_config.t_min
is not None:
1461 _write_property(fh,
"t_min", properties_config.t_min)
1462 if properties_config.t_max
is not None:
1463 _write_property(fh,
"t_max", properties_config.t_max)
1465 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1467 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1):
1468 """Write HiPS MOC file.
1472 hips_base_path : `lsst.resources.ResourcePath`
1473 ResourcePath to top of HiPS tree. File will be written as
1474 to this path as ``Moc.fits``.
1476 Maximum HEALPix order.
1477 pixels : `np.ndarray`
1478 Array of pixels covered.
1479 min_uniq_order : `int`, optional
1480 Minimum HEALPix order for looking for fully covered pixels.
1488 uniq = 4*(4**max_order) + pixels
1491 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32)
1492 hspmap[pixels] = 1.0
1495 for uniq_order
in range(max_order - 1, min_uniq_order - 1, -1):
1496 hspmap = hspmap.degrade(2**uniq_order, reduction=
"sum")
1497 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order))
1499 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero()
1500 if covered.size == 0:
1504 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered]
1507 uniq = np.unique(uniq)
1510 tbl = np.zeros(uniq.size, dtype=[(
"UNIQ",
"i8")])
1513 order = np.log2(tbl[
"UNIQ"]//4).astype(np.int32)//2
1514 moc_order = np.max(order)
1516 hdu = fits.BinTableHDU(tbl)
1517 hdu.header[
"PIXTYPE"] =
"HEALPIX"
1518 hdu.header[
"ORDERING"] =
"NUNIQ"
1519 hdu.header[
"COORDSYS"] =
"C"
1520 hdu.header[
"MOCORDER"] = moc_order
1521 hdu.header[
"MOCTOOL"] =
"lsst.pipe.tasks.hips.GenerateHipsTask"
1523 uri = hips_base_path.join(
"Moc.fits")
1525 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1526 hdu.writeto(temporary_uri.ospath)
1528 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1530 def _write_allsky_file(self, hips_base_path, allsky_order):
1531 """Write an Allsky.png file.
1535 hips_base_path : `lsst.resources.ResourcePath`
1536 Resource path to the base of the HiPS directory tree.
1537 allsky_order : `int`
1538 HEALPix order of the minimum order to make allsky file.
1540 tile_size = self.config.allsky_tilesize
1552 n_tiles = hpg.nside_to_npixel(hpg.order_to_nside(allsky_order))
1553 n_tiles_wide = int(np.floor(np.sqrt(n_tiles)))
1554 n_tiles_high = int(np.ceil(n_tiles / n_tiles_wide))
1558 allsky_order_uri = hips_base_path.join(f
"Norder{allsky_order}", forceDirectory=
True)
1559 pixel_regex = re.compile(
r"Npix([0-9]+)\.png$")
1561 ResourcePath.findFileResources(
1562 candidates=[allsky_order_uri],
1563 file_filter=pixel_regex,
1567 for png_uri
in png_uris:
1568 matches = re.match(pixel_regex, png_uri.basename())
1569 pix_num = int(matches.group(1))
1570 tile_image = Image.open(io.BytesIO(png_uri.read()))
1571 row = math.floor(pix_num//n_tiles_wide)
1572 column = pix_num % n_tiles_wide
1573 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size)
1574 tile_image_shrunk = tile_image.resize((tile_size, tile_size))
1576 if allsky_image
is None:
1577 allsky_image = Image.new(
1579 (n_tiles_wide*tile_size, n_tiles_high*tile_size),
1581 allsky_image.paste(tile_image_shrunk, box)
1583 uri = allsky_order_uri.join(
"Allsky.png")
1585 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1586 allsky_image.save(temporary_uri.ospath)
1588 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1590 def _get_dir_number(self, pixel):
1591 """Compute the directory number from a pixel.
1596 HEALPix pixel number.
1601 HiPS directory number.
1603 return (pixel//10000)*10000
1606class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections,
1607 dimensions=(
"instrument", ),
1608 defaultTemplates={
"coaddName":
"deep"}):
1609 hips_exposure_handles = pipeBase.connectionTypes.Input(
1610 doc=
"HiPS-compatible HPX images.",
1611 name=
"{coaddName}Coadd_hpx",
1612 storageClass=
"ExposureF",
1613 dimensions=(
"healpix11",
"band"),
1619class GenerateColorHipsConfig(GenerateHipsConfig,
1620 pipelineConnections=GenerateColorHipsConnections):
1621 """Configuration parameters for GenerateColorHipsTask."""
1622 blue_channel_band = pexConfig.Field(
1623 doc=
"Band to use for blue channel of color pngs.",
1627 green_channel_band = pexConfig.Field(
1628 doc=
"Band to use for green channel of color pngs.",
1632 red_channel_band = pexConfig.Field(
1633 doc=
"Band to use for red channel of color pngs.",
1637 png_color_asinh_minimum = pexConfig.Field(
1638 doc=
"AsinhMapping intensity to be mapped to black for color png scaling.",
1642 png_color_asinh_stretch = pexConfig.Field(
1643 doc=
"AsinhMapping linear stretch for color png scaling.",
1647 png_color_asinh_softening = pexConfig.Field(
1648 doc=
"AsinhMapping softening parameter (Q) for color png scaling.",
1654class GenerateColorHipsTask(GenerateHipsTask):
1655 """Task for making a HiPS tree with color pngs."""
1656 ConfigClass = GenerateColorHipsConfig
1657 _DefaultName =
"generateColorHips"
1660 def _check_data_bands(self, data_bands):
1661 """Check the data for configured bands.
1663 Warn if any color bands are missing data.
1667 data_bands : `set` [`str`]
1668 Bands from the input data.
1672 bands : `list` [`str`]
1673 List of bands in bgr color order.
1675 if len(data_bands) == 0:
1676 raise RuntimeError(
"GenerateColorHipsTask must have data from at least one band.")
1678 if self.config.blue_channel_band
not in data_bands:
1680 "Color png blue_channel_band %s not in dataset.",
1681 self.config.blue_channel_band
1683 if self.config.green_channel_band
not in data_bands:
1685 "Color png green_channel_band %s not in dataset.",
1686 self.config.green_channel_band
1688 if self.config.red_channel_band
not in data_bands:
1690 "Color png red_channel_band %s not in dataset.",
1691 self.config.red_channel_band
1695 self.config.blue_channel_band,
1696 self.config.green_channel_band,
1697 self.config.red_channel_band,