22"""Tasks for making and manipulating HIPS images."""
24__all__ = [
"HighResolutionHipsTask",
"HighResolutionHipsConfig",
"HighResolutionHipsConnections",
25 "GenerateHipsTask",
"GenerateHipsConfig",
"GenerateColorHipsTask",
"GenerateColorHipsConfig"]
27from collections
import defaultdict
35from datetime
import datetime
37import healsparse
as hsp
38from astropy.io
import fits
39from astropy.visualization.lupton_rgb
import AsinhMapping
43from lsst.utils.timer
import timeMethod
44from lsst.daf.butler
import Butler, DatasetRef, Quantum, SkyPixDimension
52from lsst.resources
import ResourcePath
56 dimensions=(
"healpix9",
"band"),
57 defaultTemplates={
"coaddName":
"deep"}):
58 coadd_exposure_handles = pipeBase.connectionTypes.Input(
59 doc=
"Coadded exposures to convert to HIPS format.",
60 name=
"{coaddName}Coadd_calexp",
61 storageClass=
"ExposureF",
62 dimensions=(
"tract",
"patch",
"skymap",
"band"),
66 hips_exposures = pipeBase.connectionTypes.Output(
67 doc=
"HiPS-compatible HPX image.",
68 name=
"{coaddName}Coadd_hpx",
69 storageClass=
"ExposureF",
70 dimensions=(
"healpix11",
"band"),
74 def __init__(self, *, config=None):
75 super().__init__(config=config)
78 for dim
in self.dimensions:
80 if quantum_order
is not None:
81 raise ValueError(
"Must not specify more than one quantum healpix dimension.")
82 quantum_order = int(dim.split(
"healpix")[1])
83 if quantum_order
is None:
84 raise ValueError(
"Must specify a healpix dimension in quantum dimensions.")
86 if quantum_order > config.hips_order:
87 raise ValueError(
"Quantum healpix dimension order must not be greater than hips_order")
90 for dim
in self.hips_exposures.dimensions:
93 raise ValueError(
"Must not specify more than one healpix dimension.")
94 order = int(dim.split(
"healpix")[1])
96 raise ValueError(
"Must specify a healpix dimension in hips_exposure dimensions.")
98 if order != config.hips_order:
99 raise ValueError(
"healpix dimension order must match config.hips_order.")
102class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig,
103 pipelineConnections=HighResolutionHipsConnections):
104 """Configuration parameters for HighResolutionHipsTask.
108 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to
109 2**hips_order. Each cell
is 'shift_order' orders deeper than the HEALPix
110 cell,
with 2**shift_order x 2**shift_order sub-pixels on a side, which
111 defines the target resolution of the HiPS image. The IVOA recommends
112 shift_order=9,
for 2**9=512 pixels on a side.
115 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf
116 shows the relationship between hips_order, number of tiles (full
117 sky coverage), cell size,
and sub-pixel size/image resolution (
with
118 the default shift_order=9):
119 +------------+-----------------+--------------+------------------+
120 | hips_order | Number of Tiles | Cell Size | Image Resolution |
121 +============+=================+==============+==================+
122 | 0 | 12 | 58.63 deg | 6.871 arcmin |
123 | 1 | 48 | 29.32 deg | 3.435 arcmin |
124 | 2 | 192 | 14.66 deg | 1.718 arcmin |
125 | 3 | 768 | 7.329 deg | 51.53 arcsec |
126 | 4 | 3072 | 3.665 deg | 25.77 arcsec |
127 | 5 | 12288 | 1.832 deg | 12.88 arcsec |
128 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec |
129 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec |
130 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec |
131 | 9 | 3145728 | 6.871 arcmin | 805.2mas |
132 | 10 | 12582912 | 3.435 arcmin | 402.6mas |
133 | 11 | 50331648 | 1.718 arcmin | 201.3mas |
134 | 12 | 201326592 | 51.53 arcsec | 100.6mas |
135 | 13 | 805306368 | 25.77 arcsec | 50.32mas |
136 +------------+-----------------+--------------+------------------+
138 hips_order = pexConfig.Field(
139 doc="HIPS image order.",
143 shift_order = pexConfig.Field(
144 doc=
"HIPS shift order (such that each tile is 2**shift_order pixels on a side)",
148 warp = pexConfig.ConfigField(
149 dtype=afwMath.Warper.ConfigClass,
150 doc=
"Warper configuration",
153 def setDefaults(self):
154 self.warp.warpingKernelName =
"lanczos5"
157class HipsTaskNameDescriptor:
158 """Descriptor used create a DefaultName that matches the order of
159 the defined dimensions in the connections
class.
164 The prefix of the Default name, to which the order will be
167 def __init__(self, prefix):
169 self._defaultName = f
"{prefix}{{}}"
172 def __get__(self, obj, klass=None):
175 "HipsTaskDescriptor was used in an unexpected context"
177 if self._order
is None:
178 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions
179 for dim
in klassDimensions:
180 if (match := re.match(
r"^healpix(\d*)$", dim))
is not None:
181 self._order = int(match.group(1))
185 "Could not find healpix dimension in connections class"
187 return self._defaultName.format(self._order)
190class HighResolutionHipsTask(pipeBase.PipelineTask):
191 """Task for making high resolution HiPS images."""
192 ConfigClass = HighResolutionHipsConfig
193 _DefaultName = HipsTaskNameDescriptor(
"highResolutionHips")
195 def __init__(self, **kwargs):
196 super().__init__(**kwargs)
197 self.warper = afwMath.Warper.fromConfig(self.config.warp)
200 def runQuantum(self, butlerQC, inputRefs, outputRefs):
201 inputs = butlerQC.get(inputRefs)
203 healpix_dim = f
"healpix{self.config.hips_order}"
205 pixels = [hips_exposure.dataId[healpix_dim]
206 for hips_exposure
in outputRefs.hips_exposures]
208 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs[
"coadd_exposure_handles"])
210 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]:
211 hips_exposure_ref
for hips_exposure_ref
in outputRefs.hips_exposures}
212 for pixel, hips_exposure
in outputs.hips_exposures.items():
213 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel])
215 def run(self, pixels, coadd_exposure_handles):
216 """Run the HighResolutionHipsTask.
220 pixels : `Iterable` [ `int` ]
221 Iterable of healpix pixels (nest ordering) to warp to.
222 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
223 Handles for the coadd exposures.
227 outputs : `lsst.pipe.base.Struct`
228 ``hips_exposures``
is a dict
with pixel (key)
and hips_exposure (value)
230 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order)
232 npix = 2**self.config.shift_order
242 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order)
243 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx)
244 exp_hpx_dict[pixel] = exp_hpx
245 warp_dict[pixel] = []
250 for handle
in coadd_exposure_handles:
251 coadd_exp = handle.get()
255 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx)
257 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs())
258 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
263 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict())
264 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter())
265 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib())
267 if warped.getBBox().getArea() == 0
or not np.any(np.isfinite(warped.image.array)):
270 "No overlap between output HPX %d and input exposure %s",
276 exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
277 warp_dict[pixel].append(exp.maskedImage)
281 stats_flags = afwMath.stringToStatisticsProperty(
"MEAN")
282 stats_ctrl = afwMath.StatisticsControl()
283 stats_ctrl.setNanSafe(
True)
284 stats_ctrl.setWeighted(
True)
285 stats_ctrl.setCalcErrorFromInputVariance(
True)
291 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask(
"NO_DATA"), np.nan)
293 if not warp_dict[pixel]:
295 self.log.debug(
"No data in HPX pixel %d", pixel)
298 exp_hpx_dict.pop(pixel)
301 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack(
305 [1.0]*len(warp_dict[pixel]),
310 return pipeBase.Struct(hips_exposures=exp_hpx_dict)
313 def build_quantum_graph_cli(cls, argv):
314 """A command-line interface entry point to `build_quantum_graph`.
315 This method provides the implementation for the
316 ``build-high-resolution-hips-qg`` script.
320 argv : `Sequence` [ `str` ]
321 Command-line arguments (e.g. ``sys.argv[1:]``).
323 parser = cls._make_cli_parser()
325 args = parser.parse_args(argv)
327 if args.subparser_name
is None:
331 pipeline = pipeBase.Pipeline.from_uri(args.pipeline)
332 expanded_pipeline = list(pipeline.toExpandedPipeline())
334 if len(expanded_pipeline) != 1:
335 raise RuntimeError(f
"Pipeline file {args.pipeline} may only contain one task.")
337 (task_def,) = expanded_pipeline
339 butler = Butler(args.butler_config, collections=args.input)
341 if args.subparser_name ==
"segment":
343 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order)
344 dataset = task_def.connections.coadd_exposure_handles.name
345 data_ids = set(butler.registry.queryDataIds(
"tract", datasets=dataset).expanded())
347 for data_id
in data_ids:
348 region = data_id.region
349 pixel_range = hpix_pixelization.envelope(region)
350 for r
in pixel_range.ranges():
351 region_pixels.extend(range(r[0], r[1]))
352 indices = np.unique(region_pixels)
354 print(f
"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:")
355 for pixel
in indices:
358 elif args.subparser_name ==
"build":
362 if args.output_run
is None:
363 if args.output
is None:
364 raise ValueError(
"At least one of --output or --output-run options is required.")
365 args.output_run =
"{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp())
367 build_ranges =
RangeSet(sorted(args.pixels))
372 "butler_argument": args.butler_config,
373 "output": args.output,
374 "output_run": args.output_run,
375 "data_query": args.where,
376 "time": f
"{datetime.now()}",
379 qg = cls.build_quantum_graph(
382 args.hpix_build_order,
385 collections=args.input,
388 qg.saveUri(args.save_qgraph)
391 def _make_cli_parser(cls):
392 """Make the command-line parser.
396 parser : `argparse.ArgumentParser`
398 parser = argparse.ArgumentParser(
400 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets."
403 subparsers = parser.add_subparsers(help=
"sub-command help", dest=
"subparser_name")
405 parser_segment = subparsers.add_parser(
"segment",
406 help=
"Determine survey segments for workflow.")
407 parser_build = subparsers.add_parser(
"build",
408 help=
"Build quantum graph for HighResolutionHipsTask")
410 for sub
in [parser_segment, parser_build]:
416 help=
"Path to data repository or butler configuration.",
423 help=
"Pipeline file, limited to one task.",
431 help=
"Input collection(s) to search for coadd exposures.",
436 "--hpix_build_order",
439 help=
"HEALPix order to segment sky for building quantum graph files.",
446 help=
"Data ID expression used when querying for input coadd datasets.",
449 parser_build.add_argument(
453 "Name of the output CHAINED collection. If this options is specified and "
454 "--output-run is not, then a new RUN collection will be created by appending "
455 "a timestamp to the value of this option."
460 parser_build.add_argument(
464 "Output RUN collection to write resulting images. If not provided "
465 "then --output must be provided and a new RUN collection will be created "
466 "by appending a timestamp to the value passed with --output."
471 parser_build.add_argument(
475 help=
"Output filename for QuantumGraph.",
478 parser_build.add_argument(
483 help=
"Pixels at --hpix_build_order to generate quantum graph.",
490 def build_quantum_graph(
500 """Generate a `QuantumGraph` for running just this task.
502 This is a temporary workaround
for incomplete butler query support
for
507 task_def : `lsst.pipe.base.TaskDef`
509 registry : `lsst.daf.butler.Registry`
510 Client
for the butler database. May be read-only.
511 constraint_order : `int`
512 HEALPix order used to contrain which quanta are generated, via
513 ``constraint_indices``. This should be a coarser grid (smaller
514 order) than the order used
for the task
's quantum and output data
515 IDs, and ideally something between the spatial scale of a patch
or
516 the data repository
's "common skypix" system (usually ``htm7``).
518 RangeSet which describes constraint pixels (HEALPix NEST, with order
519 constraint_order) to constrain generated quanta.
520 where : `str`, optional
521 A boolean `str` expression of the form accepted by
522 `Registry.queryDatasets` to constrain input datasets. This may
523 contain a constraint on tracts, patches,
or bands, but
not HEALPix
524 indices. Constraints on tracts
and patches should usually be
525 unnecessary, however - existing coadds that overlap the given
526 HEALpix indices will be selected without such a constraint,
and
527 providing one may reject some that should normally be included.
528 collections : `str`
or `Iterable` [ `str` ], optional
529 Collection
or collections to search
for input datasets,
in order.
530 If
not provided, ``registry.defaults.collections`` will be
532 metadata : `dict` [ `str`, `Any` ]
533 Graph metadata. It
is required to contain
"output_run" key
with the
534 name of the output RUN collection.
536 config = task_def.config
538 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry)
541 (input_dataset_type,) = dataset_types.inputs
546 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name]
547 incidental_output_dataset_types = dataset_types.outputs.copy()
548 incidental_output_dataset_types.remove(output_dataset_type)
549 (hpx_output_dimension,) = (d
for d
in output_dataset_type.dimensions
550 if isinstance(d, SkyPixDimension))
552 constraint_hpx_pixelization = registry.dimensions[f
"healpix{constraint_order}"].pixelization
553 common_skypix_name = registry.dimensions.commonSkyPix.name
554 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization
557 task_dimensions = registry.dimensions.extract(task_def.connections.dimensions)
558 (hpx_dimension,) = (d
for d
in task_dimensions
if d.name !=
"band")
559 hpx_pixelization = hpx_dimension.pixelization
561 if hpx_pixelization.level < constraint_order:
562 raise ValueError(f
"Quantum order {hpx_pixelization.level} must be < {constraint_order}")
563 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order))
568 for begin, end
in constraint_ranges:
569 for hpx_index
in range(begin, end):
570 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index)
571 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region)
575 for simp
in range(1, 10):
576 if len(common_skypix_ranges) < 100:
578 common_skypix_ranges.simplify(simp)
585 for n, (begin, end)
in enumerate(common_skypix_ranges):
588 where_terms.append(f
"{common_skypix_name} = cpx{n}")
589 bind[f
"cpx{n}"] = begin
591 where_terms.append(f
"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)")
592 bind[f
"cpx{n}a"] = begin
593 bind[f
"cpx{n}b"] = stop
595 where =
" OR ".join(where_terms)
597 where = f
"({where}) AND ({' OR '.join(where_terms)})"
601 input_refs = registry.queryDatasets(
605 collections=collections,
608 inputs_by_patch = defaultdict(set)
609 patch_dimensions = registry.dimensions.extract([
"patch"])
610 for input_ref
in input_refs:
611 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref)
612 if not inputs_by_patch:
613 message_body =
"\n".join(input_refs.explain_no_results())
614 raise RuntimeError(f
"No inputs found:\n{message_body}")
619 inputs_by_hpx = defaultdict(set)
620 for patch_data_id, input_refs_for_patch
in inputs_by_patch.items():
621 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region)
622 for begin, end
in patch_hpx_ranges & hpx_ranges:
623 for hpx_index
in range(begin, end):
624 inputs_by_hpx[hpx_index].update(input_refs_for_patch)
627 for hpx_index, input_refs_for_hpx_index
in inputs_by_hpx.items():
629 input_refs_by_band = defaultdict(list)
630 for input_ref
in input_refs_for_hpx_index:
631 input_refs_by_band[input_ref.dataId[
"band"]].append(input_ref)
633 for band, input_refs_for_band
in input_refs_by_band.items():
634 data_id = registry.expandDataId({hpx_dimension: hpx_index,
"band": band})
636 hpx_pixel_ranges =
RangeSet(hpx_index)
637 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level))
639 for begin, end
in hpx_output_ranges:
640 for hpx_output_index
in range(begin, end):
641 output_data_ids.append(
642 registry.expandDataId({hpx_output_dimension: hpx_output_index,
"band": band})
644 output_run = metadata[
"output_run"]
646 dt: [DatasetRef(dt, data_id, run=output_run)]
for dt
in incidental_output_dataset_types
648 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id, run=output_run)
649 for data_id
in output_data_ids]
652 taskName=task_def.taskName,
653 taskClass=task_def.taskClass,
656 inputs={input_dataset_type: input_refs_for_band},
662 raise RuntimeError(
"Given constraints yielded empty quantum graph.")
664 return pipeBase.QuantumGraph(quanta={task_def: quanta}, metadata=metadata)
667class HipsPropertiesSpectralTerm(pexConfig.Config):
668 lambda_min = pexConfig.Field(
669 doc=
"Minimum wavelength (nm)",
672 lambda_max = pexConfig.Field(
673 doc=
"Maximum wavelength (nm)",
678class HipsPropertiesConfig(pexConfig.Config):
679 """Configuration parameters for writing a HiPS properties file."""
680 creator_did_template = pexConfig.Field(
681 doc=(
"Unique identifier of the HiPS - Format: IVOID. "
682 "Use ``{band}`` to substitute the band name."),
686 obs_collection = pexConfig.Field(
687 doc=
"Short name of original data set - Format: one word",
691 obs_description_template = pexConfig.Field(
692 doc=(
"Data set description - Format: free text, longer free text "
693 "description of the dataset. Use ``{band}`` to substitute "
697 prov_progenitor = pexConfig.ListField(
698 doc=
"Provenance of the original data - Format: free text",
702 obs_title_template = pexConfig.Field(
703 doc=(
"Data set title format: free text, but should be short. "
704 "Use ``{band}`` to substitute the band name."),
708 spectral_ranges = pexConfig.ConfigDictField(
709 doc=(
"Mapping from band to lambda_min, lamba_max (nm). May be approximate."),
711 itemtype=HipsPropertiesSpectralTerm,
714 initial_ra = pexConfig.Field(
715 doc=
"Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.",
719 initial_dec = pexConfig.Field(
720 doc=
"Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.",
724 initial_fov = pexConfig.Field(
725 doc=
"Initial field-of-view (deg). If not set will use ~1 healpix tile.",
729 obs_ack = pexConfig.Field(
730 doc=
"Observation acknowledgements (free text).",
734 t_min = pexConfig.Field(
735 doc=
"Time (MJD) of earliest observation included in HiPS",
739 t_max = pexConfig.Field(
740 doc=
"Time (MJD) of latest observation included in HiPS",
748 if self.obs_collection
is not None:
749 if re.search(
r"\s", self.obs_collection):
750 raise ValueError(
"obs_collection cannot contain any space characters.")
752 def setDefaults(self):
755 u_term = HipsPropertiesSpectralTerm()
756 u_term.lambda_min = 330.
757 u_term.lambda_max = 400.
758 self.spectral_ranges[
"u"] = u_term
759 g_term = HipsPropertiesSpectralTerm()
760 g_term.lambda_min = 402.
761 g_term.lambda_max = 552.
762 self.spectral_ranges[
"g"] = g_term
763 r_term = HipsPropertiesSpectralTerm()
764 r_term.lambda_min = 552.
765 r_term.lambda_max = 691.
766 self.spectral_ranges[
"r"] = r_term
767 i_term = HipsPropertiesSpectralTerm()
768 i_term.lambda_min = 691.
769 i_term.lambda_max = 818.
770 self.spectral_ranges[
"i"] = i_term
771 z_term = HipsPropertiesSpectralTerm()
772 z_term.lambda_min = 818.
773 z_term.lambda_max = 922.
774 self.spectral_ranges[
"z"] = z_term
775 y_term = HipsPropertiesSpectralTerm()
776 y_term.lambda_min = 970.
777 y_term.lambda_max = 1060.
778 self.spectral_ranges[
"y"] = y_term
781class GenerateHipsConnections(pipeBase.PipelineTaskConnections,
782 dimensions=(
"instrument",
"band"),
783 defaultTemplates={
"coaddName":
"deep"}):
784 hips_exposure_handles = pipeBase.connectionTypes.Input(
785 doc=
"HiPS-compatible HPX images.",
786 name=
"{coaddName}Coadd_hpx",
787 storageClass=
"ExposureF",
788 dimensions=(
"healpix11",
"band"),
794class GenerateHipsConfig(pipeBase.PipelineTaskConfig,
795 pipelineConnections=GenerateHipsConnections):
796 """Configuration parameters for GenerateHipsTask."""
801 hips_base_uri = pexConfig.Field(
802 doc=
"URI to HiPS base for output.",
806 min_order = pexConfig.Field(
807 doc=
"Minimum healpix order for HiPS tree.",
811 properties = pexConfig.ConfigField(
812 dtype=HipsPropertiesConfig,
813 doc=
"Configuration for properties file.",
815 allsky_tilesize = pexConfig.Field(
817 doc=
"Allsky.png tile size. Must be power of 2.",
820 png_gray_asinh_minimum = pexConfig.Field(
821 doc=
"AsinhMapping intensity to be mapped to black for grayscale png scaling.",
825 png_gray_asinh_stretch = pexConfig.Field(
826 doc=
"AsinhMapping linear stretch for grayscale png scaling.",
830 png_gray_asinh_softening = pexConfig.Field(
831 doc=
"AsinhMapping softening parameter (Q) for grayscale png scaling.",
837class GenerateHipsTask(pipeBase.PipelineTask):
838 """Task for making a HiPS tree with FITS and grayscale PNGs."""
839 ConfigClass = GenerateHipsConfig
840 _DefaultName =
"generateHips"
844 def runQuantum(self, butlerQC, inputRefs, outputRefs):
845 inputs = butlerQC.get(inputRefs)
847 dims = inputRefs.hips_exposure_handles[0].dataId.names
851 order = int(dim.split(
"healpix")[1])
855 raise RuntimeError(
"Could not determine healpix order for input exposures.")
857 hips_exposure_handle_dict = {
858 (hips_exposure_handle.dataId[healpix_dim],
859 hips_exposure_handle.dataId[
"band"]): hips_exposure_handle
860 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]
863 data_bands = {hips_exposure_handle.dataId[
"band"]
864 for hips_exposure_handle
in inputs[
"hips_exposure_handles"]}
865 bands = self._check_data_bands(data_bands)
870 hips_exposure_handle_dict=hips_exposure_handle_dict,
871 do_color=self.color_task,
874 def _check_data_bands(self, data_bands):
875 """Check that the data has only a single band.
879 data_bands : `set` [`str`]
880 Bands from the input data.
884 bands : `list` [`str`]
885 List of single band to process.
889 RuntimeError
if there
is not exactly one band.
891 if len(data_bands) != 1:
892 raise RuntimeError(
"GenerateHipsTask can only use data from a single band.")
894 return list(data_bands)
897 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False):
898 """Run the GenerateHipsTask.
902 bands : `list [ `str` ]
903 List of bands to be processed (or single band).
905 HEALPix order of the maximum (native) HPX exposures.
906 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`}
907 Dict of handles
for the HiPS high-resolution exposures.
908 Key
is (pixel number, ``band``).
909 do_color : `bool`, optional
910 Do color pngs instead of per-band grayscale.
912 min_order = self.config.min_order
915 png_grayscale_mapping = AsinhMapping(
916 self.config.png_gray_asinh_minimum,
917 self.config.png_gray_asinh_stretch,
918 Q=self.config.png_gray_asinh_softening,
921 png_color_mapping = AsinhMapping(
922 self.config.png_color_asinh_minimum,
923 self.config.png_color_asinh_stretch,
924 Q=self.config.png_color_asinh_softening,
927 bcb = self.config.blue_channel_band
928 gcb = self.config.green_channel_band
929 rcb = self.config.red_channel_band
930 colorstr = f
"{bcb}{gcb}{rcb}"
933 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=
True)
937 pixels = np.unique(np.array([pixel
for pixel, _
in hips_exposure_handle_dict.keys()]))
940 pixels = np.append(pixels, [0])
944 pixels_shifted[max_order] = pixels
945 for order
in range(max_order - 1, min_order - 1, -1):
946 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2)
949 for order
in range(min_order, max_order + 1):
950 pixels_shifted[order][-1] = -1
953 exp0 = list(hips_exposure_handle_dict.values())[0].get()
954 bbox = exp0.getBBox()
955 npix = bbox.getWidth()
956 shift_order = int(np.round(np.log2(npix)))
963 for order
in range(min_order, max_order + 1):
964 exp = exp0.Factory(bbox=bbox)
965 exp.image.array[:, :] = np.nan
966 exposures[(band, order)] = exp
969 for pixel_counter, pixel
in enumerate(pixels[:-1]):
970 self.log.debug(
"Working on high resolution pixel %d", pixel)
976 if (pixel, band)
in hips_exposure_handle_dict:
977 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get()
983 for order
in range(max_order, min_order - 1, -1):
984 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]:
992 self._write_hips_image(
993 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
995 pixels_shifted[order][pixel_counter],
996 exposures[(band, order)].image,
997 png_grayscale_mapping,
998 shift_order=shift_order,
1002 self._write_hips_color_png(
1003 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1005 pixels_shifted[order][pixel_counter],
1006 exposures[(self.config.red_channel_band, order)].image,
1007 exposures[(self.config.green_channel_band, order)].image,
1008 exposures[(self.config.blue_channel_band, order)].image,
1012 log_level = self.log.INFO
if order == (max_order - 3)
else self.log.DEBUG
1015 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)",
1018 pixels_shifted[order][pixel_counter],
1024 if order == min_order:
1026 exposures[(band, order)].image.array[:, :] = np.nan
1031 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2)
1032 with warnings.catch_warnings():
1033 warnings.simplefilter(
"ignore")
1034 binned_image_arr = np.nanmean(arr, axis=(1, 3))
1038 sub_index = (pixels_shifted[order][pixel_counter]
1039 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2))
1042 exp = exposures[(band, order - 1)]
1046 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr
1047 elif sub_index == 1:
1048 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr
1049 elif sub_index == 2:
1050 exp.image.array[npix//2:, npix//2:] = binned_image_arr
1051 elif sub_index == 3:
1052 exp.image.array[0: npix//2, npix//2:] = binned_image_arr
1055 raise ValueError(
"Illegal pixel sub index")
1058 if order < max_order:
1059 exposures[(band, order)].image.array[:, :] = np.nan
1064 band_pixels = np.array([pixel
1065 for pixel, band_
in hips_exposure_handle_dict.keys()
1067 band_pixels = np.sort(band_pixels)
1069 self._write_properties_and_moc(
1070 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1078 self._write_allsky_file(
1079 hips_base_path.join(f
"band_{band}", forceDirectory=
True),
1083 self._write_properties_and_moc(
1084 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1092 self._write_allsky_file(
1093 hips_base_path.join(f
"color_{colorstr}", forceDirectory=
True),
1097 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9):
1098 """Write a HiPS image.
1102 hips_base_path : `lsst.resources.ResourcePath`
1103 Resource path to the base of the HiPS directory tree.
1105 HEALPix order of the HiPS image to write.
1107 HEALPix pixel of the HiPS image.
1110 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1111 Mapping to convert image to scaled png.
1112 shift_order : `int`, optional
1120 dir_number = self._get_dir_number(pixel)
1121 hips_dir = hips_base_path.join(
1129 wcs = makeHpxWcs(order, pixel, shift_order=shift_order)
1131 uri = hips_dir.join(f
"Npix{pixel}.fits")
1133 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1134 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata())
1136 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1140 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8)
1141 vals[~np.isfinite(image.array) | (image.array < 0)] = 0
1142 im = Image.fromarray(vals[::-1, :],
"L")
1144 uri = hips_dir.join(f
"Npix{pixel}.png")
1146 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1147 im.save(temporary_uri.ospath)
1149 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1151 def _write_hips_color_png(
1161 """Write a color png HiPS image.
1165 hips_base_path : `lsst.resources.ResourcePath`
1166 Resource path to the base of the HiPS directory tree.
1168 HEALPix order of the HiPS image to write.
1170 HEALPix pixel of the HiPS image.
1172 Input for red channel of output png.
1174 Input
for green channel of output png.
1176 Input
for blue channel of output png.
1177 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1178 Mapping to convert image to scaled png.
1185 dir_number = self._get_dir_number(pixel)
1186 hips_dir = hips_base_path.join(
1195 arr_red = image_red.array.copy()
1196 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0]
1197 arr_green = image_green.array.copy()
1198 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1]
1199 arr_blue = image_blue.array.copy()
1200 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2]
1202 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue)
1204 im = Image.fromarray(image_array[::-1, :, :], mode=
"RGB")
1206 uri = hips_dir.join(f
"Npix{pixel}.png")
1208 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1209 im.save(temporary_uri.ospath)
1211 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1213 def _write_properties_and_moc(
1223 """Write HiPS properties file and MOC.
1227 hips_base_path : : `lsst.resources.ResourcePath`
1228 Resource path to the base of the HiPS directory tree.
1230 Maximum HEALPix order.
1231 pixels : `np.ndarray` (N,)
1232 Array of pixels used.
1234 Sample HPX exposure used for generating HiPS tiles.
1240 Is band multiband / color?
1242 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels)
1244 initial_ra = self.config.properties.initial_ra
1245 initial_dec = self.config.properties.initial_dec
1246 initial_fov = self.config.properties.initial_fov
1248 if initial_ra
is None or initial_dec
is None or initial_fov
is None:
1251 temp_pixels = pixels.copy()
1252 if temp_pixels.size % 2 == 0:
1253 temp_pixels = np.append(temp_pixels, [temp_pixels[0]])
1254 medpix = int(np.median(temp_pixels))
1255 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix)
1256 _initial_fov = hpg.nside_to_resolution(2**max_order, units=
'arcminutes')/60.
1258 if initial_ra
is None or initial_dec
is None:
1259 initial_ra = _initial_ra
1260 initial_dec = _initial_dec
1261 if initial_fov
is None:
1262 initial_fov = _initial_fov
1264 self._write_hips_properties_file(
1266 self.config.properties,
1279 self._write_hips_moc_file(
1285 def _write_hips_properties_file(
1299 """Write HiPS properties file.
1303 hips_base_path : `lsst.resources.ResourcePath`
1304 ResourcePath at top of HiPS tree. File will be written
1305 to this path as ``properties``.
1306 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig`
1307 Configuration
for properties values.
1309 Name of band(s)
for HiPS tree.
1311 Is multiband / color?
1313 Sample HPX exposure used
for generating HiPS tiles.
1315 Maximum HEALPix order.
1319 Coverage area
in square degrees.
1320 initial_ra : `float`
1321 Initial HiPS RA position (degrees).
1322 initial_dec : `float`
1323 Initial HiPS Dec position (degrees).
1324 initial_fov : `float`
1325 Initial HiPS display size (degrees).
1331 def _write_property(fh, name, value):
1332 """Write a property name/value to a file handle.
1336 fh : file handle (blah)
1345 if re.search(
r"\s", name):
1346 raise ValueError(f
"``{name}`` cannot contain any space characters.")
1348 raise ValueError(f
"``{name}`` cannot contain an ``=``")
1350 fh.write(f
"{name:25}= {value}\n")
1352 if exposure.image.array.dtype == np.dtype(
"float32"):
1354 elif exposure.image.array.dtype == np.dtype(
"float64"):
1356 elif exposure.image.array.dtype == np.dtype(
"int32"):
1359 date_iso8601 = datetime.utcnow().isoformat(timespec=
"seconds") +
"Z"
1360 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units=
'degrees')
1362 uri = hips_base_path.join(
"properties")
1363 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1364 with open(temporary_uri.ospath,
"w")
as fh:
1368 properties_config.creator_did_template.format(band=band),
1370 if properties_config.obs_collection
is not None:
1371 _write_property(fh,
"obs_collection", properties_config.obs_collection)
1375 properties_config.obs_title_template.format(band=band),
1377 if properties_config.obs_description_template
is not None:
1381 properties_config.obs_description_template.format(band=band),
1383 if len(properties_config.prov_progenitor) > 0:
1384 for prov_progenitor
in properties_config.prov_progenitor:
1385 _write_property(fh,
"prov_progenitor", prov_progenitor)
1386 if properties_config.obs_ack
is not None:
1387 _write_property(fh,
"obs_ack", properties_config.obs_ack)
1388 _write_property(fh,
"obs_regime",
"Optical")
1389 _write_property(fh,
"data_pixel_bitpix",
str(bitpix))
1390 _write_property(fh,
"dataproduct_type",
"image")
1391 _write_property(fh,
"moc_sky_fraction",
str(area/41253.))
1392 _write_property(fh,
"data_ucd",
"phot.flux")
1393 _write_property(fh,
"hips_creation_date", date_iso8601)
1394 _write_property(fh,
"hips_builder",
"lsst.pipe.tasks.hips.GenerateHipsTask")
1395 _write_property(fh,
"hips_creator",
"Vera C. Rubin Observatory")
1396 _write_property(fh,
"hips_version",
"1.4")
1397 _write_property(fh,
"hips_release_date", date_iso8601)
1398 _write_property(fh,
"hips_frame",
"equatorial")
1399 _write_property(fh,
"hips_order",
str(max_order))
1400 _write_property(fh,
"hips_tile_width",
str(exposure.getBBox().getWidth()))
1401 _write_property(fh,
"hips_status",
"private master clonableOnce")
1403 _write_property(fh,
"hips_tile_format",
"png")
1404 _write_property(fh,
"dataproduct_subtype",
"color")
1406 _write_property(fh,
"hips_tile_format",
"png fits")
1407 _write_property(fh,
"hips_pixel_bitpix",
str(bitpix))
1408 _write_property(fh,
"hips_pixel_scale",
str(pixel_scale))
1409 _write_property(fh,
"hips_initial_ra",
str(initial_ra))
1410 _write_property(fh,
"hips_initial_dec",
str(initial_dec))
1411 _write_property(fh,
"hips_initial_fov",
str(initial_fov))
1413 if self.config.blue_channel_band
in properties_config.spectral_ranges:
1414 em_min = properties_config.spectral_ranges[
1415 self.config.blue_channel_band
1418 self.log.warning(
"blue band %s not in self.config.spectral_ranges.", band)
1420 if self.config.red_channel_band
in properties_config.spectral_ranges:
1421 em_max = properties_config.spectral_ranges[
1422 self.config.red_channel_band
1425 self.log.warning(
"red band %s not in self.config.spectral_ranges.", band)
1428 if band
in properties_config.spectral_ranges:
1429 em_min = properties_config.spectral_ranges[band].lambda_min/1e9
1430 em_max = properties_config.spectral_ranges[band].lambda_max/1e9
1432 self.log.warning(
"band %s not in self.config.spectral_ranges.", band)
1435 _write_property(fh,
"em_min",
str(em_min))
1436 _write_property(fh,
"em_max",
str(em_max))
1437 if properties_config.t_min
is not None:
1438 _write_property(fh,
"t_min", properties_config.t_min)
1439 if properties_config.t_max
is not None:
1440 _write_property(fh,
"t_max", properties_config.t_max)
1442 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1444 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1):
1445 """Write HiPS MOC file.
1449 hips_base_path : `lsst.resources.ResourcePath`
1450 ResourcePath to top of HiPS tree. File will be written as
1451 to this path
as ``Moc.fits``.
1453 Maximum HEALPix order.
1454 pixels : `np.ndarray`
1455 Array of pixels covered.
1456 min_uniq_order : `int`, optional
1457 Minimum HEALPix order
for looking
for fully covered pixels.
1465 uniq = 4*(4**max_order) + pixels
1468 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32)
1469 hspmap[pixels] = 1.0
1472 for uniq_order
in range(max_order - 1, min_uniq_order - 1, -1):
1473 hspmap = hspmap.degrade(2**uniq_order, reduction=
"sum")
1474 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order))
1476 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero()
1477 if covered.size == 0:
1481 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered]
1484 uniq = np.unique(uniq)
1487 tbl = np.zeros(uniq.size, dtype=[(
"UNIQ",
"i8")])
1490 order = np.log2(tbl[
"UNIQ"]//4).astype(np.int32)//2
1491 moc_order = np.max(order)
1493 hdu = fits.BinTableHDU(tbl)
1494 hdu.header[
"PIXTYPE"] =
"HEALPIX"
1495 hdu.header[
"ORDERING"] =
"NUNIQ"
1496 hdu.header[
"COORDSYS"] =
"C"
1497 hdu.header[
"MOCORDER"] = moc_order
1498 hdu.header[
"MOCTOOL"] =
"lsst.pipe.tasks.hips.GenerateHipsTask"
1500 uri = hips_base_path.join(
"Moc.fits")
1502 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1503 hdu.writeto(temporary_uri.ospath)
1505 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1507 def _write_allsky_file(self, hips_base_path, allsky_order):
1508 """Write an Allsky.png file.
1512 hips_base_path : `lsst.resources.ResourcePath`
1513 Resource path to the base of the HiPS directory tree.
1514 allsky_order : `int`
1515 HEALPix order of the minimum order to make allsky file.
1517 tile_size = self.config.allsky_tilesize
1518 n_tiles_per_side = int(np.sqrt(hpg.nside_to_npixel(hpg.order_to_nside(allsky_order))))
1522 allsky_order_uri = hips_base_path.join(f
"Norder{allsky_order}", forceDirectory=
True)
1523 pixel_regex = re.compile(
r"Npix([0-9]+)\.png$")
1525 ResourcePath.findFileResources(
1526 candidates=[allsky_order_uri],
1527 file_filter=pixel_regex,
1531 for png_uri
in png_uris:
1532 matches = re.match(pixel_regex, png_uri.basename())
1533 pix_num = int(matches.group(1))
1534 tile_image = Image.open(io.BytesIO(png_uri.read()))
1535 row = math.floor(pix_num//n_tiles_per_side)
1536 column = pix_num % n_tiles_per_side
1537 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size)
1538 tile_image_shrunk = tile_image.resize((tile_size, tile_size))
1540 if allsky_image
is None:
1541 allsky_image = Image.new(
1543 (n_tiles_per_side*tile_size, n_tiles_per_side*tile_size),
1545 allsky_image.paste(tile_image_shrunk, box)
1547 uri = allsky_order_uri.join(
"Allsky.png")
1549 with ResourcePath.temporary_uri(suffix=uri.getExtension())
as temporary_uri:
1550 allsky_image.save(temporary_uri.ospath)
1552 uri.transfer_from(temporary_uri, transfer=
"copy", overwrite=
True)
1554 def _get_dir_number(self, pixel):
1555 """Compute the directory number from a pixel.
1560 HEALPix pixel number.
1565 HiPS directory number.
1567 return (pixel//10000)*10000
1570class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections,
1571 dimensions=(
"instrument", ),
1572 defaultTemplates={
"coaddName":
"deep"}):
1573 hips_exposure_handles = pipeBase.connectionTypes.Input(
1574 doc=
"HiPS-compatible HPX images.",
1575 name=
"{coaddName}Coadd_hpx",
1576 storageClass=
"ExposureF",
1577 dimensions=(
"healpix11",
"band"),
1583class GenerateColorHipsConfig(GenerateHipsConfig,
1584 pipelineConnections=GenerateColorHipsConnections):
1585 """Configuration parameters for GenerateColorHipsTask."""
1586 blue_channel_band = pexConfig.Field(
1587 doc=
"Band to use for blue channel of color pngs.",
1591 green_channel_band = pexConfig.Field(
1592 doc=
"Band to use for green channel of color pngs.",
1596 red_channel_band = pexConfig.Field(
1597 doc=
"Band to use for red channel of color pngs.",
1601 png_color_asinh_minimum = pexConfig.Field(
1602 doc=
"AsinhMapping intensity to be mapped to black for color png scaling.",
1606 png_color_asinh_stretch = pexConfig.Field(
1607 doc=
"AsinhMapping linear stretch for color png scaling.",
1611 png_color_asinh_softening = pexConfig.Field(
1612 doc=
"AsinhMapping softening parameter (Q) for color png scaling.",
1618class GenerateColorHipsTask(GenerateHipsTask):
1619 """Task for making a HiPS tree with color pngs."""
1620 ConfigClass = GenerateColorHipsConfig
1621 _DefaultName =
"generateColorHips"
1624 def _check_data_bands(self, data_bands):
1625 """Check the data for configured bands.
1627 Warn if any color bands are missing data.
1631 data_bands : `set` [`str`]
1632 Bands
from the input data.
1636 bands : `list` [`str`]
1637 List of bands
in bgr color order.
1639 if len(data_bands) == 0:
1640 raise RuntimeError(
"GenerateColorHipsTask must have data from at least one band.")
1642 if self.config.blue_channel_band
not in data_bands:
1644 "Color png blue_channel_band %s not in dataset.",
1645 self.config.blue_channel_band
1647 if self.config.green_channel_band
not in data_bands:
1649 "Color png green_channel_band %s not in dataset.",
1650 self.config.green_channel_band
1652 if self.config.red_channel_band
not in data_bands:
1654 "Color png red_channel_band %s not in dataset.",
1655 self.config.red_channel_band
1659 self.config.blue_channel_band,
1660 self.config.green_channel_band,
1661 self.config.red_channel_band,