Coverage for python/lsst/pipe/tasks/hips.py: 14%
617 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-26 11:30 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-26 11:30 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Tasks for making and manipulating HIPS images."""
24__all__ = ["HighResolutionHipsTask", "HighResolutionHipsConfig", "HighResolutionHipsConnections",
25 "GenerateHipsTask", "GenerateHipsConfig", "GenerateColorHipsTask", "GenerateColorHipsConfig"]
27from collections import defaultdict
28import numpy as np
29import argparse
30import io
31import sys
32import re
33import warnings
34import math
35from datetime import datetime
36import hpgeom as hpg
37import healsparse as hsp
38from astropy.io import fits
39from astropy.visualization.lupton_rgb import AsinhMapping
40from PIL import Image
42from lsst.sphgeom import RangeSet, HealpixPixelization
43from lsst.utils.timer import timeMethod
44from lsst.daf.butler import Butler, DataCoordinate, DatasetRef, Quantum
45import lsst.pex.config as pexConfig
46import lsst.pipe.base as pipeBase
47import lsst.afw.geom as afwGeom
48import lsst.afw.math as afwMath
49import lsst.afw.image as afwImage
50import lsst.geom as geom
51from lsst.afw.geom import makeHpxWcs
52from lsst.resources import ResourcePath
54from .healSparseMapping import _is_power_of_two
57class HighResolutionHipsConnections(pipeBase.PipelineTaskConnections,
58 dimensions=("healpix9", "band"),
59 defaultTemplates={"coaddName": "deep"}):
60 coadd_exposure_handles = pipeBase.connectionTypes.Input(
61 doc="Coadded exposures to convert to HIPS format.",
62 name="{coaddName}Coadd_calexp",
63 storageClass="ExposureF",
64 dimensions=("tract", "patch", "skymap", "band"),
65 multiple=True,
66 deferLoad=True,
67 )
68 hips_exposures = pipeBase.connectionTypes.Output(
69 doc="HiPS-compatible HPX image.",
70 name="{coaddName}Coadd_hpx",
71 storageClass="ExposureF",
72 dimensions=("healpix11", "band"),
73 multiple=True,
74 )
76 def __init__(self, *, config=None):
77 super().__init__(config=config)
79 quantum_order = None
80 for dim in self.dimensions:
81 if "healpix" in dim:
82 if quantum_order is not None:
83 raise ValueError("Must not specify more than one quantum healpix dimension.")
84 quantum_order = int(dim.split("healpix")[1])
85 if quantum_order is None:
86 raise ValueError("Must specify a healpix dimension in quantum dimensions.")
88 if quantum_order > config.hips_order:
89 raise ValueError("Quantum healpix dimension order must not be greater than hips_order")
91 order = None
92 for dim in self.hips_exposures.dimensions:
93 if "healpix" in dim:
94 if order is not None:
95 raise ValueError("Must not specify more than one healpix dimension.")
96 order = int(dim.split("healpix")[1])
97 if order is None:
98 raise ValueError("Must specify a healpix dimension in hips_exposure dimensions.")
100 if order != config.hips_order:
101 raise ValueError("healpix dimension order must match config.hips_order.")
104class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig,
105 pipelineConnections=HighResolutionHipsConnections):
106 """Configuration parameters for HighResolutionHipsTask.
108 Notes
109 -----
110 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to
111 2**hips_order. Each cell is 'shift_order' orders deeper than the HEALPix
112 cell, with 2**shift_order x 2**shift_order sub-pixels on a side, which
113 defines the target resolution of the HiPS image. The IVOA recommends
114 shift_order=9, for 2**9=512 pixels on a side.
116 Table 5 from
117 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf
118 shows the relationship between hips_order, number of tiles (full
119 sky coverage), cell size, and sub-pixel size/image resolution (with
120 the default shift_order=9):
121 +------------+-----------------+--------------+------------------+
122 | hips_order | Number of Tiles | Cell Size | Image Resolution |
123 +============+=================+==============+==================+
124 | 0 | 12 | 58.63 deg | 6.871 arcmin |
125 | 1 | 48 | 29.32 deg | 3.435 arcmin |
126 | 2 | 192 | 14.66 deg | 1.718 arcmin |
127 | 3 | 768 | 7.329 deg | 51.53 arcsec |
128 | 4 | 3072 | 3.665 deg | 25.77 arcsec |
129 | 5 | 12288 | 1.832 deg | 12.88 arcsec |
130 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec |
131 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec |
132 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec |
133 | 9 | 3145728 | 6.871 arcmin | 805.2mas |
134 | 10 | 12582912 | 3.435 arcmin | 402.6mas |
135 | 11 | 50331648 | 1.718 arcmin | 201.3mas |
136 | 12 | 201326592 | 51.53 arcsec | 100.6mas |
137 | 13 | 805306368 | 25.77 arcsec | 50.32mas |
138 +------------+-----------------+--------------+------------------+
139 """
140 hips_order = pexConfig.Field(
141 doc="HIPS image order.",
142 dtype=int,
143 default=11,
144 )
145 shift_order = pexConfig.Field(
146 doc="HIPS shift order (such that each tile is 2**shift_order pixels on a side)",
147 dtype=int,
148 default=9,
149 )
150 warp = pexConfig.ConfigField(
151 dtype=afwMath.Warper.ConfigClass,
152 doc="Warper configuration",
153 )
155 def setDefaults(self):
156 self.warp.warpingKernelName = "lanczos5"
159class HipsTaskNameDescriptor:
160 """Descriptor used create a DefaultName that matches the order of
161 the defined dimensions in the connections class.
163 Parameters
164 ----------
165 prefix : `str`
166 The prefix of the Default name, to which the order will be
167 appended.
168 """
169 def __init__(self, prefix):
170 # create a defaultName template
171 self._defaultName = f"{prefix}{{}}"
172 self._order = None
174 def __get__(self, obj, klass=None):
175 if klass is None:
176 raise RuntimeError(
177 "HipsTaskDescriptor was used in an unexpected context"
178 )
179 if self._order is None:
180 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions
181 for dim in klassDimensions:
182 if (match := re.match(r"^healpix(\d*)$", dim)) is not None:
183 self._order = int(match.group(1))
184 break
185 else:
186 raise RuntimeError(
187 "Could not find healpix dimension in connections class"
188 )
189 return self._defaultName.format(self._order)
192class HighResolutionHipsTask(pipeBase.PipelineTask):
193 """Task for making high resolution HiPS images."""
194 ConfigClass = HighResolutionHipsConfig
195 _DefaultName = HipsTaskNameDescriptor("highResolutionHips")
197 def __init__(self, **kwargs):
198 super().__init__(**kwargs)
199 self.warper = afwMath.Warper.fromConfig(self.config.warp)
201 @timeMethod
202 def runQuantum(self, butlerQC, inputRefs, outputRefs):
203 inputs = butlerQC.get(inputRefs)
205 healpix_dim = f"healpix{self.config.hips_order}"
207 pixels = [hips_exposure.dataId[healpix_dim]
208 for hips_exposure in outputRefs.hips_exposures]
210 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs["coadd_exposure_handles"])
212 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]:
213 hips_exposure_ref for hips_exposure_ref in outputRefs.hips_exposures}
214 for pixel, hips_exposure in outputs.hips_exposures.items():
215 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel])
217 def run(self, pixels, coadd_exposure_handles):
218 """Run the HighResolutionHipsTask.
220 Parameters
221 ----------
222 pixels : `Iterable` [ `int` ]
223 Iterable of healpix pixels (nest ordering) to warp to.
224 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
225 Handles for the coadd exposures.
227 Returns
228 -------
229 outputs : `lsst.pipe.base.Struct`
230 ``hips_exposures`` is a dict with pixel (key) and hips_exposure (value)
231 """
232 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order)
234 npix = 2**self.config.shift_order
235 bbox_hpx = geom.Box2I(corner=geom.Point2I(0, 0),
236 dimensions=geom.Extent2I(npix, npix))
238 # For each healpix pixel we will create an empty exposure with the
239 # correct HPX WCS. We furthermore create a dict to hold each of
240 # the warps that will go into each HPX exposure.
241 exp_hpx_dict = {}
242 warp_dict = {}
243 for pixel in pixels:
244 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order)
245 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx)
246 exp_hpx_dict[pixel] = exp_hpx
247 warp_dict[pixel] = []
249 first_handle = True
250 # Loop over input coadd exposures to minimize i/o (this speeds things
251 # up by ~8x to batch together pixels that overlap a given coadd).
252 for handle in coadd_exposure_handles:
253 coadd_exp = handle.get()
255 # For each pixel, warp the coadd to the HPX WCS for the pixel.
256 for pixel in pixels:
257 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx)
259 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs())
260 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
262 if first_handle:
263 # Make sure the mask planes, filter, and photocalib of the output
264 # exposure match the (first) input exposure.
265 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict())
266 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter())
267 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib())
269 if warped.getBBox().getArea() == 0 or not np.any(np.isfinite(warped.image.array)):
270 # There is no overlap, skip.
271 self.log.debug(
272 "No overlap between output HPX %d and input exposure %s",
273 pixel,
274 handle.dataId
275 )
276 continue
278 exp.maskedImage.assign(warped.maskedImage, warped.getBBox())
279 warp_dict[pixel].append(exp.maskedImage)
281 first_handle = False
283 stats_flags = afwMath.stringToStatisticsProperty("MEAN")
284 stats_ctrl = afwMath.StatisticsControl()
285 stats_ctrl.setNanSafe(True)
286 stats_ctrl.setWeighted(True)
287 stats_ctrl.setCalcErrorFromInputVariance(True)
289 # Loop over pixels and combine the warps for each pixel.
290 # The combination is done with a simple mean for pixels that
291 # overlap in neighboring patches.
292 for pixel in pixels:
293 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan)
295 if not warp_dict[pixel]:
296 # Nothing in this pixel
297 self.log.debug("No data in HPX pixel %d", pixel)
298 # Remove the pixel from the output, no need to persist an
299 # empty exposure.
300 exp_hpx_dict.pop(pixel)
301 continue
303 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack(
304 warp_dict[pixel],
305 stats_flags,
306 stats_ctrl,
307 [1.0]*len(warp_dict[pixel]),
308 clipped=0,
309 maskMap=[]
310 )
312 return pipeBase.Struct(hips_exposures=exp_hpx_dict)
314 @classmethod
315 def build_quantum_graph_cli(cls, argv):
316 """A command-line interface entry point to `build_quantum_graph`.
317 This method provides the implementation for the
318 ``build-high-resolution-hips-qg`` script.
320 Parameters
321 ----------
322 argv : `Sequence` [ `str` ]
323 Command-line arguments (e.g. ``sys.argv[1:]``).
324 """
325 parser = cls._make_cli_parser()
327 args = parser.parse_args(argv)
329 if args.subparser_name is None:
330 parser.print_help()
331 sys.exit(1)
333 pipeline = pipeBase.Pipeline.from_uri(args.pipeline)
334 expanded_pipeline = list(pipeline.toExpandedPipeline())
336 if len(expanded_pipeline) != 1:
337 raise RuntimeError(f"Pipeline file {args.pipeline} may only contain one task.")
339 (task_def,) = expanded_pipeline
341 butler = Butler(args.butler_config, collections=args.input)
343 if args.subparser_name == "segment":
344 # Do the segmentation
345 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order)
346 dataset = task_def.connections.coadd_exposure_handles.name
347 data_ids = set(butler.registry.queryDataIds("tract", datasets=dataset).expanded())
348 region_pixels = []
349 for data_id in data_ids:
350 region = data_id.region
351 pixel_range = hpix_pixelization.envelope(region)
352 for r in pixel_range.ranges():
353 region_pixels.extend(range(r[0], r[1]))
354 indices = np.unique(region_pixels)
356 print(f"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:")
357 for pixel in indices:
358 print(pixel)
360 elif args.subparser_name == "build":
361 # Build the quantum graph.
363 # Figure out collection names.
364 if args.output_run is None:
365 if args.output is None:
366 raise ValueError("At least one of --output or --output-run options is required.")
367 args.output_run = "{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp())
369 build_ranges = RangeSet(sorted(args.pixels))
371 # Metadata includes a subset of attributes defined in CmdLineFwk.
372 metadata = {
373 "input": args.input,
374 "butler_argument": args.butler_config,
375 "output": args.output,
376 "output_run": args.output_run,
377 "data_query": args.where,
378 "time": f"{datetime.now()}",
379 }
381 qg = cls.build_quantum_graph(
382 task_def,
383 butler.registry,
384 args.hpix_build_order,
385 build_ranges,
386 where=args.where,
387 collections=args.input,
388 metadata=metadata,
389 )
390 qg.saveUri(args.save_qgraph)
392 @classmethod
393 def _make_cli_parser(cls):
394 """Make the command-line parser.
396 Returns
397 -------
398 parser : `argparse.ArgumentParser`
399 """
400 parser = argparse.ArgumentParser(
401 description=(
402 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets."
403 ),
404 )
405 subparsers = parser.add_subparsers(help="sub-command help", dest="subparser_name")
407 parser_segment = subparsers.add_parser("segment",
408 help="Determine survey segments for workflow.")
409 parser_build = subparsers.add_parser("build",
410 help="Build quantum graph for HighResolutionHipsTask")
412 for sub in [parser_segment, parser_build]:
413 # These arguments are in common.
414 sub.add_argument(
415 "-b",
416 "--butler-config",
417 type=str,
418 help="Path to data repository or butler configuration.",
419 required=True,
420 )
421 sub.add_argument(
422 "-p",
423 "--pipeline",
424 type=str,
425 help="Pipeline file, limited to one task.",
426 required=True,
427 )
428 sub.add_argument(
429 "-i",
430 "--input",
431 type=str,
432 nargs="+",
433 help="Input collection(s) to search for coadd exposures.",
434 required=True,
435 )
436 sub.add_argument(
437 "-o",
438 "--hpix_build_order",
439 type=int,
440 default=1,
441 help="HEALPix order to segment sky for building quantum graph files.",
442 )
443 sub.add_argument(
444 "-w",
445 "--where",
446 type=str,
447 default=None,
448 help="Data ID expression used when querying for input coadd datasets.",
449 )
451 parser_build.add_argument(
452 "--output",
453 type=str,
454 help=(
455 "Name of the output CHAINED collection. If this options is specified and "
456 "--output-run is not, then a new RUN collection will be created by appending "
457 "a timestamp to the value of this option."
458 ),
459 default=None,
460 metavar="COLL",
461 )
462 parser_build.add_argument(
463 "--output-run",
464 type=str,
465 help=(
466 "Output RUN collection to write resulting images. If not provided "
467 "then --output must be provided and a new RUN collection will be created "
468 "by appending a timestamp to the value passed with --output."
469 ),
470 default=None,
471 metavar="RUN",
472 )
473 parser_build.add_argument(
474 "-q",
475 "--save-qgraph",
476 type=str,
477 help="Output filename for QuantumGraph.",
478 required=True,
479 )
480 parser_build.add_argument(
481 "-P",
482 "--pixels",
483 type=int,
484 nargs="+",
485 help="Pixels at --hpix_build_order to generate quantum graph.",
486 required=True,
487 )
489 return parser
491 @classmethod
492 def build_quantum_graph(
493 cls,
494 task_def,
495 registry,
496 constraint_order,
497 constraint_ranges,
498 where=None,
499 collections=None,
500 metadata=None,
501 ):
502 """Generate a `QuantumGraph` for running just this task.
504 This is a temporary workaround for incomplete butler query support for
505 HEALPix dimensions.
507 Parameters
508 ----------
509 task_def : `lsst.pipe.base.TaskDef`
510 Task definition.
511 registry : `lsst.daf.butler.Registry`
512 Client for the butler database. May be read-only.
513 constraint_order : `int`
514 HEALPix order used to contrain which quanta are generated, via
515 ``constraint_indices``. This should be a coarser grid (smaller
516 order) than the order used for the task's quantum and output data
517 IDs, and ideally something between the spatial scale of a patch or
518 the data repository's "common skypix" system (usually ``htm7``).
519 constraint_ranges : `lsst.sphgeom.RangeSet`
520 RangeSet which describes constraint pixels (HEALPix NEST, with order
521 constraint_order) to constrain generated quanta.
522 where : `str`, optional
523 A boolean `str` expression of the form accepted by
524 `Registry.queryDatasets` to constrain input datasets. This may
525 contain a constraint on tracts, patches, or bands, but not HEALPix
526 indices. Constraints on tracts and patches should usually be
527 unnecessary, however - existing coadds that overlap the given
528 HEALpix indices will be selected without such a constraint, and
529 providing one may reject some that should normally be included.
530 collections : `str` or `Iterable` [ `str` ], optional
531 Collection or collections to search for input datasets, in order.
532 If not provided, ``registry.defaults.collections`` will be
533 searched.
534 metadata : `dict` [ `str`, `Any` ]
535 Graph metadata. It is required to contain "output_run" key with the
536 name of the output RUN collection.
537 """
538 config = task_def.config
540 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry)
541 # Since we know this is the only task in the pipeline, we know there
542 # is only one overall input and one overall output.
543 (input_dataset_type,) = dataset_types.inputs
545 # Extract the main output dataset type (which needs multiple
546 # DatasetRefs, and tells us the output HPX level), and make a set of
547 # what remains for more mechanical handling later.
548 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name]
549 incidental_output_dataset_types = dataset_types.outputs.copy()
550 incidental_output_dataset_types.remove(output_dataset_type)
551 (hpx_output_dimension,) = (
552 registry.dimensions.skypix_dimensions[d] for d in output_dataset_type.dimensions.skypix.names
553 )
555 constraint_hpx_pixelization = registry.dimensions[f"healpix{constraint_order}"].pixelization
556 common_skypix_name = registry.dimensions.commonSkyPix.name
557 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization
559 # We will need all the pixels at the quantum resolution as well
560 task_dimensions = registry.dimensions.conform(task_def.connections.dimensions)
561 (hpx_dimension,) = (
562 registry.dimensions.skypix_dimensions[d] for d in task_dimensions.names if d != "band"
563 )
564 hpx_pixelization = hpx_dimension.pixelization
566 if hpx_pixelization.level < constraint_order:
567 raise ValueError(f"Quantum order {hpx_pixelization.level} must be < {constraint_order}")
568 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order))
570 # We can be generous in looking for pixels here, because we constraint by actual
571 # patch regions below.
572 common_skypix_ranges = RangeSet()
573 for begin, end in constraint_ranges:
574 for hpx_index in range(begin, end):
575 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index)
576 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region)
578 # To keep the query from getting out of hand (and breaking) we simplify until we have fewer
579 # than 100 ranges which seems to work fine.
580 for simp in range(1, 10):
581 if len(common_skypix_ranges) < 100:
582 break
583 common_skypix_ranges.simplify(simp)
585 # Use that RangeSet to assemble a WHERE constraint expression. This
586 # could definitely get too big if the "constraint healpix" order is too
587 # fine.
588 where_terms = []
589 bind = {}
590 for n, (begin, end) in enumerate(common_skypix_ranges):
591 stop = end - 1 # registry range syntax is inclusive
592 if begin == stop:
593 where_terms.append(f"{common_skypix_name} = cpx{n}")
594 bind[f"cpx{n}"] = begin
595 else:
596 where_terms.append(f"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)")
597 bind[f"cpx{n}a"] = begin
598 bind[f"cpx{n}b"] = stop
599 if where is None:
600 where = " OR ".join(where_terms)
601 else:
602 where = f"({where}) AND ({' OR '.join(where_terms)})"
603 # Query for input datasets with this constraint, and ask for expanded
604 # data IDs because we want regions. Immediately group this by patch so
605 # we don't do later geometric stuff n_bands more times than we need to.
606 input_refs = registry.queryDatasets(
607 input_dataset_type,
608 where=where,
609 findFirst=True,
610 collections=collections,
611 bind=bind
612 ).expanded()
613 inputs_by_patch = defaultdict(set)
614 patch_dimensions = registry.dimensions.conform(["patch"])
615 for input_ref in input_refs:
616 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref)
617 if not inputs_by_patch:
618 message_body = "\n".join(input_refs.explain_no_results())
619 raise RuntimeError(f"No inputs found:\n{message_body}")
621 # Iterate over patches and compute the set of output healpix pixels
622 # that overlap each one. Use that to associate inputs with output
623 # pixels, but only for the output pixels we've already identified.
624 inputs_by_hpx = defaultdict(set)
625 for patch_data_id, input_refs_for_patch in inputs_by_patch.items():
626 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region)
627 for begin, end in patch_hpx_ranges & hpx_ranges:
628 for hpx_index in range(begin, end):
629 inputs_by_hpx[hpx_index].update(input_refs_for_patch)
630 # Iterate over the dict we just created and create the actual quanta.
631 quanta = []
632 output_run = metadata["output_run"]
633 for hpx_index, input_refs_for_hpx_index in inputs_by_hpx.items():
634 # Group inputs by band.
635 input_refs_by_band = defaultdict(list)
636 for input_ref in input_refs_for_hpx_index:
637 input_refs_by_band[input_ref.dataId["band"]].append(input_ref)
638 # Iterate over bands to make quanta.
639 for band, input_refs_for_band in input_refs_by_band.items():
640 data_id = registry.expandDataId({hpx_dimension: hpx_index, "band": band})
642 hpx_pixel_ranges = RangeSet(hpx_index)
643 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level))
644 output_data_ids = []
645 for begin, end in hpx_output_ranges:
646 for hpx_output_index in range(begin, end):
647 output_data_ids.append(
648 registry.expandDataId({hpx_output_dimension: hpx_output_index, "band": band})
649 )
650 outputs = {
651 dt: [DatasetRef(dt, data_id, run=output_run)] for dt in incidental_output_dataset_types
652 }
653 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id, run=output_run)
654 for data_id in output_data_ids]
655 quanta.append(
656 Quantum(
657 taskName=task_def.taskName,
658 taskClass=task_def.taskClass,
659 dataId=data_id,
660 initInputs={},
661 inputs={input_dataset_type: input_refs_for_band},
662 outputs=outputs,
663 )
664 )
666 if len(quanta) == 0:
667 raise RuntimeError("Given constraints yielded empty quantum graph.")
669 # Define initOutputs refs.
670 empty_data_id = DataCoordinate.make_empty(registry.dimensions)
671 init_outputs = {}
672 global_init_outputs = []
673 if config_dataset_type := dataset_types.initOutputs.get(task_def.configDatasetName):
674 init_outputs[task_def] = [DatasetRef(config_dataset_type, empty_data_id, run=output_run)]
675 packages_dataset_name = pipeBase.PipelineDatasetTypes.packagesDatasetName
676 if packages_dataset_type := dataset_types.initOutputs.get(packages_dataset_name):
677 global_init_outputs.append(DatasetRef(packages_dataset_type, empty_data_id, run=output_run))
679 return pipeBase.QuantumGraph(
680 quanta={task_def: quanta},
681 initOutputs=init_outputs,
682 globalInitOutputs=global_init_outputs,
683 metadata=metadata,
684 )
687class HipsPropertiesSpectralTerm(pexConfig.Config):
688 lambda_min = pexConfig.Field(
689 doc="Minimum wavelength (nm)",
690 dtype=float,
691 )
692 lambda_max = pexConfig.Field(
693 doc="Maximum wavelength (nm)",
694 dtype=float,
695 )
698class HipsPropertiesConfig(pexConfig.Config):
699 """Configuration parameters for writing a HiPS properties file."""
700 creator_did_template = pexConfig.Field(
701 doc=("Unique identifier of the HiPS - Format: IVOID. "
702 "Use ``{band}`` to substitute the band name."),
703 dtype=str,
704 optional=False,
705 )
706 obs_collection = pexConfig.Field(
707 doc="Short name of original data set - Format: one word",
708 dtype=str,
709 optional=True,
710 )
711 obs_description_template = pexConfig.Field(
712 doc=("Data set description - Format: free text, longer free text "
713 "description of the dataset. Use ``{band}`` to substitute "
714 "the band name."),
715 dtype=str,
716 )
717 prov_progenitor = pexConfig.ListField(
718 doc="Provenance of the original data - Format: free text",
719 dtype=str,
720 default=[],
721 )
722 obs_title_template = pexConfig.Field(
723 doc=("Data set title format: free text, but should be short. "
724 "Use ``{band}`` to substitute the band name."),
725 dtype=str,
726 optional=False,
727 )
728 spectral_ranges = pexConfig.ConfigDictField(
729 doc=("Mapping from band to lambda_min, lamba_max (nm). May be approximate."),
730 keytype=str,
731 itemtype=HipsPropertiesSpectralTerm,
732 default={},
733 )
734 initial_ra = pexConfig.Field(
735 doc="Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.",
736 dtype=float,
737 optional=True,
738 )
739 initial_dec = pexConfig.Field(
740 doc="Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.",
741 dtype=float,
742 optional=True,
743 )
744 initial_fov = pexConfig.Field(
745 doc="Initial field-of-view (deg). If not set will use ~1 healpix tile.",
746 dtype=float,
747 optional=True,
748 )
749 obs_ack = pexConfig.Field(
750 doc="Observation acknowledgements (free text).",
751 dtype=str,
752 optional=True,
753 )
754 t_min = pexConfig.Field(
755 doc="Time (MJD) of earliest observation included in HiPS",
756 dtype=float,
757 optional=True,
758 )
759 t_max = pexConfig.Field(
760 doc="Time (MJD) of latest observation included in HiPS",
761 dtype=float,
762 optional=True,
763 )
765 def validate(self):
766 super().validate()
768 if self.obs_collection is not None:
769 if re.search(r"\s", self.obs_collection):
770 raise ValueError("obs_collection cannot contain any space characters.")
772 def setDefaults(self):
773 # Values here taken from
774 # https://github.com/lsst-dm/dax_obscore/blob/44ac15029136e2ec15/configs/dp02.yaml#L46
775 u_term = HipsPropertiesSpectralTerm()
776 u_term.lambda_min = 330.
777 u_term.lambda_max = 400.
778 self.spectral_ranges["u"] = u_term
779 g_term = HipsPropertiesSpectralTerm()
780 g_term.lambda_min = 402.
781 g_term.lambda_max = 552.
782 self.spectral_ranges["g"] = g_term
783 r_term = HipsPropertiesSpectralTerm()
784 r_term.lambda_min = 552.
785 r_term.lambda_max = 691.
786 self.spectral_ranges["r"] = r_term
787 i_term = HipsPropertiesSpectralTerm()
788 i_term.lambda_min = 691.
789 i_term.lambda_max = 818.
790 self.spectral_ranges["i"] = i_term
791 z_term = HipsPropertiesSpectralTerm()
792 z_term.lambda_min = 818.
793 z_term.lambda_max = 922.
794 self.spectral_ranges["z"] = z_term
795 y_term = HipsPropertiesSpectralTerm()
796 y_term.lambda_min = 970.
797 y_term.lambda_max = 1060.
798 self.spectral_ranges["y"] = y_term
801class GenerateHipsConnections(pipeBase.PipelineTaskConnections,
802 dimensions=("instrument", "band"),
803 defaultTemplates={"coaddName": "deep"}):
804 hips_exposure_handles = pipeBase.connectionTypes.Input(
805 doc="HiPS-compatible HPX images.",
806 name="{coaddName}Coadd_hpx",
807 storageClass="ExposureF",
808 dimensions=("healpix11", "band"),
809 multiple=True,
810 deferLoad=True,
811 )
814class GenerateHipsConfig(pipeBase.PipelineTaskConfig,
815 pipelineConnections=GenerateHipsConnections):
816 """Configuration parameters for GenerateHipsTask."""
817 # WARNING: In general PipelineTasks are not allowed to do any outputs
818 # outside of the butler. This task has been given (temporary)
819 # Special Dispensation because of the nature of HiPS outputs until
820 # a more controlled solution can be found.
821 hips_base_uri = pexConfig.Field(
822 doc="URI to HiPS base for output.",
823 dtype=str,
824 optional=False,
825 )
826 min_order = pexConfig.Field(
827 doc="Minimum healpix order for HiPS tree.",
828 dtype=int,
829 default=3,
830 )
831 properties = pexConfig.ConfigField(
832 dtype=HipsPropertiesConfig,
833 doc="Configuration for properties file.",
834 )
835 allsky_tilesize = pexConfig.Field(
836 dtype=int,
837 doc="Allsky tile size; must be power of 2. HiPS standard recommends 64x64 tiles.",
838 default=64,
839 check=_is_power_of_two,
840 )
841 png_gray_asinh_minimum = pexConfig.Field(
842 doc="AsinhMapping intensity to be mapped to black for grayscale png scaling.",
843 dtype=float,
844 default=0.0,
845 )
846 png_gray_asinh_stretch = pexConfig.Field(
847 doc="AsinhMapping linear stretch for grayscale png scaling.",
848 dtype=float,
849 default=2.0,
850 )
851 png_gray_asinh_softening = pexConfig.Field(
852 doc="AsinhMapping softening parameter (Q) for grayscale png scaling.",
853 dtype=float,
854 default=8.0,
855 )
858class GenerateHipsTask(pipeBase.PipelineTask):
859 """Task for making a HiPS tree with FITS and grayscale PNGs."""
860 ConfigClass = GenerateHipsConfig
861 _DefaultName = "generateHips"
862 color_task = False
864 @timeMethod
865 def runQuantum(self, butlerQC, inputRefs, outputRefs):
866 inputs = butlerQC.get(inputRefs)
868 dims = inputRefs.hips_exposure_handles[0].dataId.dimensions.names
869 order = None
870 for dim in dims:
871 if "healpix" in dim:
872 order = int(dim.split("healpix")[1])
873 healpix_dim = dim
874 break
875 else:
876 raise RuntimeError("Could not determine healpix order for input exposures.")
878 hips_exposure_handle_dict = {
879 (hips_exposure_handle.dataId[healpix_dim],
880 hips_exposure_handle.dataId["band"]): hips_exposure_handle
881 for hips_exposure_handle in inputs["hips_exposure_handles"]
882 }
884 data_bands = {hips_exposure_handle.dataId["band"]
885 for hips_exposure_handle in inputs["hips_exposure_handles"]}
886 bands = self._check_data_bands(data_bands)
888 self.run(
889 bands=bands,
890 max_order=order,
891 hips_exposure_handle_dict=hips_exposure_handle_dict,
892 do_color=self.color_task,
893 )
895 def _check_data_bands(self, data_bands):
896 """Check that the data has only a single band.
898 Parameters
899 ----------
900 data_bands : `set` [`str`]
901 Bands from the input data.
903 Returns
904 -------
905 bands : `list` [`str`]
906 List of single band to process.
908 Raises
909 ------
910 RuntimeError if there is not exactly one band.
911 """
912 if len(data_bands) != 1:
913 raise RuntimeError("GenerateHipsTask can only use data from a single band.")
915 return list(data_bands)
917 @timeMethod
918 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False):
919 """Run the GenerateHipsTask.
921 Parameters
922 ----------
923 bands : `list [ `str` ]
924 List of bands to be processed (or single band).
925 max_order : `int`
926 HEALPix order of the maximum (native) HPX exposures.
927 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`}
928 Dict of handles for the HiPS high-resolution exposures.
929 Key is (pixel number, ``band``).
930 do_color : `bool`, optional
931 Do color pngs instead of per-band grayscale.
932 """
933 min_order = self.config.min_order
935 if not do_color:
936 png_grayscale_mapping = AsinhMapping(
937 self.config.png_gray_asinh_minimum,
938 self.config.png_gray_asinh_stretch,
939 Q=self.config.png_gray_asinh_softening,
940 )
941 else:
942 png_color_mapping = AsinhMapping(
943 self.config.png_color_asinh_minimum,
944 self.config.png_color_asinh_stretch,
945 Q=self.config.png_color_asinh_softening,
946 )
948 bcb = self.config.blue_channel_band
949 gcb = self.config.green_channel_band
950 rcb = self.config.red_channel_band
951 colorstr = f"{bcb}{gcb}{rcb}"
953 # The base path is based on the hips_base_uri.
954 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=True)
956 # We need to unique-ify the pixels because they show up for multiple bands.
957 # The output of this is a sorted array.
958 pixels = np.unique(np.array([pixel for pixel, _ in hips_exposure_handle_dict.keys()]))
960 # Add a "gutter" pixel at the end. Start with 0 which maps to 0 always.
961 pixels = np.append(pixels, [0])
963 # Convert the pixels to each order that will be generated.
964 pixels_shifted = {}
965 pixels_shifted[max_order] = pixels
966 for order in range(max_order - 1, min_order - 1, -1):
967 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2)
969 # And set the gutter to an illegal pixel value.
970 for order in range(min_order, max_order + 1):
971 pixels_shifted[order][-1] = -1
973 # Read in the first pixel for determining image properties.
974 exp0 = list(hips_exposure_handle_dict.values())[0].get()
975 bbox = exp0.getBBox()
976 npix = bbox.getWidth()
977 shift_order = int(np.round(np.log2(npix)))
979 # Create blank exposures for each level, including the highest order.
980 # We also make sure we create blank exposures for any bands used in the color
981 # PNGs, even if they aren't available.
982 exposures = {}
983 for band in bands:
984 for order in range(min_order, max_order + 1):
985 exp = exp0.Factory(bbox=bbox)
986 exp.image.array[:, :] = np.nan
987 exposures[(band, order)] = exp
989 # Loop over all pixels, avoiding the gutter.
990 for pixel_counter, pixel in enumerate(pixels[:-1]):
991 self.log.debug("Working on high resolution pixel %d", pixel)
992 for band in bands:
993 # Read all the exposures here for the highest order.
994 # There will always be at least one band with a HiPS image available
995 # at the highest order. However, for color images it is possible that
996 # not all bands have coverage so we require this check.
997 if (pixel, band) in hips_exposure_handle_dict:
998 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get()
1000 # Go up the HiPS tree.
1001 # We only write pixels and rebin to fill the parent pixel when we are
1002 # done with a current pixel, which is determined if the next pixel
1003 # has a different pixel number.
1004 for order in range(max_order, min_order - 1, -1):
1005 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]:
1006 # This order is not done, and so none of the other orders will be.
1007 break
1009 # We can now write out the images for each band.
1010 # Note this will always trigger at the max order where each pixel is unique.
1011 if not do_color:
1012 for band in bands:
1013 self._write_hips_image(
1014 hips_base_path.join(f"band_{band}", forceDirectory=True),
1015 order,
1016 pixels_shifted[order][pixel_counter],
1017 exposures[(band, order)].image,
1018 png_grayscale_mapping,
1019 shift_order=shift_order,
1020 )
1021 else:
1022 # Make a color png.
1023 self._write_hips_color_png(
1024 hips_base_path.join(f"color_{colorstr}", forceDirectory=True),
1025 order,
1026 pixels_shifted[order][pixel_counter],
1027 exposures[(self.config.red_channel_band, order)].image,
1028 exposures[(self.config.green_channel_band, order)].image,
1029 exposures[(self.config.blue_channel_band, order)].image,
1030 png_color_mapping,
1031 )
1033 log_level = self.log.INFO if order == (max_order - 3) else self.log.DEBUG
1034 self.log.log(
1035 log_level,
1036 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)",
1037 ",".join(bands),
1038 order,
1039 pixels_shifted[order][pixel_counter],
1040 pixel_counter,
1041 len(pixels) - 1,
1042 )
1044 # When we are at the top of the tree, erase top level images and continue.
1045 if order == min_order:
1046 for band in bands:
1047 exposures[(band, order)].image.array[:, :] = np.nan
1048 continue
1050 # Now average the images for each band.
1051 for band in bands:
1052 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2)
1053 with warnings.catch_warnings():
1054 warnings.simplefilter("ignore")
1055 binned_image_arr = np.nanmean(arr, axis=(1, 3))
1057 # Fill the next level up. We figure out which of the four
1058 # sub-pixels the current pixel occupies.
1059 sub_index = (pixels_shifted[order][pixel_counter]
1060 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2))
1062 # Fill exposure at the next level up.
1063 exp = exposures[(band, order - 1)]
1065 # Fill the correct subregion.
1066 if sub_index == 0:
1067 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr
1068 elif sub_index == 1:
1069 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr
1070 elif sub_index == 2:
1071 exp.image.array[npix//2:, npix//2:] = binned_image_arr
1072 elif sub_index == 3:
1073 exp.image.array[0: npix//2, npix//2:] = binned_image_arr
1074 else:
1075 # This should be impossible.
1076 raise ValueError("Illegal pixel sub index")
1078 # Erase the previous exposure.
1079 if order < max_order:
1080 exposures[(band, order)].image.array[:, :] = np.nan
1082 # Write the properties files and MOCs.
1083 if not do_color:
1084 for band in bands:
1085 band_pixels = np.array([pixel
1086 for pixel, band_ in hips_exposure_handle_dict.keys()
1087 if band_ == band])
1088 band_pixels = np.sort(band_pixels)
1090 self._write_properties_and_moc(
1091 hips_base_path.join(f"band_{band}", forceDirectory=True),
1092 max_order,
1093 band_pixels,
1094 exp0,
1095 shift_order,
1096 band,
1097 False,
1098 )
1099 self._write_allsky_file(
1100 hips_base_path.join(f"band_{band}", forceDirectory=True),
1101 min_order,
1102 )
1103 else:
1104 self._write_properties_and_moc(
1105 hips_base_path.join(f"color_{colorstr}", forceDirectory=True),
1106 max_order,
1107 pixels[:-1],
1108 exp0,
1109 shift_order,
1110 colorstr,
1111 True,
1112 )
1113 self._write_allsky_file(
1114 hips_base_path.join(f"color_{colorstr}", forceDirectory=True),
1115 min_order,
1116 )
1118 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9):
1119 """Write a HiPS image.
1121 Parameters
1122 ----------
1123 hips_base_path : `lsst.resources.ResourcePath`
1124 Resource path to the base of the HiPS directory tree.
1125 order : `int`
1126 HEALPix order of the HiPS image to write.
1127 pixel : `int`
1128 HEALPix pixel of the HiPS image.
1129 image : `lsst.afw.image.Image`
1130 Image to write.
1131 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1132 Mapping to convert image to scaled png.
1133 shift_order : `int`, optional
1134 HPX shift_order.
1135 """
1136 # WARNING: In general PipelineTasks are not allowed to do any outputs
1137 # outside of the butler. This task has been given (temporary)
1138 # Special Dispensation because of the nature of HiPS outputs until
1139 # a more controlled solution can be found.
1141 dir_number = self._get_dir_number(pixel)
1142 hips_dir = hips_base_path.join(
1143 f"Norder{order}",
1144 forceDirectory=True
1145 ).join(
1146 f"Dir{dir_number}",
1147 forceDirectory=True
1148 )
1150 wcs = makeHpxWcs(order, pixel, shift_order=shift_order)
1152 uri = hips_dir.join(f"Npix{pixel}.fits")
1154 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1155 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata())
1157 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1159 # And make a grayscale png as well
1161 with np.errstate(invalid="ignore"):
1162 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8)
1164 vals[~np.isfinite(image.array) | (image.array < 0)] = 0
1165 im = Image.fromarray(vals[::-1, :], "L")
1167 uri = hips_dir.join(f"Npix{pixel}.png")
1169 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1170 im.save(temporary_uri.ospath)
1172 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1174 def _write_hips_color_png(
1175 self,
1176 hips_base_path,
1177 order,
1178 pixel,
1179 image_red,
1180 image_green,
1181 image_blue,
1182 png_mapping,
1183 ):
1184 """Write a color png HiPS image.
1186 Parameters
1187 ----------
1188 hips_base_path : `lsst.resources.ResourcePath`
1189 Resource path to the base of the HiPS directory tree.
1190 order : `int`
1191 HEALPix order of the HiPS image to write.
1192 pixel : `int`
1193 HEALPix pixel of the HiPS image.
1194 image_red : `lsst.afw.image.Image`
1195 Input for red channel of output png.
1196 image_green : `lsst.afw.image.Image`
1197 Input for green channel of output png.
1198 image_blue : `lsst.afw.image.Image`
1199 Input for blue channel of output png.
1200 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping`
1201 Mapping to convert image to scaled png.
1202 """
1203 # WARNING: In general PipelineTasks are not allowed to do any outputs
1204 # outside of the butler. This task has been given (temporary)
1205 # Special Dispensation because of the nature of HiPS outputs until
1206 # a more controlled solution can be found.
1208 dir_number = self._get_dir_number(pixel)
1209 hips_dir = hips_base_path.join(
1210 f"Norder{order}",
1211 forceDirectory=True
1212 ).join(
1213 f"Dir{dir_number}",
1214 forceDirectory=True
1215 )
1217 # We need to convert nans to the minimum values in the mapping.
1218 arr_red = image_red.array.copy()
1219 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0]
1220 arr_green = image_green.array.copy()
1221 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1]
1222 arr_blue = image_blue.array.copy()
1223 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2]
1225 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue)
1227 im = Image.fromarray(image_array[::-1, :, :], mode="RGB")
1229 uri = hips_dir.join(f"Npix{pixel}.png")
1231 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1232 im.save(temporary_uri.ospath)
1234 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1236 def _write_properties_and_moc(
1237 self,
1238 hips_base_path,
1239 max_order,
1240 pixels,
1241 exposure,
1242 shift_order,
1243 band,
1244 multiband
1245 ):
1246 """Write HiPS properties file and MOC.
1248 Parameters
1249 ----------
1250 hips_base_path : : `lsst.resources.ResourcePath`
1251 Resource path to the base of the HiPS directory tree.
1252 max_order : `int`
1253 Maximum HEALPix order.
1254 pixels : `np.ndarray` (N,)
1255 Array of pixels used.
1256 exposure : `lsst.afw.image.Exposure`
1257 Sample HPX exposure used for generating HiPS tiles.
1258 shift_order : `int`
1259 HPX shift order.
1260 band : `str`
1261 Band (or color).
1262 multiband : `bool`
1263 Is band multiband / color?
1264 """
1265 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels)
1267 initial_ra = self.config.properties.initial_ra
1268 initial_dec = self.config.properties.initial_dec
1269 initial_fov = self.config.properties.initial_fov
1271 if initial_ra is None or initial_dec is None or initial_fov is None:
1272 # We want to point to an arbitrary pixel in the footprint.
1273 # Just take the median pixel value for simplicity.
1274 temp_pixels = pixels.copy()
1275 if temp_pixels.size % 2 == 0:
1276 temp_pixels = np.append(temp_pixels, [temp_pixels[0]])
1277 medpix = int(np.median(temp_pixels))
1278 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix)
1279 _initial_fov = hpg.nside_to_resolution(2**max_order, units='arcminutes')/60.
1281 if initial_ra is None or initial_dec is None:
1282 initial_ra = _initial_ra
1283 initial_dec = _initial_dec
1284 if initial_fov is None:
1285 initial_fov = _initial_fov
1287 self._write_hips_properties_file(
1288 hips_base_path,
1289 self.config.properties,
1290 band,
1291 multiband,
1292 exposure,
1293 max_order,
1294 shift_order,
1295 area,
1296 initial_ra,
1297 initial_dec,
1298 initial_fov,
1299 )
1301 # Write the MOC coverage
1302 self._write_hips_moc_file(
1303 hips_base_path,
1304 max_order,
1305 pixels,
1306 )
1308 def _write_hips_properties_file(
1309 self,
1310 hips_base_path,
1311 properties_config,
1312 band,
1313 multiband,
1314 exposure,
1315 max_order,
1316 shift_order,
1317 area,
1318 initial_ra,
1319 initial_dec,
1320 initial_fov
1321 ):
1322 """Write HiPS properties file.
1324 Parameters
1325 ----------
1326 hips_base_path : `lsst.resources.ResourcePath`
1327 ResourcePath at top of HiPS tree. File will be written
1328 to this path as ``properties``.
1329 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig`
1330 Configuration for properties values.
1331 band : `str`
1332 Name of band(s) for HiPS tree.
1333 multiband : `bool`
1334 Is multiband / color?
1335 exposure : `lsst.afw.image.Exposure`
1336 Sample HPX exposure used for generating HiPS tiles.
1337 max_order : `int`
1338 Maximum HEALPix order.
1339 shift_order : `int`
1340 HPX shift order.
1341 area : `float`
1342 Coverage area in square degrees.
1343 initial_ra : `float`
1344 Initial HiPS RA position (degrees).
1345 initial_dec : `float`
1346 Initial HiPS Dec position (degrees).
1347 initial_fov : `float`
1348 Initial HiPS display size (degrees).
1349 """
1350 # WARNING: In general PipelineTasks are not allowed to do any outputs
1351 # outside of the butler. This task has been given (temporary)
1352 # Special Dispensation because of the nature of HiPS outputs until
1353 # a more controlled solution can be found.
1354 def _write_property(fh, name, value):
1355 """Write a property name/value to a file handle.
1357 Parameters
1358 ----------
1359 fh : file handle (blah)
1360 Open for writing.
1361 name : `str`
1362 Name of property
1363 value : `str`
1364 Value of property
1365 """
1366 # This ensures that the name has no spaces or space-like characters,
1367 # per the HiPS standard.
1368 if re.search(r"\s", name):
1369 raise ValueError(f"``{name}`` cannot contain any space characters.")
1370 if "=" in name:
1371 raise ValueError(f"``{name}`` cannot contain an ``=``")
1373 fh.write(f"{name:25}= {value}\n")
1375 if exposure.image.array.dtype == np.dtype("float32"):
1376 bitpix = -32
1377 elif exposure.image.array.dtype == np.dtype("float64"):
1378 bitpix = -64
1379 elif exposure.image.array.dtype == np.dtype("int32"):
1380 bitpix = 32
1382 date_iso8601 = datetime.utcnow().isoformat(timespec="seconds") + "Z"
1383 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units='degrees')
1385 uri = hips_base_path.join("properties")
1386 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1387 with open(temporary_uri.ospath, "w") as fh:
1388 _write_property(
1389 fh,
1390 "creator_did",
1391 properties_config.creator_did_template.format(band=band),
1392 )
1393 if properties_config.obs_collection is not None:
1394 _write_property(fh, "obs_collection", properties_config.obs_collection)
1395 _write_property(
1396 fh,
1397 "obs_title",
1398 properties_config.obs_title_template.format(band=band),
1399 )
1400 if properties_config.obs_description_template is not None:
1401 _write_property(
1402 fh,
1403 "obs_description",
1404 properties_config.obs_description_template.format(band=band),
1405 )
1406 if len(properties_config.prov_progenitor) > 0:
1407 for prov_progenitor in properties_config.prov_progenitor:
1408 _write_property(fh, "prov_progenitor", prov_progenitor)
1409 if properties_config.obs_ack is not None:
1410 _write_property(fh, "obs_ack", properties_config.obs_ack)
1411 _write_property(fh, "obs_regime", "Optical")
1412 _write_property(fh, "data_pixel_bitpix", str(bitpix))
1413 _write_property(fh, "dataproduct_type", "image")
1414 _write_property(fh, "moc_sky_fraction", str(area/41253.))
1415 _write_property(fh, "data_ucd", "phot.flux")
1416 _write_property(fh, "hips_creation_date", date_iso8601)
1417 _write_property(fh, "hips_builder", "lsst.pipe.tasks.hips.GenerateHipsTask")
1418 _write_property(fh, "hips_creator", "Vera C. Rubin Observatory")
1419 _write_property(fh, "hips_version", "1.4")
1420 _write_property(fh, "hips_release_date", date_iso8601)
1421 _write_property(fh, "hips_frame", "equatorial")
1422 _write_property(fh, "hips_order", str(max_order))
1423 _write_property(fh, "hips_tile_width", str(exposure.getBBox().getWidth()))
1424 _write_property(fh, "hips_status", "private master clonableOnce")
1425 if multiband:
1426 _write_property(fh, "hips_tile_format", "png")
1427 _write_property(fh, "dataproduct_subtype", "color")
1428 else:
1429 _write_property(fh, "hips_tile_format", "png fits")
1430 _write_property(fh, "hips_pixel_bitpix", str(bitpix))
1431 _write_property(fh, "hips_pixel_scale", str(pixel_scale))
1432 _write_property(fh, "hips_initial_ra", str(initial_ra))
1433 _write_property(fh, "hips_initial_dec", str(initial_dec))
1434 _write_property(fh, "hips_initial_fov", str(initial_fov))
1435 if multiband:
1436 if self.config.blue_channel_band in properties_config.spectral_ranges:
1437 em_min = properties_config.spectral_ranges[
1438 self.config.blue_channel_band
1439 ].lambda_min/1e9
1440 else:
1441 self.log.warning("blue band %s not in self.config.spectral_ranges.", band)
1442 em_min = 3e-7
1443 if self.config.red_channel_band in properties_config.spectral_ranges:
1444 em_max = properties_config.spectral_ranges[
1445 self.config.red_channel_band
1446 ].lambda_max/1e9
1447 else:
1448 self.log.warning("red band %s not in self.config.spectral_ranges.", band)
1449 em_max = 1e-6
1450 else:
1451 if band in properties_config.spectral_ranges:
1452 em_min = properties_config.spectral_ranges[band].lambda_min/1e9
1453 em_max = properties_config.spectral_ranges[band].lambda_max/1e9
1454 else:
1455 self.log.warning("band %s not in self.config.spectral_ranges.", band)
1456 em_min = 3e-7
1457 em_max = 1e-6
1458 _write_property(fh, "em_min", str(em_min))
1459 _write_property(fh, "em_max", str(em_max))
1460 if properties_config.t_min is not None:
1461 _write_property(fh, "t_min", properties_config.t_min)
1462 if properties_config.t_max is not None:
1463 _write_property(fh, "t_max", properties_config.t_max)
1465 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1467 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1):
1468 """Write HiPS MOC file.
1470 Parameters
1471 ----------
1472 hips_base_path : `lsst.resources.ResourcePath`
1473 ResourcePath to top of HiPS tree. File will be written as
1474 to this path as ``Moc.fits``.
1475 max_order : `int`
1476 Maximum HEALPix order.
1477 pixels : `np.ndarray`
1478 Array of pixels covered.
1479 min_uniq_order : `int`, optional
1480 Minimum HEALPix order for looking for fully covered pixels.
1481 """
1482 # WARNING: In general PipelineTasks are not allowed to do any outputs
1483 # outside of the butler. This task has been given (temporary)
1484 # Special Dispensation because of the nature of HiPS outputs until
1485 # a more controlled solution can be found.
1487 # Make the initial list of UNIQ pixels
1488 uniq = 4*(4**max_order) + pixels
1490 # Make a healsparse map which provides easy degrade/comparisons.
1491 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32)
1492 hspmap[pixels] = 1.0
1494 # Loop over orders, degrade each time, and look for pixels with full coverage.
1495 for uniq_order in range(max_order - 1, min_uniq_order - 1, -1):
1496 hspmap = hspmap.degrade(2**uniq_order, reduction="sum")
1497 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order))
1498 # Check if any of the pixels at uniq_order have full coverage.
1499 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero()
1500 if covered.size == 0:
1501 # No pixels at uniq_order are fully covered, we're done.
1502 break
1503 # Replace the UNIQ pixels that are fully covered.
1504 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered]
1506 # Remove duplicate pixels.
1507 uniq = np.unique(uniq)
1509 # Output to fits.
1510 tbl = np.zeros(uniq.size, dtype=[("UNIQ", "i8")])
1511 tbl["UNIQ"] = uniq
1513 order = np.log2(tbl["UNIQ"]//4).astype(np.int32)//2
1514 moc_order = np.max(order)
1516 hdu = fits.BinTableHDU(tbl)
1517 hdu.header["PIXTYPE"] = "HEALPIX"
1518 hdu.header["ORDERING"] = "NUNIQ"
1519 hdu.header["COORDSYS"] = "C"
1520 hdu.header["MOCORDER"] = moc_order
1521 hdu.header["MOCTOOL"] = "lsst.pipe.tasks.hips.GenerateHipsTask"
1523 uri = hips_base_path.join("Moc.fits")
1525 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1526 hdu.writeto(temporary_uri.ospath)
1528 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1530 def _write_allsky_file(self, hips_base_path, allsky_order):
1531 """Write an Allsky.png file.
1533 Parameters
1534 ----------
1535 hips_base_path : `lsst.resources.ResourcePath`
1536 Resource path to the base of the HiPS directory tree.
1537 allsky_order : `int`
1538 HEALPix order of the minimum order to make allsky file.
1539 """
1540 tile_size = self.config.allsky_tilesize
1542 # The Allsky file format is described in
1543 # https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf
1544 # From S4.3.2:
1545 # The Allsky file is built as an array of tiles, stored side by side in
1546 # the left-to-right order. The width of this array must be the square
1547 # root of the number of the tiles of the order. For instance, the width
1548 # of this array at order 3 is 27 ( (int)sqrt(768) ). To avoid having a
1549 # too large Allsky file, the resolution of each tile may be reduced but
1550 # must stay a power of two (typically 64x64 pixels rather than 512x512).
1552 n_tiles = hpg.nside_to_npixel(hpg.order_to_nside(allsky_order))
1553 n_tiles_wide = int(np.floor(np.sqrt(n_tiles)))
1554 n_tiles_high = int(np.ceil(n_tiles / n_tiles_wide))
1556 allsky_image = None
1558 allsky_order_uri = hips_base_path.join(f"Norder{allsky_order}", forceDirectory=True)
1559 pixel_regex = re.compile(r"Npix([0-9]+)\.png$")
1560 png_uris = list(
1561 ResourcePath.findFileResources(
1562 candidates=[allsky_order_uri],
1563 file_filter=pixel_regex,
1564 )
1565 )
1567 for png_uri in png_uris:
1568 matches = re.match(pixel_regex, png_uri.basename())
1569 pix_num = int(matches.group(1))
1570 tile_image = Image.open(io.BytesIO(png_uri.read()))
1571 row = math.floor(pix_num//n_tiles_wide)
1572 column = pix_num % n_tiles_wide
1573 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size)
1574 tile_image_shrunk = tile_image.resize((tile_size, tile_size))
1576 if allsky_image is None:
1577 allsky_image = Image.new(
1578 tile_image.mode,
1579 (n_tiles_wide*tile_size, n_tiles_high*tile_size),
1580 )
1581 allsky_image.paste(tile_image_shrunk, box)
1583 uri = allsky_order_uri.join("Allsky.png")
1585 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri:
1586 allsky_image.save(temporary_uri.ospath)
1588 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True)
1590 def _get_dir_number(self, pixel):
1591 """Compute the directory number from a pixel.
1593 Parameters
1594 ----------
1595 pixel : `int`
1596 HEALPix pixel number.
1598 Returns
1599 -------
1600 dir_number : `int`
1601 HiPS directory number.
1602 """
1603 return (pixel//10000)*10000
1606class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections,
1607 dimensions=("instrument", ),
1608 defaultTemplates={"coaddName": "deep"}):
1609 hips_exposure_handles = pipeBase.connectionTypes.Input(
1610 doc="HiPS-compatible HPX images.",
1611 name="{coaddName}Coadd_hpx",
1612 storageClass="ExposureF",
1613 dimensions=("healpix11", "band"),
1614 multiple=True,
1615 deferLoad=True,
1616 )
1619class GenerateColorHipsConfig(GenerateHipsConfig,
1620 pipelineConnections=GenerateColorHipsConnections):
1621 """Configuration parameters for GenerateColorHipsTask."""
1622 blue_channel_band = pexConfig.Field(
1623 doc="Band to use for blue channel of color pngs.",
1624 dtype=str,
1625 default="g",
1626 )
1627 green_channel_band = pexConfig.Field(
1628 doc="Band to use for green channel of color pngs.",
1629 dtype=str,
1630 default="r",
1631 )
1632 red_channel_band = pexConfig.Field(
1633 doc="Band to use for red channel of color pngs.",
1634 dtype=str,
1635 default="i",
1636 )
1637 png_color_asinh_minimum = pexConfig.Field(
1638 doc="AsinhMapping intensity to be mapped to black for color png scaling.",
1639 dtype=float,
1640 default=0.0,
1641 )
1642 png_color_asinh_stretch = pexConfig.Field(
1643 doc="AsinhMapping linear stretch for color png scaling.",
1644 dtype=float,
1645 default=5.0,
1646 )
1647 png_color_asinh_softening = pexConfig.Field(
1648 doc="AsinhMapping softening parameter (Q) for color png scaling.",
1649 dtype=float,
1650 default=8.0,
1651 )
1654class GenerateColorHipsTask(GenerateHipsTask):
1655 """Task for making a HiPS tree with color pngs."""
1656 ConfigClass = GenerateColorHipsConfig
1657 _DefaultName = "generateColorHips"
1658 color_task = True
1660 def _check_data_bands(self, data_bands):
1661 """Check the data for configured bands.
1663 Warn if any color bands are missing data.
1665 Parameters
1666 ----------
1667 data_bands : `set` [`str`]
1668 Bands from the input data.
1670 Returns
1671 -------
1672 bands : `list` [`str`]
1673 List of bands in bgr color order.
1674 """
1675 if len(data_bands) == 0:
1676 raise RuntimeError("GenerateColorHipsTask must have data from at least one band.")
1678 if self.config.blue_channel_band not in data_bands:
1679 self.log.warning(
1680 "Color png blue_channel_band %s not in dataset.",
1681 self.config.blue_channel_band
1682 )
1683 if self.config.green_channel_band not in data_bands:
1684 self.log.warning(
1685 "Color png green_channel_band %s not in dataset.",
1686 self.config.green_channel_band
1687 )
1688 if self.config.red_channel_band not in data_bands:
1689 self.log.warning(
1690 "Color png red_channel_band %s not in dataset.",
1691 self.config.red_channel_band
1692 )
1694 bands = [
1695 self.config.blue_channel_band,
1696 self.config.green_channel_band,
1697 self.config.red_channel_band,
1698 ]
1700 return bands