Coverage for python/lsst/pipe/tasks/hips.py: 14%

619 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-10 12:19 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Tasks for making and manipulating HIPS images.""" 

23 

24__all__ = ["HighResolutionHipsTask", "HighResolutionHipsConfig", "HighResolutionHipsConnections", 

25 "HighResolutionHipsQuantumGraphBuilder", 

26 "GenerateHipsTask", "GenerateHipsConfig", "GenerateColorHipsTask", "GenerateColorHipsConfig"] 

27 

28from collections import defaultdict 

29import numpy as np 

30import argparse 

31import io 

32import sys 

33import re 

34import warnings 

35import math 

36from datetime import datetime 

37import hpgeom as hpg 

38import healsparse as hsp 

39from astropy.io import fits 

40from astropy.visualization.lupton_rgb import AsinhMapping 

41from PIL import Image 

42 

43from lsst.sphgeom import RangeSet, HealpixPixelization 

44from lsst.utils.timer import timeMethod 

45from lsst.daf.butler import Butler 

46import lsst.pex.config as pexConfig 

47import lsst.pipe.base as pipeBase 

48from lsst.pipe.base.quantum_graph_builder import QuantumGraphBuilder 

49from lsst.pipe.base.quantum_graph_skeleton import QuantumGraphSkeleton, DatasetKey 

50import lsst.afw.geom as afwGeom 

51import lsst.afw.math as afwMath 

52import lsst.afw.image as afwImage 

53import lsst.geom as geom 

54from lsst.afw.geom import makeHpxWcs 

55from lsst.resources import ResourcePath 

56 

57from .healSparseMapping import _is_power_of_two 

58 

59 

60class HighResolutionHipsConnections(pipeBase.PipelineTaskConnections, 

61 dimensions=("healpix9", "band"), 

62 defaultTemplates={"coaddName": "deep"}): 

63 coadd_exposure_handles = pipeBase.connectionTypes.Input( 

64 doc="Coadded exposures to convert to HIPS format.", 

65 name="{coaddName}Coadd_calexp", 

66 storageClass="ExposureF", 

67 dimensions=("tract", "patch", "skymap", "band"), 

68 multiple=True, 

69 deferLoad=True, 

70 ) 

71 hips_exposures = pipeBase.connectionTypes.Output( 

72 doc="HiPS-compatible HPX image.", 

73 name="{coaddName}Coadd_hpx", 

74 storageClass="ExposureF", 

75 dimensions=("healpix11", "band"), 

76 multiple=True, 

77 ) 

78 

79 def __init__(self, *, config=None): 

80 super().__init__(config=config) 

81 

82 quantum_order = None 

83 for dim in self.dimensions: 

84 if "healpix" in dim: 

85 if quantum_order is not None: 

86 raise ValueError("Must not specify more than one quantum healpix dimension.") 

87 quantum_order = int(dim.split("healpix")[1]) 

88 if quantum_order is None: 

89 raise ValueError("Must specify a healpix dimension in quantum dimensions.") 

90 

91 if quantum_order > config.hips_order: 

92 raise ValueError("Quantum healpix dimension order must not be greater than hips_order") 

93 

94 order = None 

95 for dim in self.hips_exposures.dimensions: 

96 if "healpix" in dim: 

97 if order is not None: 

98 raise ValueError("Must not specify more than one healpix dimension.") 

99 order = int(dim.split("healpix")[1]) 

100 if order is None: 

101 raise ValueError("Must specify a healpix dimension in hips_exposure dimensions.") 

102 

103 if order != config.hips_order: 

104 raise ValueError("healpix dimension order must match config.hips_order.") 

105 

106 

107class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig, 

108 pipelineConnections=HighResolutionHipsConnections): 

109 """Configuration parameters for HighResolutionHipsTask. 

110 

111 Notes 

112 ----- 

113 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to 

114 2**hips_order. Each cell is 'shift_order' orders deeper than the HEALPix 

115 cell, with 2**shift_order x 2**shift_order sub-pixels on a side, which 

116 defines the target resolution of the HiPS image. The IVOA recommends 

117 shift_order=9, for 2**9=512 pixels on a side. 

118 

119 Table 5 from 

120 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf 

121 shows the relationship between hips_order, number of tiles (full 

122 sky coverage), cell size, and sub-pixel size/image resolution (with 

123 the default shift_order=9): 

124 +------------+-----------------+--------------+------------------+ 

125 | hips_order | Number of Tiles | Cell Size | Image Resolution | 

126 +============+=================+==============+==================+ 

127 | 0 | 12 | 58.63 deg | 6.871 arcmin | 

128 | 1 | 48 | 29.32 deg | 3.435 arcmin | 

129 | 2 | 192 | 14.66 deg | 1.718 arcmin | 

130 | 3 | 768 | 7.329 deg | 51.53 arcsec | 

131 | 4 | 3072 | 3.665 deg | 25.77 arcsec | 

132 | 5 | 12288 | 1.832 deg | 12.88 arcsec | 

133 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec | 

134 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec | 

135 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec | 

136 | 9 | 3145728 | 6.871 arcmin | 805.2mas | 

137 | 10 | 12582912 | 3.435 arcmin | 402.6mas | 

138 | 11 | 50331648 | 1.718 arcmin | 201.3mas | 

139 | 12 | 201326592 | 51.53 arcsec | 100.6mas | 

140 | 13 | 805306368 | 25.77 arcsec | 50.32mas | 

141 +------------+-----------------+--------------+------------------+ 

142 """ 

143 hips_order = pexConfig.Field( 

144 doc="HIPS image order.", 

145 dtype=int, 

146 default=11, 

147 ) 

148 shift_order = pexConfig.Field( 

149 doc="HIPS shift order (such that each tile is 2**shift_order pixels on a side)", 

150 dtype=int, 

151 default=9, 

152 ) 

153 warp = pexConfig.ConfigField( 

154 dtype=afwMath.Warper.ConfigClass, 

155 doc="Warper configuration", 

156 ) 

157 

158 def setDefaults(self): 

159 self.warp.warpingKernelName = "lanczos5" 

160 

161 

162class HipsTaskNameDescriptor: 

163 """Descriptor used create a DefaultName that matches the order of 

164 the defined dimensions in the connections class. 

165 

166 Parameters 

167 ---------- 

168 prefix : `str` 

169 The prefix of the Default name, to which the order will be 

170 appended. 

171 """ 

172 def __init__(self, prefix): 

173 # create a defaultName template 

174 self._defaultName = f"{prefix}{{}}" 

175 self._order = None 

176 

177 def __get__(self, obj, klass=None): 

178 if klass is None: 

179 raise RuntimeError( 

180 "HipsTaskDescriptor was used in an unexpected context" 

181 ) 

182 if self._order is None: 

183 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions 

184 for dim in klassDimensions: 

185 if (match := re.match(r"^healpix(\d*)$", dim)) is not None: 

186 self._order = int(match.group(1)) 

187 break 

188 else: 

189 raise RuntimeError( 

190 "Could not find healpix dimension in connections class" 

191 ) 

192 return self._defaultName.format(self._order) 

193 

194 

195class HighResolutionHipsTask(pipeBase.PipelineTask): 

196 """Task for making high resolution HiPS images.""" 

197 ConfigClass = HighResolutionHipsConfig 

198 _DefaultName = HipsTaskNameDescriptor("highResolutionHips") 

199 

200 def __init__(self, **kwargs): 

201 super().__init__(**kwargs) 

202 self.warper = afwMath.Warper.fromConfig(self.config.warp) 

203 

204 @timeMethod 

205 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

206 inputs = butlerQC.get(inputRefs) 

207 

208 healpix_dim = f"healpix{self.config.hips_order}" 

209 

210 pixels = [hips_exposure.dataId[healpix_dim] 

211 for hips_exposure in outputRefs.hips_exposures] 

212 

213 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs["coadd_exposure_handles"]) 

214 

215 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]: 

216 hips_exposure_ref for hips_exposure_ref in outputRefs.hips_exposures} 

217 for pixel, hips_exposure in outputs.hips_exposures.items(): 

218 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel]) 

219 

220 def run(self, pixels, coadd_exposure_handles): 

221 """Run the HighResolutionHipsTask. 

222 

223 Parameters 

224 ---------- 

225 pixels : `Iterable` [ `int` ] 

226 Iterable of healpix pixels (nest ordering) to warp to. 

227 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

228 Handles for the coadd exposures. 

229 

230 Returns 

231 ------- 

232 outputs : `lsst.pipe.base.Struct` 

233 ``hips_exposures`` is a dict with pixel (key) and hips_exposure (value) 

234 """ 

235 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order) 

236 

237 npix = 2**self.config.shift_order 

238 bbox_hpx = geom.Box2I(corner=geom.Point2I(0, 0), 

239 dimensions=geom.Extent2I(npix, npix)) 

240 

241 # For each healpix pixel we will create an empty exposure with the 

242 # correct HPX WCS. We furthermore create a dict to hold each of 

243 # the warps that will go into each HPX exposure. 

244 exp_hpx_dict = {} 

245 warp_dict = {} 

246 for pixel in pixels: 

247 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order) 

248 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx) 

249 exp_hpx_dict[pixel] = exp_hpx 

250 warp_dict[pixel] = [] 

251 

252 first_handle = True 

253 # Loop over input coadd exposures to minimize i/o (this speeds things 

254 # up by ~8x to batch together pixels that overlap a given coadd). 

255 for handle in coadd_exposure_handles: 

256 coadd_exp = handle.get() 

257 

258 # For each pixel, warp the coadd to the HPX WCS for the pixel. 

259 for pixel in pixels: 

260 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx) 

261 

262 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs()) 

263 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

264 

265 if first_handle: 

266 # Make sure the mask planes, filter, and photocalib of the output 

267 # exposure match the (first) input exposure. 

268 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict()) 

269 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter()) 

270 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib()) 

271 

272 if warped.getBBox().getArea() == 0 or not np.any(np.isfinite(warped.image.array)): 

273 # There is no overlap, skip. 

274 self.log.debug( 

275 "No overlap between output HPX %d and input exposure %s", 

276 pixel, 

277 handle.dataId 

278 ) 

279 continue 

280 

281 exp.maskedImage.assign(warped.maskedImage, warped.getBBox()) 

282 warp_dict[pixel].append(exp.maskedImage) 

283 

284 first_handle = False 

285 

286 stats_flags = afwMath.stringToStatisticsProperty("MEAN") 

287 stats_ctrl = afwMath.StatisticsControl() 

288 stats_ctrl.setNanSafe(True) 

289 stats_ctrl.setWeighted(True) 

290 stats_ctrl.setCalcErrorFromInputVariance(True) 

291 

292 # Loop over pixels and combine the warps for each pixel. 

293 # The combination is done with a simple mean for pixels that 

294 # overlap in neighboring patches. 

295 for pixel in pixels: 

296 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

297 

298 if not warp_dict[pixel]: 

299 # Nothing in this pixel 

300 self.log.debug("No data in HPX pixel %d", pixel) 

301 # Remove the pixel from the output, no need to persist an 

302 # empty exposure. 

303 exp_hpx_dict.pop(pixel) 

304 continue 

305 

306 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack( 

307 warp_dict[pixel], 

308 stats_flags, 

309 stats_ctrl, 

310 [1.0]*len(warp_dict[pixel]), 

311 clipped=0, 

312 maskMap=[] 

313 ) 

314 

315 return pipeBase.Struct(hips_exposures=exp_hpx_dict) 

316 

317 @classmethod 

318 def build_quantum_graph_cli(cls, argv): 

319 """A command-line interface entry point to `build_quantum_graph`. 

320 This method provides the implementation for the 

321 ``build-high-resolution-hips-qg`` script. 

322 

323 Parameters 

324 ---------- 

325 argv : `Sequence` [ `str` ] 

326 Command-line arguments (e.g. ``sys.argv[1:]``). 

327 """ 

328 parser = cls._make_cli_parser() 

329 

330 args = parser.parse_args(argv) 

331 

332 if args.subparser_name is None: 

333 parser.print_help() 

334 sys.exit(1) 

335 

336 pipeline = pipeBase.Pipeline.from_uri(args.pipeline) 

337 pipeline_graph = pipeline.to_graph() 

338 

339 if len(pipeline_graph.tasks) != 1: 

340 raise RuntimeError(f"Pipeline file {args.pipeline} may only contain one task.") 

341 

342 (task_node,) = pipeline_graph.tasks.values() 

343 

344 butler = Butler(args.butler_config, collections=args.input) 

345 

346 if args.subparser_name == "segment": 

347 # Do the segmentation 

348 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order) 

349 dataset = task_node.inputs["coadd_exposure_handles"].dataset_type_name 

350 data_ids = set(butler.registry.queryDataIds("tract", datasets=dataset).expanded()) 

351 region_pixels = [] 

352 for data_id in data_ids: 

353 region = data_id.region 

354 pixel_range = hpix_pixelization.envelope(region) 

355 for r in pixel_range.ranges(): 

356 region_pixels.extend(range(r[0], r[1])) 

357 indices = np.unique(region_pixels) 

358 

359 print(f"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:") 

360 for pixel in indices: 

361 print(pixel) 

362 

363 elif args.subparser_name == "build": 

364 # Build the quantum graph. 

365 

366 # Figure out collection names. 

367 if args.output_run is None: 

368 if args.output is None: 

369 raise ValueError("At least one of --output or --output-run options is required.") 

370 args.output_run = "{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp()) 

371 

372 build_ranges = RangeSet(sorted(args.pixels)) 

373 

374 # Metadata includes a subset of attributes defined in CmdLineFwk. 

375 metadata = { 

376 "input": args.input, 

377 "butler_argument": args.butler_config, 

378 "output": args.output, 

379 "output_run": args.output_run, 

380 "data_query": args.where, 

381 "time": f"{datetime.now()}", 

382 } 

383 

384 builder = HighResolutionHipsQuantumGraphBuilder( 

385 pipeline_graph, 

386 butler, 

387 input_collections=args.input, 

388 output_run=args.output_run, 

389 constraint_order=args.hpix_build_order, 

390 constraint_ranges=build_ranges, 

391 where=args.where, 

392 ) 

393 qg = builder.build(metadata, attach_datastore_records=True) 

394 qg.saveUri(args.save_qgraph) 

395 

396 @classmethod 

397 def _make_cli_parser(cls): 

398 """Make the command-line parser. 

399 

400 Returns 

401 ------- 

402 parser : `argparse.ArgumentParser` 

403 """ 

404 parser = argparse.ArgumentParser( 

405 description=( 

406 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets." 

407 ), 

408 ) 

409 subparsers = parser.add_subparsers(help="sub-command help", dest="subparser_name") 

410 

411 parser_segment = subparsers.add_parser("segment", 

412 help="Determine survey segments for workflow.") 

413 parser_build = subparsers.add_parser("build", 

414 help="Build quantum graph for HighResolutionHipsTask") 

415 

416 for sub in [parser_segment, parser_build]: 

417 # These arguments are in common. 

418 sub.add_argument( 

419 "-b", 

420 "--butler-config", 

421 type=str, 

422 help="Path to data repository or butler configuration.", 

423 required=True, 

424 ) 

425 sub.add_argument( 

426 "-p", 

427 "--pipeline", 

428 type=str, 

429 help="Pipeline file, limited to one task.", 

430 required=True, 

431 ) 

432 sub.add_argument( 

433 "-i", 

434 "--input", 

435 type=str, 

436 nargs="+", 

437 help="Input collection(s) to search for coadd exposures.", 

438 required=True, 

439 ) 

440 sub.add_argument( 

441 "-o", 

442 "--hpix_build_order", 

443 type=int, 

444 default=1, 

445 help="HEALPix order to segment sky for building quantum graph files.", 

446 ) 

447 sub.add_argument( 

448 "-w", 

449 "--where", 

450 type=str, 

451 default=None, 

452 help="Data ID expression used when querying for input coadd datasets.", 

453 ) 

454 

455 parser_build.add_argument( 

456 "--output", 

457 type=str, 

458 help=( 

459 "Name of the output CHAINED collection. If this options is specified and " 

460 "--output-run is not, then a new RUN collection will be created by appending " 

461 "a timestamp to the value of this option." 

462 ), 

463 default=None, 

464 metavar="COLL", 

465 ) 

466 parser_build.add_argument( 

467 "--output-run", 

468 type=str, 

469 help=( 

470 "Output RUN collection to write resulting images. If not provided " 

471 "then --output must be provided and a new RUN collection will be created " 

472 "by appending a timestamp to the value passed with --output." 

473 ), 

474 default=None, 

475 metavar="RUN", 

476 ) 

477 parser_build.add_argument( 

478 "-q", 

479 "--save-qgraph", 

480 type=str, 

481 help="Output filename for QuantumGraph.", 

482 required=True, 

483 ) 

484 parser_build.add_argument( 

485 "-P", 

486 "--pixels", 

487 type=int, 

488 nargs="+", 

489 help="Pixels at --hpix_build_order to generate quantum graph.", 

490 required=True, 

491 ) 

492 

493 return parser 

494 

495 

496class HighResolutionHipsQuantumGraphBuilder(QuantumGraphBuilder): 

497 """A custom a `lsst.pipe.base.QuantumGraphBuilder` for running 

498 `HighResolutionHipsTask` only. 

499 

500 This is a temporary workaround for incomplete butler query support for 

501 HEALPix dimensions. 

502 

503 Parameters 

504 ---------- 

505 pipeline_graph : `lsst.pipe.base.PipelineGraph` 

506 Pipeline graph with exactly one task, which must be a configuration 

507 of `HighResolutionHipsTask`. 

508 butler : `lsst.daf.butler.Butler` 

509 Client for the butler data repository. May be read-only. 

510 input_collections : `str` or `Iterable` [ `str` ], optional 

511 Collection or collections to search for input datasets, in order. 

512 If not provided, ``butler.collections`` will be searched. 

513 output_run : `str`, optional 

514 Name of the output collection. If not provided, ``butler.run`` will 

515 be used. 

516 constraint_order : `int` 

517 HEALPix order used to constrain which quanta are generated, via 

518 ``constraint_indices``. This should be a coarser grid (smaller 

519 order) than the order used for the task's quantum and output data 

520 IDs, and ideally something between the spatial scale of a patch or 

521 the data repository's "common skypix" system (usually ``htm7``). 

522 constraint_ranges : `lsst.sphgeom.RangeSet` 

523 RangeSet that describes constraint pixels (HEALPix NEST, with order 

524 ``constraint_order``) to constrain generated quanta. 

525 where : `str`, optional 

526 A boolean `str` expression of the form accepted by 

527 `Registry.queryDatasets` to constrain input datasets. This may 

528 contain a constraint on tracts, patches, or bands, but not HEALPix 

529 indices. Constraints on tracts and patches should usually be 

530 unnecessary, however - existing coadds that overlap the given 

531 HEALpix indices will be selected without such a constraint, and 

532 providing one may reject some that should normally be included. 

533 """ 

534 

535 def __init__( 

536 self, 

537 pipeline_graph, 

538 butler, 

539 *, 

540 input_collections=None, 

541 output_run=None, 

542 constraint_order, 

543 constraint_ranges, 

544 where="", 

545 ): 

546 super().__init__(pipeline_graph, butler, input_collections=input_collections, output_run=output_run) 

547 self.constraint_order = constraint_order 

548 self.constraint_ranges = constraint_ranges 

549 self.where = where 

550 

551 def process_subgraph(self, subgraph): 

552 # Docstring inherited. 

553 (task_node,) = subgraph.tasks.values() 

554 

555 # Since we know this is the only task in the pipeline, we know there 

556 # is only one overall input and one regular output. 

557 (input_dataset_type_node,) = subgraph.inputs_of(task_node.label).values() 

558 assert input_dataset_type_node is not None, "PipelineGraph should be resolved by base class." 

559 (output_edge,) = task_node.outputs.values() 

560 output_dataset_type_node = subgraph.dataset_types[output_edge.parent_dataset_type_name] 

561 (hpx_output_dimension,) = ( 

562 self.butler.dimensions.skypix_dimensions[d] 

563 for d in output_dataset_type_node.dimensions.skypix.names 

564 ) 

565 constraint_hpx_pixelization = ( 

566 self.butler.dimensions.skypix_dimensions[f"healpix{self.constraint_order}"].pixelization 

567 ) 

568 common_skypix_name = self.butler.dimensions.commonSkyPix.name 

569 common_skypix_pixelization = self.butler.dimensions.commonSkyPix.pixelization 

570 

571 # We will need all the pixels at the quantum resolution as well 

572 (hpx_dimension,) = ( 

573 self.butler.dimensions.skypix_dimensions[d] for d in task_node.dimensions.names if d != "band" 

574 ) 

575 hpx_pixelization = hpx_dimension.pixelization 

576 if hpx_pixelization.level < self.constraint_order: 

577 raise ValueError(f"Quantum order {hpx_pixelization.level} must be < {self.constraint_order}") 

578 hpx_ranges = self.constraint_ranges.scaled(4**(hpx_pixelization.level - self.constraint_order)) 

579 

580 # We can be generous in looking for pixels here, because we constrain 

581 # by actual patch regions below. 

582 common_skypix_ranges = RangeSet() 

583 for begin, end in self.constraint_ranges: 

584 for hpx_index in range(begin, end): 

585 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index) 

586 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region) 

587 

588 # To keep the query from getting out of hand (and breaking) we simplify 

589 # until we have fewer than 100 ranges which seems to work fine. 

590 for simp in range(1, 10): 

591 if len(common_skypix_ranges) < 100: 

592 break 

593 common_skypix_ranges.simplify(simp) 

594 

595 # Use that RangeSet to assemble a WHERE constraint expression. This 

596 # could definitely get too big if the "constraint healpix" order is too 

597 # fine. 

598 where_terms = [] 

599 bind = {} 

600 for n, (begin, end) in enumerate(common_skypix_ranges): 

601 stop = end - 1 # registry range syntax is inclusive 

602 if begin == stop: 

603 where_terms.append(f"{common_skypix_name} = cpx{n}") 

604 bind[f"cpx{n}"] = begin 

605 else: 

606 where_terms.append(f"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)") 

607 bind[f"cpx{n}a"] = begin 

608 bind[f"cpx{n}b"] = stop 

609 if not self.where: 

610 where = " OR ".join(where_terms) 

611 else: 

612 where = f"({self.where}) AND ({' OR '.join(where_terms)})" 

613 # Query for input datasets with this constraint, and ask for expanded 

614 # data IDs because we want regions. Immediately group this by patch so 

615 # we don't do later geometric stuff n_bands more times than we need to. 

616 input_refs = self.butler.registry.queryDatasets( 

617 input_dataset_type_node.dataset_type, 

618 where=where, 

619 findFirst=True, 

620 collections=self.input_collections, 

621 bind=bind 

622 ).expanded() 

623 inputs_by_patch = defaultdict(set) 

624 patch_dimensions = self.butler.dimensions.conform(["patch"]) 

625 for input_ref in input_refs: 

626 dataset_key = DatasetKey(input_ref.datasetType.name, input_ref.dataId.required_values) 

627 self.existing_datasets.inputs[dataset_key] = input_ref 

628 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(dataset_key) 

629 if not inputs_by_patch: 

630 message_body = "\n".join(input_refs.explain_no_results()) 

631 raise RuntimeError(f"No inputs found:\n{message_body}") 

632 

633 # Iterate over patches and compute the set of output healpix pixels 

634 # that overlap each one. Use that to associate inputs with output 

635 # pixels, but only for the output pixels we've already identified. 

636 inputs_by_hpx = defaultdict(set) 

637 for patch_data_id, input_keys_for_patch in inputs_by_patch.items(): 

638 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region) 

639 for begin, end in patch_hpx_ranges & hpx_ranges: 

640 for hpx_index in range(begin, end): 

641 inputs_by_hpx[hpx_index].update(input_keys_for_patch) 

642 

643 # Iterate over the dict we just created and create preliminary quanta. 

644 skeleton = QuantumGraphSkeleton([task_node.label]) 

645 for hpx_index, input_keys_for_hpx_index in inputs_by_hpx.items(): 

646 # Group inputs by band. 

647 input_keys_by_band = defaultdict(list) 

648 for input_key in input_keys_for_hpx_index: 

649 input_ref = self.existing_datasets.inputs[input_key] 

650 input_keys_by_band[input_ref.dataId["band"]].append(input_key) 

651 # Iterate over bands to make quanta. 

652 for band, input_keys_for_band in input_keys_by_band.items(): 

653 data_id = self.butler.registry.expandDataId({hpx_dimension.name: hpx_index, "band": band}) 

654 quantum_key = skeleton.add_quantum_node(task_node.label, data_id) 

655 # Add inputs to the skelton 

656 skeleton.add_input_edges(quantum_key, input_keys_for_band) 

657 # Add the regular outputs. 

658 hpx_pixel_ranges = RangeSet(hpx_index) 

659 hpx_output_ranges = hpx_pixel_ranges.scaled( 

660 4**(task_node.config.hips_order - hpx_pixelization.level) 

661 ) 

662 for begin, end in hpx_output_ranges: 

663 for hpx_output_index in range(begin, end): 

664 dataset_key = skeleton.add_dataset_node( 

665 output_dataset_type_node.name, 

666 self.butler.registry.expandDataId( 

667 {hpx_output_dimension: hpx_output_index, "band": band} 

668 ), 

669 ) 

670 skeleton.add_output_edge(quantum_key, dataset_key) 

671 # Add auxiliary outputs (log, metadata). 

672 for write_edge in task_node.iter_all_outputs(): 

673 if write_edge.connection_name == output_edge.connection_name: 

674 continue 

675 dataset_key = skeleton.add_dataset_node(write_edge.parent_dataset_type_name, data_id) 

676 skeleton.add_output_edge(quantum_key, dataset_key) 

677 return skeleton 

678 

679 

680class HipsPropertiesSpectralTerm(pexConfig.Config): 

681 lambda_min = pexConfig.Field( 

682 doc="Minimum wavelength (nm)", 

683 dtype=float, 

684 ) 

685 lambda_max = pexConfig.Field( 

686 doc="Maximum wavelength (nm)", 

687 dtype=float, 

688 ) 

689 

690 

691class HipsPropertiesConfig(pexConfig.Config): 

692 """Configuration parameters for writing a HiPS properties file.""" 

693 creator_did_template = pexConfig.Field( 

694 doc=("Unique identifier of the HiPS - Format: IVOID. " 

695 "Use ``{band}`` to substitute the band name."), 

696 dtype=str, 

697 optional=False, 

698 ) 

699 obs_collection = pexConfig.Field( 

700 doc="Short name of original data set - Format: one word", 

701 dtype=str, 

702 optional=True, 

703 ) 

704 obs_description_template = pexConfig.Field( 

705 doc=("Data set description - Format: free text, longer free text " 

706 "description of the dataset. Use ``{band}`` to substitute " 

707 "the band name."), 

708 dtype=str, 

709 ) 

710 prov_progenitor = pexConfig.ListField( 

711 doc="Provenance of the original data - Format: free text", 

712 dtype=str, 

713 default=[], 

714 ) 

715 obs_title_template = pexConfig.Field( 

716 doc=("Data set title format: free text, but should be short. " 

717 "Use ``{band}`` to substitute the band name."), 

718 dtype=str, 

719 optional=False, 

720 ) 

721 spectral_ranges = pexConfig.ConfigDictField( 

722 doc=("Mapping from band to lambda_min, lamba_max (nm). May be approximate."), 

723 keytype=str, 

724 itemtype=HipsPropertiesSpectralTerm, 

725 default={}, 

726 ) 

727 initial_ra = pexConfig.Field( 

728 doc="Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

729 dtype=float, 

730 optional=True, 

731 ) 

732 initial_dec = pexConfig.Field( 

733 doc="Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

734 dtype=float, 

735 optional=True, 

736 ) 

737 initial_fov = pexConfig.Field( 

738 doc="Initial field-of-view (deg). If not set will use ~1 healpix tile.", 

739 dtype=float, 

740 optional=True, 

741 ) 

742 obs_ack = pexConfig.Field( 

743 doc="Observation acknowledgements (free text).", 

744 dtype=str, 

745 optional=True, 

746 ) 

747 t_min = pexConfig.Field( 

748 doc="Time (MJD) of earliest observation included in HiPS", 

749 dtype=float, 

750 optional=True, 

751 ) 

752 t_max = pexConfig.Field( 

753 doc="Time (MJD) of latest observation included in HiPS", 

754 dtype=float, 

755 optional=True, 

756 ) 

757 

758 def validate(self): 

759 super().validate() 

760 

761 if self.obs_collection is not None: 

762 if re.search(r"\s", self.obs_collection): 

763 raise ValueError("obs_collection cannot contain any space characters.") 

764 

765 def setDefaults(self): 

766 # Values here taken from 

767 # https://github.com/lsst-dm/dax_obscore/blob/44ac15029136e2ec15/configs/dp02.yaml#L46 

768 u_term = HipsPropertiesSpectralTerm() 

769 u_term.lambda_min = 330. 

770 u_term.lambda_max = 400. 

771 self.spectral_ranges["u"] = u_term 

772 g_term = HipsPropertiesSpectralTerm() 

773 g_term.lambda_min = 402. 

774 g_term.lambda_max = 552. 

775 self.spectral_ranges["g"] = g_term 

776 r_term = HipsPropertiesSpectralTerm() 

777 r_term.lambda_min = 552. 

778 r_term.lambda_max = 691. 

779 self.spectral_ranges["r"] = r_term 

780 i_term = HipsPropertiesSpectralTerm() 

781 i_term.lambda_min = 691. 

782 i_term.lambda_max = 818. 

783 self.spectral_ranges["i"] = i_term 

784 z_term = HipsPropertiesSpectralTerm() 

785 z_term.lambda_min = 818. 

786 z_term.lambda_max = 922. 

787 self.spectral_ranges["z"] = z_term 

788 y_term = HipsPropertiesSpectralTerm() 

789 y_term.lambda_min = 970. 

790 y_term.lambda_max = 1060. 

791 self.spectral_ranges["y"] = y_term 

792 

793 

794class GenerateHipsConnections(pipeBase.PipelineTaskConnections, 

795 dimensions=("instrument", "band"), 

796 defaultTemplates={"coaddName": "deep"}): 

797 hips_exposure_handles = pipeBase.connectionTypes.Input( 

798 doc="HiPS-compatible HPX images.", 

799 name="{coaddName}Coadd_hpx", 

800 storageClass="ExposureF", 

801 dimensions=("healpix11", "band"), 

802 multiple=True, 

803 deferLoad=True, 

804 ) 

805 

806 

807class GenerateHipsConfig(pipeBase.PipelineTaskConfig, 

808 pipelineConnections=GenerateHipsConnections): 

809 """Configuration parameters for GenerateHipsTask.""" 

810 # WARNING: In general PipelineTasks are not allowed to do any outputs 

811 # outside of the butler. This task has been given (temporary) 

812 # Special Dispensation because of the nature of HiPS outputs until 

813 # a more controlled solution can be found. 

814 hips_base_uri = pexConfig.Field( 

815 doc="URI to HiPS base for output.", 

816 dtype=str, 

817 optional=False, 

818 ) 

819 min_order = pexConfig.Field( 

820 doc="Minimum healpix order for HiPS tree.", 

821 dtype=int, 

822 default=3, 

823 ) 

824 properties = pexConfig.ConfigField( 

825 dtype=HipsPropertiesConfig, 

826 doc="Configuration for properties file.", 

827 ) 

828 allsky_tilesize = pexConfig.Field( 

829 dtype=int, 

830 doc="Allsky tile size; must be power of 2. HiPS standard recommends 64x64 tiles.", 

831 default=64, 

832 check=_is_power_of_two, 

833 ) 

834 png_gray_asinh_minimum = pexConfig.Field( 

835 doc="AsinhMapping intensity to be mapped to black for grayscale png scaling.", 

836 dtype=float, 

837 default=0.0, 

838 ) 

839 png_gray_asinh_stretch = pexConfig.Field( 

840 doc="AsinhMapping linear stretch for grayscale png scaling.", 

841 dtype=float, 

842 default=2.0, 

843 ) 

844 png_gray_asinh_softening = pexConfig.Field( 

845 doc="AsinhMapping softening parameter (Q) for grayscale png scaling.", 

846 dtype=float, 

847 default=8.0, 

848 ) 

849 

850 

851class GenerateHipsTask(pipeBase.PipelineTask): 

852 """Task for making a HiPS tree with FITS and grayscale PNGs.""" 

853 ConfigClass = GenerateHipsConfig 

854 _DefaultName = "generateHips" 

855 color_task = False 

856 

857 @timeMethod 

858 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

859 inputs = butlerQC.get(inputRefs) 

860 

861 dims = inputRefs.hips_exposure_handles[0].dataId.dimensions.names 

862 order = None 

863 for dim in dims: 

864 if "healpix" in dim: 

865 order = int(dim.split("healpix")[1]) 

866 healpix_dim = dim 

867 break 

868 else: 

869 raise RuntimeError("Could not determine healpix order for input exposures.") 

870 

871 hips_exposure_handle_dict = { 

872 (hips_exposure_handle.dataId[healpix_dim], 

873 hips_exposure_handle.dataId["band"]): hips_exposure_handle 

874 for hips_exposure_handle in inputs["hips_exposure_handles"] 

875 } 

876 

877 data_bands = {hips_exposure_handle.dataId["band"] 

878 for hips_exposure_handle in inputs["hips_exposure_handles"]} 

879 bands = self._check_data_bands(data_bands) 

880 

881 self.run( 

882 bands=bands, 

883 max_order=order, 

884 hips_exposure_handle_dict=hips_exposure_handle_dict, 

885 do_color=self.color_task, 

886 ) 

887 

888 def _check_data_bands(self, data_bands): 

889 """Check that the data has only a single band. 

890 

891 Parameters 

892 ---------- 

893 data_bands : `set` [`str`] 

894 Bands from the input data. 

895 

896 Returns 

897 ------- 

898 bands : `list` [`str`] 

899 List of single band to process. 

900 

901 Raises 

902 ------ 

903 RuntimeError if there is not exactly one band. 

904 """ 

905 if len(data_bands) != 1: 

906 raise RuntimeError("GenerateHipsTask can only use data from a single band.") 

907 

908 return list(data_bands) 

909 

910 @timeMethod 

911 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False): 

912 """Run the GenerateHipsTask. 

913 

914 Parameters 

915 ---------- 

916 bands : `list [ `str` ] 

917 List of bands to be processed (or single band). 

918 max_order : `int` 

919 HEALPix order of the maximum (native) HPX exposures. 

920 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`} 

921 Dict of handles for the HiPS high-resolution exposures. 

922 Key is (pixel number, ``band``). 

923 do_color : `bool`, optional 

924 Do color pngs instead of per-band grayscale. 

925 """ 

926 min_order = self.config.min_order 

927 

928 if not do_color: 

929 png_grayscale_mapping = AsinhMapping( 

930 self.config.png_gray_asinh_minimum, 

931 self.config.png_gray_asinh_stretch, 

932 Q=self.config.png_gray_asinh_softening, 

933 ) 

934 else: 

935 png_color_mapping = AsinhMapping( 

936 self.config.png_color_asinh_minimum, 

937 self.config.png_color_asinh_stretch, 

938 Q=self.config.png_color_asinh_softening, 

939 ) 

940 

941 bcb = self.config.blue_channel_band 

942 gcb = self.config.green_channel_band 

943 rcb = self.config.red_channel_band 

944 colorstr = f"{bcb}{gcb}{rcb}" 

945 

946 # The base path is based on the hips_base_uri. 

947 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=True) 

948 

949 # We need to unique-ify the pixels because they show up for multiple bands. 

950 # The output of this is a sorted array. 

951 pixels = np.unique(np.array([pixel for pixel, _ in hips_exposure_handle_dict.keys()])) 

952 

953 # Add a "gutter" pixel at the end. Start with 0 which maps to 0 always. 

954 pixels = np.append(pixels, [0]) 

955 

956 # Convert the pixels to each order that will be generated. 

957 pixels_shifted = {} 

958 pixels_shifted[max_order] = pixels 

959 for order in range(max_order - 1, min_order - 1, -1): 

960 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2) 

961 

962 # And set the gutter to an illegal pixel value. 

963 for order in range(min_order, max_order + 1): 

964 pixels_shifted[order][-1] = -1 

965 

966 # Read in the first pixel for determining image properties. 

967 exp0 = list(hips_exposure_handle_dict.values())[0].get() 

968 bbox = exp0.getBBox() 

969 npix = bbox.getWidth() 

970 shift_order = int(np.round(np.log2(npix))) 

971 

972 # Create blank exposures for each level, including the highest order. 

973 # We also make sure we create blank exposures for any bands used in the color 

974 # PNGs, even if they aren't available. 

975 exposures = {} 

976 for band in bands: 

977 for order in range(min_order, max_order + 1): 

978 exp = exp0.Factory(bbox=bbox) 

979 exp.image.array[:, :] = np.nan 

980 exposures[(band, order)] = exp 

981 

982 # Loop over all pixels, avoiding the gutter. 

983 for pixel_counter, pixel in enumerate(pixels[:-1]): 

984 self.log.debug("Working on high resolution pixel %d", pixel) 

985 for band in bands: 

986 # Read all the exposures here for the highest order. 

987 # There will always be at least one band with a HiPS image available 

988 # at the highest order. However, for color images it is possible that 

989 # not all bands have coverage so we require this check. 

990 if (pixel, band) in hips_exposure_handle_dict: 

991 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get() 

992 

993 # Go up the HiPS tree. 

994 # We only write pixels and rebin to fill the parent pixel when we are 

995 # done with a current pixel, which is determined if the next pixel 

996 # has a different pixel number. 

997 for order in range(max_order, min_order - 1, -1): 

998 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]: 

999 # This order is not done, and so none of the other orders will be. 

1000 break 

1001 

1002 # We can now write out the images for each band. 

1003 # Note this will always trigger at the max order where each pixel is unique. 

1004 if not do_color: 

1005 for band in bands: 

1006 self._write_hips_image( 

1007 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1008 order, 

1009 pixels_shifted[order][pixel_counter], 

1010 exposures[(band, order)].image, 

1011 png_grayscale_mapping, 

1012 shift_order=shift_order, 

1013 ) 

1014 else: 

1015 # Make a color png. 

1016 self._write_hips_color_png( 

1017 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1018 order, 

1019 pixels_shifted[order][pixel_counter], 

1020 exposures[(self.config.red_channel_band, order)].image, 

1021 exposures[(self.config.green_channel_band, order)].image, 

1022 exposures[(self.config.blue_channel_band, order)].image, 

1023 png_color_mapping, 

1024 ) 

1025 

1026 log_level = self.log.INFO if order == (max_order - 3) else self.log.DEBUG 

1027 self.log.log( 

1028 log_level, 

1029 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)", 

1030 ",".join(bands), 

1031 order, 

1032 pixels_shifted[order][pixel_counter], 

1033 pixel_counter, 

1034 len(pixels) - 1, 

1035 ) 

1036 

1037 # When we are at the top of the tree, erase top level images and continue. 

1038 if order == min_order: 

1039 for band in bands: 

1040 exposures[(band, order)].image.array[:, :] = np.nan 

1041 continue 

1042 

1043 # Now average the images for each band. 

1044 for band in bands: 

1045 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2) 

1046 with warnings.catch_warnings(): 

1047 warnings.simplefilter("ignore") 

1048 binned_image_arr = np.nanmean(arr, axis=(1, 3)) 

1049 

1050 # Fill the next level up. We figure out which of the four 

1051 # sub-pixels the current pixel occupies. 

1052 sub_index = (pixels_shifted[order][pixel_counter] 

1053 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2)) 

1054 

1055 # Fill exposure at the next level up. 

1056 exp = exposures[(band, order - 1)] 

1057 

1058 # Fill the correct subregion. 

1059 if sub_index == 0: 

1060 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr 

1061 elif sub_index == 1: 

1062 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr 

1063 elif sub_index == 2: 

1064 exp.image.array[npix//2:, npix//2:] = binned_image_arr 

1065 elif sub_index == 3: 

1066 exp.image.array[0: npix//2, npix//2:] = binned_image_arr 

1067 else: 

1068 # This should be impossible. 

1069 raise ValueError("Illegal pixel sub index") 

1070 

1071 # Erase the previous exposure. 

1072 if order < max_order: 

1073 exposures[(band, order)].image.array[:, :] = np.nan 

1074 

1075 # Write the properties files and MOCs. 

1076 if not do_color: 

1077 for band in bands: 

1078 band_pixels = np.array([pixel 

1079 for pixel, band_ in hips_exposure_handle_dict.keys() 

1080 if band_ == band]) 

1081 band_pixels = np.sort(band_pixels) 

1082 

1083 self._write_properties_and_moc( 

1084 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1085 max_order, 

1086 band_pixels, 

1087 exp0, 

1088 shift_order, 

1089 band, 

1090 False, 

1091 ) 

1092 self._write_allsky_file( 

1093 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1094 min_order, 

1095 ) 

1096 else: 

1097 self._write_properties_and_moc( 

1098 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1099 max_order, 

1100 pixels[:-1], 

1101 exp0, 

1102 shift_order, 

1103 colorstr, 

1104 True, 

1105 ) 

1106 self._write_allsky_file( 

1107 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1108 min_order, 

1109 ) 

1110 

1111 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9): 

1112 """Write a HiPS image. 

1113 

1114 Parameters 

1115 ---------- 

1116 hips_base_path : `lsst.resources.ResourcePath` 

1117 Resource path to the base of the HiPS directory tree. 

1118 order : `int` 

1119 HEALPix order of the HiPS image to write. 

1120 pixel : `int` 

1121 HEALPix pixel of the HiPS image. 

1122 image : `lsst.afw.image.Image` 

1123 Image to write. 

1124 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1125 Mapping to convert image to scaled png. 

1126 shift_order : `int`, optional 

1127 HPX shift_order. 

1128 """ 

1129 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1130 # outside of the butler. This task has been given (temporary) 

1131 # Special Dispensation because of the nature of HiPS outputs until 

1132 # a more controlled solution can be found. 

1133 

1134 dir_number = self._get_dir_number(pixel) 

1135 hips_dir = hips_base_path.join( 

1136 f"Norder{order}", 

1137 forceDirectory=True 

1138 ).join( 

1139 f"Dir{dir_number}", 

1140 forceDirectory=True 

1141 ) 

1142 

1143 wcs = makeHpxWcs(order, pixel, shift_order=shift_order) 

1144 

1145 uri = hips_dir.join(f"Npix{pixel}.fits") 

1146 

1147 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1148 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata()) 

1149 

1150 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1151 

1152 # And make a grayscale png as well 

1153 

1154 with np.errstate(invalid="ignore"): 

1155 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8) 

1156 

1157 vals[~np.isfinite(image.array) | (image.array < 0)] = 0 

1158 im = Image.fromarray(vals[::-1, :], "L") 

1159 

1160 uri = hips_dir.join(f"Npix{pixel}.png") 

1161 

1162 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1163 im.save(temporary_uri.ospath) 

1164 

1165 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1166 

1167 def _write_hips_color_png( 

1168 self, 

1169 hips_base_path, 

1170 order, 

1171 pixel, 

1172 image_red, 

1173 image_green, 

1174 image_blue, 

1175 png_mapping, 

1176 ): 

1177 """Write a color png HiPS image. 

1178 

1179 Parameters 

1180 ---------- 

1181 hips_base_path : `lsst.resources.ResourcePath` 

1182 Resource path to the base of the HiPS directory tree. 

1183 order : `int` 

1184 HEALPix order of the HiPS image to write. 

1185 pixel : `int` 

1186 HEALPix pixel of the HiPS image. 

1187 image_red : `lsst.afw.image.Image` 

1188 Input for red channel of output png. 

1189 image_green : `lsst.afw.image.Image` 

1190 Input for green channel of output png. 

1191 image_blue : `lsst.afw.image.Image` 

1192 Input for blue channel of output png. 

1193 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1194 Mapping to convert image to scaled png. 

1195 """ 

1196 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1197 # outside of the butler. This task has been given (temporary) 

1198 # Special Dispensation because of the nature of HiPS outputs until 

1199 # a more controlled solution can be found. 

1200 

1201 dir_number = self._get_dir_number(pixel) 

1202 hips_dir = hips_base_path.join( 

1203 f"Norder{order}", 

1204 forceDirectory=True 

1205 ).join( 

1206 f"Dir{dir_number}", 

1207 forceDirectory=True 

1208 ) 

1209 

1210 # We need to convert nans to the minimum values in the mapping. 

1211 arr_red = image_red.array.copy() 

1212 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0] 

1213 arr_green = image_green.array.copy() 

1214 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1] 

1215 arr_blue = image_blue.array.copy() 

1216 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2] 

1217 

1218 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue) 

1219 

1220 im = Image.fromarray(image_array[::-1, :, :], mode="RGB") 

1221 

1222 uri = hips_dir.join(f"Npix{pixel}.png") 

1223 

1224 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1225 im.save(temporary_uri.ospath) 

1226 

1227 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1228 

1229 def _write_properties_and_moc( 

1230 self, 

1231 hips_base_path, 

1232 max_order, 

1233 pixels, 

1234 exposure, 

1235 shift_order, 

1236 band, 

1237 multiband 

1238 ): 

1239 """Write HiPS properties file and MOC. 

1240 

1241 Parameters 

1242 ---------- 

1243 hips_base_path : : `lsst.resources.ResourcePath` 

1244 Resource path to the base of the HiPS directory tree. 

1245 max_order : `int` 

1246 Maximum HEALPix order. 

1247 pixels : `np.ndarray` (N,) 

1248 Array of pixels used. 

1249 exposure : `lsst.afw.image.Exposure` 

1250 Sample HPX exposure used for generating HiPS tiles. 

1251 shift_order : `int` 

1252 HPX shift order. 

1253 band : `str` 

1254 Band (or color). 

1255 multiband : `bool` 

1256 Is band multiband / color? 

1257 """ 

1258 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels) 

1259 

1260 initial_ra = self.config.properties.initial_ra 

1261 initial_dec = self.config.properties.initial_dec 

1262 initial_fov = self.config.properties.initial_fov 

1263 

1264 if initial_ra is None or initial_dec is None or initial_fov is None: 

1265 # We want to point to an arbitrary pixel in the footprint. 

1266 # Just take the median pixel value for simplicity. 

1267 temp_pixels = pixels.copy() 

1268 if temp_pixels.size % 2 == 0: 

1269 temp_pixels = np.append(temp_pixels, [temp_pixels[0]]) 

1270 medpix = int(np.median(temp_pixels)) 

1271 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix) 

1272 _initial_fov = hpg.nside_to_resolution(2**max_order, units='arcminutes')/60. 

1273 

1274 if initial_ra is None or initial_dec is None: 

1275 initial_ra = _initial_ra 

1276 initial_dec = _initial_dec 

1277 if initial_fov is None: 

1278 initial_fov = _initial_fov 

1279 

1280 self._write_hips_properties_file( 

1281 hips_base_path, 

1282 self.config.properties, 

1283 band, 

1284 multiband, 

1285 exposure, 

1286 max_order, 

1287 shift_order, 

1288 area, 

1289 initial_ra, 

1290 initial_dec, 

1291 initial_fov, 

1292 ) 

1293 

1294 # Write the MOC coverage 

1295 self._write_hips_moc_file( 

1296 hips_base_path, 

1297 max_order, 

1298 pixels, 

1299 ) 

1300 

1301 def _write_hips_properties_file( 

1302 self, 

1303 hips_base_path, 

1304 properties_config, 

1305 band, 

1306 multiband, 

1307 exposure, 

1308 max_order, 

1309 shift_order, 

1310 area, 

1311 initial_ra, 

1312 initial_dec, 

1313 initial_fov 

1314 ): 

1315 """Write HiPS properties file. 

1316 

1317 Parameters 

1318 ---------- 

1319 hips_base_path : `lsst.resources.ResourcePath` 

1320 ResourcePath at top of HiPS tree. File will be written 

1321 to this path as ``properties``. 

1322 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig` 

1323 Configuration for properties values. 

1324 band : `str` 

1325 Name of band(s) for HiPS tree. 

1326 multiband : `bool` 

1327 Is multiband / color? 

1328 exposure : `lsst.afw.image.Exposure` 

1329 Sample HPX exposure used for generating HiPS tiles. 

1330 max_order : `int` 

1331 Maximum HEALPix order. 

1332 shift_order : `int` 

1333 HPX shift order. 

1334 area : `float` 

1335 Coverage area in square degrees. 

1336 initial_ra : `float` 

1337 Initial HiPS RA position (degrees). 

1338 initial_dec : `float` 

1339 Initial HiPS Dec position (degrees). 

1340 initial_fov : `float` 

1341 Initial HiPS display size (degrees). 

1342 """ 

1343 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1344 # outside of the butler. This task has been given (temporary) 

1345 # Special Dispensation because of the nature of HiPS outputs until 

1346 # a more controlled solution can be found. 

1347 def _write_property(fh, name, value): 

1348 """Write a property name/value to a file handle. 

1349 

1350 Parameters 

1351 ---------- 

1352 fh : file handle (blah) 

1353 Open for writing. 

1354 name : `str` 

1355 Name of property 

1356 value : `str` 

1357 Value of property 

1358 """ 

1359 # This ensures that the name has no spaces or space-like characters, 

1360 # per the HiPS standard. 

1361 if re.search(r"\s", name): 

1362 raise ValueError(f"``{name}`` cannot contain any space characters.") 

1363 if "=" in name: 

1364 raise ValueError(f"``{name}`` cannot contain an ``=``") 

1365 

1366 fh.write(f"{name:25}= {value}\n") 

1367 

1368 if exposure.image.array.dtype == np.dtype("float32"): 

1369 bitpix = -32 

1370 elif exposure.image.array.dtype == np.dtype("float64"): 

1371 bitpix = -64 

1372 elif exposure.image.array.dtype == np.dtype("int32"): 

1373 bitpix = 32 

1374 

1375 date_iso8601 = datetime.utcnow().isoformat(timespec="seconds") + "Z" 

1376 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units='degrees') 

1377 

1378 uri = hips_base_path.join("properties") 

1379 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1380 with open(temporary_uri.ospath, "w") as fh: 

1381 _write_property( 

1382 fh, 

1383 "creator_did", 

1384 properties_config.creator_did_template.format(band=band), 

1385 ) 

1386 if properties_config.obs_collection is not None: 

1387 _write_property(fh, "obs_collection", properties_config.obs_collection) 

1388 _write_property( 

1389 fh, 

1390 "obs_title", 

1391 properties_config.obs_title_template.format(band=band), 

1392 ) 

1393 if properties_config.obs_description_template is not None: 

1394 _write_property( 

1395 fh, 

1396 "obs_description", 

1397 properties_config.obs_description_template.format(band=band), 

1398 ) 

1399 if len(properties_config.prov_progenitor) > 0: 

1400 for prov_progenitor in properties_config.prov_progenitor: 

1401 _write_property(fh, "prov_progenitor", prov_progenitor) 

1402 if properties_config.obs_ack is not None: 

1403 _write_property(fh, "obs_ack", properties_config.obs_ack) 

1404 _write_property(fh, "obs_regime", "Optical") 

1405 _write_property(fh, "data_pixel_bitpix", str(bitpix)) 

1406 _write_property(fh, "dataproduct_type", "image") 

1407 _write_property(fh, "moc_sky_fraction", str(area/41253.)) 

1408 _write_property(fh, "data_ucd", "phot.flux") 

1409 _write_property(fh, "hips_creation_date", date_iso8601) 

1410 _write_property(fh, "hips_builder", "lsst.pipe.tasks.hips.GenerateHipsTask") 

1411 _write_property(fh, "hips_creator", "Vera C. Rubin Observatory") 

1412 _write_property(fh, "hips_version", "1.4") 

1413 _write_property(fh, "hips_release_date", date_iso8601) 

1414 _write_property(fh, "hips_frame", "equatorial") 

1415 _write_property(fh, "hips_order", str(max_order)) 

1416 _write_property(fh, "hips_tile_width", str(exposure.getBBox().getWidth())) 

1417 _write_property(fh, "hips_status", "private master clonableOnce") 

1418 if multiband: 

1419 _write_property(fh, "hips_tile_format", "png") 

1420 _write_property(fh, "dataproduct_subtype", "color") 

1421 else: 

1422 _write_property(fh, "hips_tile_format", "png fits") 

1423 _write_property(fh, "hips_pixel_bitpix", str(bitpix)) 

1424 _write_property(fh, "hips_pixel_scale", str(pixel_scale)) 

1425 _write_property(fh, "hips_initial_ra", str(initial_ra)) 

1426 _write_property(fh, "hips_initial_dec", str(initial_dec)) 

1427 _write_property(fh, "hips_initial_fov", str(initial_fov)) 

1428 if multiband: 

1429 if self.config.blue_channel_band in properties_config.spectral_ranges: 

1430 em_min = properties_config.spectral_ranges[ 

1431 self.config.blue_channel_band 

1432 ].lambda_min/1e9 

1433 else: 

1434 self.log.warning("blue band %s not in self.config.spectral_ranges.", band) 

1435 em_min = 3e-7 

1436 if self.config.red_channel_band in properties_config.spectral_ranges: 

1437 em_max = properties_config.spectral_ranges[ 

1438 self.config.red_channel_band 

1439 ].lambda_max/1e9 

1440 else: 

1441 self.log.warning("red band %s not in self.config.spectral_ranges.", band) 

1442 em_max = 1e-6 

1443 else: 

1444 if band in properties_config.spectral_ranges: 

1445 em_min = properties_config.spectral_ranges[band].lambda_min/1e9 

1446 em_max = properties_config.spectral_ranges[band].lambda_max/1e9 

1447 else: 

1448 self.log.warning("band %s not in self.config.spectral_ranges.", band) 

1449 em_min = 3e-7 

1450 em_max = 1e-6 

1451 _write_property(fh, "em_min", str(em_min)) 

1452 _write_property(fh, "em_max", str(em_max)) 

1453 if properties_config.t_min is not None: 

1454 _write_property(fh, "t_min", properties_config.t_min) 

1455 if properties_config.t_max is not None: 

1456 _write_property(fh, "t_max", properties_config.t_max) 

1457 

1458 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1459 

1460 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1): 

1461 """Write HiPS MOC file. 

1462 

1463 Parameters 

1464 ---------- 

1465 hips_base_path : `lsst.resources.ResourcePath` 

1466 ResourcePath to top of HiPS tree. File will be written as 

1467 to this path as ``Moc.fits``. 

1468 max_order : `int` 

1469 Maximum HEALPix order. 

1470 pixels : `np.ndarray` 

1471 Array of pixels covered. 

1472 min_uniq_order : `int`, optional 

1473 Minimum HEALPix order for looking for fully covered pixels. 

1474 """ 

1475 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1476 # outside of the butler. This task has been given (temporary) 

1477 # Special Dispensation because of the nature of HiPS outputs until 

1478 # a more controlled solution can be found. 

1479 

1480 # Make the initial list of UNIQ pixels 

1481 uniq = 4*(4**max_order) + pixels 

1482 

1483 # Make a healsparse map which provides easy degrade/comparisons. 

1484 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32) 

1485 hspmap[pixels] = 1.0 

1486 

1487 # Loop over orders, degrade each time, and look for pixels with full coverage. 

1488 for uniq_order in range(max_order - 1, min_uniq_order - 1, -1): 

1489 hspmap = hspmap.degrade(2**uniq_order, reduction="sum") 

1490 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order)) 

1491 # Check if any of the pixels at uniq_order have full coverage. 

1492 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero() 

1493 if covered.size == 0: 

1494 # No pixels at uniq_order are fully covered, we're done. 

1495 break 

1496 # Replace the UNIQ pixels that are fully covered. 

1497 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered] 

1498 

1499 # Remove duplicate pixels. 

1500 uniq = np.unique(uniq) 

1501 

1502 # Output to fits. 

1503 tbl = np.zeros(uniq.size, dtype=[("UNIQ", "i8")]) 

1504 tbl["UNIQ"] = uniq 

1505 

1506 order = np.log2(tbl["UNIQ"]//4).astype(np.int32)//2 

1507 moc_order = np.max(order) 

1508 

1509 hdu = fits.BinTableHDU(tbl) 

1510 hdu.header["PIXTYPE"] = "HEALPIX" 

1511 hdu.header["ORDERING"] = "NUNIQ" 

1512 hdu.header["COORDSYS"] = "C" 

1513 hdu.header["MOCORDER"] = moc_order 

1514 hdu.header["MOCTOOL"] = "lsst.pipe.tasks.hips.GenerateHipsTask" 

1515 

1516 uri = hips_base_path.join("Moc.fits") 

1517 

1518 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1519 hdu.writeto(temporary_uri.ospath) 

1520 

1521 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1522 

1523 def _write_allsky_file(self, hips_base_path, allsky_order): 

1524 """Write an Allsky.png file. 

1525 

1526 Parameters 

1527 ---------- 

1528 hips_base_path : `lsst.resources.ResourcePath` 

1529 Resource path to the base of the HiPS directory tree. 

1530 allsky_order : `int` 

1531 HEALPix order of the minimum order to make allsky file. 

1532 """ 

1533 tile_size = self.config.allsky_tilesize 

1534 

1535 # The Allsky file format is described in 

1536 # https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf 

1537 # From S4.3.2: 

1538 # The Allsky file is built as an array of tiles, stored side by side in 

1539 # the left-to-right order. The width of this array must be the square 

1540 # root of the number of the tiles of the order. For instance, the width 

1541 # of this array at order 3 is 27 ( (int)sqrt(768) ). To avoid having a 

1542 # too large Allsky file, the resolution of each tile may be reduced but 

1543 # must stay a power of two (typically 64x64 pixels rather than 512x512). 

1544 

1545 n_tiles = hpg.nside_to_npixel(hpg.order_to_nside(allsky_order)) 

1546 n_tiles_wide = int(np.floor(np.sqrt(n_tiles))) 

1547 n_tiles_high = int(np.ceil(n_tiles / n_tiles_wide)) 

1548 

1549 allsky_image = None 

1550 

1551 allsky_order_uri = hips_base_path.join(f"Norder{allsky_order}", forceDirectory=True) 

1552 pixel_regex = re.compile(r"Npix([0-9]+)\.png$") 

1553 png_uris = list( 

1554 ResourcePath.findFileResources( 

1555 candidates=[allsky_order_uri], 

1556 file_filter=pixel_regex, 

1557 ) 

1558 ) 

1559 

1560 for png_uri in png_uris: 

1561 matches = re.match(pixel_regex, png_uri.basename()) 

1562 pix_num = int(matches.group(1)) 

1563 tile_image = Image.open(io.BytesIO(png_uri.read())) 

1564 row = math.floor(pix_num//n_tiles_wide) 

1565 column = pix_num % n_tiles_wide 

1566 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size) 

1567 tile_image_shrunk = tile_image.resize((tile_size, tile_size)) 

1568 

1569 if allsky_image is None: 

1570 allsky_image = Image.new( 

1571 tile_image.mode, 

1572 (n_tiles_wide*tile_size, n_tiles_high*tile_size), 

1573 ) 

1574 allsky_image.paste(tile_image_shrunk, box) 

1575 

1576 uri = allsky_order_uri.join("Allsky.png") 

1577 

1578 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1579 allsky_image.save(temporary_uri.ospath) 

1580 

1581 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1582 

1583 def _get_dir_number(self, pixel): 

1584 """Compute the directory number from a pixel. 

1585 

1586 Parameters 

1587 ---------- 

1588 pixel : `int` 

1589 HEALPix pixel number. 

1590 

1591 Returns 

1592 ------- 

1593 dir_number : `int` 

1594 HiPS directory number. 

1595 """ 

1596 return (pixel//10000)*10000 

1597 

1598 

1599class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections, 

1600 dimensions=("instrument", ), 

1601 defaultTemplates={"coaddName": "deep"}): 

1602 hips_exposure_handles = pipeBase.connectionTypes.Input( 

1603 doc="HiPS-compatible HPX images.", 

1604 name="{coaddName}Coadd_hpx", 

1605 storageClass="ExposureF", 

1606 dimensions=("healpix11", "band"), 

1607 multiple=True, 

1608 deferLoad=True, 

1609 ) 

1610 

1611 

1612class GenerateColorHipsConfig(GenerateHipsConfig, 

1613 pipelineConnections=GenerateColorHipsConnections): 

1614 """Configuration parameters for GenerateColorHipsTask.""" 

1615 blue_channel_band = pexConfig.Field( 

1616 doc="Band to use for blue channel of color pngs.", 

1617 dtype=str, 

1618 default="g", 

1619 ) 

1620 green_channel_band = pexConfig.Field( 

1621 doc="Band to use for green channel of color pngs.", 

1622 dtype=str, 

1623 default="r", 

1624 ) 

1625 red_channel_band = pexConfig.Field( 

1626 doc="Band to use for red channel of color pngs.", 

1627 dtype=str, 

1628 default="i", 

1629 ) 

1630 png_color_asinh_minimum = pexConfig.Field( 

1631 doc="AsinhMapping intensity to be mapped to black for color png scaling.", 

1632 dtype=float, 

1633 default=0.0, 

1634 ) 

1635 png_color_asinh_stretch = pexConfig.Field( 

1636 doc="AsinhMapping linear stretch for color png scaling.", 

1637 dtype=float, 

1638 default=5.0, 

1639 ) 

1640 png_color_asinh_softening = pexConfig.Field( 

1641 doc="AsinhMapping softening parameter (Q) for color png scaling.", 

1642 dtype=float, 

1643 default=8.0, 

1644 ) 

1645 

1646 

1647class GenerateColorHipsTask(GenerateHipsTask): 

1648 """Task for making a HiPS tree with color pngs.""" 

1649 ConfigClass = GenerateColorHipsConfig 

1650 _DefaultName = "generateColorHips" 

1651 color_task = True 

1652 

1653 def _check_data_bands(self, data_bands): 

1654 """Check the data for configured bands. 

1655 

1656 Warn if any color bands are missing data. 

1657 

1658 Parameters 

1659 ---------- 

1660 data_bands : `set` [`str`] 

1661 Bands from the input data. 

1662 

1663 Returns 

1664 ------- 

1665 bands : `list` [`str`] 

1666 List of bands in bgr color order. 

1667 """ 

1668 if len(data_bands) == 0: 

1669 raise RuntimeError("GenerateColorHipsTask must have data from at least one band.") 

1670 

1671 if self.config.blue_channel_band not in data_bands: 

1672 self.log.warning( 

1673 "Color png blue_channel_band %s not in dataset.", 

1674 self.config.blue_channel_band 

1675 ) 

1676 if self.config.green_channel_band not in data_bands: 

1677 self.log.warning( 

1678 "Color png green_channel_band %s not in dataset.", 

1679 self.config.green_channel_band 

1680 ) 

1681 if self.config.red_channel_band not in data_bands: 

1682 self.log.warning( 

1683 "Color png red_channel_band %s not in dataset.", 

1684 self.config.red_channel_band 

1685 ) 

1686 

1687 bands = [ 

1688 self.config.blue_channel_band, 

1689 self.config.green_channel_band, 

1690 self.config.red_channel_band, 

1691 ] 

1692 

1693 return bands