Coverage for python/lsst/pipe/tasks/hips.py: 14%

617 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-29 10:48 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Tasks for making and manipulating HIPS images.""" 

23 

24__all__ = ["HighResolutionHipsTask", "HighResolutionHipsConfig", "HighResolutionHipsConnections", 

25 "GenerateHipsTask", "GenerateHipsConfig", "GenerateColorHipsTask", "GenerateColorHipsConfig"] 

26 

27from collections import defaultdict 

28import numpy as np 

29import argparse 

30import io 

31import sys 

32import re 

33import warnings 

34import math 

35from datetime import datetime 

36import hpgeom as hpg 

37import healsparse as hsp 

38from astropy.io import fits 

39from astropy.visualization.lupton_rgb import AsinhMapping 

40from PIL import Image 

41 

42from lsst.sphgeom import RangeSet, HealpixPixelization 

43from lsst.utils.timer import timeMethod 

44from lsst.daf.butler import Butler, DataCoordinate, DatasetRef, Quantum 

45import lsst.pex.config as pexConfig 

46import lsst.pipe.base as pipeBase 

47import lsst.afw.geom as afwGeom 

48import lsst.afw.math as afwMath 

49import lsst.afw.image as afwImage 

50import lsst.geom as geom 

51from lsst.afw.geom import makeHpxWcs 

52from lsst.resources import ResourcePath 

53 

54from .healSparseMapping import _is_power_of_two 

55 

56 

57class HighResolutionHipsConnections(pipeBase.PipelineTaskConnections, 

58 dimensions=("healpix9", "band"), 

59 defaultTemplates={"coaddName": "deep"}): 

60 coadd_exposure_handles = pipeBase.connectionTypes.Input( 

61 doc="Coadded exposures to convert to HIPS format.", 

62 name="{coaddName}Coadd_calexp", 

63 storageClass="ExposureF", 

64 dimensions=("tract", "patch", "skymap", "band"), 

65 multiple=True, 

66 deferLoad=True, 

67 ) 

68 hips_exposures = pipeBase.connectionTypes.Output( 

69 doc="HiPS-compatible HPX image.", 

70 name="{coaddName}Coadd_hpx", 

71 storageClass="ExposureF", 

72 dimensions=("healpix11", "band"), 

73 multiple=True, 

74 ) 

75 

76 def __init__(self, *, config=None): 

77 super().__init__(config=config) 

78 

79 quantum_order = None 

80 for dim in self.dimensions: 

81 if "healpix" in dim: 

82 if quantum_order is not None: 

83 raise ValueError("Must not specify more than one quantum healpix dimension.") 

84 quantum_order = int(dim.split("healpix")[1]) 

85 if quantum_order is None: 

86 raise ValueError("Must specify a healpix dimension in quantum dimensions.") 

87 

88 if quantum_order > config.hips_order: 

89 raise ValueError("Quantum healpix dimension order must not be greater than hips_order") 

90 

91 order = None 

92 for dim in self.hips_exposures.dimensions: 

93 if "healpix" in dim: 

94 if order is not None: 

95 raise ValueError("Must not specify more than one healpix dimension.") 

96 order = int(dim.split("healpix")[1]) 

97 if order is None: 

98 raise ValueError("Must specify a healpix dimension in hips_exposure dimensions.") 

99 

100 if order != config.hips_order: 

101 raise ValueError("healpix dimension order must match config.hips_order.") 

102 

103 

104class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig, 

105 pipelineConnections=HighResolutionHipsConnections): 

106 """Configuration parameters for HighResolutionHipsTask. 

107 

108 Notes 

109 ----- 

110 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to 

111 2**hips_order. Each cell is 'shift_order' orders deeper than the HEALPix 

112 cell, with 2**shift_order x 2**shift_order sub-pixels on a side, which 

113 defines the target resolution of the HiPS image. The IVOA recommends 

114 shift_order=9, for 2**9=512 pixels on a side. 

115 

116 Table 5 from 

117 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf 

118 shows the relationship between hips_order, number of tiles (full 

119 sky coverage), cell size, and sub-pixel size/image resolution (with 

120 the default shift_order=9): 

121 +------------+-----------------+--------------+------------------+ 

122 | hips_order | Number of Tiles | Cell Size | Image Resolution | 

123 +============+=================+==============+==================+ 

124 | 0 | 12 | 58.63 deg | 6.871 arcmin | 

125 | 1 | 48 | 29.32 deg | 3.435 arcmin | 

126 | 2 | 192 | 14.66 deg | 1.718 arcmin | 

127 | 3 | 768 | 7.329 deg | 51.53 arcsec | 

128 | 4 | 3072 | 3.665 deg | 25.77 arcsec | 

129 | 5 | 12288 | 1.832 deg | 12.88 arcsec | 

130 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec | 

131 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec | 

132 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec | 

133 | 9 | 3145728 | 6.871 arcmin | 805.2mas | 

134 | 10 | 12582912 | 3.435 arcmin | 402.6mas | 

135 | 11 | 50331648 | 1.718 arcmin | 201.3mas | 

136 | 12 | 201326592 | 51.53 arcsec | 100.6mas | 

137 | 13 | 805306368 | 25.77 arcsec | 50.32mas | 

138 +------------+-----------------+--------------+------------------+ 

139 """ 

140 hips_order = pexConfig.Field( 

141 doc="HIPS image order.", 

142 dtype=int, 

143 default=11, 

144 ) 

145 shift_order = pexConfig.Field( 

146 doc="HIPS shift order (such that each tile is 2**shift_order pixels on a side)", 

147 dtype=int, 

148 default=9, 

149 ) 

150 warp = pexConfig.ConfigField( 

151 dtype=afwMath.Warper.ConfigClass, 

152 doc="Warper configuration", 

153 ) 

154 

155 def setDefaults(self): 

156 self.warp.warpingKernelName = "lanczos5" 

157 

158 

159class HipsTaskNameDescriptor: 

160 """Descriptor used create a DefaultName that matches the order of 

161 the defined dimensions in the connections class. 

162 

163 Parameters 

164 ---------- 

165 prefix : `str` 

166 The prefix of the Default name, to which the order will be 

167 appended. 

168 """ 

169 def __init__(self, prefix): 

170 # create a defaultName template 

171 self._defaultName = f"{prefix}{{}}" 

172 self._order = None 

173 

174 def __get__(self, obj, klass=None): 

175 if klass is None: 

176 raise RuntimeError( 

177 "HipsTaskDescriptor was used in an unexpected context" 

178 ) 

179 if self._order is None: 

180 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions 

181 for dim in klassDimensions: 

182 if (match := re.match(r"^healpix(\d*)$", dim)) is not None: 

183 self._order = int(match.group(1)) 

184 break 

185 else: 

186 raise RuntimeError( 

187 "Could not find healpix dimension in connections class" 

188 ) 

189 return self._defaultName.format(self._order) 

190 

191 

192class HighResolutionHipsTask(pipeBase.PipelineTask): 

193 """Task for making high resolution HiPS images.""" 

194 ConfigClass = HighResolutionHipsConfig 

195 _DefaultName = HipsTaskNameDescriptor("highResolutionHips") 

196 

197 def __init__(self, **kwargs): 

198 super().__init__(**kwargs) 

199 self.warper = afwMath.Warper.fromConfig(self.config.warp) 

200 

201 @timeMethod 

202 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

203 inputs = butlerQC.get(inputRefs) 

204 

205 healpix_dim = f"healpix{self.config.hips_order}" 

206 

207 pixels = [hips_exposure.dataId[healpix_dim] 

208 for hips_exposure in outputRefs.hips_exposures] 

209 

210 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs["coadd_exposure_handles"]) 

211 

212 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]: 

213 hips_exposure_ref for hips_exposure_ref in outputRefs.hips_exposures} 

214 for pixel, hips_exposure in outputs.hips_exposures.items(): 

215 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel]) 

216 

217 def run(self, pixels, coadd_exposure_handles): 

218 """Run the HighResolutionHipsTask. 

219 

220 Parameters 

221 ---------- 

222 pixels : `Iterable` [ `int` ] 

223 Iterable of healpix pixels (nest ordering) to warp to. 

224 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

225 Handles for the coadd exposures. 

226 

227 Returns 

228 ------- 

229 outputs : `lsst.pipe.base.Struct` 

230 ``hips_exposures`` is a dict with pixel (key) and hips_exposure (value) 

231 """ 

232 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order) 

233 

234 npix = 2**self.config.shift_order 

235 bbox_hpx = geom.Box2I(corner=geom.Point2I(0, 0), 

236 dimensions=geom.Extent2I(npix, npix)) 

237 

238 # For each healpix pixel we will create an empty exposure with the 

239 # correct HPX WCS. We furthermore create a dict to hold each of 

240 # the warps that will go into each HPX exposure. 

241 exp_hpx_dict = {} 

242 warp_dict = {} 

243 for pixel in pixels: 

244 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order) 

245 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx) 

246 exp_hpx_dict[pixel] = exp_hpx 

247 warp_dict[pixel] = [] 

248 

249 first_handle = True 

250 # Loop over input coadd exposures to minimize i/o (this speeds things 

251 # up by ~8x to batch together pixels that overlap a given coadd). 

252 for handle in coadd_exposure_handles: 

253 coadd_exp = handle.get() 

254 

255 # For each pixel, warp the coadd to the HPX WCS for the pixel. 

256 for pixel in pixels: 

257 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx) 

258 

259 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs()) 

260 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

261 

262 if first_handle: 

263 # Make sure the mask planes, filter, and photocalib of the output 

264 # exposure match the (first) input exposure. 

265 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict()) 

266 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter()) 

267 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib()) 

268 

269 if warped.getBBox().getArea() == 0 or not np.any(np.isfinite(warped.image.array)): 

270 # There is no overlap, skip. 

271 self.log.debug( 

272 "No overlap between output HPX %d and input exposure %s", 

273 pixel, 

274 handle.dataId 

275 ) 

276 continue 

277 

278 exp.maskedImage.assign(warped.maskedImage, warped.getBBox()) 

279 warp_dict[pixel].append(exp.maskedImage) 

280 

281 first_handle = False 

282 

283 stats_flags = afwMath.stringToStatisticsProperty("MEAN") 

284 stats_ctrl = afwMath.StatisticsControl() 

285 stats_ctrl.setNanSafe(True) 

286 stats_ctrl.setWeighted(True) 

287 stats_ctrl.setCalcErrorFromInputVariance(True) 

288 

289 # Loop over pixels and combine the warps for each pixel. 

290 # The combination is done with a simple mean for pixels that 

291 # overlap in neighboring patches. 

292 for pixel in pixels: 

293 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

294 

295 if not warp_dict[pixel]: 

296 # Nothing in this pixel 

297 self.log.debug("No data in HPX pixel %d", pixel) 

298 # Remove the pixel from the output, no need to persist an 

299 # empty exposure. 

300 exp_hpx_dict.pop(pixel) 

301 continue 

302 

303 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack( 

304 warp_dict[pixel], 

305 stats_flags, 

306 stats_ctrl, 

307 [1.0]*len(warp_dict[pixel]), 

308 clipped=0, 

309 maskMap=[] 

310 ) 

311 

312 return pipeBase.Struct(hips_exposures=exp_hpx_dict) 

313 

314 @classmethod 

315 def build_quantum_graph_cli(cls, argv): 

316 """A command-line interface entry point to `build_quantum_graph`. 

317 This method provides the implementation for the 

318 ``build-high-resolution-hips-qg`` script. 

319 

320 Parameters 

321 ---------- 

322 argv : `Sequence` [ `str` ] 

323 Command-line arguments (e.g. ``sys.argv[1:]``). 

324 """ 

325 parser = cls._make_cli_parser() 

326 

327 args = parser.parse_args(argv) 

328 

329 if args.subparser_name is None: 

330 parser.print_help() 

331 sys.exit(1) 

332 

333 pipeline = pipeBase.Pipeline.from_uri(args.pipeline) 

334 expanded_pipeline = list(pipeline.toExpandedPipeline()) 

335 

336 if len(expanded_pipeline) != 1: 

337 raise RuntimeError(f"Pipeline file {args.pipeline} may only contain one task.") 

338 

339 (task_def,) = expanded_pipeline 

340 

341 butler = Butler(args.butler_config, collections=args.input) 

342 

343 if args.subparser_name == "segment": 

344 # Do the segmentation 

345 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order) 

346 dataset = task_def.connections.coadd_exposure_handles.name 

347 data_ids = set(butler.registry.queryDataIds("tract", datasets=dataset).expanded()) 

348 region_pixels = [] 

349 for data_id in data_ids: 

350 region = data_id.region 

351 pixel_range = hpix_pixelization.envelope(region) 

352 for r in pixel_range.ranges(): 

353 region_pixels.extend(range(r[0], r[1])) 

354 indices = np.unique(region_pixels) 

355 

356 print(f"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:") 

357 for pixel in indices: 

358 print(pixel) 

359 

360 elif args.subparser_name == "build": 

361 # Build the quantum graph. 

362 

363 # Figure out collection names. 

364 if args.output_run is None: 

365 if args.output is None: 

366 raise ValueError("At least one of --output or --output-run options is required.") 

367 args.output_run = "{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp()) 

368 

369 build_ranges = RangeSet(sorted(args.pixels)) 

370 

371 # Metadata includes a subset of attributes defined in CmdLineFwk. 

372 metadata = { 

373 "input": args.input, 

374 "butler_argument": args.butler_config, 

375 "output": args.output, 

376 "output_run": args.output_run, 

377 "data_query": args.where, 

378 "time": f"{datetime.now()}", 

379 } 

380 

381 qg = cls.build_quantum_graph( 

382 task_def, 

383 butler.registry, 

384 args.hpix_build_order, 

385 build_ranges, 

386 where=args.where, 

387 collections=args.input, 

388 metadata=metadata, 

389 ) 

390 qg.saveUri(args.save_qgraph) 

391 

392 @classmethod 

393 def _make_cli_parser(cls): 

394 """Make the command-line parser. 

395 

396 Returns 

397 ------- 

398 parser : `argparse.ArgumentParser` 

399 """ 

400 parser = argparse.ArgumentParser( 

401 description=( 

402 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets." 

403 ), 

404 ) 

405 subparsers = parser.add_subparsers(help="sub-command help", dest="subparser_name") 

406 

407 parser_segment = subparsers.add_parser("segment", 

408 help="Determine survey segments for workflow.") 

409 parser_build = subparsers.add_parser("build", 

410 help="Build quantum graph for HighResolutionHipsTask") 

411 

412 for sub in [parser_segment, parser_build]: 

413 # These arguments are in common. 

414 sub.add_argument( 

415 "-b", 

416 "--butler-config", 

417 type=str, 

418 help="Path to data repository or butler configuration.", 

419 required=True, 

420 ) 

421 sub.add_argument( 

422 "-p", 

423 "--pipeline", 

424 type=str, 

425 help="Pipeline file, limited to one task.", 

426 required=True, 

427 ) 

428 sub.add_argument( 

429 "-i", 

430 "--input", 

431 type=str, 

432 nargs="+", 

433 help="Input collection(s) to search for coadd exposures.", 

434 required=True, 

435 ) 

436 sub.add_argument( 

437 "-o", 

438 "--hpix_build_order", 

439 type=int, 

440 default=1, 

441 help="HEALPix order to segment sky for building quantum graph files.", 

442 ) 

443 sub.add_argument( 

444 "-w", 

445 "--where", 

446 type=str, 

447 default=None, 

448 help="Data ID expression used when querying for input coadd datasets.", 

449 ) 

450 

451 parser_build.add_argument( 

452 "--output", 

453 type=str, 

454 help=( 

455 "Name of the output CHAINED collection. If this options is specified and " 

456 "--output-run is not, then a new RUN collection will be created by appending " 

457 "a timestamp to the value of this option." 

458 ), 

459 default=None, 

460 metavar="COLL", 

461 ) 

462 parser_build.add_argument( 

463 "--output-run", 

464 type=str, 

465 help=( 

466 "Output RUN collection to write resulting images. If not provided " 

467 "then --output must be provided and a new RUN collection will be created " 

468 "by appending a timestamp to the value passed with --output." 

469 ), 

470 default=None, 

471 metavar="RUN", 

472 ) 

473 parser_build.add_argument( 

474 "-q", 

475 "--save-qgraph", 

476 type=str, 

477 help="Output filename for QuantumGraph.", 

478 required=True, 

479 ) 

480 parser_build.add_argument( 

481 "-P", 

482 "--pixels", 

483 type=int, 

484 nargs="+", 

485 help="Pixels at --hpix_build_order to generate quantum graph.", 

486 required=True, 

487 ) 

488 

489 return parser 

490 

491 @classmethod 

492 def build_quantum_graph( 

493 cls, 

494 task_def, 

495 registry, 

496 constraint_order, 

497 constraint_ranges, 

498 where=None, 

499 collections=None, 

500 metadata=None, 

501 ): 

502 """Generate a `QuantumGraph` for running just this task. 

503 

504 This is a temporary workaround for incomplete butler query support for 

505 HEALPix dimensions. 

506 

507 Parameters 

508 ---------- 

509 task_def : `lsst.pipe.base.TaskDef` 

510 Task definition. 

511 registry : `lsst.daf.butler.Registry` 

512 Client for the butler database. May be read-only. 

513 constraint_order : `int` 

514 HEALPix order used to contrain which quanta are generated, via 

515 ``constraint_indices``. This should be a coarser grid (smaller 

516 order) than the order used for the task's quantum and output data 

517 IDs, and ideally something between the spatial scale of a patch or 

518 the data repository's "common skypix" system (usually ``htm7``). 

519 constraint_ranges : `lsst.sphgeom.RangeSet` 

520 RangeSet which describes constraint pixels (HEALPix NEST, with order 

521 constraint_order) to constrain generated quanta. 

522 where : `str`, optional 

523 A boolean `str` expression of the form accepted by 

524 `Registry.queryDatasets` to constrain input datasets. This may 

525 contain a constraint on tracts, patches, or bands, but not HEALPix 

526 indices. Constraints on tracts and patches should usually be 

527 unnecessary, however - existing coadds that overlap the given 

528 HEALpix indices will be selected without such a constraint, and 

529 providing one may reject some that should normally be included. 

530 collections : `str` or `Iterable` [ `str` ], optional 

531 Collection or collections to search for input datasets, in order. 

532 If not provided, ``registry.defaults.collections`` will be 

533 searched. 

534 metadata : `dict` [ `str`, `Any` ] 

535 Graph metadata. It is required to contain "output_run" key with the 

536 name of the output RUN collection. 

537 """ 

538 config = task_def.config 

539 

540 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry) 

541 # Since we know this is the only task in the pipeline, we know there 

542 # is only one overall input and one overall output. 

543 (input_dataset_type,) = dataset_types.inputs 

544 

545 # Extract the main output dataset type (which needs multiple 

546 # DatasetRefs, and tells us the output HPX level), and make a set of 

547 # what remains for more mechanical handling later. 

548 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name] 

549 incidental_output_dataset_types = dataset_types.outputs.copy() 

550 incidental_output_dataset_types.remove(output_dataset_type) 

551 (hpx_output_dimension,) = ( 

552 registry.dimensions.skypix_dimensions[d] for d in output_dataset_type.dimensions.skypix.names 

553 ) 

554 

555 constraint_hpx_pixelization = registry.dimensions[f"healpix{constraint_order}"].pixelization 

556 common_skypix_name = registry.dimensions.commonSkyPix.name 

557 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization 

558 

559 # We will need all the pixels at the quantum resolution as well 

560 task_dimensions = registry.dimensions.conform(task_def.connections.dimensions) 

561 (hpx_dimension,) = ( 

562 registry.dimensions.skypix_dimensions[d] for d in task_dimensions.names if d != "band" 

563 ) 

564 hpx_pixelization = hpx_dimension.pixelization 

565 

566 if hpx_pixelization.level < constraint_order: 

567 raise ValueError(f"Quantum order {hpx_pixelization.level} must be < {constraint_order}") 

568 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order)) 

569 

570 # We can be generous in looking for pixels here, because we constraint by actual 

571 # patch regions below. 

572 common_skypix_ranges = RangeSet() 

573 for begin, end in constraint_ranges: 

574 for hpx_index in range(begin, end): 

575 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index) 

576 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region) 

577 

578 # To keep the query from getting out of hand (and breaking) we simplify until we have fewer 

579 # than 100 ranges which seems to work fine. 

580 for simp in range(1, 10): 

581 if len(common_skypix_ranges) < 100: 

582 break 

583 common_skypix_ranges.simplify(simp) 

584 

585 # Use that RangeSet to assemble a WHERE constraint expression. This 

586 # could definitely get too big if the "constraint healpix" order is too 

587 # fine. 

588 where_terms = [] 

589 bind = {} 

590 for n, (begin, end) in enumerate(common_skypix_ranges): 

591 stop = end - 1 # registry range syntax is inclusive 

592 if begin == stop: 

593 where_terms.append(f"{common_skypix_name} = cpx{n}") 

594 bind[f"cpx{n}"] = begin 

595 else: 

596 where_terms.append(f"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)") 

597 bind[f"cpx{n}a"] = begin 

598 bind[f"cpx{n}b"] = stop 

599 if where is None: 

600 where = " OR ".join(where_terms) 

601 else: 

602 where = f"({where}) AND ({' OR '.join(where_terms)})" 

603 # Query for input datasets with this constraint, and ask for expanded 

604 # data IDs because we want regions. Immediately group this by patch so 

605 # we don't do later geometric stuff n_bands more times than we need to. 

606 input_refs = registry.queryDatasets( 

607 input_dataset_type, 

608 where=where, 

609 findFirst=True, 

610 collections=collections, 

611 bind=bind 

612 ).expanded() 

613 inputs_by_patch = defaultdict(set) 

614 patch_dimensions = registry.dimensions.conform(["patch"]) 

615 for input_ref in input_refs: 

616 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref) 

617 if not inputs_by_patch: 

618 message_body = "\n".join(input_refs.explain_no_results()) 

619 raise RuntimeError(f"No inputs found:\n{message_body}") 

620 

621 # Iterate over patches and compute the set of output healpix pixels 

622 # that overlap each one. Use that to associate inputs with output 

623 # pixels, but only for the output pixels we've already identified. 

624 inputs_by_hpx = defaultdict(set) 

625 for patch_data_id, input_refs_for_patch in inputs_by_patch.items(): 

626 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region) 

627 for begin, end in patch_hpx_ranges & hpx_ranges: 

628 for hpx_index in range(begin, end): 

629 inputs_by_hpx[hpx_index].update(input_refs_for_patch) 

630 # Iterate over the dict we just created and create the actual quanta. 

631 quanta = [] 

632 output_run = metadata["output_run"] 

633 for hpx_index, input_refs_for_hpx_index in inputs_by_hpx.items(): 

634 # Group inputs by band. 

635 input_refs_by_band = defaultdict(list) 

636 for input_ref in input_refs_for_hpx_index: 

637 input_refs_by_band[input_ref.dataId["band"]].append(input_ref) 

638 # Iterate over bands to make quanta. 

639 for band, input_refs_for_band in input_refs_by_band.items(): 

640 data_id = registry.expandDataId({hpx_dimension: hpx_index, "band": band}) 

641 

642 hpx_pixel_ranges = RangeSet(hpx_index) 

643 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level)) 

644 output_data_ids = [] 

645 for begin, end in hpx_output_ranges: 

646 for hpx_output_index in range(begin, end): 

647 output_data_ids.append( 

648 registry.expandDataId({hpx_output_dimension: hpx_output_index, "band": band}) 

649 ) 

650 outputs = { 

651 dt: [DatasetRef(dt, data_id, run=output_run)] for dt in incidental_output_dataset_types 

652 } 

653 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id, run=output_run) 

654 for data_id in output_data_ids] 

655 quanta.append( 

656 Quantum( 

657 taskName=task_def.taskName, 

658 taskClass=task_def.taskClass, 

659 dataId=data_id, 

660 initInputs={}, 

661 inputs={input_dataset_type: input_refs_for_band}, 

662 outputs=outputs, 

663 ) 

664 ) 

665 

666 if len(quanta) == 0: 

667 raise RuntimeError("Given constraints yielded empty quantum graph.") 

668 

669 # Define initOutputs refs. 

670 empty_data_id = DataCoordinate.make_empty(registry.dimensions) 

671 init_outputs = {} 

672 global_init_outputs = [] 

673 if config_dataset_type := dataset_types.initOutputs.get(task_def.configDatasetName): 

674 init_outputs[task_def] = [DatasetRef(config_dataset_type, empty_data_id, run=output_run)] 

675 packages_dataset_name = pipeBase.PipelineDatasetTypes.packagesDatasetName 

676 if packages_dataset_type := dataset_types.initOutputs.get(packages_dataset_name): 

677 global_init_outputs.append(DatasetRef(packages_dataset_type, empty_data_id, run=output_run)) 

678 

679 return pipeBase.QuantumGraph( 

680 quanta={task_def: quanta}, 

681 initOutputs=init_outputs, 

682 globalInitOutputs=global_init_outputs, 

683 metadata=metadata, 

684 ) 

685 

686 

687class HipsPropertiesSpectralTerm(pexConfig.Config): 

688 lambda_min = pexConfig.Field( 

689 doc="Minimum wavelength (nm)", 

690 dtype=float, 

691 ) 

692 lambda_max = pexConfig.Field( 

693 doc="Maximum wavelength (nm)", 

694 dtype=float, 

695 ) 

696 

697 

698class HipsPropertiesConfig(pexConfig.Config): 

699 """Configuration parameters for writing a HiPS properties file.""" 

700 creator_did_template = pexConfig.Field( 

701 doc=("Unique identifier of the HiPS - Format: IVOID. " 

702 "Use ``{band}`` to substitute the band name."), 

703 dtype=str, 

704 optional=False, 

705 ) 

706 obs_collection = pexConfig.Field( 

707 doc="Short name of original data set - Format: one word", 

708 dtype=str, 

709 optional=True, 

710 ) 

711 obs_description_template = pexConfig.Field( 

712 doc=("Data set description - Format: free text, longer free text " 

713 "description of the dataset. Use ``{band}`` to substitute " 

714 "the band name."), 

715 dtype=str, 

716 ) 

717 prov_progenitor = pexConfig.ListField( 

718 doc="Provenance of the original data - Format: free text", 

719 dtype=str, 

720 default=[], 

721 ) 

722 obs_title_template = pexConfig.Field( 

723 doc=("Data set title format: free text, but should be short. " 

724 "Use ``{band}`` to substitute the band name."), 

725 dtype=str, 

726 optional=False, 

727 ) 

728 spectral_ranges = pexConfig.ConfigDictField( 

729 doc=("Mapping from band to lambda_min, lamba_max (nm). May be approximate."), 

730 keytype=str, 

731 itemtype=HipsPropertiesSpectralTerm, 

732 default={}, 

733 ) 

734 initial_ra = pexConfig.Field( 

735 doc="Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

736 dtype=float, 

737 optional=True, 

738 ) 

739 initial_dec = pexConfig.Field( 

740 doc="Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

741 dtype=float, 

742 optional=True, 

743 ) 

744 initial_fov = pexConfig.Field( 

745 doc="Initial field-of-view (deg). If not set will use ~1 healpix tile.", 

746 dtype=float, 

747 optional=True, 

748 ) 

749 obs_ack = pexConfig.Field( 

750 doc="Observation acknowledgements (free text).", 

751 dtype=str, 

752 optional=True, 

753 ) 

754 t_min = pexConfig.Field( 

755 doc="Time (MJD) of earliest observation included in HiPS", 

756 dtype=float, 

757 optional=True, 

758 ) 

759 t_max = pexConfig.Field( 

760 doc="Time (MJD) of latest observation included in HiPS", 

761 dtype=float, 

762 optional=True, 

763 ) 

764 

765 def validate(self): 

766 super().validate() 

767 

768 if self.obs_collection is not None: 

769 if re.search(r"\s", self.obs_collection): 

770 raise ValueError("obs_collection cannot contain any space characters.") 

771 

772 def setDefaults(self): 

773 # Values here taken from 

774 # https://github.com/lsst-dm/dax_obscore/blob/44ac15029136e2ec15/configs/dp02.yaml#L46 

775 u_term = HipsPropertiesSpectralTerm() 

776 u_term.lambda_min = 330. 

777 u_term.lambda_max = 400. 

778 self.spectral_ranges["u"] = u_term 

779 g_term = HipsPropertiesSpectralTerm() 

780 g_term.lambda_min = 402. 

781 g_term.lambda_max = 552. 

782 self.spectral_ranges["g"] = g_term 

783 r_term = HipsPropertiesSpectralTerm() 

784 r_term.lambda_min = 552. 

785 r_term.lambda_max = 691. 

786 self.spectral_ranges["r"] = r_term 

787 i_term = HipsPropertiesSpectralTerm() 

788 i_term.lambda_min = 691. 

789 i_term.lambda_max = 818. 

790 self.spectral_ranges["i"] = i_term 

791 z_term = HipsPropertiesSpectralTerm() 

792 z_term.lambda_min = 818. 

793 z_term.lambda_max = 922. 

794 self.spectral_ranges["z"] = z_term 

795 y_term = HipsPropertiesSpectralTerm() 

796 y_term.lambda_min = 970. 

797 y_term.lambda_max = 1060. 

798 self.spectral_ranges["y"] = y_term 

799 

800 

801class GenerateHipsConnections(pipeBase.PipelineTaskConnections, 

802 dimensions=("instrument", "band"), 

803 defaultTemplates={"coaddName": "deep"}): 

804 hips_exposure_handles = pipeBase.connectionTypes.Input( 

805 doc="HiPS-compatible HPX images.", 

806 name="{coaddName}Coadd_hpx", 

807 storageClass="ExposureF", 

808 dimensions=("healpix11", "band"), 

809 multiple=True, 

810 deferLoad=True, 

811 ) 

812 

813 

814class GenerateHipsConfig(pipeBase.PipelineTaskConfig, 

815 pipelineConnections=GenerateHipsConnections): 

816 """Configuration parameters for GenerateHipsTask.""" 

817 # WARNING: In general PipelineTasks are not allowed to do any outputs 

818 # outside of the butler. This task has been given (temporary) 

819 # Special Dispensation because of the nature of HiPS outputs until 

820 # a more controlled solution can be found. 

821 hips_base_uri = pexConfig.Field( 

822 doc="URI to HiPS base for output.", 

823 dtype=str, 

824 optional=False, 

825 ) 

826 min_order = pexConfig.Field( 

827 doc="Minimum healpix order for HiPS tree.", 

828 dtype=int, 

829 default=3, 

830 ) 

831 properties = pexConfig.ConfigField( 

832 dtype=HipsPropertiesConfig, 

833 doc="Configuration for properties file.", 

834 ) 

835 allsky_tilesize = pexConfig.Field( 

836 dtype=int, 

837 doc="Allsky tile size; must be power of 2. HiPS standard recommends 64x64 tiles.", 

838 default=64, 

839 check=_is_power_of_two, 

840 ) 

841 png_gray_asinh_minimum = pexConfig.Field( 

842 doc="AsinhMapping intensity to be mapped to black for grayscale png scaling.", 

843 dtype=float, 

844 default=0.0, 

845 ) 

846 png_gray_asinh_stretch = pexConfig.Field( 

847 doc="AsinhMapping linear stretch for grayscale png scaling.", 

848 dtype=float, 

849 default=2.0, 

850 ) 

851 png_gray_asinh_softening = pexConfig.Field( 

852 doc="AsinhMapping softening parameter (Q) for grayscale png scaling.", 

853 dtype=float, 

854 default=8.0, 

855 ) 

856 

857 

858class GenerateHipsTask(pipeBase.PipelineTask): 

859 """Task for making a HiPS tree with FITS and grayscale PNGs.""" 

860 ConfigClass = GenerateHipsConfig 

861 _DefaultName = "generateHips" 

862 color_task = False 

863 

864 @timeMethod 

865 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

866 inputs = butlerQC.get(inputRefs) 

867 

868 dims = inputRefs.hips_exposure_handles[0].dataId.dimensions.names 

869 order = None 

870 for dim in dims: 

871 if "healpix" in dim: 

872 order = int(dim.split("healpix")[1]) 

873 healpix_dim = dim 

874 break 

875 else: 

876 raise RuntimeError("Could not determine healpix order for input exposures.") 

877 

878 hips_exposure_handle_dict = { 

879 (hips_exposure_handle.dataId[healpix_dim], 

880 hips_exposure_handle.dataId["band"]): hips_exposure_handle 

881 for hips_exposure_handle in inputs["hips_exposure_handles"] 

882 } 

883 

884 data_bands = {hips_exposure_handle.dataId["band"] 

885 for hips_exposure_handle in inputs["hips_exposure_handles"]} 

886 bands = self._check_data_bands(data_bands) 

887 

888 self.run( 

889 bands=bands, 

890 max_order=order, 

891 hips_exposure_handle_dict=hips_exposure_handle_dict, 

892 do_color=self.color_task, 

893 ) 

894 

895 def _check_data_bands(self, data_bands): 

896 """Check that the data has only a single band. 

897 

898 Parameters 

899 ---------- 

900 data_bands : `set` [`str`] 

901 Bands from the input data. 

902 

903 Returns 

904 ------- 

905 bands : `list` [`str`] 

906 List of single band to process. 

907 

908 Raises 

909 ------ 

910 RuntimeError if there is not exactly one band. 

911 """ 

912 if len(data_bands) != 1: 

913 raise RuntimeError("GenerateHipsTask can only use data from a single band.") 

914 

915 return list(data_bands) 

916 

917 @timeMethod 

918 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False): 

919 """Run the GenerateHipsTask. 

920 

921 Parameters 

922 ---------- 

923 bands : `list [ `str` ] 

924 List of bands to be processed (or single band). 

925 max_order : `int` 

926 HEALPix order of the maximum (native) HPX exposures. 

927 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`} 

928 Dict of handles for the HiPS high-resolution exposures. 

929 Key is (pixel number, ``band``). 

930 do_color : `bool`, optional 

931 Do color pngs instead of per-band grayscale. 

932 """ 

933 min_order = self.config.min_order 

934 

935 if not do_color: 

936 png_grayscale_mapping = AsinhMapping( 

937 self.config.png_gray_asinh_minimum, 

938 self.config.png_gray_asinh_stretch, 

939 Q=self.config.png_gray_asinh_softening, 

940 ) 

941 else: 

942 png_color_mapping = AsinhMapping( 

943 self.config.png_color_asinh_minimum, 

944 self.config.png_color_asinh_stretch, 

945 Q=self.config.png_color_asinh_softening, 

946 ) 

947 

948 bcb = self.config.blue_channel_band 

949 gcb = self.config.green_channel_band 

950 rcb = self.config.red_channel_band 

951 colorstr = f"{bcb}{gcb}{rcb}" 

952 

953 # The base path is based on the hips_base_uri. 

954 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=True) 

955 

956 # We need to unique-ify the pixels because they show up for multiple bands. 

957 # The output of this is a sorted array. 

958 pixels = np.unique(np.array([pixel for pixel, _ in hips_exposure_handle_dict.keys()])) 

959 

960 # Add a "gutter" pixel at the end. Start with 0 which maps to 0 always. 

961 pixels = np.append(pixels, [0]) 

962 

963 # Convert the pixels to each order that will be generated. 

964 pixels_shifted = {} 

965 pixels_shifted[max_order] = pixels 

966 for order in range(max_order - 1, min_order - 1, -1): 

967 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2) 

968 

969 # And set the gutter to an illegal pixel value. 

970 for order in range(min_order, max_order + 1): 

971 pixels_shifted[order][-1] = -1 

972 

973 # Read in the first pixel for determining image properties. 

974 exp0 = list(hips_exposure_handle_dict.values())[0].get() 

975 bbox = exp0.getBBox() 

976 npix = bbox.getWidth() 

977 shift_order = int(np.round(np.log2(npix))) 

978 

979 # Create blank exposures for each level, including the highest order. 

980 # We also make sure we create blank exposures for any bands used in the color 

981 # PNGs, even if they aren't available. 

982 exposures = {} 

983 for band in bands: 

984 for order in range(min_order, max_order + 1): 

985 exp = exp0.Factory(bbox=bbox) 

986 exp.image.array[:, :] = np.nan 

987 exposures[(band, order)] = exp 

988 

989 # Loop over all pixels, avoiding the gutter. 

990 for pixel_counter, pixel in enumerate(pixels[:-1]): 

991 self.log.debug("Working on high resolution pixel %d", pixel) 

992 for band in bands: 

993 # Read all the exposures here for the highest order. 

994 # There will always be at least one band with a HiPS image available 

995 # at the highest order. However, for color images it is possible that 

996 # not all bands have coverage so we require this check. 

997 if (pixel, band) in hips_exposure_handle_dict: 

998 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get() 

999 

1000 # Go up the HiPS tree. 

1001 # We only write pixels and rebin to fill the parent pixel when we are 

1002 # done with a current pixel, which is determined if the next pixel 

1003 # has a different pixel number. 

1004 for order in range(max_order, min_order - 1, -1): 

1005 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]: 

1006 # This order is not done, and so none of the other orders will be. 

1007 break 

1008 

1009 # We can now write out the images for each band. 

1010 # Note this will always trigger at the max order where each pixel is unique. 

1011 if not do_color: 

1012 for band in bands: 

1013 self._write_hips_image( 

1014 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1015 order, 

1016 pixels_shifted[order][pixel_counter], 

1017 exposures[(band, order)].image, 

1018 png_grayscale_mapping, 

1019 shift_order=shift_order, 

1020 ) 

1021 else: 

1022 # Make a color png. 

1023 self._write_hips_color_png( 

1024 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1025 order, 

1026 pixels_shifted[order][pixel_counter], 

1027 exposures[(self.config.red_channel_band, order)].image, 

1028 exposures[(self.config.green_channel_band, order)].image, 

1029 exposures[(self.config.blue_channel_band, order)].image, 

1030 png_color_mapping, 

1031 ) 

1032 

1033 log_level = self.log.INFO if order == (max_order - 3) else self.log.DEBUG 

1034 self.log.log( 

1035 log_level, 

1036 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)", 

1037 ",".join(bands), 

1038 order, 

1039 pixels_shifted[order][pixel_counter], 

1040 pixel_counter, 

1041 len(pixels) - 1, 

1042 ) 

1043 

1044 # When we are at the top of the tree, erase top level images and continue. 

1045 if order == min_order: 

1046 for band in bands: 

1047 exposures[(band, order)].image.array[:, :] = np.nan 

1048 continue 

1049 

1050 # Now average the images for each band. 

1051 for band in bands: 

1052 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2) 

1053 with warnings.catch_warnings(): 

1054 warnings.simplefilter("ignore") 

1055 binned_image_arr = np.nanmean(arr, axis=(1, 3)) 

1056 

1057 # Fill the next level up. We figure out which of the four 

1058 # sub-pixels the current pixel occupies. 

1059 sub_index = (pixels_shifted[order][pixel_counter] 

1060 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2)) 

1061 

1062 # Fill exposure at the next level up. 

1063 exp = exposures[(band, order - 1)] 

1064 

1065 # Fill the correct subregion. 

1066 if sub_index == 0: 

1067 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr 

1068 elif sub_index == 1: 

1069 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr 

1070 elif sub_index == 2: 

1071 exp.image.array[npix//2:, npix//2:] = binned_image_arr 

1072 elif sub_index == 3: 

1073 exp.image.array[0: npix//2, npix//2:] = binned_image_arr 

1074 else: 

1075 # This should be impossible. 

1076 raise ValueError("Illegal pixel sub index") 

1077 

1078 # Erase the previous exposure. 

1079 if order < max_order: 

1080 exposures[(band, order)].image.array[:, :] = np.nan 

1081 

1082 # Write the properties files and MOCs. 

1083 if not do_color: 

1084 for band in bands: 

1085 band_pixels = np.array([pixel 

1086 for pixel, band_ in hips_exposure_handle_dict.keys() 

1087 if band_ == band]) 

1088 band_pixels = np.sort(band_pixels) 

1089 

1090 self._write_properties_and_moc( 

1091 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1092 max_order, 

1093 band_pixels, 

1094 exp0, 

1095 shift_order, 

1096 band, 

1097 False, 

1098 ) 

1099 self._write_allsky_file( 

1100 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1101 min_order, 

1102 ) 

1103 else: 

1104 self._write_properties_and_moc( 

1105 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1106 max_order, 

1107 pixels[:-1], 

1108 exp0, 

1109 shift_order, 

1110 colorstr, 

1111 True, 

1112 ) 

1113 self._write_allsky_file( 

1114 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1115 min_order, 

1116 ) 

1117 

1118 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9): 

1119 """Write a HiPS image. 

1120 

1121 Parameters 

1122 ---------- 

1123 hips_base_path : `lsst.resources.ResourcePath` 

1124 Resource path to the base of the HiPS directory tree. 

1125 order : `int` 

1126 HEALPix order of the HiPS image to write. 

1127 pixel : `int` 

1128 HEALPix pixel of the HiPS image. 

1129 image : `lsst.afw.image.Image` 

1130 Image to write. 

1131 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1132 Mapping to convert image to scaled png. 

1133 shift_order : `int`, optional 

1134 HPX shift_order. 

1135 """ 

1136 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1137 # outside of the butler. This task has been given (temporary) 

1138 # Special Dispensation because of the nature of HiPS outputs until 

1139 # a more controlled solution can be found. 

1140 

1141 dir_number = self._get_dir_number(pixel) 

1142 hips_dir = hips_base_path.join( 

1143 f"Norder{order}", 

1144 forceDirectory=True 

1145 ).join( 

1146 f"Dir{dir_number}", 

1147 forceDirectory=True 

1148 ) 

1149 

1150 wcs = makeHpxWcs(order, pixel, shift_order=shift_order) 

1151 

1152 uri = hips_dir.join(f"Npix{pixel}.fits") 

1153 

1154 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1155 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata()) 

1156 

1157 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1158 

1159 # And make a grayscale png as well 

1160 

1161 with np.errstate(invalid="ignore"): 

1162 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8) 

1163 

1164 vals[~np.isfinite(image.array) | (image.array < 0)] = 0 

1165 im = Image.fromarray(vals[::-1, :], "L") 

1166 

1167 uri = hips_dir.join(f"Npix{pixel}.png") 

1168 

1169 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1170 im.save(temporary_uri.ospath) 

1171 

1172 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1173 

1174 def _write_hips_color_png( 

1175 self, 

1176 hips_base_path, 

1177 order, 

1178 pixel, 

1179 image_red, 

1180 image_green, 

1181 image_blue, 

1182 png_mapping, 

1183 ): 

1184 """Write a color png HiPS image. 

1185 

1186 Parameters 

1187 ---------- 

1188 hips_base_path : `lsst.resources.ResourcePath` 

1189 Resource path to the base of the HiPS directory tree. 

1190 order : `int` 

1191 HEALPix order of the HiPS image to write. 

1192 pixel : `int` 

1193 HEALPix pixel of the HiPS image. 

1194 image_red : `lsst.afw.image.Image` 

1195 Input for red channel of output png. 

1196 image_green : `lsst.afw.image.Image` 

1197 Input for green channel of output png. 

1198 image_blue : `lsst.afw.image.Image` 

1199 Input for blue channel of output png. 

1200 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1201 Mapping to convert image to scaled png. 

1202 """ 

1203 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1204 # outside of the butler. This task has been given (temporary) 

1205 # Special Dispensation because of the nature of HiPS outputs until 

1206 # a more controlled solution can be found. 

1207 

1208 dir_number = self._get_dir_number(pixel) 

1209 hips_dir = hips_base_path.join( 

1210 f"Norder{order}", 

1211 forceDirectory=True 

1212 ).join( 

1213 f"Dir{dir_number}", 

1214 forceDirectory=True 

1215 ) 

1216 

1217 # We need to convert nans to the minimum values in the mapping. 

1218 arr_red = image_red.array.copy() 

1219 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0] 

1220 arr_green = image_green.array.copy() 

1221 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1] 

1222 arr_blue = image_blue.array.copy() 

1223 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2] 

1224 

1225 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue) 

1226 

1227 im = Image.fromarray(image_array[::-1, :, :], mode="RGB") 

1228 

1229 uri = hips_dir.join(f"Npix{pixel}.png") 

1230 

1231 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1232 im.save(temporary_uri.ospath) 

1233 

1234 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1235 

1236 def _write_properties_and_moc( 

1237 self, 

1238 hips_base_path, 

1239 max_order, 

1240 pixels, 

1241 exposure, 

1242 shift_order, 

1243 band, 

1244 multiband 

1245 ): 

1246 """Write HiPS properties file and MOC. 

1247 

1248 Parameters 

1249 ---------- 

1250 hips_base_path : : `lsst.resources.ResourcePath` 

1251 Resource path to the base of the HiPS directory tree. 

1252 max_order : `int` 

1253 Maximum HEALPix order. 

1254 pixels : `np.ndarray` (N,) 

1255 Array of pixels used. 

1256 exposure : `lsst.afw.image.Exposure` 

1257 Sample HPX exposure used for generating HiPS tiles. 

1258 shift_order : `int` 

1259 HPX shift order. 

1260 band : `str` 

1261 Band (or color). 

1262 multiband : `bool` 

1263 Is band multiband / color? 

1264 """ 

1265 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels) 

1266 

1267 initial_ra = self.config.properties.initial_ra 

1268 initial_dec = self.config.properties.initial_dec 

1269 initial_fov = self.config.properties.initial_fov 

1270 

1271 if initial_ra is None or initial_dec is None or initial_fov is None: 

1272 # We want to point to an arbitrary pixel in the footprint. 

1273 # Just take the median pixel value for simplicity. 

1274 temp_pixels = pixels.copy() 

1275 if temp_pixels.size % 2 == 0: 

1276 temp_pixels = np.append(temp_pixels, [temp_pixels[0]]) 

1277 medpix = int(np.median(temp_pixels)) 

1278 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix) 

1279 _initial_fov = hpg.nside_to_resolution(2**max_order, units='arcminutes')/60. 

1280 

1281 if initial_ra is None or initial_dec is None: 

1282 initial_ra = _initial_ra 

1283 initial_dec = _initial_dec 

1284 if initial_fov is None: 

1285 initial_fov = _initial_fov 

1286 

1287 self._write_hips_properties_file( 

1288 hips_base_path, 

1289 self.config.properties, 

1290 band, 

1291 multiband, 

1292 exposure, 

1293 max_order, 

1294 shift_order, 

1295 area, 

1296 initial_ra, 

1297 initial_dec, 

1298 initial_fov, 

1299 ) 

1300 

1301 # Write the MOC coverage 

1302 self._write_hips_moc_file( 

1303 hips_base_path, 

1304 max_order, 

1305 pixels, 

1306 ) 

1307 

1308 def _write_hips_properties_file( 

1309 self, 

1310 hips_base_path, 

1311 properties_config, 

1312 band, 

1313 multiband, 

1314 exposure, 

1315 max_order, 

1316 shift_order, 

1317 area, 

1318 initial_ra, 

1319 initial_dec, 

1320 initial_fov 

1321 ): 

1322 """Write HiPS properties file. 

1323 

1324 Parameters 

1325 ---------- 

1326 hips_base_path : `lsst.resources.ResourcePath` 

1327 ResourcePath at top of HiPS tree. File will be written 

1328 to this path as ``properties``. 

1329 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig` 

1330 Configuration for properties values. 

1331 band : `str` 

1332 Name of band(s) for HiPS tree. 

1333 multiband : `bool` 

1334 Is multiband / color? 

1335 exposure : `lsst.afw.image.Exposure` 

1336 Sample HPX exposure used for generating HiPS tiles. 

1337 max_order : `int` 

1338 Maximum HEALPix order. 

1339 shift_order : `int` 

1340 HPX shift order. 

1341 area : `float` 

1342 Coverage area in square degrees. 

1343 initial_ra : `float` 

1344 Initial HiPS RA position (degrees). 

1345 initial_dec : `float` 

1346 Initial HiPS Dec position (degrees). 

1347 initial_fov : `float` 

1348 Initial HiPS display size (degrees). 

1349 """ 

1350 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1351 # outside of the butler. This task has been given (temporary) 

1352 # Special Dispensation because of the nature of HiPS outputs until 

1353 # a more controlled solution can be found. 

1354 def _write_property(fh, name, value): 

1355 """Write a property name/value to a file handle. 

1356 

1357 Parameters 

1358 ---------- 

1359 fh : file handle (blah) 

1360 Open for writing. 

1361 name : `str` 

1362 Name of property 

1363 value : `str` 

1364 Value of property 

1365 """ 

1366 # This ensures that the name has no spaces or space-like characters, 

1367 # per the HiPS standard. 

1368 if re.search(r"\s", name): 

1369 raise ValueError(f"``{name}`` cannot contain any space characters.") 

1370 if "=" in name: 

1371 raise ValueError(f"``{name}`` cannot contain an ``=``") 

1372 

1373 fh.write(f"{name:25}= {value}\n") 

1374 

1375 if exposure.image.array.dtype == np.dtype("float32"): 

1376 bitpix = -32 

1377 elif exposure.image.array.dtype == np.dtype("float64"): 

1378 bitpix = -64 

1379 elif exposure.image.array.dtype == np.dtype("int32"): 

1380 bitpix = 32 

1381 

1382 date_iso8601 = datetime.utcnow().isoformat(timespec="seconds") + "Z" 

1383 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units='degrees') 

1384 

1385 uri = hips_base_path.join("properties") 

1386 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1387 with open(temporary_uri.ospath, "w") as fh: 

1388 _write_property( 

1389 fh, 

1390 "creator_did", 

1391 properties_config.creator_did_template.format(band=band), 

1392 ) 

1393 if properties_config.obs_collection is not None: 

1394 _write_property(fh, "obs_collection", properties_config.obs_collection) 

1395 _write_property( 

1396 fh, 

1397 "obs_title", 

1398 properties_config.obs_title_template.format(band=band), 

1399 ) 

1400 if properties_config.obs_description_template is not None: 

1401 _write_property( 

1402 fh, 

1403 "obs_description", 

1404 properties_config.obs_description_template.format(band=band), 

1405 ) 

1406 if len(properties_config.prov_progenitor) > 0: 

1407 for prov_progenitor in properties_config.prov_progenitor: 

1408 _write_property(fh, "prov_progenitor", prov_progenitor) 

1409 if properties_config.obs_ack is not None: 

1410 _write_property(fh, "obs_ack", properties_config.obs_ack) 

1411 _write_property(fh, "obs_regime", "Optical") 

1412 _write_property(fh, "data_pixel_bitpix", str(bitpix)) 

1413 _write_property(fh, "dataproduct_type", "image") 

1414 _write_property(fh, "moc_sky_fraction", str(area/41253.)) 

1415 _write_property(fh, "data_ucd", "phot.flux") 

1416 _write_property(fh, "hips_creation_date", date_iso8601) 

1417 _write_property(fh, "hips_builder", "lsst.pipe.tasks.hips.GenerateHipsTask") 

1418 _write_property(fh, "hips_creator", "Vera C. Rubin Observatory") 

1419 _write_property(fh, "hips_version", "1.4") 

1420 _write_property(fh, "hips_release_date", date_iso8601) 

1421 _write_property(fh, "hips_frame", "equatorial") 

1422 _write_property(fh, "hips_order", str(max_order)) 

1423 _write_property(fh, "hips_tile_width", str(exposure.getBBox().getWidth())) 

1424 _write_property(fh, "hips_status", "private master clonableOnce") 

1425 if multiband: 

1426 _write_property(fh, "hips_tile_format", "png") 

1427 _write_property(fh, "dataproduct_subtype", "color") 

1428 else: 

1429 _write_property(fh, "hips_tile_format", "png fits") 

1430 _write_property(fh, "hips_pixel_bitpix", str(bitpix)) 

1431 _write_property(fh, "hips_pixel_scale", str(pixel_scale)) 

1432 _write_property(fh, "hips_initial_ra", str(initial_ra)) 

1433 _write_property(fh, "hips_initial_dec", str(initial_dec)) 

1434 _write_property(fh, "hips_initial_fov", str(initial_fov)) 

1435 if multiband: 

1436 if self.config.blue_channel_band in properties_config.spectral_ranges: 

1437 em_min = properties_config.spectral_ranges[ 

1438 self.config.blue_channel_band 

1439 ].lambda_min/1e9 

1440 else: 

1441 self.log.warning("blue band %s not in self.config.spectral_ranges.", band) 

1442 em_min = 3e-7 

1443 if self.config.red_channel_band in properties_config.spectral_ranges: 

1444 em_max = properties_config.spectral_ranges[ 

1445 self.config.red_channel_band 

1446 ].lambda_max/1e9 

1447 else: 

1448 self.log.warning("red band %s not in self.config.spectral_ranges.", band) 

1449 em_max = 1e-6 

1450 else: 

1451 if band in properties_config.spectral_ranges: 

1452 em_min = properties_config.spectral_ranges[band].lambda_min/1e9 

1453 em_max = properties_config.spectral_ranges[band].lambda_max/1e9 

1454 else: 

1455 self.log.warning("band %s not in self.config.spectral_ranges.", band) 

1456 em_min = 3e-7 

1457 em_max = 1e-6 

1458 _write_property(fh, "em_min", str(em_min)) 

1459 _write_property(fh, "em_max", str(em_max)) 

1460 if properties_config.t_min is not None: 

1461 _write_property(fh, "t_min", properties_config.t_min) 

1462 if properties_config.t_max is not None: 

1463 _write_property(fh, "t_max", properties_config.t_max) 

1464 

1465 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1466 

1467 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1): 

1468 """Write HiPS MOC file. 

1469 

1470 Parameters 

1471 ---------- 

1472 hips_base_path : `lsst.resources.ResourcePath` 

1473 ResourcePath to top of HiPS tree. File will be written as 

1474 to this path as ``Moc.fits``. 

1475 max_order : `int` 

1476 Maximum HEALPix order. 

1477 pixels : `np.ndarray` 

1478 Array of pixels covered. 

1479 min_uniq_order : `int`, optional 

1480 Minimum HEALPix order for looking for fully covered pixels. 

1481 """ 

1482 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1483 # outside of the butler. This task has been given (temporary) 

1484 # Special Dispensation because of the nature of HiPS outputs until 

1485 # a more controlled solution can be found. 

1486 

1487 # Make the initial list of UNIQ pixels 

1488 uniq = 4*(4**max_order) + pixels 

1489 

1490 # Make a healsparse map which provides easy degrade/comparisons. 

1491 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32) 

1492 hspmap[pixels] = 1.0 

1493 

1494 # Loop over orders, degrade each time, and look for pixels with full coverage. 

1495 for uniq_order in range(max_order - 1, min_uniq_order - 1, -1): 

1496 hspmap = hspmap.degrade(2**uniq_order, reduction="sum") 

1497 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order)) 

1498 # Check if any of the pixels at uniq_order have full coverage. 

1499 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero() 

1500 if covered.size == 0: 

1501 # No pixels at uniq_order are fully covered, we're done. 

1502 break 

1503 # Replace the UNIQ pixels that are fully covered. 

1504 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered] 

1505 

1506 # Remove duplicate pixels. 

1507 uniq = np.unique(uniq) 

1508 

1509 # Output to fits. 

1510 tbl = np.zeros(uniq.size, dtype=[("UNIQ", "i8")]) 

1511 tbl["UNIQ"] = uniq 

1512 

1513 order = np.log2(tbl["UNIQ"]//4).astype(np.int32)//2 

1514 moc_order = np.max(order) 

1515 

1516 hdu = fits.BinTableHDU(tbl) 

1517 hdu.header["PIXTYPE"] = "HEALPIX" 

1518 hdu.header["ORDERING"] = "NUNIQ" 

1519 hdu.header["COORDSYS"] = "C" 

1520 hdu.header["MOCORDER"] = moc_order 

1521 hdu.header["MOCTOOL"] = "lsst.pipe.tasks.hips.GenerateHipsTask" 

1522 

1523 uri = hips_base_path.join("Moc.fits") 

1524 

1525 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1526 hdu.writeto(temporary_uri.ospath) 

1527 

1528 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1529 

1530 def _write_allsky_file(self, hips_base_path, allsky_order): 

1531 """Write an Allsky.png file. 

1532 

1533 Parameters 

1534 ---------- 

1535 hips_base_path : `lsst.resources.ResourcePath` 

1536 Resource path to the base of the HiPS directory tree. 

1537 allsky_order : `int` 

1538 HEALPix order of the minimum order to make allsky file. 

1539 """ 

1540 tile_size = self.config.allsky_tilesize 

1541 

1542 # The Allsky file format is described in 

1543 # https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf 

1544 # From S4.3.2: 

1545 # The Allsky file is built as an array of tiles, stored side by side in 

1546 # the left-to-right order. The width of this array must be the square 

1547 # root of the number of the tiles of the order. For instance, the width 

1548 # of this array at order 3 is 27 ( (int)sqrt(768) ). To avoid having a 

1549 # too large Allsky file, the resolution of each tile may be reduced but 

1550 # must stay a power of two (typically 64x64 pixels rather than 512x512). 

1551 

1552 n_tiles = hpg.nside_to_npixel(hpg.order_to_nside(allsky_order)) 

1553 n_tiles_wide = int(np.floor(np.sqrt(n_tiles))) 

1554 n_tiles_high = int(np.ceil(n_tiles / n_tiles_wide)) 

1555 

1556 allsky_image = None 

1557 

1558 allsky_order_uri = hips_base_path.join(f"Norder{allsky_order}", forceDirectory=True) 

1559 pixel_regex = re.compile(r"Npix([0-9]+)\.png$") 

1560 png_uris = list( 

1561 ResourcePath.findFileResources( 

1562 candidates=[allsky_order_uri], 

1563 file_filter=pixel_regex, 

1564 ) 

1565 ) 

1566 

1567 for png_uri in png_uris: 

1568 matches = re.match(pixel_regex, png_uri.basename()) 

1569 pix_num = int(matches.group(1)) 

1570 tile_image = Image.open(io.BytesIO(png_uri.read())) 

1571 row = math.floor(pix_num//n_tiles_wide) 

1572 column = pix_num % n_tiles_wide 

1573 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size) 

1574 tile_image_shrunk = tile_image.resize((tile_size, tile_size)) 

1575 

1576 if allsky_image is None: 

1577 allsky_image = Image.new( 

1578 tile_image.mode, 

1579 (n_tiles_wide*tile_size, n_tiles_high*tile_size), 

1580 ) 

1581 allsky_image.paste(tile_image_shrunk, box) 

1582 

1583 uri = allsky_order_uri.join("Allsky.png") 

1584 

1585 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1586 allsky_image.save(temporary_uri.ospath) 

1587 

1588 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1589 

1590 def _get_dir_number(self, pixel): 

1591 """Compute the directory number from a pixel. 

1592 

1593 Parameters 

1594 ---------- 

1595 pixel : `int` 

1596 HEALPix pixel number. 

1597 

1598 Returns 

1599 ------- 

1600 dir_number : `int` 

1601 HiPS directory number. 

1602 """ 

1603 return (pixel//10000)*10000 

1604 

1605 

1606class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections, 

1607 dimensions=("instrument", ), 

1608 defaultTemplates={"coaddName": "deep"}): 

1609 hips_exposure_handles = pipeBase.connectionTypes.Input( 

1610 doc="HiPS-compatible HPX images.", 

1611 name="{coaddName}Coadd_hpx", 

1612 storageClass="ExposureF", 

1613 dimensions=("healpix11", "band"), 

1614 multiple=True, 

1615 deferLoad=True, 

1616 ) 

1617 

1618 

1619class GenerateColorHipsConfig(GenerateHipsConfig, 

1620 pipelineConnections=GenerateColorHipsConnections): 

1621 """Configuration parameters for GenerateColorHipsTask.""" 

1622 blue_channel_band = pexConfig.Field( 

1623 doc="Band to use for blue channel of color pngs.", 

1624 dtype=str, 

1625 default="g", 

1626 ) 

1627 green_channel_band = pexConfig.Field( 

1628 doc="Band to use for green channel of color pngs.", 

1629 dtype=str, 

1630 default="r", 

1631 ) 

1632 red_channel_band = pexConfig.Field( 

1633 doc="Band to use for red channel of color pngs.", 

1634 dtype=str, 

1635 default="i", 

1636 ) 

1637 png_color_asinh_minimum = pexConfig.Field( 

1638 doc="AsinhMapping intensity to be mapped to black for color png scaling.", 

1639 dtype=float, 

1640 default=0.0, 

1641 ) 

1642 png_color_asinh_stretch = pexConfig.Field( 

1643 doc="AsinhMapping linear stretch for color png scaling.", 

1644 dtype=float, 

1645 default=5.0, 

1646 ) 

1647 png_color_asinh_softening = pexConfig.Field( 

1648 doc="AsinhMapping softening parameter (Q) for color png scaling.", 

1649 dtype=float, 

1650 default=8.0, 

1651 ) 

1652 

1653 

1654class GenerateColorHipsTask(GenerateHipsTask): 

1655 """Task for making a HiPS tree with color pngs.""" 

1656 ConfigClass = GenerateColorHipsConfig 

1657 _DefaultName = "generateColorHips" 

1658 color_task = True 

1659 

1660 def _check_data_bands(self, data_bands): 

1661 """Check the data for configured bands. 

1662 

1663 Warn if any color bands are missing data. 

1664 

1665 Parameters 

1666 ---------- 

1667 data_bands : `set` [`str`] 

1668 Bands from the input data. 

1669 

1670 Returns 

1671 ------- 

1672 bands : `list` [`str`] 

1673 List of bands in bgr color order. 

1674 """ 

1675 if len(data_bands) == 0: 

1676 raise RuntimeError("GenerateColorHipsTask must have data from at least one band.") 

1677 

1678 if self.config.blue_channel_band not in data_bands: 

1679 self.log.warning( 

1680 "Color png blue_channel_band %s not in dataset.", 

1681 self.config.blue_channel_band 

1682 ) 

1683 if self.config.green_channel_band not in data_bands: 

1684 self.log.warning( 

1685 "Color png green_channel_band %s not in dataset.", 

1686 self.config.green_channel_band 

1687 ) 

1688 if self.config.red_channel_band not in data_bands: 

1689 self.log.warning( 

1690 "Color png red_channel_band %s not in dataset.", 

1691 self.config.red_channel_band 

1692 ) 

1693 

1694 bands = [ 

1695 self.config.blue_channel_band, 

1696 self.config.green_channel_band, 

1697 self.config.red_channel_band, 

1698 ] 

1699 

1700 return bands