Coverage for python/lsst/pipe/tasks/hips.py: 14%

614 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-11 10:41 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Tasks for making and manipulating HIPS images.""" 

23 

24__all__ = ["HighResolutionHipsTask", "HighResolutionHipsConfig", "HighResolutionHipsConnections", 

25 "GenerateHipsTask", "GenerateHipsConfig", "GenerateColorHipsTask", "GenerateColorHipsConfig"] 

26 

27from collections import defaultdict 

28import numpy as np 

29import argparse 

30import io 

31import sys 

32import re 

33import warnings 

34import math 

35from datetime import datetime 

36import hpgeom as hpg 

37import healsparse as hsp 

38from astropy.io import fits 

39from astropy.visualization.lupton_rgb import AsinhMapping 

40from PIL import Image 

41 

42from lsst.sphgeom import RangeSet, HealpixPixelization 

43from lsst.utils.timer import timeMethod 

44from lsst.daf.butler import Butler, DataCoordinate, DatasetRef, Quantum, SkyPixDimension 

45import lsst.pex.config as pexConfig 

46import lsst.pipe.base as pipeBase 

47import lsst.afw.geom as afwGeom 

48import lsst.afw.math as afwMath 

49import lsst.afw.image as afwImage 

50import lsst.geom as geom 

51from lsst.afw.geom import makeHpxWcs 

52from lsst.resources import ResourcePath 

53 

54 

55class HighResolutionHipsConnections(pipeBase.PipelineTaskConnections, 

56 dimensions=("healpix9", "band"), 

57 defaultTemplates={"coaddName": "deep"}): 

58 coadd_exposure_handles = pipeBase.connectionTypes.Input( 

59 doc="Coadded exposures to convert to HIPS format.", 

60 name="{coaddName}Coadd_calexp", 

61 storageClass="ExposureF", 

62 dimensions=("tract", "patch", "skymap", "band"), 

63 multiple=True, 

64 deferLoad=True, 

65 ) 

66 hips_exposures = pipeBase.connectionTypes.Output( 

67 doc="HiPS-compatible HPX image.", 

68 name="{coaddName}Coadd_hpx", 

69 storageClass="ExposureF", 

70 dimensions=("healpix11", "band"), 

71 multiple=True, 

72 ) 

73 

74 def __init__(self, *, config=None): 

75 super().__init__(config=config) 

76 

77 quantum_order = None 

78 for dim in self.dimensions: 

79 if "healpix" in dim: 

80 if quantum_order is not None: 

81 raise ValueError("Must not specify more than one quantum healpix dimension.") 

82 quantum_order = int(dim.split("healpix")[1]) 

83 if quantum_order is None: 

84 raise ValueError("Must specify a healpix dimension in quantum dimensions.") 

85 

86 if quantum_order > config.hips_order: 

87 raise ValueError("Quantum healpix dimension order must not be greater than hips_order") 

88 

89 order = None 

90 for dim in self.hips_exposures.dimensions: 

91 if "healpix" in dim: 

92 if order is not None: 

93 raise ValueError("Must not specify more than one healpix dimension.") 

94 order = int(dim.split("healpix")[1]) 

95 if order is None: 

96 raise ValueError("Must specify a healpix dimension in hips_exposure dimensions.") 

97 

98 if order != config.hips_order: 

99 raise ValueError("healpix dimension order must match config.hips_order.") 

100 

101 

102class HighResolutionHipsConfig(pipeBase.PipelineTaskConfig, 

103 pipelineConnections=HighResolutionHipsConnections): 

104 """Configuration parameters for HighResolutionHipsTask. 

105 

106 Notes 

107 ----- 

108 A HiPS image covers one HEALPix cell, with the HEALPix nside equal to 

109 2**hips_order. Each cell is 'shift_order' orders deeper than the HEALPix 

110 cell, with 2**shift_order x 2**shift_order sub-pixels on a side, which 

111 defines the target resolution of the HiPS image. The IVOA recommends 

112 shift_order=9, for 2**9=512 pixels on a side. 

113 

114 Table 5 from 

115 https://www.ivoa.net/documents/HiPS/20170519/REC-HIPS-1.0-20170519.pdf 

116 shows the relationship between hips_order, number of tiles (full 

117 sky coverage), cell size, and sub-pixel size/image resolution (with 

118 the default shift_order=9): 

119 +------------+-----------------+--------------+------------------+ 

120 | hips_order | Number of Tiles | Cell Size | Image Resolution | 

121 +============+=================+==============+==================+ 

122 | 0 | 12 | 58.63 deg | 6.871 arcmin | 

123 | 1 | 48 | 29.32 deg | 3.435 arcmin | 

124 | 2 | 192 | 14.66 deg | 1.718 arcmin | 

125 | 3 | 768 | 7.329 deg | 51.53 arcsec | 

126 | 4 | 3072 | 3.665 deg | 25.77 arcsec | 

127 | 5 | 12288 | 1.832 deg | 12.88 arcsec | 

128 | 6 | 49152 | 54.97 arcmin | 6.442 arcsec | 

129 | 7 | 196608 | 27.48 arcmin | 3.221 arcsec | 

130 | 8 | 786432 | 13.74 arcmin | 1.61 arcsec | 

131 | 9 | 3145728 | 6.871 arcmin | 805.2mas | 

132 | 10 | 12582912 | 3.435 arcmin | 402.6mas | 

133 | 11 | 50331648 | 1.718 arcmin | 201.3mas | 

134 | 12 | 201326592 | 51.53 arcsec | 100.6mas | 

135 | 13 | 805306368 | 25.77 arcsec | 50.32mas | 

136 +------------+-----------------+--------------+------------------+ 

137 """ 

138 hips_order = pexConfig.Field( 

139 doc="HIPS image order.", 

140 dtype=int, 

141 default=11, 

142 ) 

143 shift_order = pexConfig.Field( 

144 doc="HIPS shift order (such that each tile is 2**shift_order pixels on a side)", 

145 dtype=int, 

146 default=9, 

147 ) 

148 warp = pexConfig.ConfigField( 

149 dtype=afwMath.Warper.ConfigClass, 

150 doc="Warper configuration", 

151 ) 

152 

153 def setDefaults(self): 

154 self.warp.warpingKernelName = "lanczos5" 

155 

156 

157class HipsTaskNameDescriptor: 

158 """Descriptor used create a DefaultName that matches the order of 

159 the defined dimensions in the connections class. 

160 

161 Parameters 

162 ---------- 

163 prefix : `str` 

164 The prefix of the Default name, to which the order will be 

165 appended. 

166 """ 

167 def __init__(self, prefix): 

168 # create a defaultName template 

169 self._defaultName = f"{prefix}{{}}" 

170 self._order = None 

171 

172 def __get__(self, obj, klass=None): 

173 if klass is None: 

174 raise RuntimeError( 

175 "HipsTaskDescriptor was used in an unexpected context" 

176 ) 

177 if self._order is None: 

178 klassDimensions = klass.ConfigClass.ConnectionsClass.dimensions 

179 for dim in klassDimensions: 

180 if (match := re.match(r"^healpix(\d*)$", dim)) is not None: 

181 self._order = int(match.group(1)) 

182 break 

183 else: 

184 raise RuntimeError( 

185 "Could not find healpix dimension in connections class" 

186 ) 

187 return self._defaultName.format(self._order) 

188 

189 

190class HighResolutionHipsTask(pipeBase.PipelineTask): 

191 """Task for making high resolution HiPS images.""" 

192 ConfigClass = HighResolutionHipsConfig 

193 _DefaultName = HipsTaskNameDescriptor("highResolutionHips") 

194 

195 def __init__(self, **kwargs): 

196 super().__init__(**kwargs) 

197 self.warper = afwMath.Warper.fromConfig(self.config.warp) 

198 

199 @timeMethod 

200 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

201 inputs = butlerQC.get(inputRefs) 

202 

203 healpix_dim = f"healpix{self.config.hips_order}" 

204 

205 pixels = [hips_exposure.dataId[healpix_dim] 

206 for hips_exposure in outputRefs.hips_exposures] 

207 

208 outputs = self.run(pixels=pixels, coadd_exposure_handles=inputs["coadd_exposure_handles"]) 

209 

210 hips_exposure_ref_dict = {hips_exposure_ref.dataId[healpix_dim]: 

211 hips_exposure_ref for hips_exposure_ref in outputRefs.hips_exposures} 

212 for pixel, hips_exposure in outputs.hips_exposures.items(): 

213 butlerQC.put(hips_exposure, hips_exposure_ref_dict[pixel]) 

214 

215 def run(self, pixels, coadd_exposure_handles): 

216 """Run the HighResolutionHipsTask. 

217 

218 Parameters 

219 ---------- 

220 pixels : `Iterable` [ `int` ] 

221 Iterable of healpix pixels (nest ordering) to warp to. 

222 coadd_exposure_handles : `list` [`lsst.daf.butler.DeferredDatasetHandle`] 

223 Handles for the coadd exposures. 

224 

225 Returns 

226 ------- 

227 outputs : `lsst.pipe.base.Struct` 

228 ``hips_exposures`` is a dict with pixel (key) and hips_exposure (value) 

229 """ 

230 self.log.info("Generating HPX images for %d pixels at order %d", len(pixels), self.config.hips_order) 

231 

232 npix = 2**self.config.shift_order 

233 bbox_hpx = geom.Box2I(corner=geom.Point2I(0, 0), 

234 dimensions=geom.Extent2I(npix, npix)) 

235 

236 # For each healpix pixel we will create an empty exposure with the 

237 # correct HPX WCS. We furthermore create a dict to hold each of 

238 # the warps that will go into each HPX exposure. 

239 exp_hpx_dict = {} 

240 warp_dict = {} 

241 for pixel in pixels: 

242 wcs_hpx = afwGeom.makeHpxWcs(self.config.hips_order, pixel, shift_order=self.config.shift_order) 

243 exp_hpx = afwImage.ExposureF(bbox_hpx, wcs_hpx) 

244 exp_hpx_dict[pixel] = exp_hpx 

245 warp_dict[pixel] = [] 

246 

247 first_handle = True 

248 # Loop over input coadd exposures to minimize i/o (this speeds things 

249 # up by ~8x to batch together pixels that overlap a given coadd). 

250 for handle in coadd_exposure_handles: 

251 coadd_exp = handle.get() 

252 

253 # For each pixel, warp the coadd to the HPX WCS for the pixel. 

254 for pixel in pixels: 

255 warped = self.warper.warpExposure(exp_hpx_dict[pixel].getWcs(), coadd_exp, maxBBox=bbox_hpx) 

256 

257 exp = afwImage.ExposureF(exp_hpx_dict[pixel].getBBox(), exp_hpx_dict[pixel].getWcs()) 

258 exp.maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

259 

260 if first_handle: 

261 # Make sure the mask planes, filter, and photocalib of the output 

262 # exposure match the (first) input exposure. 

263 exp_hpx_dict[pixel].mask.conformMaskPlanes(coadd_exp.mask.getMaskPlaneDict()) 

264 exp_hpx_dict[pixel].setFilter(coadd_exp.getFilter()) 

265 exp_hpx_dict[pixel].setPhotoCalib(coadd_exp.getPhotoCalib()) 

266 

267 if warped.getBBox().getArea() == 0 or not np.any(np.isfinite(warped.image.array)): 

268 # There is no overlap, skip. 

269 self.log.debug( 

270 "No overlap between output HPX %d and input exposure %s", 

271 pixel, 

272 handle.dataId 

273 ) 

274 continue 

275 

276 exp.maskedImage.assign(warped.maskedImage, warped.getBBox()) 

277 warp_dict[pixel].append(exp.maskedImage) 

278 

279 first_handle = False 

280 

281 stats_flags = afwMath.stringToStatisticsProperty("MEAN") 

282 stats_ctrl = afwMath.StatisticsControl() 

283 stats_ctrl.setNanSafe(True) 

284 stats_ctrl.setWeighted(True) 

285 stats_ctrl.setCalcErrorFromInputVariance(True) 

286 

287 # Loop over pixels and combine the warps for each pixel. 

288 # The combination is done with a simple mean for pixels that 

289 # overlap in neighboring patches. 

290 for pixel in pixels: 

291 exp_hpx_dict[pixel].maskedImage.set(np.nan, afwImage.Mask.getPlaneBitMask("NO_DATA"), np.nan) 

292 

293 if not warp_dict[pixel]: 

294 # Nothing in this pixel 

295 self.log.debug("No data in HPX pixel %d", pixel) 

296 # Remove the pixel from the output, no need to persist an 

297 # empty exposure. 

298 exp_hpx_dict.pop(pixel) 

299 continue 

300 

301 exp_hpx_dict[pixel].maskedImage = afwMath.statisticsStack( 

302 warp_dict[pixel], 

303 stats_flags, 

304 stats_ctrl, 

305 [1.0]*len(warp_dict[pixel]), 

306 clipped=0, 

307 maskMap=[] 

308 ) 

309 

310 return pipeBase.Struct(hips_exposures=exp_hpx_dict) 

311 

312 @classmethod 

313 def build_quantum_graph_cli(cls, argv): 

314 """A command-line interface entry point to `build_quantum_graph`. 

315 This method provides the implementation for the 

316 ``build-high-resolution-hips-qg`` script. 

317 

318 Parameters 

319 ---------- 

320 argv : `Sequence` [ `str` ] 

321 Command-line arguments (e.g. ``sys.argv[1:]``). 

322 """ 

323 parser = cls._make_cli_parser() 

324 

325 args = parser.parse_args(argv) 

326 

327 if args.subparser_name is None: 

328 parser.print_help() 

329 sys.exit(1) 

330 

331 pipeline = pipeBase.Pipeline.from_uri(args.pipeline) 

332 expanded_pipeline = list(pipeline.toExpandedPipeline()) 

333 

334 if len(expanded_pipeline) != 1: 

335 raise RuntimeError(f"Pipeline file {args.pipeline} may only contain one task.") 

336 

337 (task_def,) = expanded_pipeline 

338 

339 butler = Butler(args.butler_config, collections=args.input) 

340 

341 if args.subparser_name == "segment": 

342 # Do the segmentation 

343 hpix_pixelization = HealpixPixelization(level=args.hpix_build_order) 

344 dataset = task_def.connections.coadd_exposure_handles.name 

345 data_ids = set(butler.registry.queryDataIds("tract", datasets=dataset).expanded()) 

346 region_pixels = [] 

347 for data_id in data_ids: 

348 region = data_id.region 

349 pixel_range = hpix_pixelization.envelope(region) 

350 for r in pixel_range.ranges(): 

351 region_pixels.extend(range(r[0], r[1])) 

352 indices = np.unique(region_pixels) 

353 

354 print(f"Pixels to run at HEALPix order --hpix_build_order {args.hpix_build_order}:") 

355 for pixel in indices: 

356 print(pixel) 

357 

358 elif args.subparser_name == "build": 

359 # Build the quantum graph. 

360 

361 # Figure out collection names. 

362 if args.output_run is None: 

363 if args.output is None: 

364 raise ValueError("At least one of --output or --output-run options is required.") 

365 args.output_run = "{}/{}".format(args.output, pipeBase.Instrument.makeCollectionTimestamp()) 

366 

367 build_ranges = RangeSet(sorted(args.pixels)) 

368 

369 # Metadata includes a subset of attributes defined in CmdLineFwk. 

370 metadata = { 

371 "input": args.input, 

372 "butler_argument": args.butler_config, 

373 "output": args.output, 

374 "output_run": args.output_run, 

375 "data_query": args.where, 

376 "time": f"{datetime.now()}", 

377 } 

378 

379 qg = cls.build_quantum_graph( 

380 task_def, 

381 butler.registry, 

382 args.hpix_build_order, 

383 build_ranges, 

384 where=args.where, 

385 collections=args.input, 

386 metadata=metadata, 

387 ) 

388 qg.saveUri(args.save_qgraph) 

389 

390 @classmethod 

391 def _make_cli_parser(cls): 

392 """Make the command-line parser. 

393 

394 Returns 

395 ------- 

396 parser : `argparse.ArgumentParser` 

397 """ 

398 parser = argparse.ArgumentParser( 

399 description=( 

400 "Build a QuantumGraph that runs HighResolutionHipsTask on existing coadd datasets." 

401 ), 

402 ) 

403 subparsers = parser.add_subparsers(help="sub-command help", dest="subparser_name") 

404 

405 parser_segment = subparsers.add_parser("segment", 

406 help="Determine survey segments for workflow.") 

407 parser_build = subparsers.add_parser("build", 

408 help="Build quantum graph for HighResolutionHipsTask") 

409 

410 for sub in [parser_segment, parser_build]: 

411 # These arguments are in common. 

412 sub.add_argument( 

413 "-b", 

414 "--butler-config", 

415 type=str, 

416 help="Path to data repository or butler configuration.", 

417 required=True, 

418 ) 

419 sub.add_argument( 

420 "-p", 

421 "--pipeline", 

422 type=str, 

423 help="Pipeline file, limited to one task.", 

424 required=True, 

425 ) 

426 sub.add_argument( 

427 "-i", 

428 "--input", 

429 type=str, 

430 nargs="+", 

431 help="Input collection(s) to search for coadd exposures.", 

432 required=True, 

433 ) 

434 sub.add_argument( 

435 "-o", 

436 "--hpix_build_order", 

437 type=int, 

438 default=1, 

439 help="HEALPix order to segment sky for building quantum graph files.", 

440 ) 

441 sub.add_argument( 

442 "-w", 

443 "--where", 

444 type=str, 

445 default=None, 

446 help="Data ID expression used when querying for input coadd datasets.", 

447 ) 

448 

449 parser_build.add_argument( 

450 "--output", 

451 type=str, 

452 help=( 

453 "Name of the output CHAINED collection. If this options is specified and " 

454 "--output-run is not, then a new RUN collection will be created by appending " 

455 "a timestamp to the value of this option." 

456 ), 

457 default=None, 

458 metavar="COLL", 

459 ) 

460 parser_build.add_argument( 

461 "--output-run", 

462 type=str, 

463 help=( 

464 "Output RUN collection to write resulting images. If not provided " 

465 "then --output must be provided and a new RUN collection will be created " 

466 "by appending a timestamp to the value passed with --output." 

467 ), 

468 default=None, 

469 metavar="RUN", 

470 ) 

471 parser_build.add_argument( 

472 "-q", 

473 "--save-qgraph", 

474 type=str, 

475 help="Output filename for QuantumGraph.", 

476 required=True, 

477 ) 

478 parser_build.add_argument( 

479 "-P", 

480 "--pixels", 

481 type=int, 

482 nargs="+", 

483 help="Pixels at --hpix_build_order to generate quantum graph.", 

484 required=True, 

485 ) 

486 

487 return parser 

488 

489 @classmethod 

490 def build_quantum_graph( 

491 cls, 

492 task_def, 

493 registry, 

494 constraint_order, 

495 constraint_ranges, 

496 where=None, 

497 collections=None, 

498 metadata=None, 

499 ): 

500 """Generate a `QuantumGraph` for running just this task. 

501 

502 This is a temporary workaround for incomplete butler query support for 

503 HEALPix dimensions. 

504 

505 Parameters 

506 ---------- 

507 task_def : `lsst.pipe.base.TaskDef` 

508 Task definition. 

509 registry : `lsst.daf.butler.Registry` 

510 Client for the butler database. May be read-only. 

511 constraint_order : `int` 

512 HEALPix order used to contrain which quanta are generated, via 

513 ``constraint_indices``. This should be a coarser grid (smaller 

514 order) than the order used for the task's quantum and output data 

515 IDs, and ideally something between the spatial scale of a patch or 

516 the data repository's "common skypix" system (usually ``htm7``). 

517 constraint_ranges : `lsst.sphgeom.RangeSet` 

518 RangeSet which describes constraint pixels (HEALPix NEST, with order 

519 constraint_order) to constrain generated quanta. 

520 where : `str`, optional 

521 A boolean `str` expression of the form accepted by 

522 `Registry.queryDatasets` to constrain input datasets. This may 

523 contain a constraint on tracts, patches, or bands, but not HEALPix 

524 indices. Constraints on tracts and patches should usually be 

525 unnecessary, however - existing coadds that overlap the given 

526 HEALpix indices will be selected without such a constraint, and 

527 providing one may reject some that should normally be included. 

528 collections : `str` or `Iterable` [ `str` ], optional 

529 Collection or collections to search for input datasets, in order. 

530 If not provided, ``registry.defaults.collections`` will be 

531 searched. 

532 metadata : `dict` [ `str`, `Any` ] 

533 Graph metadata. It is required to contain "output_run" key with the 

534 name of the output RUN collection. 

535 """ 

536 config = task_def.config 

537 

538 dataset_types = pipeBase.PipelineDatasetTypes.fromPipeline(pipeline=[task_def], registry=registry) 

539 # Since we know this is the only task in the pipeline, we know there 

540 # is only one overall input and one overall output. 

541 (input_dataset_type,) = dataset_types.inputs 

542 

543 # Extract the main output dataset type (which needs multiple 

544 # DatasetRefs, and tells us the output HPX level), and make a set of 

545 # what remains for more mechanical handling later. 

546 output_dataset_type = dataset_types.outputs[task_def.connections.hips_exposures.name] 

547 incidental_output_dataset_types = dataset_types.outputs.copy() 

548 incidental_output_dataset_types.remove(output_dataset_type) 

549 (hpx_output_dimension,) = (d for d in output_dataset_type.dimensions 

550 if isinstance(d, SkyPixDimension)) 

551 

552 constraint_hpx_pixelization = registry.dimensions[f"healpix{constraint_order}"].pixelization 

553 common_skypix_name = registry.dimensions.commonSkyPix.name 

554 common_skypix_pixelization = registry.dimensions.commonSkyPix.pixelization 

555 

556 # We will need all the pixels at the quantum resolution as well 

557 task_dimensions = registry.dimensions.extract(task_def.connections.dimensions) 

558 (hpx_dimension,) = (d for d in task_dimensions if d.name != "band") 

559 hpx_pixelization = hpx_dimension.pixelization 

560 

561 if hpx_pixelization.level < constraint_order: 

562 raise ValueError(f"Quantum order {hpx_pixelization.level} must be < {constraint_order}") 

563 hpx_ranges = constraint_ranges.scaled(4**(hpx_pixelization.level - constraint_order)) 

564 

565 # We can be generous in looking for pixels here, because we constraint by actual 

566 # patch regions below. 

567 common_skypix_ranges = RangeSet() 

568 for begin, end in constraint_ranges: 

569 for hpx_index in range(begin, end): 

570 constraint_hpx_region = constraint_hpx_pixelization.pixel(hpx_index) 

571 common_skypix_ranges |= common_skypix_pixelization.envelope(constraint_hpx_region) 

572 

573 # To keep the query from getting out of hand (and breaking) we simplify until we have fewer 

574 # than 100 ranges which seems to work fine. 

575 for simp in range(1, 10): 

576 if len(common_skypix_ranges) < 100: 

577 break 

578 common_skypix_ranges.simplify(simp) 

579 

580 # Use that RangeSet to assemble a WHERE constraint expression. This 

581 # could definitely get too big if the "constraint healpix" order is too 

582 # fine. 

583 where_terms = [] 

584 bind = {} 

585 for n, (begin, end) in enumerate(common_skypix_ranges): 

586 stop = end - 1 # registry range syntax is inclusive 

587 if begin == stop: 

588 where_terms.append(f"{common_skypix_name} = cpx{n}") 

589 bind[f"cpx{n}"] = begin 

590 else: 

591 where_terms.append(f"({common_skypix_name} >= cpx{n}a AND {common_skypix_name} <= cpx{n}b)") 

592 bind[f"cpx{n}a"] = begin 

593 bind[f"cpx{n}b"] = stop 

594 if where is None: 

595 where = " OR ".join(where_terms) 

596 else: 

597 where = f"({where}) AND ({' OR '.join(where_terms)})" 

598 # Query for input datasets with this constraint, and ask for expanded 

599 # data IDs because we want regions. Immediately group this by patch so 

600 # we don't do later geometric stuff n_bands more times than we need to. 

601 input_refs = registry.queryDatasets( 

602 input_dataset_type, 

603 where=where, 

604 findFirst=True, 

605 collections=collections, 

606 bind=bind 

607 ).expanded() 

608 inputs_by_patch = defaultdict(set) 

609 patch_dimensions = registry.dimensions.extract(["patch"]) 

610 for input_ref in input_refs: 

611 inputs_by_patch[input_ref.dataId.subset(patch_dimensions)].add(input_ref) 

612 if not inputs_by_patch: 

613 message_body = "\n".join(input_refs.explain_no_results()) 

614 raise RuntimeError(f"No inputs found:\n{message_body}") 

615 

616 # Iterate over patches and compute the set of output healpix pixels 

617 # that overlap each one. Use that to associate inputs with output 

618 # pixels, but only for the output pixels we've already identified. 

619 inputs_by_hpx = defaultdict(set) 

620 for patch_data_id, input_refs_for_patch in inputs_by_patch.items(): 

621 patch_hpx_ranges = hpx_pixelization.envelope(patch_data_id.region) 

622 for begin, end in patch_hpx_ranges & hpx_ranges: 

623 for hpx_index in range(begin, end): 

624 inputs_by_hpx[hpx_index].update(input_refs_for_patch) 

625 # Iterate over the dict we just created and create the actual quanta. 

626 quanta = [] 

627 output_run = metadata["output_run"] 

628 for hpx_index, input_refs_for_hpx_index in inputs_by_hpx.items(): 

629 # Group inputs by band. 

630 input_refs_by_band = defaultdict(list) 

631 for input_ref in input_refs_for_hpx_index: 

632 input_refs_by_band[input_ref.dataId["band"]].append(input_ref) 

633 # Iterate over bands to make quanta. 

634 for band, input_refs_for_band in input_refs_by_band.items(): 

635 data_id = registry.expandDataId({hpx_dimension: hpx_index, "band": band}) 

636 

637 hpx_pixel_ranges = RangeSet(hpx_index) 

638 hpx_output_ranges = hpx_pixel_ranges.scaled(4**(config.hips_order - hpx_pixelization.level)) 

639 output_data_ids = [] 

640 for begin, end in hpx_output_ranges: 

641 for hpx_output_index in range(begin, end): 

642 output_data_ids.append( 

643 registry.expandDataId({hpx_output_dimension: hpx_output_index, "band": band}) 

644 ) 

645 outputs = { 

646 dt: [DatasetRef(dt, data_id, run=output_run)] for dt in incidental_output_dataset_types 

647 } 

648 outputs[output_dataset_type] = [DatasetRef(output_dataset_type, data_id, run=output_run) 

649 for data_id in output_data_ids] 

650 quanta.append( 

651 Quantum( 

652 taskName=task_def.taskName, 

653 taskClass=task_def.taskClass, 

654 dataId=data_id, 

655 initInputs={}, 

656 inputs={input_dataset_type: input_refs_for_band}, 

657 outputs=outputs, 

658 ) 

659 ) 

660 

661 if len(quanta) == 0: 

662 raise RuntimeError("Given constraints yielded empty quantum graph.") 

663 

664 # Define initOutputs refs. 

665 empty_data_id = DataCoordinate.makeEmpty(registry.dimensions) 

666 init_outputs = {} 

667 global_init_outputs = [] 

668 if config_dataset_type := dataset_types.initOutputs.get(task_def.configDatasetName): 

669 init_outputs[task_def] = [DatasetRef(config_dataset_type, empty_data_id, run=output_run)] 

670 packages_dataset_name = pipeBase.PipelineDatasetTypes.packagesDatasetName 

671 if packages_dataset_type := dataset_types.initOutputs.get(packages_dataset_name): 

672 global_init_outputs.append(DatasetRef(packages_dataset_type, empty_data_id, run=output_run)) 

673 

674 return pipeBase.QuantumGraph( 

675 quanta={task_def: quanta}, 

676 initOutputs=init_outputs, 

677 globalInitOutputs=global_init_outputs, 

678 metadata=metadata, 

679 ) 

680 

681 

682class HipsPropertiesSpectralTerm(pexConfig.Config): 

683 lambda_min = pexConfig.Field( 

684 doc="Minimum wavelength (nm)", 

685 dtype=float, 

686 ) 

687 lambda_max = pexConfig.Field( 

688 doc="Maximum wavelength (nm)", 

689 dtype=float, 

690 ) 

691 

692 

693class HipsPropertiesConfig(pexConfig.Config): 

694 """Configuration parameters for writing a HiPS properties file.""" 

695 creator_did_template = pexConfig.Field( 

696 doc=("Unique identifier of the HiPS - Format: IVOID. " 

697 "Use ``{band}`` to substitute the band name."), 

698 dtype=str, 

699 optional=False, 

700 ) 

701 obs_collection = pexConfig.Field( 

702 doc="Short name of original data set - Format: one word", 

703 dtype=str, 

704 optional=True, 

705 ) 

706 obs_description_template = pexConfig.Field( 

707 doc=("Data set description - Format: free text, longer free text " 

708 "description of the dataset. Use ``{band}`` to substitute " 

709 "the band name."), 

710 dtype=str, 

711 ) 

712 prov_progenitor = pexConfig.ListField( 

713 doc="Provenance of the original data - Format: free text", 

714 dtype=str, 

715 default=[], 

716 ) 

717 obs_title_template = pexConfig.Field( 

718 doc=("Data set title format: free text, but should be short. " 

719 "Use ``{band}`` to substitute the band name."), 

720 dtype=str, 

721 optional=False, 

722 ) 

723 spectral_ranges = pexConfig.ConfigDictField( 

724 doc=("Mapping from band to lambda_min, lamba_max (nm). May be approximate."), 

725 keytype=str, 

726 itemtype=HipsPropertiesSpectralTerm, 

727 default={}, 

728 ) 

729 initial_ra = pexConfig.Field( 

730 doc="Initial RA (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

731 dtype=float, 

732 optional=True, 

733 ) 

734 initial_dec = pexConfig.Field( 

735 doc="Initial Declination (deg) (default for HiPS viewer). If not set will use a point in MOC.", 

736 dtype=float, 

737 optional=True, 

738 ) 

739 initial_fov = pexConfig.Field( 

740 doc="Initial field-of-view (deg). If not set will use ~1 healpix tile.", 

741 dtype=float, 

742 optional=True, 

743 ) 

744 obs_ack = pexConfig.Field( 

745 doc="Observation acknowledgements (free text).", 

746 dtype=str, 

747 optional=True, 

748 ) 

749 t_min = pexConfig.Field( 

750 doc="Time (MJD) of earliest observation included in HiPS", 

751 dtype=float, 

752 optional=True, 

753 ) 

754 t_max = pexConfig.Field( 

755 doc="Time (MJD) of latest observation included in HiPS", 

756 dtype=float, 

757 optional=True, 

758 ) 

759 

760 def validate(self): 

761 super().validate() 

762 

763 if self.obs_collection is not None: 

764 if re.search(r"\s", self.obs_collection): 

765 raise ValueError("obs_collection cannot contain any space characters.") 

766 

767 def setDefaults(self): 

768 # Values here taken from 

769 # https://github.com/lsst-dm/dax_obscore/blob/44ac15029136e2ec15/configs/dp02.yaml#L46 

770 u_term = HipsPropertiesSpectralTerm() 

771 u_term.lambda_min = 330. 

772 u_term.lambda_max = 400. 

773 self.spectral_ranges["u"] = u_term 

774 g_term = HipsPropertiesSpectralTerm() 

775 g_term.lambda_min = 402. 

776 g_term.lambda_max = 552. 

777 self.spectral_ranges["g"] = g_term 

778 r_term = HipsPropertiesSpectralTerm() 

779 r_term.lambda_min = 552. 

780 r_term.lambda_max = 691. 

781 self.spectral_ranges["r"] = r_term 

782 i_term = HipsPropertiesSpectralTerm() 

783 i_term.lambda_min = 691. 

784 i_term.lambda_max = 818. 

785 self.spectral_ranges["i"] = i_term 

786 z_term = HipsPropertiesSpectralTerm() 

787 z_term.lambda_min = 818. 

788 z_term.lambda_max = 922. 

789 self.spectral_ranges["z"] = z_term 

790 y_term = HipsPropertiesSpectralTerm() 

791 y_term.lambda_min = 970. 

792 y_term.lambda_max = 1060. 

793 self.spectral_ranges["y"] = y_term 

794 

795 

796class GenerateHipsConnections(pipeBase.PipelineTaskConnections, 

797 dimensions=("instrument", "band"), 

798 defaultTemplates={"coaddName": "deep"}): 

799 hips_exposure_handles = pipeBase.connectionTypes.Input( 

800 doc="HiPS-compatible HPX images.", 

801 name="{coaddName}Coadd_hpx", 

802 storageClass="ExposureF", 

803 dimensions=("healpix11", "band"), 

804 multiple=True, 

805 deferLoad=True, 

806 ) 

807 

808 

809class GenerateHipsConfig(pipeBase.PipelineTaskConfig, 

810 pipelineConnections=GenerateHipsConnections): 

811 """Configuration parameters for GenerateHipsTask.""" 

812 # WARNING: In general PipelineTasks are not allowed to do any outputs 

813 # outside of the butler. This task has been given (temporary) 

814 # Special Dispensation because of the nature of HiPS outputs until 

815 # a more controlled solution can be found. 

816 hips_base_uri = pexConfig.Field( 

817 doc="URI to HiPS base for output.", 

818 dtype=str, 

819 optional=False, 

820 ) 

821 min_order = pexConfig.Field( 

822 doc="Minimum healpix order for HiPS tree.", 

823 dtype=int, 

824 default=3, 

825 ) 

826 properties = pexConfig.ConfigField( 

827 dtype=HipsPropertiesConfig, 

828 doc="Configuration for properties file.", 

829 ) 

830 allsky_tilesize = pexConfig.Field( 

831 dtype=int, 

832 doc="Allsky.png tile size. Must be power of 2.", 

833 default=512, 

834 ) 

835 png_gray_asinh_minimum = pexConfig.Field( 

836 doc="AsinhMapping intensity to be mapped to black for grayscale png scaling.", 

837 dtype=float, 

838 default=0.0, 

839 ) 

840 png_gray_asinh_stretch = pexConfig.Field( 

841 doc="AsinhMapping linear stretch for grayscale png scaling.", 

842 dtype=float, 

843 default=2.0, 

844 ) 

845 png_gray_asinh_softening = pexConfig.Field( 

846 doc="AsinhMapping softening parameter (Q) for grayscale png scaling.", 

847 dtype=float, 

848 default=8.0, 

849 ) 

850 

851 

852class GenerateHipsTask(pipeBase.PipelineTask): 

853 """Task for making a HiPS tree with FITS and grayscale PNGs.""" 

854 ConfigClass = GenerateHipsConfig 

855 _DefaultName = "generateHips" 

856 color_task = False 

857 

858 @timeMethod 

859 def runQuantum(self, butlerQC, inputRefs, outputRefs): 

860 inputs = butlerQC.get(inputRefs) 

861 

862 dims = inputRefs.hips_exposure_handles[0].dataId.names 

863 order = None 

864 for dim in dims: 

865 if "healpix" in dim: 

866 order = int(dim.split("healpix")[1]) 

867 healpix_dim = dim 

868 break 

869 else: 

870 raise RuntimeError("Could not determine healpix order for input exposures.") 

871 

872 hips_exposure_handle_dict = { 

873 (hips_exposure_handle.dataId[healpix_dim], 

874 hips_exposure_handle.dataId["band"]): hips_exposure_handle 

875 for hips_exposure_handle in inputs["hips_exposure_handles"] 

876 } 

877 

878 data_bands = {hips_exposure_handle.dataId["band"] 

879 for hips_exposure_handle in inputs["hips_exposure_handles"]} 

880 bands = self._check_data_bands(data_bands) 

881 

882 self.run( 

883 bands=bands, 

884 max_order=order, 

885 hips_exposure_handle_dict=hips_exposure_handle_dict, 

886 do_color=self.color_task, 

887 ) 

888 

889 def _check_data_bands(self, data_bands): 

890 """Check that the data has only a single band. 

891 

892 Parameters 

893 ---------- 

894 data_bands : `set` [`str`] 

895 Bands from the input data. 

896 

897 Returns 

898 ------- 

899 bands : `list` [`str`] 

900 List of single band to process. 

901 

902 Raises 

903 ------ 

904 RuntimeError if there is not exactly one band. 

905 """ 

906 if len(data_bands) != 1: 

907 raise RuntimeError("GenerateHipsTask can only use data from a single band.") 

908 

909 return list(data_bands) 

910 

911 @timeMethod 

912 def run(self, bands, max_order, hips_exposure_handle_dict, do_color=False): 

913 """Run the GenerateHipsTask. 

914 

915 Parameters 

916 ---------- 

917 bands : `list [ `str` ] 

918 List of bands to be processed (or single band). 

919 max_order : `int` 

920 HEALPix order of the maximum (native) HPX exposures. 

921 hips_exposure_handle_dict : `dict` {`int`: `lsst.daf.butler.DeferredDatasetHandle`} 

922 Dict of handles for the HiPS high-resolution exposures. 

923 Key is (pixel number, ``band``). 

924 do_color : `bool`, optional 

925 Do color pngs instead of per-band grayscale. 

926 """ 

927 min_order = self.config.min_order 

928 

929 if not do_color: 

930 png_grayscale_mapping = AsinhMapping( 

931 self.config.png_gray_asinh_minimum, 

932 self.config.png_gray_asinh_stretch, 

933 Q=self.config.png_gray_asinh_softening, 

934 ) 

935 else: 

936 png_color_mapping = AsinhMapping( 

937 self.config.png_color_asinh_minimum, 

938 self.config.png_color_asinh_stretch, 

939 Q=self.config.png_color_asinh_softening, 

940 ) 

941 

942 bcb = self.config.blue_channel_band 

943 gcb = self.config.green_channel_band 

944 rcb = self.config.red_channel_band 

945 colorstr = f"{bcb}{gcb}{rcb}" 

946 

947 # The base path is based on the hips_base_uri. 

948 hips_base_path = ResourcePath(self.config.hips_base_uri, forceDirectory=True) 

949 

950 # We need to unique-ify the pixels because they show up for multiple bands. 

951 # The output of this is a sorted array. 

952 pixels = np.unique(np.array([pixel for pixel, _ in hips_exposure_handle_dict.keys()])) 

953 

954 # Add a "gutter" pixel at the end. Start with 0 which maps to 0 always. 

955 pixels = np.append(pixels, [0]) 

956 

957 # Convert the pixels to each order that will be generated. 

958 pixels_shifted = {} 

959 pixels_shifted[max_order] = pixels 

960 for order in range(max_order - 1, min_order - 1, -1): 

961 pixels_shifted[order] = np.right_shift(pixels_shifted[order + 1], 2) 

962 

963 # And set the gutter to an illegal pixel value. 

964 for order in range(min_order, max_order + 1): 

965 pixels_shifted[order][-1] = -1 

966 

967 # Read in the first pixel for determining image properties. 

968 exp0 = list(hips_exposure_handle_dict.values())[0].get() 

969 bbox = exp0.getBBox() 

970 npix = bbox.getWidth() 

971 shift_order = int(np.round(np.log2(npix))) 

972 

973 # Create blank exposures for each level, including the highest order. 

974 # We also make sure we create blank exposures for any bands used in the color 

975 # PNGs, even if they aren't available. 

976 exposures = {} 

977 for band in bands: 

978 for order in range(min_order, max_order + 1): 

979 exp = exp0.Factory(bbox=bbox) 

980 exp.image.array[:, :] = np.nan 

981 exposures[(band, order)] = exp 

982 

983 # Loop over all pixels, avoiding the gutter. 

984 for pixel_counter, pixel in enumerate(pixels[:-1]): 

985 self.log.debug("Working on high resolution pixel %d", pixel) 

986 for band in bands: 

987 # Read all the exposures here for the highest order. 

988 # There will always be at least one band with a HiPS image available 

989 # at the highest order. However, for color images it is possible that 

990 # not all bands have coverage so we require this check. 

991 if (pixel, band) in hips_exposure_handle_dict: 

992 exposures[(band, max_order)] = hips_exposure_handle_dict[(pixel, band)].get() 

993 

994 # Go up the HiPS tree. 

995 # We only write pixels and rebin to fill the parent pixel when we are 

996 # done with a current pixel, which is determined if the next pixel 

997 # has a different pixel number. 

998 for order in range(max_order, min_order - 1, -1): 

999 if pixels_shifted[order][pixel_counter + 1] == pixels_shifted[order][pixel_counter]: 

1000 # This order is not done, and so none of the other orders will be. 

1001 break 

1002 

1003 # We can now write out the images for each band. 

1004 # Note this will always trigger at the max order where each pixel is unique. 

1005 if not do_color: 

1006 for band in bands: 

1007 self._write_hips_image( 

1008 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1009 order, 

1010 pixels_shifted[order][pixel_counter], 

1011 exposures[(band, order)].image, 

1012 png_grayscale_mapping, 

1013 shift_order=shift_order, 

1014 ) 

1015 else: 

1016 # Make a color png. 

1017 self._write_hips_color_png( 

1018 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1019 order, 

1020 pixels_shifted[order][pixel_counter], 

1021 exposures[(self.config.red_channel_band, order)].image, 

1022 exposures[(self.config.green_channel_band, order)].image, 

1023 exposures[(self.config.blue_channel_band, order)].image, 

1024 png_color_mapping, 

1025 ) 

1026 

1027 log_level = self.log.INFO if order == (max_order - 3) else self.log.DEBUG 

1028 self.log.log( 

1029 log_level, 

1030 "Completed HiPS generation for %s, order %d, pixel %d (%d/%d)", 

1031 ",".join(bands), 

1032 order, 

1033 pixels_shifted[order][pixel_counter], 

1034 pixel_counter, 

1035 len(pixels) - 1, 

1036 ) 

1037 

1038 # When we are at the top of the tree, erase top level images and continue. 

1039 if order == min_order: 

1040 for band in bands: 

1041 exposures[(band, order)].image.array[:, :] = np.nan 

1042 continue 

1043 

1044 # Now average the images for each band. 

1045 for band in bands: 

1046 arr = exposures[(band, order)].image.array.reshape(npix//2, 2, npix//2, 2) 

1047 with warnings.catch_warnings(): 

1048 warnings.simplefilter("ignore") 

1049 binned_image_arr = np.nanmean(arr, axis=(1, 3)) 

1050 

1051 # Fill the next level up. We figure out which of the four 

1052 # sub-pixels the current pixel occupies. 

1053 sub_index = (pixels_shifted[order][pixel_counter] 

1054 - np.left_shift(pixels_shifted[order - 1][pixel_counter], 2)) 

1055 

1056 # Fill exposure at the next level up. 

1057 exp = exposures[(band, order - 1)] 

1058 

1059 # Fill the correct subregion. 

1060 if sub_index == 0: 

1061 exp.image.array[npix//2:, 0: npix//2] = binned_image_arr 

1062 elif sub_index == 1: 

1063 exp.image.array[0: npix//2, 0: npix//2] = binned_image_arr 

1064 elif sub_index == 2: 

1065 exp.image.array[npix//2:, npix//2:] = binned_image_arr 

1066 elif sub_index == 3: 

1067 exp.image.array[0: npix//2, npix//2:] = binned_image_arr 

1068 else: 

1069 # This should be impossible. 

1070 raise ValueError("Illegal pixel sub index") 

1071 

1072 # Erase the previous exposure. 

1073 if order < max_order: 

1074 exposures[(band, order)].image.array[:, :] = np.nan 

1075 

1076 # Write the properties files and MOCs. 

1077 if not do_color: 

1078 for band in bands: 

1079 band_pixels = np.array([pixel 

1080 for pixel, band_ in hips_exposure_handle_dict.keys() 

1081 if band_ == band]) 

1082 band_pixels = np.sort(band_pixels) 

1083 

1084 self._write_properties_and_moc( 

1085 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1086 max_order, 

1087 band_pixels, 

1088 exp0, 

1089 shift_order, 

1090 band, 

1091 False, 

1092 ) 

1093 self._write_allsky_file( 

1094 hips_base_path.join(f"band_{band}", forceDirectory=True), 

1095 min_order, 

1096 ) 

1097 else: 

1098 self._write_properties_and_moc( 

1099 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1100 max_order, 

1101 pixels[:-1], 

1102 exp0, 

1103 shift_order, 

1104 colorstr, 

1105 True, 

1106 ) 

1107 self._write_allsky_file( 

1108 hips_base_path.join(f"color_{colorstr}", forceDirectory=True), 

1109 min_order, 

1110 ) 

1111 

1112 def _write_hips_image(self, hips_base_path, order, pixel, image, png_mapping, shift_order=9): 

1113 """Write a HiPS image. 

1114 

1115 Parameters 

1116 ---------- 

1117 hips_base_path : `lsst.resources.ResourcePath` 

1118 Resource path to the base of the HiPS directory tree. 

1119 order : `int` 

1120 HEALPix order of the HiPS image to write. 

1121 pixel : `int` 

1122 HEALPix pixel of the HiPS image. 

1123 image : `lsst.afw.image.Image` 

1124 Image to write. 

1125 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1126 Mapping to convert image to scaled png. 

1127 shift_order : `int`, optional 

1128 HPX shift_order. 

1129 """ 

1130 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1131 # outside of the butler. This task has been given (temporary) 

1132 # Special Dispensation because of the nature of HiPS outputs until 

1133 # a more controlled solution can be found. 

1134 

1135 dir_number = self._get_dir_number(pixel) 

1136 hips_dir = hips_base_path.join( 

1137 f"Norder{order}", 

1138 forceDirectory=True 

1139 ).join( 

1140 f"Dir{dir_number}", 

1141 forceDirectory=True 

1142 ) 

1143 

1144 wcs = makeHpxWcs(order, pixel, shift_order=shift_order) 

1145 

1146 uri = hips_dir.join(f"Npix{pixel}.fits") 

1147 

1148 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1149 image.writeFits(temporary_uri.ospath, metadata=wcs.getFitsMetadata()) 

1150 

1151 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1152 

1153 # And make a grayscale png as well 

1154 

1155 with np.errstate(invalid="ignore"): 

1156 vals = 255 - png_mapping.map_intensity_to_uint8(image.array).astype(np.uint8) 

1157 

1158 vals[~np.isfinite(image.array) | (image.array < 0)] = 0 

1159 im = Image.fromarray(vals[::-1, :], "L") 

1160 

1161 uri = hips_dir.join(f"Npix{pixel}.png") 

1162 

1163 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1164 im.save(temporary_uri.ospath) 

1165 

1166 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1167 

1168 def _write_hips_color_png( 

1169 self, 

1170 hips_base_path, 

1171 order, 

1172 pixel, 

1173 image_red, 

1174 image_green, 

1175 image_blue, 

1176 png_mapping, 

1177 ): 

1178 """Write a color png HiPS image. 

1179 

1180 Parameters 

1181 ---------- 

1182 hips_base_path : `lsst.resources.ResourcePath` 

1183 Resource path to the base of the HiPS directory tree. 

1184 order : `int` 

1185 HEALPix order of the HiPS image to write. 

1186 pixel : `int` 

1187 HEALPix pixel of the HiPS image. 

1188 image_red : `lsst.afw.image.Image` 

1189 Input for red channel of output png. 

1190 image_green : `lsst.afw.image.Image` 

1191 Input for green channel of output png. 

1192 image_blue : `lsst.afw.image.Image` 

1193 Input for blue channel of output png. 

1194 png_mapping : `astropy.visualization.lupton_rgb.AsinhMapping` 

1195 Mapping to convert image to scaled png. 

1196 """ 

1197 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1198 # outside of the butler. This task has been given (temporary) 

1199 # Special Dispensation because of the nature of HiPS outputs until 

1200 # a more controlled solution can be found. 

1201 

1202 dir_number = self._get_dir_number(pixel) 

1203 hips_dir = hips_base_path.join( 

1204 f"Norder{order}", 

1205 forceDirectory=True 

1206 ).join( 

1207 f"Dir{dir_number}", 

1208 forceDirectory=True 

1209 ) 

1210 

1211 # We need to convert nans to the minimum values in the mapping. 

1212 arr_red = image_red.array.copy() 

1213 arr_red[np.isnan(arr_red)] = png_mapping.minimum[0] 

1214 arr_green = image_green.array.copy() 

1215 arr_green[np.isnan(arr_green)] = png_mapping.minimum[1] 

1216 arr_blue = image_blue.array.copy() 

1217 arr_blue[np.isnan(arr_blue)] = png_mapping.minimum[2] 

1218 

1219 image_array = png_mapping.make_rgb_image(arr_red, arr_green, arr_blue) 

1220 

1221 im = Image.fromarray(image_array[::-1, :, :], mode="RGB") 

1222 

1223 uri = hips_dir.join(f"Npix{pixel}.png") 

1224 

1225 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1226 im.save(temporary_uri.ospath) 

1227 

1228 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1229 

1230 def _write_properties_and_moc( 

1231 self, 

1232 hips_base_path, 

1233 max_order, 

1234 pixels, 

1235 exposure, 

1236 shift_order, 

1237 band, 

1238 multiband 

1239 ): 

1240 """Write HiPS properties file and MOC. 

1241 

1242 Parameters 

1243 ---------- 

1244 hips_base_path : : `lsst.resources.ResourcePath` 

1245 Resource path to the base of the HiPS directory tree. 

1246 max_order : `int` 

1247 Maximum HEALPix order. 

1248 pixels : `np.ndarray` (N,) 

1249 Array of pixels used. 

1250 exposure : `lsst.afw.image.Exposure` 

1251 Sample HPX exposure used for generating HiPS tiles. 

1252 shift_order : `int` 

1253 HPX shift order. 

1254 band : `str` 

1255 Band (or color). 

1256 multiband : `bool` 

1257 Is band multiband / color? 

1258 """ 

1259 area = hpg.nside_to_pixel_area(2**max_order, degrees=True)*len(pixels) 

1260 

1261 initial_ra = self.config.properties.initial_ra 

1262 initial_dec = self.config.properties.initial_dec 

1263 initial_fov = self.config.properties.initial_fov 

1264 

1265 if initial_ra is None or initial_dec is None or initial_fov is None: 

1266 # We want to point to an arbitrary pixel in the footprint. 

1267 # Just take the median pixel value for simplicity. 

1268 temp_pixels = pixels.copy() 

1269 if temp_pixels.size % 2 == 0: 

1270 temp_pixels = np.append(temp_pixels, [temp_pixels[0]]) 

1271 medpix = int(np.median(temp_pixels)) 

1272 _initial_ra, _initial_dec = hpg.pixel_to_angle(2**max_order, medpix) 

1273 _initial_fov = hpg.nside_to_resolution(2**max_order, units='arcminutes')/60. 

1274 

1275 if initial_ra is None or initial_dec is None: 

1276 initial_ra = _initial_ra 

1277 initial_dec = _initial_dec 

1278 if initial_fov is None: 

1279 initial_fov = _initial_fov 

1280 

1281 self._write_hips_properties_file( 

1282 hips_base_path, 

1283 self.config.properties, 

1284 band, 

1285 multiband, 

1286 exposure, 

1287 max_order, 

1288 shift_order, 

1289 area, 

1290 initial_ra, 

1291 initial_dec, 

1292 initial_fov, 

1293 ) 

1294 

1295 # Write the MOC coverage 

1296 self._write_hips_moc_file( 

1297 hips_base_path, 

1298 max_order, 

1299 pixels, 

1300 ) 

1301 

1302 def _write_hips_properties_file( 

1303 self, 

1304 hips_base_path, 

1305 properties_config, 

1306 band, 

1307 multiband, 

1308 exposure, 

1309 max_order, 

1310 shift_order, 

1311 area, 

1312 initial_ra, 

1313 initial_dec, 

1314 initial_fov 

1315 ): 

1316 """Write HiPS properties file. 

1317 

1318 Parameters 

1319 ---------- 

1320 hips_base_path : `lsst.resources.ResourcePath` 

1321 ResourcePath at top of HiPS tree. File will be written 

1322 to this path as ``properties``. 

1323 properties_config : `lsst.pipe.tasks.hips.HipsPropertiesConfig` 

1324 Configuration for properties values. 

1325 band : `str` 

1326 Name of band(s) for HiPS tree. 

1327 multiband : `bool` 

1328 Is multiband / color? 

1329 exposure : `lsst.afw.image.Exposure` 

1330 Sample HPX exposure used for generating HiPS tiles. 

1331 max_order : `int` 

1332 Maximum HEALPix order. 

1333 shift_order : `int` 

1334 HPX shift order. 

1335 area : `float` 

1336 Coverage area in square degrees. 

1337 initial_ra : `float` 

1338 Initial HiPS RA position (degrees). 

1339 initial_dec : `float` 

1340 Initial HiPS Dec position (degrees). 

1341 initial_fov : `float` 

1342 Initial HiPS display size (degrees). 

1343 """ 

1344 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1345 # outside of the butler. This task has been given (temporary) 

1346 # Special Dispensation because of the nature of HiPS outputs until 

1347 # a more controlled solution can be found. 

1348 def _write_property(fh, name, value): 

1349 """Write a property name/value to a file handle. 

1350 

1351 Parameters 

1352 ---------- 

1353 fh : file handle (blah) 

1354 Open for writing. 

1355 name : `str` 

1356 Name of property 

1357 value : `str` 

1358 Value of property 

1359 """ 

1360 # This ensures that the name has no spaces or space-like characters, 

1361 # per the HiPS standard. 

1362 if re.search(r"\s", name): 

1363 raise ValueError(f"``{name}`` cannot contain any space characters.") 

1364 if "=" in name: 

1365 raise ValueError(f"``{name}`` cannot contain an ``=``") 

1366 

1367 fh.write(f"{name:25}= {value}\n") 

1368 

1369 if exposure.image.array.dtype == np.dtype("float32"): 

1370 bitpix = -32 

1371 elif exposure.image.array.dtype == np.dtype("float64"): 

1372 bitpix = -64 

1373 elif exposure.image.array.dtype == np.dtype("int32"): 

1374 bitpix = 32 

1375 

1376 date_iso8601 = datetime.utcnow().isoformat(timespec="seconds") + "Z" 

1377 pixel_scale = hpg.nside_to_resolution(2**(max_order + shift_order), units='degrees') 

1378 

1379 uri = hips_base_path.join("properties") 

1380 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1381 with open(temporary_uri.ospath, "w") as fh: 

1382 _write_property( 

1383 fh, 

1384 "creator_did", 

1385 properties_config.creator_did_template.format(band=band), 

1386 ) 

1387 if properties_config.obs_collection is not None: 

1388 _write_property(fh, "obs_collection", properties_config.obs_collection) 

1389 _write_property( 

1390 fh, 

1391 "obs_title", 

1392 properties_config.obs_title_template.format(band=band), 

1393 ) 

1394 if properties_config.obs_description_template is not None: 

1395 _write_property( 

1396 fh, 

1397 "obs_description", 

1398 properties_config.obs_description_template.format(band=band), 

1399 ) 

1400 if len(properties_config.prov_progenitor) > 0: 

1401 for prov_progenitor in properties_config.prov_progenitor: 

1402 _write_property(fh, "prov_progenitor", prov_progenitor) 

1403 if properties_config.obs_ack is not None: 

1404 _write_property(fh, "obs_ack", properties_config.obs_ack) 

1405 _write_property(fh, "obs_regime", "Optical") 

1406 _write_property(fh, "data_pixel_bitpix", str(bitpix)) 

1407 _write_property(fh, "dataproduct_type", "image") 

1408 _write_property(fh, "moc_sky_fraction", str(area/41253.)) 

1409 _write_property(fh, "data_ucd", "phot.flux") 

1410 _write_property(fh, "hips_creation_date", date_iso8601) 

1411 _write_property(fh, "hips_builder", "lsst.pipe.tasks.hips.GenerateHipsTask") 

1412 _write_property(fh, "hips_creator", "Vera C. Rubin Observatory") 

1413 _write_property(fh, "hips_version", "1.4") 

1414 _write_property(fh, "hips_release_date", date_iso8601) 

1415 _write_property(fh, "hips_frame", "equatorial") 

1416 _write_property(fh, "hips_order", str(max_order)) 

1417 _write_property(fh, "hips_tile_width", str(exposure.getBBox().getWidth())) 

1418 _write_property(fh, "hips_status", "private master clonableOnce") 

1419 if multiband: 

1420 _write_property(fh, "hips_tile_format", "png") 

1421 _write_property(fh, "dataproduct_subtype", "color") 

1422 else: 

1423 _write_property(fh, "hips_tile_format", "png fits") 

1424 _write_property(fh, "hips_pixel_bitpix", str(bitpix)) 

1425 _write_property(fh, "hips_pixel_scale", str(pixel_scale)) 

1426 _write_property(fh, "hips_initial_ra", str(initial_ra)) 

1427 _write_property(fh, "hips_initial_dec", str(initial_dec)) 

1428 _write_property(fh, "hips_initial_fov", str(initial_fov)) 

1429 if multiband: 

1430 if self.config.blue_channel_band in properties_config.spectral_ranges: 

1431 em_min = properties_config.spectral_ranges[ 

1432 self.config.blue_channel_band 

1433 ].lambda_min/1e9 

1434 else: 

1435 self.log.warning("blue band %s not in self.config.spectral_ranges.", band) 

1436 em_min = 3e-7 

1437 if self.config.red_channel_band in properties_config.spectral_ranges: 

1438 em_max = properties_config.spectral_ranges[ 

1439 self.config.red_channel_band 

1440 ].lambda_max/1e9 

1441 else: 

1442 self.log.warning("red band %s not in self.config.spectral_ranges.", band) 

1443 em_max = 1e-6 

1444 else: 

1445 if band in properties_config.spectral_ranges: 

1446 em_min = properties_config.spectral_ranges[band].lambda_min/1e9 

1447 em_max = properties_config.spectral_ranges[band].lambda_max/1e9 

1448 else: 

1449 self.log.warning("band %s not in self.config.spectral_ranges.", band) 

1450 em_min = 3e-7 

1451 em_max = 1e-6 

1452 _write_property(fh, "em_min", str(em_min)) 

1453 _write_property(fh, "em_max", str(em_max)) 

1454 if properties_config.t_min is not None: 

1455 _write_property(fh, "t_min", properties_config.t_min) 

1456 if properties_config.t_max is not None: 

1457 _write_property(fh, "t_max", properties_config.t_max) 

1458 

1459 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1460 

1461 def _write_hips_moc_file(self, hips_base_path, max_order, pixels, min_uniq_order=1): 

1462 """Write HiPS MOC file. 

1463 

1464 Parameters 

1465 ---------- 

1466 hips_base_path : `lsst.resources.ResourcePath` 

1467 ResourcePath to top of HiPS tree. File will be written as 

1468 to this path as ``Moc.fits``. 

1469 max_order : `int` 

1470 Maximum HEALPix order. 

1471 pixels : `np.ndarray` 

1472 Array of pixels covered. 

1473 min_uniq_order : `int`, optional 

1474 Minimum HEALPix order for looking for fully covered pixels. 

1475 """ 

1476 # WARNING: In general PipelineTasks are not allowed to do any outputs 

1477 # outside of the butler. This task has been given (temporary) 

1478 # Special Dispensation because of the nature of HiPS outputs until 

1479 # a more controlled solution can be found. 

1480 

1481 # Make the initial list of UNIQ pixels 

1482 uniq = 4*(4**max_order) + pixels 

1483 

1484 # Make a healsparse map which provides easy degrade/comparisons. 

1485 hspmap = hsp.HealSparseMap.make_empty(2**min_uniq_order, 2**max_order, dtype=np.float32) 

1486 hspmap[pixels] = 1.0 

1487 

1488 # Loop over orders, degrade each time, and look for pixels with full coverage. 

1489 for uniq_order in range(max_order - 1, min_uniq_order - 1, -1): 

1490 hspmap = hspmap.degrade(2**uniq_order, reduction="sum") 

1491 pix_shift = np.right_shift(pixels, 2*(max_order - uniq_order)) 

1492 # Check if any of the pixels at uniq_order have full coverage. 

1493 covered, = np.isclose(hspmap[pix_shift], 4**(max_order - uniq_order)).nonzero() 

1494 if covered.size == 0: 

1495 # No pixels at uniq_order are fully covered, we're done. 

1496 break 

1497 # Replace the UNIQ pixels that are fully covered. 

1498 uniq[covered] = 4*(4**uniq_order) + pix_shift[covered] 

1499 

1500 # Remove duplicate pixels. 

1501 uniq = np.unique(uniq) 

1502 

1503 # Output to fits. 

1504 tbl = np.zeros(uniq.size, dtype=[("UNIQ", "i8")]) 

1505 tbl["UNIQ"] = uniq 

1506 

1507 order = np.log2(tbl["UNIQ"]//4).astype(np.int32)//2 

1508 moc_order = np.max(order) 

1509 

1510 hdu = fits.BinTableHDU(tbl) 

1511 hdu.header["PIXTYPE"] = "HEALPIX" 

1512 hdu.header["ORDERING"] = "NUNIQ" 

1513 hdu.header["COORDSYS"] = "C" 

1514 hdu.header["MOCORDER"] = moc_order 

1515 hdu.header["MOCTOOL"] = "lsst.pipe.tasks.hips.GenerateHipsTask" 

1516 

1517 uri = hips_base_path.join("Moc.fits") 

1518 

1519 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1520 hdu.writeto(temporary_uri.ospath) 

1521 

1522 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1523 

1524 def _write_allsky_file(self, hips_base_path, allsky_order): 

1525 """Write an Allsky.png file. 

1526 

1527 Parameters 

1528 ---------- 

1529 hips_base_path : `lsst.resources.ResourcePath` 

1530 Resource path to the base of the HiPS directory tree. 

1531 allsky_order : `int` 

1532 HEALPix order of the minimum order to make allsky file. 

1533 """ 

1534 tile_size = self.config.allsky_tilesize 

1535 n_tiles_per_side = int(np.sqrt(hpg.nside_to_npixel(hpg.order_to_nside(allsky_order)))) 

1536 

1537 allsky_image = None 

1538 

1539 allsky_order_uri = hips_base_path.join(f"Norder{allsky_order}", forceDirectory=True) 

1540 pixel_regex = re.compile(r"Npix([0-9]+)\.png$") 

1541 png_uris = list( 

1542 ResourcePath.findFileResources( 

1543 candidates=[allsky_order_uri], 

1544 file_filter=pixel_regex, 

1545 ) 

1546 ) 

1547 

1548 for png_uri in png_uris: 

1549 matches = re.match(pixel_regex, png_uri.basename()) 

1550 pix_num = int(matches.group(1)) 

1551 tile_image = Image.open(io.BytesIO(png_uri.read())) 

1552 row = math.floor(pix_num//n_tiles_per_side) 

1553 column = pix_num % n_tiles_per_side 

1554 box = (column*tile_size, row*tile_size, (column + 1)*tile_size, (row + 1)*tile_size) 

1555 tile_image_shrunk = tile_image.resize((tile_size, tile_size)) 

1556 

1557 if allsky_image is None: 

1558 allsky_image = Image.new( 

1559 tile_image.mode, 

1560 (n_tiles_per_side*tile_size, n_tiles_per_side*tile_size), 

1561 ) 

1562 allsky_image.paste(tile_image_shrunk, box) 

1563 

1564 uri = allsky_order_uri.join("Allsky.png") 

1565 

1566 with ResourcePath.temporary_uri(suffix=uri.getExtension()) as temporary_uri: 

1567 allsky_image.save(temporary_uri.ospath) 

1568 

1569 uri.transfer_from(temporary_uri, transfer="copy", overwrite=True) 

1570 

1571 def _get_dir_number(self, pixel): 

1572 """Compute the directory number from a pixel. 

1573 

1574 Parameters 

1575 ---------- 

1576 pixel : `int` 

1577 HEALPix pixel number. 

1578 

1579 Returns 

1580 ------- 

1581 dir_number : `int` 

1582 HiPS directory number. 

1583 """ 

1584 return (pixel//10000)*10000 

1585 

1586 

1587class GenerateColorHipsConnections(pipeBase.PipelineTaskConnections, 

1588 dimensions=("instrument", ), 

1589 defaultTemplates={"coaddName": "deep"}): 

1590 hips_exposure_handles = pipeBase.connectionTypes.Input( 

1591 doc="HiPS-compatible HPX images.", 

1592 name="{coaddName}Coadd_hpx", 

1593 storageClass="ExposureF", 

1594 dimensions=("healpix11", "band"), 

1595 multiple=True, 

1596 deferLoad=True, 

1597 ) 

1598 

1599 

1600class GenerateColorHipsConfig(GenerateHipsConfig, 

1601 pipelineConnections=GenerateColorHipsConnections): 

1602 """Configuration parameters for GenerateColorHipsTask.""" 

1603 blue_channel_band = pexConfig.Field( 

1604 doc="Band to use for blue channel of color pngs.", 

1605 dtype=str, 

1606 default="g", 

1607 ) 

1608 green_channel_band = pexConfig.Field( 

1609 doc="Band to use for green channel of color pngs.", 

1610 dtype=str, 

1611 default="r", 

1612 ) 

1613 red_channel_band = pexConfig.Field( 

1614 doc="Band to use for red channel of color pngs.", 

1615 dtype=str, 

1616 default="i", 

1617 ) 

1618 png_color_asinh_minimum = pexConfig.Field( 

1619 doc="AsinhMapping intensity to be mapped to black for color png scaling.", 

1620 dtype=float, 

1621 default=0.0, 

1622 ) 

1623 png_color_asinh_stretch = pexConfig.Field( 

1624 doc="AsinhMapping linear stretch for color png scaling.", 

1625 dtype=float, 

1626 default=5.0, 

1627 ) 

1628 png_color_asinh_softening = pexConfig.Field( 

1629 doc="AsinhMapping softening parameter (Q) for color png scaling.", 

1630 dtype=float, 

1631 default=8.0, 

1632 ) 

1633 

1634 

1635class GenerateColorHipsTask(GenerateHipsTask): 

1636 """Task for making a HiPS tree with color pngs.""" 

1637 ConfigClass = GenerateColorHipsConfig 

1638 _DefaultName = "generateColorHips" 

1639 color_task = True 

1640 

1641 def _check_data_bands(self, data_bands): 

1642 """Check the data for configured bands. 

1643 

1644 Warn if any color bands are missing data. 

1645 

1646 Parameters 

1647 ---------- 

1648 data_bands : `set` [`str`] 

1649 Bands from the input data. 

1650 

1651 Returns 

1652 ------- 

1653 bands : `list` [`str`] 

1654 List of bands in bgr color order. 

1655 """ 

1656 if len(data_bands) == 0: 

1657 raise RuntimeError("GenerateColorHipsTask must have data from at least one band.") 

1658 

1659 if self.config.blue_channel_band not in data_bands: 

1660 self.log.warning( 

1661 "Color png blue_channel_band %s not in dataset.", 

1662 self.config.blue_channel_band 

1663 ) 

1664 if self.config.green_channel_band not in data_bands: 

1665 self.log.warning( 

1666 "Color png green_channel_band %s not in dataset.", 

1667 self.config.green_channel_band 

1668 ) 

1669 if self.config.red_channel_band not in data_bands: 

1670 self.log.warning( 

1671 "Color png red_channel_band %s not in dataset.", 

1672 self.config.red_channel_band 

1673 ) 

1674 

1675 bands = [ 

1676 self.config.blue_channel_band, 

1677 self.config.green_channel_band, 

1678 self.config.red_channel_band, 

1679 ] 

1680 

1681 return bands