Coverage for python / lsst / cp / verify / report.py: 0%
257 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-17 09:37 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27import argparse
28import logging
29import matplotlib.colors as colors
30import numpy as np
31import os
32import re
33import yaml
35from matplotlib import cm
36from mpl_toolkits.axes_grid1 import make_axes_locatable
38from lsst.daf.butler import Butler
39from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import (
40 determine_destination_for_retrieved_artifact,
41)
42from lsst.resources import ResourcePath
43from lsst.utils import getPackageDir
44from lsst.utils.plotting import make_figure
47class CpvReporter():
48 """A class to generate calibration verification reports.
50 Parameters
51 ----------
52 repo : `str`
53 The location of the butler repository to retrieve results
54 from.
55 instrument : `str`
56 The instrument associated with this data.
57 output_path : `str`
58 The location the report will be written to.
59 collections : `list` [`str`]
60 A list of collections to search.
61 **kwargs :
62 Other keyword parameters. Currently parsed values:
64 - ``do_copy``: Should files be copied from butler (`bool`).
65 - ``do_overwrite``: Should pre-existing files be overwritten (`bool`).
66 """
68 def __init__(self, repo, output_path, collections=[], **kwargs):
69 super().__init__()
70 # Set source and destination information.
71 self.repo = repo
72 self.output_path = output_path
73 self.src_dir = os.path.join(self.output_path, "src")
74 self.collections = collections
76 # We need a log
77 self.log = logging.getLogger(__name__) if "log" not in kwargs else kwargs["log"]
78 # List of the datasets we need to process. The datasets are
79 # dictionaries of parameters, including the dataset_type_name,
80 # the source collection, the butler dataIds, file locations,
81 # etc.
82 self.datasets = []
84 # A dictionary of output html files that will be created.
85 # These are dictionaries with the key being the filename, and
86 # the values being arrays of strings that will be written
87 # one-per-line in the output html page.
88 self.out_files = {
89 'index.html': self._init_page(),
90 }
92 # Instantiate a butler for our repository.
93 self.butler = Butler(repo)
95 # Configure behavior from kwargs:
96 self.do_copy = kwargs['do_copy'] if 'do_copy' in kwargs else True
97 self.do_overwrite = kwargs['do_overwrite'] if 'do_overwrite' in kwargs else False
99 # Read dataset configuration yaml:
100 self.dataset_map = self._read_dataset_map()
102 def _read_dataset_map(self):
103 """Read dataset information from source yaml."""
104 filename = os.path.join(getPackageDir("cp_verify"),
105 "python", "lsst", "cp", "verify", "configs", "report.yaml")
106 with open(filename) as in_file:
107 return yaml.safe_load(in_file)
109 def run(self):
110 """Generate the report"""
111 # Generate directories that will hold the output products.
112 os.makedirs(self.src_dir, exist_ok=True)
114 # Copy datasets: This populates self.datasets.
115 self.copy_datasets()
117 # Parse datasets: This determines what report pages should be made.
118 self.parse_datasets()
120 # Write pages: This saves the information to disk.
121 self.write_pages()
123 def copy_datasets(self):
124 """Copy datasets, similar to butler retrieve-artifacts.
126 See Also
127 --------
128 self.fits_to_png
129 self.parq_to_html
130 self.py_to_html
131 """
132 # Based on daf_butler fileDatastore.py#L1978
133 for stage, dataset_types in self.dataset_map['stages'].items():
134 # Iterate over each stage, getting all of that stage's datasets.
135 # Convert dict dataset_types to a list of strings:
136 if not dataset_types:
137 continue
138 dataset_type_names = list(dataset_types.keys())
140 # Find all the references to these datasets in our collections
141 refs = self.butler.registry.queryDatasets(datasetType=dataset_type_names,
142 collections=self.collections,
143 where=None, findFirst=True)
145 for ref in refs:
146 # Iterate over each reference
147 locations = self.butler._datastore._get_dataset_locations_info(ref)
148 self.log.warn(f"Found {stage} {ref}")
150 # This is our dataset information:
151 dataset = {'stage': stage,
152 'ref': ref,
153 'type': ref.datasetType.name,
154 'storage_class': ref.datasetType.storageClass.name,
155 'dataId': ref.dataId,
156 'instrument': ref.dataId.get('instrument', None),
157 'exposure': ref.dataId.get('exposure', None),
158 'detector': ref.dataId.get('detector', None),
159 'physical_filter': ref.dataId.get('physical_filter', None),
160 'collection': ref.run,
161 }
163 # Do the actual copy. This may convert the butler
164 # product to something more web-accessible.
165 for location, _ in locations:
166 source_uri = location.uri
167 target_uri = determine_destination_for_retrieved_artifact(
168 ResourcePath(self.src_dir),
169 location.pathInStore,
170 False
171 )
172 dataset['uri'] = target_uri
173 willCopy = self.do_copy
175 if ((target_uri.getExtension().lower() == ".fits"
176 and dataset['storage_class'] in ('ImageF', 'ExposureF', 'ImageD', 'ExposureD'))):
177 # Convert this to a PNG image.
178 png_uri = target_uri.updatedExtension("png")
179 dataset['uri'] = png_uri
180 if os.path.exists(png_uri.path):
181 if willCopy and not self.do_overwrite:
182 willCopy = False
184 if willCopy:
185 data = self.butler.get(ref)
186 self.fits_to_png(data, png_uri, f"{ref.datasetType.name} {ref.dataId}")
187 elif (target_uri.getExtension().lower() == ".parq"
188 and dataset['storage_class'] in ('ArrowAstropy', )):
189 # Convert to html table.
190 html_uri = target_uri.updatedExtension("html")
191 dataset['uri'] = html_uri
192 if os.path.exists(html_uri.path):
193 if willCopy and not self.do_overwrite:
194 willCopy = False
196 if willCopy:
197 data = self.butler.get(ref)
198 self.parq_to_html(data, html_uri)
199 elif (target_uri.getExtension().lower() == ".py"
200 and dataset['storage_class'] in ('Config', )):
201 # Convert configs to html.
202 html_uri = target_uri.updatedExtension("html")
203 dataset['uri'] = html_uri
205 if willCopy:
206 data = self.butler.get(ref)
207 self.py_to_html(data, html_uri)
208 else:
209 if willCopy:
210 # Otherwise just copy directly.
211 target_uri.transfer_from(source_uri, transfer="copy",
212 overwrite=self.do_overwrite)
214 # Add this dataset to our list.
215 self.datasets.append(dataset)
217 def parse_datasets(self):
218 """Analyze datasets, and add information to report pages.
220 See Also
221 --------
222 self._init_page
223 self.include
224 self.navigation_init
225 self.navigation_append
226 self.title
227 self.link
228 """
229 populated_stages_list = []
231 # Make navigation page.
232 self.out_files["navigation.html"] = self._init_page("Navigation")
233 self.navigation_init()
234 self.include("navigation.html", "index.html")
236 # Make manifest page:
237 self.out_files["manifest.html"] = self._init_page("Manifest")
239 tag = self.link(self.out_files["index.html"], "Parent", "./..")
240 self.navigation_append("index.html", "Parent", tag, newColumn=True)
241 self.navigation_append("manifest.html", "Manifest", "", newColumn=False)
243 for stage in self.dataset_map['stages'].keys():
244 # Filter datasets to only those for this stage
245 dss = [x for x in self.datasets if x['stage'] == stage]
246 if len(dss) > 0:
247 populated_stages_list.append(stage)
248 else:
249 continue
250 self.log.warn(f"Parsing data for {stage}")
251 self.title(self.out_files["index.html"], stage)
253 self.out_files[f"{stage}_exp.html"] = self._init_page()
254 self.include("navigation.html", f"{stage}_exp.html")
255 self.out_files[f"{stage}_det.html"] = self._init_page()
256 self.include("navigation.html", f"{stage}_det.html")
258 # Get dimension information we'll want to use to put
259 # things on appropriate pages.
260 exps = [x for x in dss if x['exposure'] is not None]
261 dets = [x for x in dss if x['detector'] is not None]
263 for ds in sorted(exps, key=lambda x: x['exposure']):
264 self.block(ds, self.out_files[f"{stage}_exp.html"])
265 if ('.png' in ds['uri'].path):
266 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files[f"{stage}_exp.html"])
267 tag = self.link(self.out_files["index.html"], f"{stage} exposures", f"./{stage}_exp.html")
268 self.navigation_append("index.html", f"{stage} exposures", tag)
270 for ds in sorted(dets, key=lambda x: x['detector']):
271 self.block(ds, self.out_files[f"{stage}_det.html"])
272 if ('.png' in ds['uri'].path):
273 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files[f"{stage}_det.html"])
274 tag = self.link(self.out_files["index.html"], f"{stage} detectors", f"./{stage}_det.html")
275 self.navigation_append("index.html", f"{stage} detectors", tag)
277 for ds in dss:
278 if ds['storage_class'] == 'ArrowAstropy':
279 # Is a results catalog
280 self.block(ds, self.out_files["index.html"], ds['uri'])
281 self.include(ds['uri'].path, "index.html")
282 elif ds['exposure'] is None and ds['detector'] is None:
283 self.block(ds, self.out_files["index.html"])
284 if ".png" in ds['uri'].path:
285 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files["index.html"])
286 else:
287 self.include(ds['uri'].path, "index.html")
289 # Record all datasets here, and link to them directly:
290 page = self.out_files["manifest.html"]
291 page.extend(["<table width='100%'>",
292 "<tr>",
293 "<th>stage</th>", "<th>ref</th>", "<th>type</th>", "<th>storage_class</th>",
294 "<th>dataId</th>", "<th>collection</th>",
295 "</tr>"])
296 for ds in self.datasets:
297 relative_file = self.relative_file(ds['uri'].path)
298 page.extend(["<tr>",
299 f"<td>{ds['stage']}</td>", f"<td>{ds['ref']}</td>",
300 "<td>", f'<a href="{relative_file}">', f"{ds['type']}", "</a>",
301 f"<td>{ds['storage_class']}</td>",
302 f"<td>{ds['dataId']}</td>", f"<td>{ds['collection']}</td>",
303 "</tr>"])
305 # File conversion utilities:
306 @staticmethod
307 def fits_to_png(data, target_uri, title):
308 """Convert FITS to PNG, using the same scaling as on RubinTV.
310 Parameters
311 ----------
312 data : `lsst.afw.Image`, `lsst.afw.MaskedImage` or `lsst.afw.Exposure`
313 The fits image data to convert.
314 target_uri : `str`
315 Path to the PNG file to write.
316 title : `str`
317 Title to add to the figure.
319 See Also
320 --------
321 rubintv_production / mosaicing.py
322 """
323 # Get array from either an Image or an Exposure:
324 try:
325 array = data.image.array
326 except AttributeError:
327 array = data.array
329 fig = make_figure()
330 ax = fig.gca()
331 ax.clear()
332 cmap = cm.gray
333 # This was using summit_utils.getQuantiles, but that
334 # was blowing out the scaling more than I wanted.
335 q25, q50, q75 = np.nanpercentile(array, [25, 50, 75])
336 scale = 3.0 * 0.74 * (q75 - q25)
337 quantiles = np.arange(q50 - scale, q50 + scale, 2.0 * scale / cmap.N)
338 norm = colors.BoundaryNorm(quantiles, cmap.N)
340 im = ax.imshow(array, norm=norm, interpolation='None', cmap=cmap, origin='lower')
342 divider = make_axes_locatable(ax)
343 cax = divider.append_axes("right", size="5%", pad=0.05)
344 fig.suptitle(title)
345 fig.colorbar(im, cax=cax)
346 fig.tight_layout()
347 fig.savefig(target_uri.path)
349 @staticmethod
350 def parq_to_html(data, target_uri):
351 """Convert catalogs to html tables.
353 Parameters
354 ----------
355 data : `astropy.Table`
356 The table to convert to html.
357 target_uri : `str`
358 Path to the HTML file to write.
359 """
360 format_dict = {}
361 # Remove vectors, as they will be too big.
362 # Build up format dictionary.
363 columns_to_remove = []
364 for index, name in enumerate(data.dtype.names):
365 if len(data.dtype[index].shape) != 0:
366 columns_to_remove.append(name)
367 continue
368 if data.dtype[index].kind in ('f', 'c'):
369 # is float
370 format_dict[name] = "%.4g"
371 if name == 'mjd':
372 # Let's let these be long
373 format_dict[name] = "12.10f"
374 elif data.dtype[index].kind in ('i', 'u'):
375 # is int
376 format_dict[name] = "%d"
377 else:
378 format_dict[name] = "%s"
380 # Actually remove the vectors:
381 data.remove_columns(columns_to_remove)
383 # Write full table:
384 data.write(target_uri.path,
385 format='ascii.html',
386 overwrite=True,
387 formats=format_dict)
389 # TODO: Write subset tables, filtered by exposure and by detector.
390 # files_created[f"{filename_base}_exp{exp}.html"] = {'exposure': exp}
391 # files_created[f"{filename_base}_det{det}.html"] = {'detector': det}
393 def py_to_html(self, data, target_uri):
394 """Convert python configs to html.
396 Parameters
397 ----------
398 data : `Config`
399 The python file contents to write.
400 target_uri : `str`
401 Path to the HTML file to write.
402 """
403 with open(target_uri.path, 'w') as ff:
404 for line in self._init_page():
405 print(line, file=ff)
406 for line in data.saveToString().split("\n"):
407 if len(line) > 0 and line[0] == '#':
408 print("<pre class='pre-comment'>", line, "</pre>", file=ff)
409 else:
410 print("<pre>", line, "</pre>", file=ff)
411 for line in self._close_page():
412 print(line, file=ff)
414 def write_pages(self):
415 """Write all page arrays to files."""
416 for ff, contents in self.out_files.items():
417 reportOut = os.path.join(self.output_path, ff)
418 contents.extend(self._close_page())
419 with open(reportOut, 'w') as ff:
420 for line in contents:
421 print(line, file=ff)
423 # Page construction methods. These all return arrays of strings
424 # that can be appended to until we write to disk.
425 @staticmethod
426 def _init_page(title=""):
427 """Write boilerplate html for the start of a page."""
428 return [
429 "<html>", "<head>", "<style> ",
430 " * { margin: 0; padding: 0;}",
431 " .imgbox { display: grid; height: 100%; }",
432 " .pre-comment { color: red; }",
433 " td { padding: 0 15px; }",
434 " .ctable { display: table; }",
435 " .ctable tr { display: table-cell; }",
436 " .ctable tr td { display: block; }",
437 "</style>",
438 "<title>", title, "</title>",
439 '<base target="_parent">',
440 "</head>",
441 "<body>",
442 ]
444 @staticmethod
445 def _close_page():
446 """Write boilerplate html for the end of a page."""
447 return ["</body>",
448 "</html>"]
450 @staticmethod
451 def relative_file(filename):
452 """Convert absolute paths to the relative format needed for the html.
453 """
454 return re.sub(r"^.*/src/", "./src/", filename)
456 def svg_atools_latiss(self, relative_file, page):
457 """This adds an image to the html, with an SVG overlay for LATISS.
459 Parameters
460 ----------
461 relative_file : `str`
462 Relative location of the image to be added.
463 page : `list` [`str`]
464 The page to append to.
466 Notes
467 -----
468 This method assumes the image is made by analysis_tools, and
469 displays a focal plane plot for LATISS. Under these
470 assumptions, the boxes defined in SVG map to the amplifier
471 segments of the single detector.
472 """
473 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
474 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
475 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
476 '<a xlink:href="./c10.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
477 'width="176" height="692"></rect> </a>',
478 '<a xlink:href="./c11.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
479 'width="176" height="692"></rect> </a>',
480 '<a xlink:href="./c12.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
481 'width="176" height="692"></rect> </a>',
482 '<a xlink:href="./c13.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
483 'width="176" height="692"></rect> </a>',
484 '<a xlink:href="./c14.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
485 'width="176" height="692"></rect> </a>',
486 '<a xlink:href="./c15.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
487 'width="176" height="692"></rect> </a>',
488 '<a xlink:href="./c16.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
489 'width="176" height="692"></rect> </a>',
490 '<a xlink:href="./c17.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
491 'width="176" height="692"></rect> </a>',
492 '<a xlink:href="./c07.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
493 'width="176" height="692"></rect> </a>',
494 '<a xlink:href="./c06.html"> <rect x="1375" y="911" fill="#fff" opacity="0" '
495 'width="176" height="692"></rect> </a>',
496 '<a xlink:href="./c05.html"> <rect x="1197" y="911" fill="#fff" opacity="0" '
497 'width="176" height="692"></rect> </a>',
498 '<a xlink:href="./c04.html"> <rect x="1020" y="911" fill="#fff" opacity="0" '
499 'width="176" height="692"></rect> </a>',
500 '<a xlink:href="./c03.html"> <rect x="845" y="911" fill="#fff" opacity="0" '
501 'width="176" height="692"></rect> </a>',
502 '<a xlink:href="./c02.html"> <rect x="670" y="911" fill="#fff" opacity="0" '
503 'width="176" height="692"></rect> </a>',
504 '<a xlink:href="./c01.html"> <rect x="490" y="911" fill="#fff" opacity="0" '
505 'width="176" height="692"></rect> </a>',
506 '<a xlink:href="./c00.html"> <rect x="317" y="911" fill="#fff" opacity="0" '
507 'width="176" height="692"></rect> </a>',
508 '</svg>'])
510 def svg_atools_comcam(self, relative_file, page):
511 """This adds an image to the html, with an SVG overlay for ComCam.
513 Parameters
514 ----------
515 relative_file : `str`
516 Relative location of the image to be added.
517 page : `list` [`str`]
518 The page to append to.
520 Notes
521 -----
522 This method assumes the image is made by analysis_tools, and
523 displays a focal plane plot for LSSTComCam. Under these
524 assumptions, the boxes defined in SVG map map to the detectors
525 in the single raft.
526 """
527 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
528 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
529 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
530 '<a xlink:href="./S00.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
531 'width="176" height="692"></rect> </a>',
532 '<a xlink:href="./S01.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
533 'width="176" height="692"></rect> </a>',
534 '<a xlink:href="./S02.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
535 'width="176" height="692"></rect> </a>',
536 '<a xlink:href="./S10.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
537 'width="176" height="692"></rect> </a>',
538 '<a xlink:href="./S11.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
539 'width="176" height="692"></rect> </a>',
540 '<a xlink:href="./S12.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
541 'width="176" height="692"></rect> </a>',
542 '<a xlink:href="./S20.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
543 'width="176" height="692"></rect> </a>',
544 '<a xlink:href="./S21.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
545 'width="176" height="692"></rect> </a>',
546 '<a xlink:href="./S22.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
547 'width="176" height="692"></rect> </a>',
548 '</svg>'])
550 def svg_mosaic_latiss(self, relative_file, page):
551 """This adds an image to the html, with an SVG overlay for LATISS.
553 Parameters
554 ----------
555 relative_file : `str`
556 Relative location of the image to be added.
557 page : `list` [`str`]
558 The page to append to.
560 Notes
561 -----
562 This method assumes the image is a 8x8 binned focal plane
563 mosaic of a LATISS exposure made by one of the VisualizeVisit
564 tasks. Under these assumptions, the boxes defined in SVG map
565 to the amplifiers of the single detector.
566 """
567 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
568 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
569 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
570 '<a xlink:href="./c10.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
571 'width="176" height="692"></rect> </a>',
572 '<a xlink:href="./c11.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
573 'width="176" height="692"></rect> </a>',
574 '<a xlink:href="./c12.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
575 'width="176" height="692"></rect> </a>',
576 '<a xlink:href="./c13.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
577 'width="176" height="692"></rect> </a>',
578 '<a xlink:href="./c14.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
579 'width="176" height="692"></rect> </a>',
580 '<a xlink:href="./c15.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
581 'width="176" height="692"></rect> </a>',
582 '<a xlink:href="./c16.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
583 'width="176" height="692"></rect> </a>',
584 '<a xlink:href="./c17.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
585 'width="176" height="692"></rect> </a>',
586 '<a xlink:href="./c07.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
587 'width="176" height="692"></rect> </a>',
588 '<a xlink:href="./c06.html"> <rect x="1375" y="911" fill="#fff" opacity="0" '
589 'width="176" height="692"></rect> </a>',
590 '<a xlink:href="./c05.html"> <rect x="1197" y="911" fill="#fff" opacity="0" '
591 'width="176" height="692"></rect> </a>',
592 '<a xlink:href="./c04.html"> <rect x="1020" y="911" fill="#fff" opacity="0" '
593 'width="176" height="692"></rect> </a>',
594 '<a xlink:href="./c03.html"> <rect x="845" y="911" fill="#fff" opacity="0" '
595 'width="176" height="692"></rect> </a>',
596 '<a xlink:href="./c02.html"> <rect x="670" y="911" fill="#fff" opacity="0" '
597 'width="176" height="692"></rect> </a>',
598 '<a xlink:href="./c01.html"> <rect x="490" y="911" fill="#fff" opacity="0" '
599 'width="176" height="692"></rect> </a>',
600 '<a xlink:href="./c00.html"> <rect x="317" y="911" fill="#fff" opacity="0" '
601 'width="176" height="692"></rect> </a>',
602 '</svg>'])
604 def svg_mosaic_comcam(self, relative_file, page):
605 """This adds an image to the html, with an SVG overlay for ComCam.
607 Parameters
608 ----------
609 relative_file : `str`
610 Relative location of the image to be added.
611 page : `list` [`str`]
612 The page to append to.
614 Notes
615 -----
616 This method assumes the image is a 8x8 binned focal plane
617 mosaic of a ComCam exposure made by one of the VisualizeVisit
618 tasks. Under these assumptions, the boxes defined in SVG map
619 to the detectors in the single raft.
620 """
621 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
622 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 640 480">',
623 f'<image width="640" height="480" xlink:href="{relative_file}"></image>',
624 '<a xlink:href="./S00.html"> <rect x="111" y="306" fill="#fff" opacity="0"'
625 'width="128" height="128"></rect> </a>',
626 '<a xlink:href="./S01.html"> <rect x="111" y="175" fill="#fff" opacity="0"'
627 'width="128" height="128"></rect> </a>',
628 '<a xlink:href="./S02.html"> <rect x="111" y="50" fill="#fff" opacity="0" '
629 'width="128" height="128"></rect> </a>',
630 '<a xlink:href="./S10.html"> <rect x="244" y="306" fill="#fff" opacity="0" '
631 'width="128" height="128"></rect> </a>',
632 '<a xlink:href="./S11.html"> <rect x="244" y="175" fill="#fff" opacity="0" '
633 'width="128" height="128"></rect> </a>',
634 '<a xlink:href="./S12.html"> <rect x="244" y="50" fill="#fff" opacity="0" '
635 'width="128" height="128"></rect> </a>',
636 '<a xlink:href="./S20.html"> <rect x="376" y="306" fill="#fff" opacity="0" '
637 'width="128" height="128"></rect> </a>',
638 '<a xlink:href="./S21.html"> <rect x="376" y="175" fill="#fff" opacity="0" '
639 'width="128" height="128"></rect> </a>',
640 '<a xlink:href="./S22.html"> <rect x="376" y="50" fill="#fff" opacity="0" '
641 'width="128" height="128"></rect> </a>',
642 '</svg>'])
644 def image_handler(self, image_filename, instrument, page):
645 """Handle images.
647 This method attempts to find the correct way to add an image
648 to a page, with the appropriate SVG subregion information.
650 Parameters
651 ----------
652 image_filename : `str`
653 Full path to the image file to be added to a page.
654 instrument : `str`
655 Instrument name to use to identify special handling.
656 page : `list`
657 The page to append to.
659 """
660 relative_file = self.relative_file(image_filename)
661 if "Mosaic64" not in relative_file:
662 # Skip Mosaic64 products.
663 if instrument == 'LATISS':
664 if "Mosaic" in relative_file:
665 self.svg_mosaic_latiss(relative_file, page)
666 else:
667 self.svg_atools_latiss(relative_file, page)
668 elif instrument in ('LSSTComCam', 'LSSTComCamSim'):
669 if "Mosaic" in relative_file:
670 # This is a fits_to_png thing
671 self.svg_mosaic_comcam(relative_file, page)
672 else:
673 self.svg_atools_comcam(relative_file, page)
674 else:
675 # We couldn't find a magic handler, so add it
676 # manually.
677 page.append(f'<img class="center-fit" src="{relative_file}">')
679 def block(self, dataset, page, link=None, doc=None):
680 """Add a block of information, containing the dataId and collections
681 for a particular dataset.
683 Parameters
684 ----------
685 dataset : `dict`
686 The dataset containing dataId information.
687 page : `list`
688 The page to append to.
689 link : `str`, optional
690 A link to apply to the dataset type name.
691 doc : `str`, optional
692 A documentation string. If not supplied, the documentation
693 in the configuration is used.
694 """
695 if doc is None:
696 stage = dataset['stage']
697 dsType = dataset['type']
698 doc = self.dataset_map['stages'][stage][dsType].get("description", "Undocumented.")
699 page.extend(["<table width='100%'>",
700 "<tr>",
701 "<th colspan='2' align='center'>DataId:</th>",
702 "<th colspan='2' align='center'>Dataset:</th>", "</tr>"])
703 page.extend(["<tr>",
704 "<td align='right'>Instrument</td>", f"<td>{dataset['instrument']}</td>",
705 "<td align='right'>Collection</td>", f"<td>{dataset['collection']}</td>",
706 "</tr>"])
707 page.extend(["<tr>",
708 "<td align='right'>Exposure</td>", f"<td>{dataset['exposure']}</td>",
709 "<td align='right'>Stage</td>", f"<td>{dataset['stage']}</td>", "</tr>"])
710 page.extend(["<tr>",
711 "<td align='right'>Detector</td>", f"<td>{dataset['detector']}</td>"]),
713 page.extend(["<a href='" + f"{link}" + "'>" if link else ""])
714 page.extend(["<td align='right'>Dataset type</td>", f"<td>{dataset['type']} ", "</td>"])
715 page.extend(["</a>" if link else ""])
716 page.extend(["</tr>"])
717 page.extend(["<tr>",
718 "<td align='right'>Physical Filter</td>", f"<td>{dataset['physical_filter']}</td>",
719 "<td align='right'>Docstring</td>", f"<td>{doc}</td>",
720 "</tr>"])
721 page.extend(["</table>"])
723 @staticmethod
724 def link(page, text, link):
725 text_ref = text.replace(" ", "-")
726 page.extend([f"<h2 id='{text_ref}'>", f"<a href='{link}'>",
727 f"{text}" "</a>", "</h2>"])
728 return text_ref
730 def navigation_init(self):
731 page = self.out_files["navigation.html"]
732 page.extend(["<table class='ctable' width='100%'>",
733 "<tr>"])
735 def navigation_append(self, destination, text, tag, newColumn=False):
736 page = self.out_files["navigation.html"]
737 if newColumn:
738 page.extend(["</tr>", "</tr>"])
739 page.extend(["<td>",
740 f"<a href='./{destination}#{tag}'>",
741 f"{text}",
742 "</a>", "</td>"])
744 def title(self, page, text):
745 text_ref = text.replace(" ", "-")
746 page.extend(["<center>",
747 f"<h1 id={text_ref}>", text, "</h1>",
748 "</center>"])
749 self.navigation_append("index.html", text, text_ref, newColumn=True)
751 def include(self, source, target):
752 target_page = self.out_files[target]
753 relative_file = self.relative_file(source)
754 target_page.extend([f"<iframe width='100%' src='./{relative_file}'>",
755 "</iframe>"])
758def main():
759 # Could these use the click/daf_butler CLI tools?
760 parser = argparse.ArgumentParser(description="Construct a cp_verify metric report.")
761 parser.add_argument("-r", "--repository", dest="repository", default="",
762 help="Butler repository to pull results from.")
763 parser.add_argument("-O", "--output_path", dest="output_path", default="",
764 help="Output path to write report to.")
765 parser.add_argument("-c", "--collections", dest="collections", action="append", default=[],
766 help="Collections to search for results.")
767 parser.add_argument("--no_copy", dest="do_copy", action="store_false", default=True,
768 help="Skip copying files for debugging purposes.")
769 parser.add_argument("--do_overwrite", dest="do_overwrite", action="store_true", default=False,
770 help="Allow existing files to be overwritten?")
771 args = parser.parse_args()
773 reporter = CpvReporter(
774 repo=args.repository,
775 output_path=args.output_path,
776 collections=args.collections,
777 do_copy=args.do_copy,
778 do_overwrite=args.do_overwrite,
779 )
780 reporter.run()