Coverage for python/lsst/cp/verify/report.py: 0%
256 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-08 00:07 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27import argparse
28import logging
29import matplotlib.colors as colors
30import numpy as np
31import os
32import re
33import yaml
35from matplotlib import cm
36from mpl_toolkits.axes_grid1 import make_axes_locatable
38from lsst.daf.butler import Butler
39from lsst.daf.butler.datastores.file_datastore.retrieve_artifacts import (
40 determine_destination_for_retrieved_artifact,
41)
42from lsst.resources import ResourcePath
43from lsst.utils.plotting import make_figure
46class CpvReporter():
47 """A class to generate calibration verification reports.
49 Parameters
50 ----------
51 repo : `str`
52 The location of the butler repository to retrieve results
53 from.
54 instrument : `str`
55 The instrument associated with this data.
56 output_path : `str`
57 The location the report will be written to.
58 collections : `list` [`str`]
59 A list of collections to search.
60 **kwargs :
61 Other keyword parameters. Currently parsed values:
63 - ``do_copy``: Should files be copied from butler (`bool`).
64 - ``do_overwrite``: Should pre-existing files be overwritten (`bool`).
65 """
67 def __init__(self, repo, output_path, collections=[], **kwargs):
68 super().__init__()
69 # Set source and destination information.
70 self.repo = repo
71 self.output_path = output_path
72 self.src_dir = os.path.join(self.output_path, "src")
73 self.collections = collections
75 # We need a log
76 self.log = logging.getLogger(__name__) if "log" not in kwargs else kwargs["log"]
77 # List of the datasets we need to process. The datasets are
78 # dictionaries of parameters, including the dataset_type_name,
79 # the source collection, the butler dataIds, file locations,
80 # etc.
81 self.datasets = []
83 # A dictionary of output html files that will be created.
84 # These are dictionaries with the key being the filename, and
85 # the values being arrays of strings that will be written
86 # one-per-line in the output html page.
87 self.out_files = {
88 'index.html': self._init_page(),
89 }
91 # Instantiate a butler for our repository.
92 self.butler = Butler(repo)
94 # Configure behavior from kwargs:
95 self.do_copy = kwargs['do_copy'] if 'do_copy' in kwargs else True
96 self.do_overwrite = kwargs['do_overwrite'] if 'do_overwrite' in kwargs else False
98 # Read dataset configuration yaml:
99 self.dataset_map = self._read_dataset_map()
101 def _read_dataset_map(self):
102 """Read dataset information from source yaml."""
103 filename = ResourcePath("resource://lsst.cp.verify/configs/report.yaml")
104 with filename.open("r") as in_file:
105 return yaml.safe_load(in_file)
107 def run(self):
108 """Generate the report"""
109 # Generate directories that will hold the output products.
110 os.makedirs(self.src_dir, exist_ok=True)
112 # Copy datasets: This populates self.datasets.
113 self.copy_datasets()
115 # Parse datasets: This determines what report pages should be made.
116 self.parse_datasets()
118 # Write pages: This saves the information to disk.
119 self.write_pages()
121 def copy_datasets(self):
122 """Copy datasets, similar to butler retrieve-artifacts.
124 See Also
125 --------
126 self.fits_to_png
127 self.parq_to_html
128 self.py_to_html
129 """
130 # Based on daf_butler fileDatastore.py#L1978
131 for stage, dataset_types in self.dataset_map['stages'].items():
132 # Iterate over each stage, getting all of that stage's datasets.
133 # Convert dict dataset_types to a list of strings:
134 if not dataset_types:
135 continue
136 dataset_type_names = list(dataset_types.keys())
138 # Find all the references to these datasets in our collections
139 refs = self.butler.registry.queryDatasets(datasetType=dataset_type_names,
140 collections=self.collections,
141 where=None, findFirst=True)
143 for ref in refs:
144 # Iterate over each reference
145 locations = self.butler._datastore._get_dataset_locations_info(ref)
146 self.log.warn(f"Found {stage} {ref}")
148 # This is our dataset information:
149 dataset = {'stage': stage,
150 'ref': ref,
151 'type': ref.datasetType.name,
152 'storage_class': ref.datasetType.storageClass.name,
153 'dataId': ref.dataId,
154 'instrument': ref.dataId.get('instrument', None),
155 'exposure': ref.dataId.get('exposure', None),
156 'detector': ref.dataId.get('detector', None),
157 'physical_filter': ref.dataId.get('physical_filter', None),
158 'collection': ref.run,
159 }
161 # Do the actual copy. This may convert the butler
162 # product to something more web-accessible.
163 for location, _ in locations:
164 source_uri = location.uri
165 target_uri = determine_destination_for_retrieved_artifact(
166 ResourcePath(self.src_dir),
167 location.pathInStore,
168 False
169 )
170 dataset['uri'] = target_uri
171 willCopy = self.do_copy
173 if ((target_uri.getExtension().lower() == ".fits"
174 and dataset['storage_class'] in ('ImageF', 'ExposureF', 'ImageD', 'ExposureD'))):
175 # Convert this to a PNG image.
176 png_uri = target_uri.updatedExtension("png")
177 dataset['uri'] = png_uri
178 if os.path.exists(png_uri.path):
179 if willCopy and not self.do_overwrite:
180 willCopy = False
182 if willCopy:
183 data = self.butler.get(ref)
184 self.fits_to_png(data, png_uri, f"{ref.datasetType.name} {ref.dataId}")
185 elif (target_uri.getExtension().lower() == ".parq"
186 and dataset['storage_class'] in ('ArrowAstropy', )):
187 # Convert to html table.
188 html_uri = target_uri.updatedExtension("html")
189 dataset['uri'] = html_uri
190 if os.path.exists(html_uri.path):
191 if willCopy and not self.do_overwrite:
192 willCopy = False
194 if willCopy:
195 data = self.butler.get(ref)
196 self.parq_to_html(data, html_uri)
197 elif (target_uri.getExtension().lower() == ".py"
198 and dataset['storage_class'] in ('Config', )):
199 # Convert configs to html.
200 html_uri = target_uri.updatedExtension("html")
201 dataset['uri'] = html_uri
203 if willCopy:
204 data = self.butler.get(ref)
205 self.py_to_html(data, html_uri)
206 else:
207 if willCopy:
208 # Otherwise just copy directly.
209 target_uri.transfer_from(source_uri, transfer="copy",
210 overwrite=self.do_overwrite)
212 # Add this dataset to our list.
213 self.datasets.append(dataset)
215 def parse_datasets(self):
216 """Analyze datasets, and add information to report pages.
218 See Also
219 --------
220 self._init_page
221 self.include
222 self.navigation_init
223 self.navigation_append
224 self.title
225 self.link
226 """
227 populated_stages_list = []
229 # Make navigation page.
230 self.out_files["navigation.html"] = self._init_page("Navigation")
231 self.navigation_init()
232 self.include("navigation.html", "index.html")
234 # Make manifest page:
235 self.out_files["manifest.html"] = self._init_page("Manifest")
237 tag = self.link(self.out_files["index.html"], "Parent", "./..")
238 self.navigation_append("index.html", "Parent", tag, newColumn=True)
239 self.navigation_append("manifest.html", "Manifest", "", newColumn=False)
241 for stage in self.dataset_map['stages'].keys():
242 # Filter datasets to only those for this stage
243 dss = [x for x in self.datasets if x['stage'] == stage]
244 if len(dss) > 0:
245 populated_stages_list.append(stage)
246 else:
247 continue
248 self.log.warn(f"Parsing data for {stage}")
249 self.title(self.out_files["index.html"], stage)
251 self.out_files[f"{stage}_exp.html"] = self._init_page()
252 self.include("navigation.html", f"{stage}_exp.html")
253 self.out_files[f"{stage}_det.html"] = self._init_page()
254 self.include("navigation.html", f"{stage}_det.html")
256 # Get dimension information we'll want to use to put
257 # things on appropriate pages.
258 exps = [x for x in dss if x['exposure'] is not None]
259 dets = [x for x in dss if x['detector'] is not None]
261 for ds in sorted(exps, key=lambda x: x['exposure']):
262 self.block(ds, self.out_files[f"{stage}_exp.html"])
263 if ('.png' in ds['uri'].path):
264 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files[f"{stage}_exp.html"])
265 tag = self.link(self.out_files["index.html"], f"{stage} exposures", f"./{stage}_exp.html")
266 self.navigation_append("index.html", f"{stage} exposures", tag)
268 for ds in sorted(dets, key=lambda x: x['detector']):
269 self.block(ds, self.out_files[f"{stage}_det.html"])
270 if ('.png' in ds['uri'].path):
271 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files[f"{stage}_det.html"])
272 tag = self.link(self.out_files["index.html"], f"{stage} detectors", f"./{stage}_det.html")
273 self.navigation_append("index.html", f"{stage} detectors", tag)
275 for ds in dss:
276 if ds['storage_class'] == 'ArrowAstropy':
277 # Is a results catalog
278 self.block(ds, self.out_files["index.html"], ds['uri'])
279 self.include(ds['uri'].path, "index.html")
280 elif ds['exposure'] is None and ds['detector'] is None:
281 self.block(ds, self.out_files["index.html"])
282 if ".png" in ds['uri'].path:
283 self.image_handler(ds['uri'].path, ds['instrument'], self.out_files["index.html"])
284 else:
285 self.include(ds['uri'].path, "index.html")
287 # Record all datasets here, and link to them directly:
288 page = self.out_files["manifest.html"]
289 page.extend(["<table width='100%'>",
290 "<tr>",
291 "<th>stage</th>", "<th>ref</th>", "<th>type</th>", "<th>storage_class</th>",
292 "<th>dataId</th>", "<th>collection</th>",
293 "</tr>"])
294 for ds in self.datasets:
295 relative_file = self.relative_file(ds['uri'].path)
296 page.extend(["<tr>",
297 f"<td>{ds['stage']}</td>", f"<td>{ds['ref']}</td>",
298 "<td>", f'<a href="{relative_file}">', f"{ds['type']}", "</a>",
299 f"<td>{ds['storage_class']}</td>",
300 f"<td>{ds['dataId']}</td>", f"<td>{ds['collection']}</td>",
301 "</tr>"])
303 # File conversion utilities:
304 @staticmethod
305 def fits_to_png(data, target_uri, title):
306 """Convert FITS to PNG, using the same scaling as on RubinTV.
308 Parameters
309 ----------
310 data : `lsst.afw.Image`, `lsst.afw.MaskedImage` or `lsst.afw.Exposure`
311 The fits image data to convert.
312 target_uri : `str`
313 Path to the PNG file to write.
314 title : `str`
315 Title to add to the figure.
317 See Also
318 --------
319 rubintv_production / mosaicing.py
320 """
321 # Get array from either an Image or an Exposure:
322 try:
323 array = data.image.array
324 except AttributeError:
325 array = data.array
327 fig = make_figure()
328 ax = fig.gca()
329 ax.clear()
330 cmap = cm.gray
331 # This was using summit_utils.getQuantiles, but that
332 # was blowing out the scaling more than I wanted.
333 q25, q50, q75 = np.nanpercentile(array, [25, 50, 75])
334 scale = 3.0 * 0.74 * (q75 - q25)
335 quantiles = np.arange(q50 - scale, q50 + scale, 2.0 * scale / cmap.N)
336 norm = colors.BoundaryNorm(quantiles, cmap.N)
338 im = ax.imshow(array, norm=norm, interpolation='None', cmap=cmap, origin='lower')
340 divider = make_axes_locatable(ax)
341 cax = divider.append_axes("right", size="5%", pad=0.05)
342 fig.suptitle(title)
343 fig.colorbar(im, cax=cax)
344 fig.tight_layout()
345 fig.savefig(target_uri.path)
347 @staticmethod
348 def parq_to_html(data, target_uri):
349 """Convert catalogs to html tables.
351 Parameters
352 ----------
353 data : `astropy.Table`
354 The table to convert to html.
355 target_uri : `str`
356 Path to the HTML file to write.
357 """
358 format_dict = {}
359 # Remove vectors, as they will be too big.
360 # Build up format dictionary.
361 columns_to_remove = []
362 for index, name in enumerate(data.dtype.names):
363 if len(data.dtype[index].shape) != 0:
364 columns_to_remove.append(name)
365 continue
366 if data.dtype[index].kind in ('f', 'c'):
367 # is float
368 format_dict[name] = "%.4g"
369 if name == 'mjd':
370 # Let's let these be long
371 format_dict[name] = "12.10f"
372 elif data.dtype[index].kind in ('i', 'u'):
373 # is int
374 format_dict[name] = "%d"
375 else:
376 format_dict[name] = "%s"
378 # Actually remove the vectors:
379 data.remove_columns(columns_to_remove)
381 # Write full table:
382 data.write(target_uri.path,
383 format='ascii.html',
384 overwrite=True,
385 formats=format_dict)
387 # TODO: Write subset tables, filtered by exposure and by detector.
388 # files_created[f"{filename_base}_exp{exp}.html"] = {'exposure': exp}
389 # files_created[f"{filename_base}_det{det}.html"] = {'detector': det}
391 def py_to_html(self, data, target_uri):
392 """Convert python configs to html.
394 Parameters
395 ----------
396 data : `Config`
397 The python file contents to write.
398 target_uri : `str`
399 Path to the HTML file to write.
400 """
401 with open(target_uri.path, 'w') as ff:
402 for line in self._init_page():
403 print(line, file=ff)
404 for line in data.saveToString().split("\n"):
405 if len(line) > 0 and line[0] == '#':
406 print("<pre class='pre-comment'>", line, "</pre>", file=ff)
407 else:
408 print("<pre>", line, "</pre>", file=ff)
409 for line in self._close_page():
410 print(line, file=ff)
412 def write_pages(self):
413 """Write all page arrays to files."""
414 for ff, contents in self.out_files.items():
415 reportOut = os.path.join(self.output_path, ff)
416 contents.extend(self._close_page())
417 with open(reportOut, 'w') as ff:
418 for line in contents:
419 print(line, file=ff)
421 # Page construction methods. These all return arrays of strings
422 # that can be appended to until we write to disk.
423 @staticmethod
424 def _init_page(title=""):
425 """Write boilerplate html for the start of a page."""
426 return [
427 "<html>", "<head>", "<style> ",
428 " * { margin: 0; padding: 0;}",
429 " .imgbox { display: grid; height: 100%; }",
430 " .pre-comment { color: red; }",
431 " td { padding: 0 15px; }",
432 " .ctable { display: table; }",
433 " .ctable tr { display: table-cell; }",
434 " .ctable tr td { display: block; }",
435 "</style>",
436 "<title>", title, "</title>",
437 '<base target="_parent">',
438 "</head>",
439 "<body>",
440 ]
442 @staticmethod
443 def _close_page():
444 """Write boilerplate html for the end of a page."""
445 return ["</body>",
446 "</html>"]
448 @staticmethod
449 def relative_file(filename):
450 """Convert absolute paths to the relative format needed for the html.
451 """
452 return re.sub(r"^.*/src/", "./src/", filename)
454 def svg_atools_latiss(self, relative_file, page):
455 """This adds an image to the html, with an SVG overlay for LATISS.
457 Parameters
458 ----------
459 relative_file : `str`
460 Relative location of the image to be added.
461 page : `list` [`str`]
462 The page to append to.
464 Notes
465 -----
466 This method assumes the image is made by analysis_tools, and
467 displays a focal plane plot for LATISS. Under these
468 assumptions, the boxes defined in SVG map to the amplifier
469 segments of the single detector.
470 """
471 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
472 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
473 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
474 '<a xlink:href="./c10.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
475 'width="176" height="692"></rect> </a>',
476 '<a xlink:href="./c11.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
477 'width="176" height="692"></rect> </a>',
478 '<a xlink:href="./c12.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
479 'width="176" height="692"></rect> </a>',
480 '<a xlink:href="./c13.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
481 'width="176" height="692"></rect> </a>',
482 '<a xlink:href="./c14.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
483 'width="176" height="692"></rect> </a>',
484 '<a xlink:href="./c15.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
485 'width="176" height="692"></rect> </a>',
486 '<a xlink:href="./c16.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
487 'width="176" height="692"></rect> </a>',
488 '<a xlink:href="./c17.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
489 'width="176" height="692"></rect> </a>',
490 '<a xlink:href="./c07.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
491 'width="176" height="692"></rect> </a>',
492 '<a xlink:href="./c06.html"> <rect x="1375" y="911" fill="#fff" opacity="0" '
493 'width="176" height="692"></rect> </a>',
494 '<a xlink:href="./c05.html"> <rect x="1197" y="911" fill="#fff" opacity="0" '
495 'width="176" height="692"></rect> </a>',
496 '<a xlink:href="./c04.html"> <rect x="1020" y="911" fill="#fff" opacity="0" '
497 'width="176" height="692"></rect> </a>',
498 '<a xlink:href="./c03.html"> <rect x="845" y="911" fill="#fff" opacity="0" '
499 'width="176" height="692"></rect> </a>',
500 '<a xlink:href="./c02.html"> <rect x="670" y="911" fill="#fff" opacity="0" '
501 'width="176" height="692"></rect> </a>',
502 '<a xlink:href="./c01.html"> <rect x="490" y="911" fill="#fff" opacity="0" '
503 'width="176" height="692"></rect> </a>',
504 '<a xlink:href="./c00.html"> <rect x="317" y="911" fill="#fff" opacity="0" '
505 'width="176" height="692"></rect> </a>',
506 '</svg>'])
508 def svg_atools_comcam(self, relative_file, page):
509 """This adds an image to the html, with an SVG overlay for ComCam.
511 Parameters
512 ----------
513 relative_file : `str`
514 Relative location of the image to be added.
515 page : `list` [`str`]
516 The page to append to.
518 Notes
519 -----
520 This method assumes the image is made by analysis_tools, and
521 displays a focal plane plot for LSSTComCam. Under these
522 assumptions, the boxes defined in SVG map map to the detectors
523 in the single raft.
524 """
525 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
526 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
527 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
528 '<a xlink:href="./S00.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
529 'width="176" height="692"></rect> </a>',
530 '<a xlink:href="./S01.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
531 'width="176" height="692"></rect> </a>',
532 '<a xlink:href="./S02.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
533 'width="176" height="692"></rect> </a>',
534 '<a xlink:href="./S10.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
535 'width="176" height="692"></rect> </a>',
536 '<a xlink:href="./S11.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
537 'width="176" height="692"></rect> </a>',
538 '<a xlink:href="./S12.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
539 'width="176" height="692"></rect> </a>',
540 '<a xlink:href="./S20.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
541 'width="176" height="692"></rect> </a>',
542 '<a xlink:href="./S21.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
543 'width="176" height="692"></rect> </a>',
544 '<a xlink:href="./S22.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
545 'width="176" height="692"></rect> </a>',
546 '</svg>'])
548 def svg_mosaic_latiss(self, relative_file, page):
549 """This adds an image to the html, with an SVG overlay for LATISS.
551 Parameters
552 ----------
553 relative_file : `str`
554 Relative location of the image to be added.
555 page : `list` [`str`]
556 The page to append to.
558 Notes
559 -----
560 This method assumes the image is a 8x8 binned focal plane
561 mosaic of a LATISS exposure made by one of the VisualizeVisit
562 tasks. Under these assumptions, the boxes defined in SVG map
563 to the amplifiers of the single detector.
564 """
565 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
566 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 3600 1800">',
567 f'<image width="3600" height="1800" xlink:href="{relative_file}"></image>',
568 '<a xlink:href="./c10.html"> <rect x="317" y="216" fill="#fff" opacity="0" '
569 'width="176" height="692"></rect> </a>',
570 '<a xlink:href="./c11.html"> <rect x="490" y="216" fill="#fff" opacity="0" '
571 'width="176" height="692"></rect> </a>',
572 '<a xlink:href="./c12.html"> <rect x="670" y="216" fill="#fff" opacity="0" '
573 'width="176" height="692"></rect> </a>',
574 '<a xlink:href="./c13.html"> <rect x="845" y="216" fill="#fff" opacity="0" '
575 'width="176" height="692"></rect> </a>',
576 '<a xlink:href="./c14.html"> <rect x="1020" y="216" fill="#fff" opacity="0" '
577 'width="176" height="692"></rect> </a>',
578 '<a xlink:href="./c15.html"> <rect x="1197" y="216" fill="#fff" opacity="0" '
579 'width="176" height="692"></rect> </a>',
580 '<a xlink:href="./c16.html"> <rect x="1375" y="216" fill="#fff" opacity="0" '
581 'width="176" height="692"></rect> </a>',
582 '<a xlink:href="./c17.html"> <rect x="1550" y="216" fill="#fff" opacity="0" '
583 'width="176" height="692"></rect> </a>',
584 '<a xlink:href="./c07.html"> <rect x="1550" y="911" fill="#fff" opacity="0" '
585 'width="176" height="692"></rect> </a>',
586 '<a xlink:href="./c06.html"> <rect x="1375" y="911" fill="#fff" opacity="0" '
587 'width="176" height="692"></rect> </a>',
588 '<a xlink:href="./c05.html"> <rect x="1197" y="911" fill="#fff" opacity="0" '
589 'width="176" height="692"></rect> </a>',
590 '<a xlink:href="./c04.html"> <rect x="1020" y="911" fill="#fff" opacity="0" '
591 'width="176" height="692"></rect> </a>',
592 '<a xlink:href="./c03.html"> <rect x="845" y="911" fill="#fff" opacity="0" '
593 'width="176" height="692"></rect> </a>',
594 '<a xlink:href="./c02.html"> <rect x="670" y="911" fill="#fff" opacity="0" '
595 'width="176" height="692"></rect> </a>',
596 '<a xlink:href="./c01.html"> <rect x="490" y="911" fill="#fff" opacity="0" '
597 'width="176" height="692"></rect> </a>',
598 '<a xlink:href="./c00.html"> <rect x="317" y="911" fill="#fff" opacity="0" '
599 'width="176" height="692"></rect> </a>',
600 '</svg>'])
602 def svg_mosaic_comcam(self, relative_file, page):
603 """This adds an image to the html, with an SVG overlay for ComCam.
605 Parameters
606 ----------
607 relative_file : `str`
608 Relative location of the image to be added.
609 page : `list` [`str`]
610 The page to append to.
612 Notes
613 -----
614 This method assumes the image is a 8x8 binned focal plane
615 mosaic of a ComCam exposure made by one of the VisualizeVisit
616 tasks. Under these assumptions, the boxes defined in SVG map
617 to the detectors in the single raft.
618 """
619 page.extend(['<svg version="1.1" xmlns="http://www.w3.org/2000/svg" '
620 'xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="0 0 640 480">',
621 f'<image width="640" height="480" xlink:href="{relative_file}"></image>',
622 '<a xlink:href="./S00.html"> <rect x="111" y="306" fill="#fff" opacity="0"'
623 'width="128" height="128"></rect> </a>',
624 '<a xlink:href="./S01.html"> <rect x="111" y="175" fill="#fff" opacity="0"'
625 'width="128" height="128"></rect> </a>',
626 '<a xlink:href="./S02.html"> <rect x="111" y="50" fill="#fff" opacity="0" '
627 'width="128" height="128"></rect> </a>',
628 '<a xlink:href="./S10.html"> <rect x="244" y="306" fill="#fff" opacity="0" '
629 'width="128" height="128"></rect> </a>',
630 '<a xlink:href="./S11.html"> <rect x="244" y="175" fill="#fff" opacity="0" '
631 'width="128" height="128"></rect> </a>',
632 '<a xlink:href="./S12.html"> <rect x="244" y="50" fill="#fff" opacity="0" '
633 'width="128" height="128"></rect> </a>',
634 '<a xlink:href="./S20.html"> <rect x="376" y="306" fill="#fff" opacity="0" '
635 'width="128" height="128"></rect> </a>',
636 '<a xlink:href="./S21.html"> <rect x="376" y="175" fill="#fff" opacity="0" '
637 'width="128" height="128"></rect> </a>',
638 '<a xlink:href="./S22.html"> <rect x="376" y="50" fill="#fff" opacity="0" '
639 'width="128" height="128"></rect> </a>',
640 '</svg>'])
642 def image_handler(self, image_filename, instrument, page):
643 """Handle images.
645 This method attempts to find the correct way to add an image
646 to a page, with the appropriate SVG subregion information.
648 Parameters
649 ----------
650 image_filename : `str`
651 Full path to the image file to be added to a page.
652 instrument : `str`
653 Instrument name to use to identify special handling.
654 page : `list`
655 The page to append to.
657 """
658 relative_file = self.relative_file(image_filename)
659 if "Mosaic64" not in relative_file:
660 # Skip Mosaic64 products.
661 if instrument == 'LATISS':
662 if "Mosaic" in relative_file:
663 self.svg_mosaic_latiss(relative_file, page)
664 else:
665 self.svg_atools_latiss(relative_file, page)
666 elif instrument in ('LSSTComCam', 'LSSTComCamSim'):
667 if "Mosaic" in relative_file:
668 # This is a fits_to_png thing
669 self.svg_mosaic_comcam(relative_file, page)
670 else:
671 self.svg_atools_comcam(relative_file, page)
672 else:
673 # We couldn't find a magic handler, so add it
674 # manually.
675 page.append(f'<img class="center-fit" src="{relative_file}">')
677 def block(self, dataset, page, link=None, doc=None):
678 """Add a block of information, containing the dataId and collections
679 for a particular dataset.
681 Parameters
682 ----------
683 dataset : `dict`
684 The dataset containing dataId information.
685 page : `list`
686 The page to append to.
687 link : `str`, optional
688 A link to apply to the dataset type name.
689 doc : `str`, optional
690 A documentation string. If not supplied, the documentation
691 in the configuration is used.
692 """
693 if doc is None:
694 stage = dataset['stage']
695 dsType = dataset['type']
696 doc = self.dataset_map['stages'][stage][dsType].get("description", "Undocumented.")
697 page.extend(["<table width='100%'>",
698 "<tr>",
699 "<th colspan='2' align='center'>DataId:</th>",
700 "<th colspan='2' align='center'>Dataset:</th>", "</tr>"])
701 page.extend(["<tr>",
702 "<td align='right'>Instrument</td>", f"<td>{dataset['instrument']}</td>",
703 "<td align='right'>Collection</td>", f"<td>{dataset['collection']}</td>",
704 "</tr>"])
705 page.extend(["<tr>",
706 "<td align='right'>Exposure</td>", f"<td>{dataset['exposure']}</td>",
707 "<td align='right'>Stage</td>", f"<td>{dataset['stage']}</td>", "</tr>"])
708 page.extend(["<tr>",
709 "<td align='right'>Detector</td>", f"<td>{dataset['detector']}</td>"]),
711 page.extend(["<a href='" + f"{link}" + "'>" if link else ""])
712 page.extend(["<td align='right'>Dataset type</td>", f"<td>{dataset['type']} ", "</td>"])
713 page.extend(["</a>" if link else ""])
714 page.extend(["</tr>"])
715 page.extend(["<tr>",
716 "<td align='right'>Physical Filter</td>", f"<td>{dataset['physical_filter']}</td>",
717 "<td align='right'>Docstring</td>", f"<td>{doc}</td>",
718 "</tr>"])
719 page.extend(["</table>"])
721 @staticmethod
722 def link(page, text, link):
723 text_ref = text.replace(" ", "-")
724 page.extend([f"<h2 id='{text_ref}'>", f"<a href='{link}'>",
725 f"{text}" "</a>", "</h2>"])
726 return text_ref
728 def navigation_init(self):
729 page = self.out_files["navigation.html"]
730 page.extend(["<table class='ctable' width='100%'>",
731 "<tr>"])
733 def navigation_append(self, destination, text, tag, newColumn=False):
734 page = self.out_files["navigation.html"]
735 if newColumn:
736 page.extend(["</tr>", "</tr>"])
737 page.extend(["<td>",
738 f"<a href='./{destination}#{tag}'>",
739 f"{text}",
740 "</a>", "</td>"])
742 def title(self, page, text):
743 text_ref = text.replace(" ", "-")
744 page.extend(["<center>",
745 f"<h1 id={text_ref}>", text, "</h1>",
746 "</center>"])
747 self.navigation_append("index.html", text, text_ref, newColumn=True)
749 def include(self, source, target):
750 target_page = self.out_files[target]
751 relative_file = self.relative_file(source)
752 target_page.extend([f"<iframe width='100%' src='./{relative_file}'>",
753 "</iframe>"])
756def main():
757 # Could these use the click/daf_butler CLI tools?
758 parser = argparse.ArgumentParser(description="Construct a cp_verify metric report.")
759 parser.add_argument("-r", "--repository", dest="repository", default="",
760 help="Butler repository to pull results from.")
761 parser.add_argument("-O", "--output_path", dest="output_path", default="",
762 help="Output path to write report to.")
763 parser.add_argument("-c", "--collections", dest="collections", action="append", default=[],
764 help="Collections to search for results.")
765 parser.add_argument("--no_copy", dest="do_copy", action="store_false", default=True,
766 help="Skip copying files for debugging purposes.")
767 parser.add_argument("--do_overwrite", dest="do_overwrite", action="store_true", default=False,
768 help="Allow existing files to be overwritten?")
769 args = parser.parse_args()
771 reporter = CpvReporter(
772 repo=args.repository,
773 output_path=args.output_path,
774 collections=args.collections,
775 do_copy=args.do_copy,
776 do_overwrite=args.do_overwrite,
777 )
778 reporter.run()