Coverage for python/lsst/daf/butler/script/exportCalibs.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from astropy.table import Table
23import logging
24import os
25from .. import Butler, CollectionType
28log = logging.getLogger(__name__)
31def parseCalibrationCollection(registry, collection, datasetTypes):
32 """Search a calibration collection for calibration datasets.
34 Parameters
35 ----------
36 registry : `lsst.daf.butler.Registry`
37 Butler registry to use.
38 collection : `str`
39 Collection to search. This should be a CALIBRATION
40 collection.
41 datasetTypes : `list` [`lsst.daf.Butler.DatasetType`]
42 List of calibration dataset types.
44 Returns
45 -------
46 exportCollections : `list` [`str`]
47 List of collections to save on export.
48 exportDatasets : `list` [`lsst.daf.butler.queries.DatasetQueryResults`]
49 Datasets to save on export.
51 Raises
52 ------
53 RuntimeError
54 Raised if the collection to search is not a CALIBRATION collection.
55 """
56 if registry.getCollectionType(collection) != CollectionType.CALIBRATION:
57 raise RuntimeError(f"Collection {collection} is not a CALIBRATION collection.")
59 exportCollections = []
60 exportDatasets = []
61 for calibType in datasetTypes:
62 associations = registry.queryDatasetAssociations(calibType, collections=collection,
63 collectionTypes=[CollectionType.CALIBRATION])
64 for result in associations:
65 exportDatasets.append(result.ref)
66 exportCollections.append(result.ref.run)
67 return exportCollections, exportDatasets
70def exportCalibs(repo, directory, collections):
71 """Certify a set of calibrations with a validity range.
73 Parameters
74 ----------
75 repo : `str`
76 URI to the location of the repo or URI to a config file
77 describing the repo and its location.
78 directory : `str`
79 URI string of the directory to write the exported
80 calibrations.
81 collections : `list` [`str`]
82 Data collections to pull calibrations from. Must be an
83 existing `~CollectionType.CHAINED` or
84 `~CollectionType.CALIBRATION` collection.
86 Returns
87 -------
88 datasetTable : `astropy.table.Table`
89 A table containing relevant information about the calibrations
90 exported.
92 Raises
93 ------
94 RuntimeError :
95 Raised if the output directory already exists.
96 """
97 butler = Butler(repo, writeable=False)
99 calibTypes = [datasetType for datasetType in butler.registry.queryDatasetTypes(...)
100 if datasetType.isCalibration()]
102 collectionsToExport = []
103 datasetsToExport = []
105 for collection in butler.registry.queryCollections(collections, flattenChains=True, includeChains=True,
106 collectionTypes={CollectionType.CALIBRATION,
107 CollectionType.CHAINED}):
108 log.info("Checking collection: %s", collection)
110 # Get collection information.
111 collectionsToExport.append(collection)
112 collectionType = butler.registry.getCollectionType(collection)
113 if collectionType == CollectionType.CALIBRATION:
114 exportCollections, exportDatasets = parseCalibrationCollection(butler.registry,
115 collection,
116 calibTypes)
117 collectionsToExport.extend(exportCollections)
118 datasetsToExport.extend(exportDatasets)
120 if os.path.exists(directory):
121 raise RuntimeError(f"Export directory exists: {directory}")
122 os.makedirs(directory)
123 with butler.export(directory=directory, format="yaml", transfer="auto") as export:
124 collectionsToExport = list(set(collectionsToExport))
125 datasetsToExport = list(set(datasetsToExport))
127 for exportable in collectionsToExport:
128 try:
129 export.saveCollection(exportable)
130 except Exception as e:
131 log.warning("Did not save collection %s due to %s.", exportable, e)
133 log.info("Saving %d dataset(s)", len(datasetsToExport))
134 export.saveDatasets(datasetsToExport)
136 sortedDatasets = sorted(datasetsToExport, key=lambda x: x.datasetType.name)
138 requiredDimensions = set()
139 for ref in sortedDatasets:
140 requiredDimensions.update(ref.dimensions.names)
141 dimensionColumns = {dimensionName: [ref.dataId.get(dimensionName, "") for ref in sortedDatasets]
142 for dimensionName in requiredDimensions}
144 return Table({"calibrationType": [ref.datasetType.name for ref in sortedDatasets],
145 "run": [ref.run for ref in sortedDatasets],
146 **dimensionColumns
147 })