Coverage for python/lsst/obs/lsst/_ingestPhotodiode.py: 17%
81 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-27 03:54 -0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-27 03:54 -0700
1# This file is part of obs_lsst.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21__all__ = ('PhotodiodeIngestConfig', 'PhotodiodeIngestTask')
23import warnings
25from lsst.daf.butler import (
26 CollectionType,
27 DataCoordinate,
28 DatasetIdGenEnum,
29 DatasetRef,
30 DatasetType,
31 FileDataset,
32 Progress,
33 UnresolvedRefWarning,
34)
35from lsst.ip.isr import PhotodiodeCalib
36from lsst.obs.base import makeTransferChoiceField
37from lsst.obs.base.formatters.fitsGeneric import FitsGenericFormatter
38from lsst.pex.config import Config
39from lsst.pipe.base import Task
40from lsst.resources import ResourcePath
43class PhotodiodeIngestConfig(Config):
44 """Configuration class for PhotodiodeIngestTask."""
46 transfer = makeTransferChoiceField(default="copy")
48 def validate(self):
49 super().validate()
50 if self.transfer != "copy":
51 raise ValueError(f"Transfer Must be 'copy' for photodiode data. {self.transfer}")
54class PhotodiodeIngestTask(Task):
55 """Task to ingest photodiode data into a butler repository.
57 Parameters
58 ----------
59 config : `PhotodiodeIngestConfig`
60 Configuration for the task.
61 instrument : `~lsst.obs.base.Instrument`
62 The instrument these photodiode datasets are from.
63 butler : `~lsst.daf.butler.Butler`
64 Writable butler instance, with ``butler.run`` set to the
65 appropriate `~lsst.daf.butler.CollectionType.RUN` collection
66 for these datasets.
67 **kwargs
68 Additional keyword arguments.
69 """
71 ConfigClass = PhotodiodeIngestConfig
72 _DefaultName = "photodiodeIngest"
74 def getDatasetType(self):
75 """Return the DatasetType of the photodiode datasets."""
76 return DatasetType(
77 "photodiode",
78 ("instrument", "exposure"),
79 "IsrCalib",
80 universe=self.butler.registry.dimensions,
81 )
83 def __init__(self, butler, instrument, config=None, **kwargs):
84 config.validate()
85 super().__init__(config, **kwargs)
86 self.butler = butler
87 self.universe = self.butler.registry.dimensions
88 self.datasetType = self.getDatasetType()
89 self.progress = Progress(self.log.name)
90 self.instrument = instrument
91 self.camera = self.instrument.getCamera()
93 def run(self, locations, run=None, file_filter=r".*Photodiode_Readings.*txt",
94 track_file_attrs=None):
95 """Ingest photodiode data into a Butler data repository.
97 Parameters
98 ----------
99 files : iterable over `lsst.resources.ResourcePath`
100 URIs to the files to be ingested.
101 run : `str`, optional
102 Name of the RUN-type collection to write to,
103 overriding the default derived from the instrument
104 name.
105 skip_existing_exposures : `bool`, optional
106 If `True`, skip photodiodes that have already been
107 ingested (i.e. raws for which we already have a
108 dataset with the same data ID in the target
109 collection).
110 track_file_attrs : `bool`, optional
111 Control whether file attributes such as the size or
112 checksum should be tracked by the datastore. Whether
113 this parameter is honored depends on the specific
114 datastore implementation.
116 Returns
117 -------
118 refs : `list` [`lsst.daf.butler.DatasetRef`]
119 Dataset references for ingested raws.
121 Raises
122 ------
123 RuntimeError
124 Raised if the number of exposures found for a photodiode
125 file is not one
126 """
127 files = ResourcePath.findFileResources(locations, file_filter)
129 registry = self.butler.registry
130 registry.registerDatasetType(self.datasetType)
132 # Find and register run that we will ingest to.
133 if run is None:
134 run = self.instrument.makeCollectionName("calib", "photodiode")
135 registry.registerCollection(run, type=CollectionType.RUN)
137 # Use datasetIds that match the raw exposure data.
138 if self.butler.registry.supportsIdGenerationMode(DatasetIdGenEnum.DATAID_TYPE_RUN):
139 mode = DatasetIdGenEnum.DATAID_TYPE_RUN
140 else:
141 mode = DatasetIdGenEnum.UNIQUE
143 refs = []
144 numExisting = 0
145 numFailed = 0
146 for inputFile in files:
147 # Convert the file into the right class.
148 with inputFile.as_local() as localFile:
149 calib = PhotodiodeCalib.readTwoColumnPhotodiodeData(localFile.ospath)
151 dayObs = calib.getMetadata()['day_obs']
152 seqNum = calib.getMetadata()['seq_num']
154 # Find the associated exposure information.
155 whereClause = "exposure.day_obs=dayObs and exposure.seq_num=seqNum"
156 instrumentName = self.instrument.getName()
157 exposureRecords = [rec for rec in registry.queryDimensionRecords("exposure",
158 instrument=instrumentName,
159 where=whereClause,
160 bind={"dayObs": dayObs,
161 "seqNum": seqNum})]
163 nRecords = len(exposureRecords)
164 if nRecords == 1:
165 exposureId = exposureRecords[0].id
166 calib.updateMetadata(camera=self.camera, exposure=exposureId)
167 elif nRecords == 0:
168 numFailed += 1
169 self.log.warning("Skipping instrument %s and dayObs/seqNum %d %d: no exposures found.",
170 instrumentName, dayObs, seqNum)
171 continue
172 else:
173 numFailed += 1
174 self.log.warning("Multiple exposure entries found for instrument %s and "
175 "dayObs/seqNum %d %d.", instrumentName, dayObs, seqNum)
176 continue
178 # Generate the dataId for this file.
179 dataId = DataCoordinate.standardize(
180 instrument=self.instrument.getName(),
181 exposure=exposureId,
182 universe=self.universe,
183 )
185 # If this already exists, we should skip it and continue.
186 existing = {
187 ref.dataId
188 for ref in self.butler.registry.queryDatasets(self.datasetType, collections=[run],
189 dataId=dataId)
190 }
191 if existing:
192 self.log.debug("Skipping instrument %s and dayObs/seqNum %d %d: already exists in run %s.",
193 instrumentName, dayObs, seqNum, run)
194 numExisting += 1
195 continue
197 # Ingest must work from a file, but we can't use the
198 # original, as we've added new metadata and reformatted
199 # it. Write it to a temp file that we can use to ingest.
200 # If we can have the files written appropriately, this
201 # will be a direct ingest of those files.
202 with ResourcePath.temporary_uri(suffix=".fits") as tempFile:
203 calib.writeFits(tempFile.ospath)
205 with warnings.catch_warnings():
206 warnings.simplefilter("ignore", category=UnresolvedRefWarning)
207 ref = DatasetRef(self.datasetType, dataId)
208 dataset = FileDataset(path=tempFile, refs=ref, formatter=FitsGenericFormatter)
210 # No try, as if this fails, we should stop.
211 self.butler.ingest(dataset, transfer=self.config.transfer, run=run,
212 idGenerationMode=mode,
213 record_validation_info=track_file_attrs)
214 self.log.info("Photodiode %s:%d (%d/%d) ingested successfully", instrumentName, exposureId,
215 dayObs, seqNum)
216 refs.append(dataset)
218 if numExisting != 0:
219 self.log.warning("Skipped %d entries that already existed in run %s", numExisting, run)
220 if numFailed != 0:
221 raise RuntimeError(f"Failed to ingest {numFailed} entries due to missing exposure information.")
222 return refs